AMBARI-8317 AMBARI-8317 Refactor the OS-dependent Ambari Server Windows components - Part 2

+Moved LDAP and HTTPS setup routines away from ambari-server.py
+Merged ambari-server-windows.py into ambari-server.py, leaving out only the Windows service wrapper and the server start routine

ambari-server/pom.xml

@@ -349,6 +349,9 @@
                 <source>
                   <location>src/main/python/ambari-server.py</location>
                 </source>
+                <source>
+                  <location>src/main/python/ambari_server_main.py</location>
+                </source>
                 <source>
                   <location>sbin/ambari-server</location>
                   <filter>true</filter>

ambari-server/src/main/assemblies/server-windows.xml

@@ -44,7 +44,15 @@
       <outputDirectory>/ambari-server-${project.version}/keystore</outputDirectory>
     </file>
     <file>
-      <source>${basedir}/src/main/python/ambari-server-windows.py</source>
+      <source>${basedir}/src/main/python/ambari-server.py</source>
+      <outputDirectory>/ambari-server-${project.version}/sbin</outputDirectory>
+    </file>
+    <file>
+      <source>${basedir}/src/main/python/ambari_server_main.py</source>
+      <outputDirectory>/ambari-server-${project.version}/sbin</outputDirectory>
+    </file>
+    <file>
+      <source>${basedir}/src/main/python/ambari_windows_service.py</source>
       <outputDirectory>/ambari-server-${project.version}/sbin</outputDirectory>
     </file>
     <file>

ambari-server/src/main/python/ambari-server-windows.py

@@ -1,559 +0,0 @@
-#!/usr/bin/env python
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import optparse
-import os
-import sys
-import subprocess
-
-from ambari_commons.ambari_service import AmbariService
-from ambari_commons.exceptions import FatalException, NonFatalException
-from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, \
-  get_verbose, set_verbose, get_silent, set_silent, get_debug_mode, set_debug_mode_from_options
-from ambari_commons.os_utils import remove_file, set_open_files_limit
-from ambari_commons.os_windows import SvcStatusCallback
-
-from ambari_server import utils
-from ambari_server.dbConfiguration import DBMSConfigFactory
-from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
-from ambari_server.serverConfiguration import find_jdk, get_ambari_classpath, get_ambari_properties, get_conf_dir, \
-  get_value_from_properties, configDefaults, DEBUG_MODE_KEY, RESOURCES_DIR_DEFAULT, RESOURCES_DIR_PROPERTY, \
-  SERVER_OUT_FILE_KEY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, \
-  SUSPEND_START_MODE_KEY, VERBOSE_OUTPUT_KEY, PID_NAME, get_java_exe_path
-from ambari_server.serverSetup import setup, reset, is_server_running
-from ambari_server.serverUpgrade import upgrade
-from ambari_server.setupActions import SETUP_ACTION, START_ACTION, PSTART_ACTION, STOP_ACTION, RESET_ACTION, \
-  STATUS_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION, LDAP_SETUP_ACTION, SETUP_SECURITY_ACTION, ACTION_REQUIRE_RESTART
-from ambari_server.setupSecurity import setup_ambari_krb5_jaas, setup_https, setup_ldap, setup_master_key
-from ambari_server.userInput import get_validated_string_input
-
-# debug settings
-
-SERVER_START_DEBUG = False
-SUSPEND_START_MODE = False
-
-# server commands
-ambari_provider_module_option = ""
-ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE')
-
-AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information."
-
-if ambari_provider_module is not None:
-  ambari_provider_module_option = "-Dprovider.module.class=" +\
-                                  ambari_provider_module + " "
-
-SERVER_START_CMD = \
-                 "-server -XX:NewRatio=3 "\
-                 "-XX:+UseConcMarkSweepGC " +\
-                 "-XX:-UseGCOverheadLimit -XX:CMSInitiatingOccupancyFraction=60 " +\
-                 ambari_provider_module_option +\
-                 os.getenv('AMBARI_JVM_ARGS', '-Xms512m -Xmx2048m') +\
-                 " -cp {0}" +\
-                 " org.apache.ambari.server.controller.AmbariServer"
-SERVER_START_CMD_DEBUG = \
-                       "-server -XX:NewRatio=2 -XX:+UseConcMarkSweepGC " +\
-                       ambari_provider_module_option +\
-                       os.getenv('AMBARI_JVM_ARGS', '-Xms512m -Xmx2048m') +\
-                       " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,"\
-                       "server=y,suspend={1} -cp {0}" +\
-                       " org.apache.ambari.server.controller.AmbariServer"
-
-SERVER_INIT_TIMEOUT = 5
-SERVER_START_TIMEOUT = 10
-
-EXITCODE_NAME = "ambari-server.exitcode"
-
-# linux open-file limit
-ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
-ULIMIT_OPEN_FILES_DEFAULT = 10000
-
-
-class AmbariServerService(AmbariService):
-  AmbariService._svc_name_ = "Ambari Server"
-  AmbariService._svc_display_name_ = "Ambari Server"
-  AmbariService._svc_description_ = "Ambari Server"
-
-  AmbariService._AdjustServiceVersion()
-
-  # Adds the necessary script dir to the Python's modules path
-  def _adjustPythonPath(self, current_dir):
-    python_path = os.path.join(current_dir, "sbin")
-    sys.path.insert(0, python_path)
-
-  def SvcDoRun(self):
-    scmStatus = SvcStatusCallback(self)
-
-    properties = get_ambari_properties()
-    self.options.verbose = get_value_from_properties(properties, VERBOSE_OUTPUT_KEY, self.options.verbose)
-    self.options.debug = get_value_from_properties(properties, DEBUG_MODE_KEY, self.options.debug)
-    self.options.suspend_start = get_value_from_properties(properties, SUSPEND_START_MODE_KEY, self.options.suspend_start)
-
-    # set verbose
-    set_verbose(self.options.verbose)
-
-    # set silent
-    set_silent(self.options.silent)
-
-    self.redirect_output_streams()
-
-    childProc = server_process_main(self.options, scmStatus)
-
-    if not self._StopOrWaitForChildProcessToFinish(childProc):
-      return
-
-    pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
-    remove_file(pid_file_path)
-    pass
-
-  def _InitOptionsParser(self):
-    return init_options_parser()
-
-  def redirect_output_streams(self):
-    properties = get_ambari_properties()
-
-    outFilePath = properties[SERVER_OUT_FILE_KEY]
-    if (outFilePath is None or outFilePath == ""):
-      outFilePath = configDefaults.SERVER_OUT_FILE
-
-    self._RedirectOutputStreamsToFile(outFilePath)
-    pass
-
-def ctrlHandler(ctrlType):
-  AmbariServerService.DefCtrlCHandler()
-  return True
-
-def svcsetup():
-  AmbariServerService.set_ctrl_c_handler(ctrlHandler)
-  # we don't save password between 'setup' runs, so we can't run Install every time. We run 'setup' only if user and
-  # password provided or if service not installed
-  if (SERVICE_USERNAME_KEY in os.environ and SERVICE_PASSWORD_KEY in os.environ):
-    AmbariServerService.Install(username=os.environ[SERVICE_USERNAME_KEY], password=os.environ[SERVICE_PASSWORD_KEY])
-  elif AmbariServerService.QueryStatus() == "not installed":
-    AmbariServerService.Install()
-  pass
-
-#
-# Starts the Ambari Server as a standalone process.
-# args:
-#  <no arguments> = start the server as a process. For now, there is no restrictions for the number of server instances
-#     that can run like this.
-#  -s, --single-instance = Reserved for future use. When starting the server as a process, ensure only one instance of the process is running.
-#     If this is the second instance of the process, the function fails.
-#
-def start(options):
-  AmbariServerService.set_ctrl_c_handler(ctrlHandler)
-
-  #Run as a normal process. Invoke the ServiceMain directly.
-  childProc = server_process_main(options)
-
-  childProc.wait()
-
-  pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
-  remove_file(pid_file_path)
-
-#
-# Starts the Ambari Server as a service.
-# Start the server in normal mode, as a Windows service. If the Ambari server is
-#     not registered as a service, the function fails. By default, only one instance of the service can
-#     possibly run.
-#
-def svcstart():
-  AmbariServerService.Start()
-  pass
-
-def server_process_main(options, scmStatus=None):
-  # debug mode, including stop Java process at startup
-  try:
-    set_debug_mode_from_options(options)
-  except AttributeError:
-    pass
-
-  if not utils.check_reverse_lookup():
-    print_warning_msg("The hostname was not found in the reverse DNS lookup. "
-                      "This may result in incorrect behavior. "
-                      "Please check the DNS setup and fix the issue.")
-
-  properties = get_ambari_properties()
-
-  print_info_msg("Ambari Server is not running...")
-
-  conf_dir = get_conf_dir()
-  jdk_path = find_jdk()
-  if jdk_path is None:
-    err = "No JDK found, please run the \"ambari-server setup\" " \
-                    "command to install a JDK automatically or install any " \
-                    "JDK manually to " + configDefaults.JDK_INSTALL_DIR
-    raise FatalException(1, err)
-
-  # Preparations
-
-  result = ensure_dbms_is_running(options, properties, scmStatus)
-  if result == -1:
-    raise FatalException(-1, "Unable to connect to the database")
-
-  if scmStatus is not None:
-    scmStatus.reportStartPending()
-
-  ensure_resources_are_organized(properties)
-
-  if scmStatus is not None:
-    scmStatus.reportStartPending()
-
-  environ = os.environ.copy()
-  ensure_server_security_is_configured(properties, environ)
-
-  if scmStatus is not None:
-    scmStatus.reportStartPending()
-
-  conf_dir = os.path.abspath(conf_dir) + os.pathsep + get_ambari_classpath()
-  if conf_dir.find(' ') != -1:
-    conf_dir = '"' + conf_dir + '"'
-
-  java_exe = get_java_exe_path()
-  pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
-
-  debug_mode = get_debug_mode()
-  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
-  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
-
-  command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
-  suspend_mode = 'y' if suspend_start else 'n'
-  command = command_base.format(conf_dir, suspend_mode)
-  if not os.path.exists(configDefaults.PID_DIR):
-    os.makedirs(configDefaults.PID_DIR, 0755)
-
-  set_open_files_limit(get_ulimit_open_files());
-
-  #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context
-  # and the environment from the parent process.
-  param_list = java_exe + " " + command
-
-  print_info_msg("Running server: " + str(param_list))
-  procJava = subprocess.Popen(param_list, env=environ)
-
-  #wait for server process for SERVER_START_TIMEOUT seconds
-  print "Waiting for server start..."
-
-  pidJava = procJava.pid
-  if pidJava <= 0:
-    procJava.terminate()
-    exitcode = procJava.returncode
-    exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
-    utils.save_pid(exitcode, exitfile)
-
-    if scmStatus is not None:
-      scmStatus.reportStopPending()
-
-    raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
-  else:
-    utils.save_pid(pidJava, pidfile)
-    print "Server PID at: "+pidfile
-    print "Server out at: "+configDefaults.SERVER_OUT_FILE
-    print "Server log at: "+configDefaults.SERVER_LOG_FILE
-
-  if scmStatus is not None:
-    scmStatus.reportStarted()
-
-  return procJava
-
-#Check the JDBC driver status
-#If not found abort
-#Get SQL Server service status from SCM
-#If 'stopped' then start it
-#Wait until the status is 'started' or a configured timeout elapses
-#If the timeout has been reached, bail out with exception
-def ensure_dbms_is_running(options, properties, scmStatus):
-  factory = DBMSConfigFactory()
-  dbms = factory.create(options, properties, "Ambari")
-  if not dbms._is_jdbc_driver_installed(properties):
-    raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.")
-
-  dbms.ensure_dbms_is_running(options, properties, scmStatus)
-
-def ensure_resources_are_organized(properties):
-  resources_location = get_resources_location(properties)
-  resource_files_keeper = ResourceFilesKeeper(resources_location)
-  try:
-    print "Organizing resource files at {0}...".format(resources_location,
-                                                       verbose=get_verbose())
-    resource_files_keeper.perform_housekeeping()
-  except KeeperException, ex:
-    msg = "Can not organize resource files at {0}: {1}".format(
-      resources_location, str(ex))
-    raise FatalException(-1, msg)
-
-
-def ensure_server_security_is_configured(properties, environ):
-  pass
-
-
-#
-# Stops the Ambari Server.
-#
-def svcstop():
-  AmbariServerService.Stop()
-
-
-### Stack upgrade ###
-
-def get_resources_location(properties):
-  res_location = properties[RESOURCES_DIR_PROPERTY]
-  if res_location is None:
-    res_location = RESOURCES_DIR_DEFAULT
-  return res_location
-#  pass
-
-
-#
-# The Ambari Server status.
-#
-def svcstatus(options):
-  options.exit_message = None
-
-  statusStr = AmbariServerService.QueryStatus()
-  print "Ambari Server is " + statusStr
-
-
-def get_ulimit_open_files():
-  properties = get_ambari_properties()
-  if properties == -1:
-    print "Error reading ambari properties"
-    return None
-
-  open_files = int(properties[ULIMIT_OPEN_FILES_KEY])
-  if open_files > 0:
-    return open_files
-  else:
-    return ULIMIT_OPEN_FILES_DEFAULT
-
-
-def init_options_parser():
-  parser = optparse.OptionParser(usage="usage: %prog action [options] [stack_id os]", )
-  #parser.add_option('-i', '--create-db-script-file', dest="create_db_script_file",
-  #                  default="resources" + os.sep + "Ambari-DDL-SQLServer-CREATELOCAL.sql",
-  #                  help="File with database creation script")
-  parser.add_option('-f', '--init-script-file', dest="init_db_script_file",
-                    default="resources" + os.sep + "Ambari-DDL-SQLServer-CREATE.sql",
-                    help="File with database setup script")
-  parser.add_option('-r', '--drop-script-file', dest="cleanup_db_script_file",
-                    default="resources" + os.sep + "Ambari-DDL-SQLServer-DROP.sql",
-                    help="File with database cleanup script")
-  parser.add_option('-j', '--java-home', dest="java_home", default=None,
-                    help="Use specified java_home.  Must be valid on all hosts")
-  parser.add_option("-v", "--verbose",
-                    action="store_true", dest="verbose", default=False,
-                    help="Print verbose status messages")
-  parser.add_option("-s", "--silent",
-                    action="store_true", dest="silent", default=False,
-                    help="Silently accepts default prompt values")
-  parser.add_option('-g', '--debug', action="store_true", dest='debug', default=False,
-                    help="Start ambari-server in debug mode")
-  parser.add_option('-y', '--suspend-start', action="store_true", dest='suspend_start', default=False,
-                    help="Freeze ambari-server Java process at startup in debug mode")
-
-  parser.add_option('-a', '--databasehost', dest="database_host", default=None,
-                    help="Hostname of database server")
-  parser.add_option('-n', '--databaseport', dest="database_port", default=None,
-                    help="Database server listening port")
-  parser.add_option('-d', '--databasename', dest="database_name", default=None,
-                    help="Database/Schema/Service name or ServiceID")
-  parser.add_option('-w', '--windowsauth', action="store_true", dest="database_windows_auth", default=None,
-                    help="Integrated Windows authentication")
-  parser.add_option('-u', '--databaseusername', dest="database_username", default=None,
-                    help="Database user login")
-  parser.add_option('-p', '--databasepassword', dest="database_password", default=None,
-                    help="Database user password")
-  parser.add_option('--jdbc-driver', default=None, dest="jdbc_driver",
-                    help="Specifies the path to the JDBC driver JAR file for the " \
-                         "database type specified with the --jdbc-db option. Used only with --jdbc-db option.")
-  # -b, -i, -k and -x the remaining available short options
-  # -h reserved for help
-  return parser
-
-def are_cmd_line_db_args_blank(options):
-  if (options.database_host is None \
-    and options.database_name is None \
-    and options.database_windows_auth is None \
-    and options.database_username is None \
-    and options.database_password is None):
-    return True
-  return False
-
-
-def are_db_auth_options_ok(db_windows_auth, db_username, db_password):
-  if db_windows_auth is True:
-    return True
-  else:
-    if db_username is not None and db_username is not "" and db_password is not None and db_password is not "":
-      return True
-  return False
-
-def are_cmd_line_db_args_valid(options):
-  if (options.database_host is not None and options.database_host is not "" \
-      #and options.database_name is not None \         # ambari by default is ok
-      and are_db_auth_options_ok(options.database_windows_auth,
-                               options.database_username,
-                               options.database_password)):
-    return True
-  return False
-
-
-def setup_security(args):
-  need_restart = True
-  #Print menu options
-  print '=' * 75
-  print 'Choose one of the following options: '
-  print '  [1] Enable HTTPS for Ambari server.'
-  print '  [2] Encrypt passwords stored in ambari.properties file.'
-  print '  [3] Setup Ambari kerberos JAAS configuration.'
-  print '=' * 75
-  choice = get_validated_string_input('Enter choice, (1-3): ', '0', '[1-3]',
-                                      'Invalid choice', False, False)
-
-  if choice == '1':
-    need_restart = setup_https(args)
-  elif choice == '2':
-    setup_master_key()
-  elif choice == '3':
-    setup_ambari_krb5_jaas()
-  else:
-    raise FatalException('Unknown option for setup-security command.')
-
-  return need_restart
-
-#
-# Main.
-#
-def main():
-  parser = init_options_parser()
-  (options, args) = parser.parse_args()
-
-  #perform checks
-  options.warnings = []
-  options.must_set_database_options = False
-
-  if are_cmd_line_db_args_blank(options):
-    options.must_set_database_options = True
-
-  elif not are_cmd_line_db_args_valid(options):
-    parser.error('All database options should be set. Please see help for the options.')
-
-  # set verbose
-  set_verbose(options.verbose)
-
-  # set silent
-  set_silent(options.silent)
-
-  ## jdbc driver and db options validation
-  #if options.jdbc_driver is None and options.jdbc_db is not None:
-  #  parser.error("Option --jdbc-db is used only in pair with --jdbc-driver")
-  #elif options.jdbc_driver is not None and options.jdbc_db is None:
-  #  parser.error("Option --jdbc-driver is used only in pair with --jdbc-db")
-
-  if options.debug:
-    sys.frozen = 'windows_exe' # Fake py2exe so we can debug
-
-  if len(args) == 0:
-    print parser.print_help()
-    parser.error("No action entered")
-
-  action = args[0]
-
-  if action == UPGRADE_STACK_ACTION:
-    possible_args_numbers = [2,4] # OR
-  else:
-    possible_args_numbers = [1]
-
-  matches = 0
-  for args_number_required in possible_args_numbers:
-    matches += int(len(args) == args_number_required)
-
-  if matches == 0:
-    print parser.print_help()
-    possible_args = ' or '.join(str(x) for x in possible_args_numbers)
-    parser.error("Invalid number of arguments. Entered: " + str(len(args)) + ", required: " + possible_args)
-
-  options.exit_message = "Ambari Server '%s' completed successfully." % action
-  need_restart = True
-  try:
-    if action == SETUP_ACTION:
-      setup(options)
-      svcsetup()
-    elif action == START_ACTION:
-      svcstart()
-    elif action == PSTART_ACTION:
-      start(options)
-    elif action == STOP_ACTION:
-      svcstop()
-    elif action == RESET_ACTION:
-      reset(options, AmbariServerService)
-    elif action == STATUS_ACTION:
-      svcstatus(options)
-    elif action == UPGRADE_ACTION:
-      upgrade(options)
-#    elif action == UPGRADE_STACK_ACTION:
-#      stack_id = args[1]
-#      repo_url = None
-#      repo_url_os = None
-#
-#      if len(args) > 2:
-#        repo_url = args[2]
-#      if len(args) > 3:
-#        repo_url_os = args[3]
-#
-#      upgrade_stack(options, stack_id, repo_url, repo_url_os)
-    elif action == LDAP_SETUP_ACTION:
-      setup_ldap()
-    elif action == SETUP_SECURITY_ACTION:
-      need_restart = setup_security(options)
-    else:
-      parser.error("Invalid action")
-
-    if action in ACTION_REQUIRE_RESTART and need_restart:
-      status, stateDesc = is_server_running(AmbariServerService)
-      if status:
-        print 'NOTE: Restart Ambari Server to apply changes' + \
-              ' ("ambari-server restart|stop|start")'
-
-    if options.warnings:
-      for warning in options.warnings:
-        print_warning_msg(warning)
-        pass
-      options.exit_message = "Ambari Server '%s' completed with warnings." % action
-      pass
-  except FatalException as e:
-    if e.reason is not None:
-      print_error_msg("Exiting with exit code {0}. \nREASON: {1}".format(e.code, e.reason))
-    sys.exit(e.code)
-  except NonFatalException as e:
-    options.exit_message = "Ambari Server '%s' completed with warnings." % action
-    if e.reason is not None:
-      print_warning_msg(e.reason)
-
-  if options.exit_message is not None:
-    print options.exit_message
-
-
-if __name__ == "__main__":
-  try:
-    main()
-  except (KeyboardInterrupt, EOFError):
-    print("\nAborting ... Keyboard Interrupt.")
-    sys.exit(1)

ambari-server/src/main/python/ambari_server/dbConfiguration.py

@@ -17,6 +17,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import glob
 import os
 
 from ambari_commons import OSConst
@@ -26,7 +27,7 @@ from ambari_commons.os_family_impl import OsFamilyImpl
 from ambari_server.serverConfiguration import decrypt_password_for_alias, get_value_from_properties, get_is_secure, \
   is_alias_string, \
   JDBC_PASSWORD_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, PRESS_ENTER_MSG, get_ambari_properties, update_properties, \
-  RESOURCES_DIR_PROPERTY
+  RESOURCES_DIR_PROPERTY, JDBC_PATTERNS, configDefaults
 from ambari_server.userInput import get_validated_string_input
 
 
@@ -273,6 +274,9 @@ class DBMSConfigFactory(object):
     """
     pass
 
+  def get_supported_dbms(self):
+    return []
+
 #
 # Database configuration factory for Windows
 #
@@ -300,6 +304,9 @@ class DBMSConfigFactoryWindows(DBMSConfigFactory):
     from ambari_server.dbConfiguration_windows import createSQLServerConfig
     return createSQLServerConfig(options, properties, STORAGE_TYPE_REMOTE, dbId)
 
+  def get_supported_dbms(self):
+    return self.DBMS_KEYS_LIST
+
   def get_supported_jdbc_drivers(self):
     return self.DBMS_KEYS_LIST
 
@@ -481,3 +488,26 @@ def check_jdbc_drivers(args):
       db_idx += 1
   finally:
     args.database_index = db_idx_orig
+
+
+#Check the JDBC driver status
+#If not found abort
+#Get SQL Server service status from SCM
+#If 'stopped' then start it
+#Wait until the status is 'started' or a configured timeout elapses
+#If the timeout has been reached, bail out with exception
+def ensure_dbms_is_running(options, properties, scmStatus=None):
+  factory = DBMSConfigFactory()
+  dbms = factory.create(options, properties)
+  result = dbms._is_jdbc_driver_installed(properties)
+  if result == -1:
+    raise FatalException(-1, "JDBC driver is not installed. Run ambari-server setup and try again.")
+  dbms.ensure_dbms_is_running(options, properties, scmStatus)
+
+
+def ensure_jdbc_driver_is_installed(options, properties):
+  factory = DBMSConfigFactory()
+  dbms = factory.create(options, properties)
+  result = dbms._is_jdbc_driver_installed(properties)
+  if result == -1:
+    raise FatalException(-1, dbms.JDBC_DRIVER_INSTALL_MSG)

ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py

@@ -30,7 +30,7 @@ import time
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.logging_utils import get_silent, get_verbose, print_error_msg, print_info_msg, print_warning_msg
 from ambari_commons.exceptions import NonFatalException, FatalException
-from ambari_commons.os_utils import copy_files, remove_file, run_os_command, find_in_path
+from ambari_commons.os_utils import copy_files, find_in_path, is_root, remove_file, run_os_command
 from ambari_server.dbConfiguration import DBMSConfig, USERNAME_PATTERN, SETUP_DB_CONNECT_ATTEMPTS, \
     SETUP_DB_CONNECT_TIMEOUT, STORAGE_TYPE_LOCAL, DEFAULT_USERNAME, DEFAULT_PASSWORD
 from ambari_server.serverConfiguration import get_ambari_properties, get_value_from_properties, configDefaults, \
@@ -41,9 +41,8 @@ from ambari_server.serverConfiguration import get_ambari_properties, get_value_f
     JDBC_DRIVER_PROPERTY, JDBC_URL_PROPERTY, \
     JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_PASSWORD_FILE_PROPERTY, \
     JDBC_RCA_DRIVER_PROPERTY, JDBC_RCA_URL_PROPERTY, \
-    PERSISTENCE_TYPE_PROPERTY
-from ambari_server.setupSecurity import read_password, store_password_file, encrypt_password
-from ambari_server.userInput import get_YN_input, get_validated_string_input
+    PERSISTENCE_TYPE_PROPERTY, encrypt_password, store_password_file
+from ambari_server.userInput import get_YN_input, get_validated_string_input, read_password
 from ambari_server.utils import get_postgre_hba_dir, get_postgre_running_status
 
 ORACLE_DB_ID_TYPES = ["Service Name", "SID"]
@@ -365,6 +364,17 @@ class PGConfig(LinuxDBMSConfig):
   #
   # Public methods
   #
+  def ensure_dbms_is_running(self, options, properties, scmStatus=None):
+    if self._is_local_database():
+      if is_root():
+        (pg_status, retcode, out, err) = PGConfig._check_postgre_up()
+        if not retcode == 0:
+          err = 'Unable to start PostgreSQL server. Status {0}. {1}. Exiting'.format(pg_status, err)
+          raise FatalException(retcode, err)
+      else:
+        print "Unable to check PostgreSQL server status when starting " \
+              "without root privileges."
+        print "Please do not forget to start PostgreSQL server."
 
   #
   # Private implementation

ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py

@@ -26,17 +26,16 @@ from ambari_commons.logging_utils import print_info_msg, print_warning_msg
 from ambari_commons.os_utils import search_file, run_os_command
 from ambari_commons.os_windows import WinServiceController
 from ambari_commons.str_utils import compress_backslashes, ensure_double_backslashes
-from ambari_server.dbConfiguration import AMBARI_DATABASE_NAME, DEFAULT_USERNAME, DBMSConfig, DbPropKeys, DbAuthenticationKeys
+from ambari_server.dbConfiguration import AMBARI_DATABASE_NAME, DEFAULT_USERNAME, DEFAULT_PASSWORD, \
+  DBMSConfig, DbPropKeys, DbAuthenticationKeys
 from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_PATH_PROPERTY, JDBC_URL_PROPERTY, \
   JDBC_DATABASE_PROPERTY, JDBC_DATABASE_NAME_PROPERTY, \
   JDBC_HOSTNAME_PROPERTY, JDBC_PORT_PROPERTY, JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, \
   JDBC_PASSWORD_FILENAME, \
-  JDBC_RCA_DRIVER_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_RCA_DATABASE_PROPERTY, JDBC_RCA_SCHEMA_PROPERTY, \
-  JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, \
-  JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_FILENAME, JDBC_RCA_PASSWORD_ALIAS, \
+  JDBC_RCA_DRIVER_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, \
+  JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, \
   PERSISTENCE_TYPE_PROPERTY, \
-  get_value_from_properties, configDefaults
-from ambari_server.setupSecurity import encrypt_password, store_password_file
+  get_value_from_properties, configDefaults, encrypt_password, store_password_file
 from ambari_server.userInput import get_validated_string_input
 
 
@@ -339,15 +338,10 @@ class SQLServerAmbariDBConfig(SQLServerConfig):
     self.env_var_db_owner = 'AMBARIDBOWNER'
 
     # The values from options supersede the values from properties
-    if options.init_db_script_file is not None and options.init_db_script_file is not "":
-      self.init_script_file = compress_backslashes(options.init_db_script_file)
-    else:
-      self.init_script_file = "resources" + os.path.sep + "Ambari-DDL-SQLServer-CREATE.sql"
-    if options.cleanup_db_script_file is not None and options.cleanup_db_script_file is not "":
-      self.drop_tables_script_file = compress_backslashes(options.cleanup_db_script_file)
-    else:
-      self.drop_tables_script_file = "resources" + os.path.sep + "Ambari-DDL-SQLServer-DROP.sql"
-    pass
+    self.init_script_file = compress_backslashes(DBMSConfig._init_member_with_default(options, "init_db_script_file",
+        "resources" + os.path.sep + "Ambari-DDL-SQLServer-CREATE.sql"))
+    self.drop_tables_script_file = compress_backslashes(DBMSConfig._init_member_with_default(options, "cleanup_db_script_file",
+        "resources" + os.path.sep + "Ambari-DDL-SQLServer-DROP.sql"))
 
   def _setup_remote_server(self, properties):
     super(SQLServerAmbariDBConfig, self)._setup_remote_server(properties)

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -31,7 +31,7 @@ import tempfile
 from ambari_commons.exceptions import FatalException
 from ambari_commons.os_check import OSCheck, OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
-from ambari_commons.os_utils import run_os_command, search_file
+from ambari_commons.os_utils import run_os_command, search_file, set_file_permissions
 from ambari_commons.logging_utils import get_debug_mode, print_info_msg, print_warning_msg, print_error_msg, \
   set_debug_mode
 from ambari_server.properties import Properties
@@ -48,8 +48,6 @@ PID_NAME = "ambari-server.pid"
 # Non-root user setup commands
 NR_USER_PROPERTY = "ambari-server.user"
 
-# constants
-STACK_NAME_VER_SEP = "-"
 BLIND_PASSWORD = "*****"
 
 # Common messages
@@ -207,9 +205,10 @@ class ServerConfigDefaults(object):
     self.DEFAULT_VIEWS_DIR = ""
 
     #keytool commands
-    self.keytool_bin = ""
+    self.keytool_bin_subpath = ""
 
     #Standard messages
+    self.MESSAGE_SERVER_RUNNING_AS_ROOT = ""
     self.MESSAGE_ERROR_SETUP_NOT_ROOT = ""
     self.MESSAGE_ERROR_RESET_NOT_ROOT = ""
     self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = ""
@@ -260,9 +259,10 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults):
     self.DEFAULT_VIEWS_DIR = "resources\\views"
 
     #keytool commands
-    self.keytool_bin = "keytool.exe"
+    self.keytool_bin_subpath = "bin\\keytool.exe"
 
     #Standard messages
+    self.MESSAGE_SERVER_RUNNING_AS_ROOT = "Ambari Server running with 'root' privileges."
     self.MESSAGE_ERROR_SETUP_NOT_ROOT = "Ambari-server setup must be run with administrator-level privileges"
     self.MESSAGE_ERROR_RESET_NOT_ROOT = "Ambari-server reset must be run with administrator-level privileges"
     self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = "Ambari-server upgrade must be run with administrator-level privileges"
@@ -325,9 +325,10 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
     self.DEFAULT_VIEWS_DIR = "/var/lib/ambari-server/resources/views"
 
     #keytool commands
-    self.keytool_bin = "keytool"
+    self.keytool_bin_subpath = "bin/keytool"
 
     #Standard messages
+    self.MESSAGE_SERVER_RUNNING_AS_ROOT = "Ambari Server running with administrator privileges."
     self.MESSAGE_ERROR_SETUP_NOT_ROOT = "Ambari-server setup should be run with root-level privileges"
     self.MESSAGE_ERROR_RESET_NOT_ROOT = "Ambari-server reset should be run with root-level privileges"
     self.MESSAGE_ERROR_UPGRADE_NOT_ROOT = "Ambari-server upgrade must be run with root-level privileges"
@@ -543,6 +544,31 @@ def update_database_name_property(upgrade=False):
       raise FatalException(-1, err)
 
 
+def encrypt_password(alias, password):
+  properties = get_ambari_properties()
+  if properties == -1:
+    raise FatalException(1, None)
+  return get_encrypted_password(alias, password, properties)
+
+def get_encrypted_password(alias, password, properties):
+  isSecure = get_is_secure(properties)
+  (isPersisted, masterKeyFile) = get_is_persisted(properties)
+  if isSecure:
+    masterKey = None
+    if not masterKeyFile:
+      # Encryption enabled but no master key file found
+      masterKey = get_original_master_key(properties)
+
+    retCode = save_passwd_for_alias(alias, password, masterKey)
+    if retCode != 0:
+      print 'Failed to save secure password!'
+      return password
+    else:
+      return get_alias_string(alias)
+
+  return password
+
+
 def is_alias_string(passwdStr):
   regex = re.compile("\$\{alias=[\w\.]+\}")
   # Match implies string at beginning of word
@@ -552,6 +578,12 @@ def is_alias_string(passwdStr):
   else:
     return False
 
+def get_alias_string(alias):
+  return "${alias=" + alias + "}"
+
+def get_alias_from_alias_string(aliasStr):
+  return aliasStr[8:-1]
+
 def read_passwd_for_alias(alias, masterKey=""):
   if alias:
     jdk_path = find_jdk()
@@ -601,6 +633,59 @@ def decrypt_password_for_alias(properties, alias):
   else:
     return alias
 
+def save_passwd_for_alias(alias, passwd, masterKey=""):
+  if alias and passwd:
+    jdk_path = find_jdk()
+    if jdk_path is None:
+      print_error_msg("No JDK found, please run the \"setup\" "
+                      "command to install a JDK automatically or install any "
+                      "JDK manually to " + configDefaults.JDK_INSTALL_DIR)
+      return 1
+
+    if masterKey is None or masterKey == "":
+      masterKey = "None"
+
+    command = SECURITY_PROVIDER_PUT_CMD.format(get_java_exe_path(),
+                                               get_full_ambari_classpath(), alias, passwd, masterKey)
+    (retcode, stdout, stderr) = run_os_command(command)
+    print_info_msg("Return code from credential provider save passwd: " +
+                   str(retcode))
+    return retcode
+  else:
+    print_error_msg("Alias or password is unreadable.")
+
+
+def get_pass_file_path(conf_file, filename):
+  return os.path.join(os.path.dirname(conf_file), filename)
+
+def store_password_file(password, filename):
+  conf_file = find_properties_file()
+  passFilePath = get_pass_file_path(conf_file, filename)
+
+  with open(passFilePath, 'w+') as passFile:
+    passFile.write(password)
+  print_info_msg("Adjusting filesystem permissions")
+  ambari_user = read_ambari_user()
+  set_file_permissions(passFilePath, "660", ambari_user, False)
+
+  #Windows paths need double backslashes, otherwise the Ambari server deserializer will think the single \ are escape markers
+  return passFilePath.replace('\\', '\\\\')
+
+def remove_password_file(filename):
+  conf_file = find_properties_file()
+  passFilePath = os.path.join(os.path.dirname(conf_file),
+                              filename)
+
+  if os.path.exists(passFilePath):
+    try:
+      os.remove(passFilePath)
+    except Exception, e:
+      print_warning_msg('Unable to remove password file: ' + str(e))
+      return 1
+  pass
+  return 0
+
+
 def get_original_master_key(properties):
   input = True
   while(input):
@@ -739,8 +824,6 @@ def update_properties(propertyMap):
       print_error_msg('Could not read "%s": %s' % (conf_file, e))
       return -1
 
-    #for key in propertyMap.keys():
-      #properties[key] = propertyMap[key]
     for key in propertyMap.keys():
       properties.removeOldProp(key)
       properties.process_pair(key, str(propertyMap[key]))
@@ -765,7 +848,7 @@ def update_properties_2(properties, propertyMap):
       pass
 
     with open(conf_file, 'w') as file:
-      properties.store(file)
+      properties.store_ordered(file)
     pass
   pass
 
@@ -994,20 +1077,6 @@ def get_java_exe_path():
     return java_exe
   return
 
-#
-# JDBC
-#
-
-#Check if required jdbc drivers present
-def find_jdbc_driver(args):
-  if args.dbms in JDBC_PATTERNS.keys():
-    drivers = []
-    drivers.extend(glob.glob(os.path.join(configDefaults.JAVA_SHARE_PATH, JDBC_PATTERNS[args.dbms])))
-    if drivers:
-      return drivers
-    return -1
-  return 0
-
 
 #
 # Stack upgrade

ambari-server/src/main/python/ambari_server/serverSetup.py

@@ -24,7 +24,7 @@ import re
 import shutil
 import sys
 
-from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.exceptions import FatalException
 from ambari_commons.firewall import Firewall
 from ambari_commons.inet_utils import force_download_file
 from ambari_commons.logging_utils import get_silent, print_info_msg, print_warning_msg, print_error_msg
@@ -38,13 +38,16 @@ from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
   read_ambari_user, update_properties, validate_jdk, write_property, \
   JAVA_HOME, JAVA_HOME_PROPERTY, JCE_NAME_PROPERTY, JDBC_RCA_URL_PROPERTY, JDBC_URL_PROPERTY, \
   JDK_NAME_PROPERTY, JDK_RELEASES, NR_USER_PROPERTY, OS_FAMILY, OS_FAMILY_PROPERTY, OS_TYPE, OS_TYPE_PROPERTY, OS_VERSION, \
-  RESOURCES_DIR_PROPERTY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, VIEWS_DIR_PROPERTY, PID_NAME, get_is_secure, \
+  RESOURCES_DIR_PROPERTY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, VIEWS_DIR_PROPERTY, get_is_secure, \
   get_is_persisted
+from ambari_server.serverUtils import is_server_runing
 from ambari_server.setupSecurity import adjust_directory_permissions
 from ambari_server.userInput import get_YN_input, get_validated_string_input
 from ambari_server.utils import locate_file
 
 
+
+
 # selinux commands
 GET_SE_LINUX_ST_CMD = locate_file('sestatus', '/usr/sbin')
 SE_SETENFORCE_CMD = "setenforce 0"
@@ -937,15 +940,12 @@ def setup(options):
 #
 # Resets the Ambari Server.
 #
-def reset(options, serviceClass=None):
+def reset(options):
   if not is_root():
     err = configDefaults.MESSAGE_ERROR_RESET_NOT_ROOT
     raise FatalException(4, err)
 
-  if serviceClass:
-    status, stateDesc = is_server_running(serviceClass)
-  else:
-    status, stateDesc = is_server_runing()
+  status, stateDesc = is_server_runing()
   if status:
     err = 'Ambari-server must be stopped to reset'
     raise FatalException(1, err)
@@ -970,44 +970,3 @@ def reset(options, serviceClass=None):
 
   _reset_database(options)
   pass
-
-
-def is_server_running(serviceClass):
-  from ambari_commons.os_windows import SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING, \
-    SERVICE_STATUS_STOPPED, SERVICE_STATUS_NOT_INSTALLED
-
-  statusStr = serviceClass.QueryStatus()
-  if statusStr in(SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING):
-    return True, ""
-  elif statusStr == SERVICE_STATUS_STOPPED:
-    return False, SERVICE_STATUS_STOPPED
-  elif statusStr == SERVICE_STATUS_NOT_INSTALLED:
-    return False, SERVICE_STATUS_NOT_INSTALLED
-  else:
-    return False, None
-
-def is_server_runing():
-  pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
-
-  if os.path.exists(pid_file_path):
-    try:
-      f = open(pid_file_path, "r")
-    except IOError, ex:
-      raise FatalException(1, str(ex))
-
-    pid = f.readline().strip()
-
-    if not pid.isdigit():
-      err = "%s is corrupt. Removing" % (pid_file_path)
-      f.close()
-      run_os_command("rm -f " + pid_file_path)
-      raise NonFatalException(err)
-
-    f.close()
-    retcode, out, err = run_os_command("ps -p " + pid)
-    if retcode == 0:
-      return True, int(pid)
-    else:
-      return False, None
-  else:
-    return False, None

ambari-server/src/main/python/ambari_server/serverUpgrade.py

@@ -34,10 +34,9 @@ from ambari_server.serverConfiguration import configDefaults, \
   AMBARI_PROPERTIES_FILE, IS_LDAP_CONFIGURED, LDAP_PRIMARY_URL_PROPERTY, RESOURCES_DIR_PROPERTY, \
   SETUP_OR_UPGRADE_MSG
 from ambari_server.setupSecurity import adjust_directory_permissions
-
-# constants
 from ambari_server.utils import compare_versions
 
+# constants
 STACK_NAME_VER_SEP = "-"
 
 SCHEMA_UPGRADE_HELPER_CMD = "{0} -cp {1} " + \
@@ -53,13 +52,29 @@ STACK_UPGRADE_HELPER_CMD = "{0} -cp {1} " + \
 # Stack upgrade
 #
 
-def upgrade_stack(args, stack_id, repo_url=None, repo_url_os=None):
+def upgrade_stack(args):
   if not is_root():
     err = 'Ambari-server upgradestack should be run with ' \
           'root-level privileges'
     raise FatalException(4, err)
   check_database_name_property()
 
+  try:
+    stack_id = args[1]
+  except IndexError:
+    #stack_id is mandatory
+    raise FatalException("Invalid number of stack upgrade arguments")
+
+  try:
+    repo_url = args[2]
+  except IndexError:
+    repo_url = None
+
+  try:
+    repo_url_os = args[3]
+  except IndexError:
+    repo_url_os = None
+
   stack_name, stack_version = stack_id.split(STACK_NAME_VER_SEP)
   retcode = run_stack_upgrade(stack_name, stack_version, repo_url, repo_url_os)
 

ambari-server/src/main/python/ambari_server/serverUtils.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.logging_utils import get_verbose
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.os_check import OSConst
+from ambari_commons.os_utils import run_os_command
+from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
+from ambari_server.serverConfiguration import configDefaults, PID_NAME, get_ambari_properties, get_stack_location
+
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def is_server_runing():
+  pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
+
+  if os.path.exists(pid_file_path):
+    try:
+      f = open(pid_file_path, "r")
+    except IOError, ex:
+      raise FatalException(1, str(ex))
+
+    pid = f.readline().strip()
+
+    if not pid.isdigit():
+      err = "%s is corrupt. Removing" % (pid_file_path)
+      f.close()
+      run_os_command("rm -f " + pid_file_path)
+      raise NonFatalException(err)
+
+    f.close()
+    retcode, out, err = run_os_command("ps -p " + pid)
+    if retcode == 0:
+      return True, int(pid)
+    else:
+      return False, None
+  else:
+    return False, None
+
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def is_server_runing():
+  from ambari_commons.os_windows import SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING, \
+    SERVICE_STATUS_STOPPED, SERVICE_STATUS_NOT_INSTALLED
+  from ambari_windows_service import AmbariServerService
+
+  statusStr = AmbariServerService.QueryStatus()
+  if statusStr in(SERVICE_STATUS_STARTING, SERVICE_STATUS_RUNNING, SERVICE_STATUS_STOPPING):
+    return True, ""
+  elif statusStr == SERVICE_STATUS_STOPPED:
+    return False, SERVICE_STATUS_STOPPED
+  elif statusStr == SERVICE_STATUS_NOT_INSTALLED:
+    return False, SERVICE_STATUS_NOT_INSTALLED
+  else:
+    return False, None
+
+
+#
+# Performs HDP stack housekeeping
+#
+def refresh_stack_hash(properties):
+  stack_location = get_stack_location(properties)
+  # Hack: we determine resource dir as a parent dir for stack_location
+  resources_location = os.path.dirname(stack_location)
+  resource_files_keeper = ResourceFilesKeeper(resources_location)
+
+  try:
+    print "Organizing resource files at {0}...".format(resources_location,
+                                                       verbose=get_verbose())
+    resource_files_keeper.perform_housekeeping()
+  except KeeperException, ex:
+    msg = "Can not organize resource files at {0}: {1}".format(
+      resources_location, str(ex))
+    raise FatalException(-1, msg)

ambari-server/src/main/python/ambari_server/setupActions.py

@@ -36,6 +36,3 @@ ENCRYPT_PASSWORDS_ACTION = "encrypt-passwords"
 SETUP_SECURITY_ACTION = "setup-security"
 BACKUP_ACTION = "backup"
 RESTORE_ACTION = "restore"
-
-ACTION_REQUIRE_RESTART = [RESET_ACTION, UPGRADE_ACTION, UPGRADE_STACK_ACTION,
-                          SETUP_SECURITY_ACTION, LDAP_SETUP_ACTION]

ambari-server/src/main/python/ambari_server/setupHttps.py

@@ -0,0 +1,504 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import random
+import re
+import shutil
+import socket
+import string
+import datetime
+import tempfile
+import urllib2
+from ambari_commons.exceptions import FatalException, NonFatalException
+from ambari_commons.logging_utils import get_silent, print_warning_msg, print_error_msg
+from ambari_commons.os_utils import is_root, run_os_command, copy_file, set_file_permissions, remove_file
+from ambari_server.serverConfiguration import get_ambari_properties, find_properties_file, read_ambari_user, \
+    SSL_TRUSTSTORE_PASSWORD_PROPERTY, get_is_secure, decrypt_password_for_alias, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
+    SSL_TRUSTSTORE_PATH_PROPERTY, get_value_from_properties, SSL_TRUSTSTORE_TYPE_PROPERTY, find_jdk, configDefaults, \
+    get_encrypted_password, GET_FQDN_SERVICE_URL
+from ambari_server.setupSecurity import adjust_directory_permissions
+from ambari_server.userInput import get_YN_input, get_validated_string_input, read_password, get_prompt_default, \
+    get_validated_filepath_input
+
+#keytool commands
+KEYTOOL_IMPORT_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + "keytool -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt"
+KEYTOOL_DELETE_CERT_CMD = "{0}" + os.sep + "bin" + os.sep + "keytool -delete -alias '{1}' -storepass '{2}' -noprompt"
+KEYTOOL_KEYSTORE = " -keystore '{0}'"
+
+SSL_PASSWORD_FILE = "pass.txt"
+SSL_PASSIN_FILE = "passin.txt"
+
+# openssl command
+VALIDATE_KEYSTORE_CMD = "openssl pkcs12 -info -in '{0}' -password file:'{1}' -passout file:'{2}'"
+EXPRT_KSTR_CMD = "openssl pkcs12 -export -in '{0}' -inkey '{1}' -certfile '{0}' -out '{4}' -password file:'{2}' -passin file:'{3}'"
+CHANGE_KEY_PWD_CND = 'openssl rsa -in {0} -des3 -out {0}.secured -passout pass:{1}'
+GET_CRT_INFO_CMD = 'openssl x509 -dates -subject -in {0}'
+
+#keytool commands
+KEYTOOL_IMPORT_CERT_CMD = "{0} -import -alias '{1}' -storetype '{2}' -file '{3}' -storepass '{4}' -noprompt"
+KEYTOOL_DELETE_CERT_CMD = "{0} -delete -alias '{1}' -storepass '{2}' -noprompt"
+KEYTOOL_KEYSTORE = " -keystore '{0}'"
+
+SSL_KEY_DIR = 'security.server.keys_dir'
+SSL_API_PORT = 'client.api.ssl.port'
+SSL_API = 'api.ssl'
+SSL_SERVER_CERT_NAME = 'client.api.ssl.cert_name'
+SSL_SERVER_KEY_NAME = 'client.api.ssl.key_name'
+SSL_CERT_FILE_NAME = "https.crt"
+SSL_KEY_FILE_NAME = "https.key"
+SSL_KEYSTORE_FILE_NAME = "https.keystore.p12"
+SSL_KEY_PASSWORD_FILE_NAME = "https.pass.txt"
+SSL_KEY_PASSWORD_LENGTH = 50
+DEFAULT_SSL_API_PORT = 8443
+SSL_DATE_FORMAT = '%b  %d %H:%M:%S %Y GMT'
+
+#SSL certificate metainfo
+COMMON_NAME_ATTR = 'CN'
+NOT_BEFORE_ATTR = 'notBefore'
+NOT_AFTER_ATTR = 'notAfter'
+
+SRVR_TWO_WAY_SSL_PORT_PROPERTY = "security.server.two_way_ssl.port"
+SRVR_TWO_WAY_SSL_PORT = "8441"
+
+SRVR_ONE_WAY_SSL_PORT_PROPERTY = "security.server.one_way_ssl.port"
+SRVR_ONE_WAY_SSL_PORT = "8440"
+
+GANGLIA_HTTPS = 'ganglia.https'
+
+
+def get_truststore_path(properties):
+  truststore_path = properties.get_property(SSL_TRUSTSTORE_PATH_PROPERTY)
+  if not truststore_path:
+    SSL_TRUSTSTORE_PATH_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_PATH_PROPERTY)
+
+    while not truststore_path:
+      truststore_path = get_validated_string_input(
+          "Path to TrustStore file {0}:".format(get_prompt_default(SSL_TRUSTSTORE_PATH_DEFAULT)),
+          SSL_TRUSTSTORE_PATH_DEFAULT,
+          ".*", False, False)
+
+    if truststore_path:
+      properties.process_pair(SSL_TRUSTSTORE_PATH_PROPERTY, truststore_path)
+
+  return truststore_path
+
+def get_truststore_type(properties):
+  truststore_type = properties.get_property(SSL_TRUSTSTORE_TYPE_PROPERTY)
+  if not truststore_type:
+    SSL_TRUSTSTORE_TYPE_DEFAULT = get_value_from_properties(properties, SSL_TRUSTSTORE_TYPE_PROPERTY, "jks")
+
+    truststore_type = get_validated_string_input(
+        "TrustStore type [jks/jceks/pkcs12] {0}:".format(get_prompt_default(SSL_TRUSTSTORE_TYPE_DEFAULT)),
+        SSL_TRUSTSTORE_TYPE_DEFAULT,
+        "^(jks|jceks|pkcs12)?$", "Wrong type", False)
+
+    if truststore_type:
+        properties.process_pair(SSL_TRUSTSTORE_TYPE_PROPERTY, truststore_type)
+
+  return truststore_type
+
+def get_truststore_password(properties):
+  truststore_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
+  isSecure = get_is_secure(properties)
+  if truststore_password:
+    if isSecure:
+      truststore_password = decrypt_password_for_alias(properties, SSL_TRUSTSTORE_PASSWORD_ALIAS)
+  else:
+    truststore_password = read_password("", ".*", "Password for TrustStore:", "Invalid characters in password")
+    if truststore_password:
+      encrypted_password = get_encrypted_password(SSL_TRUSTSTORE_PASSWORD_ALIAS, truststore_password, properties)
+      properties.process_pair(SSL_TRUSTSTORE_PASSWORD_PROPERTY, encrypted_password)
+
+  return truststore_password
+
+def get_keytool_path(jdk_path):
+    return os.path.join(jdk_path, configDefaults.keytool_bin_subpath)
+
+def get_import_cert_command(jdk_path, alias, truststore_type, import_cert_path, truststore_path, truststore_password):
+  cmd = KEYTOOL_IMPORT_CERT_CMD.format(get_keytool_path(jdk_path), alias, truststore_type, import_cert_path, truststore_password)
+  if truststore_path:
+    cmd += KEYTOOL_KEYSTORE.format(truststore_path)
+  return cmd
+
+def get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password):
+    cmd = KEYTOOL_DELETE_CERT_CMD.format(get_keytool_path(jdk_path), alias, truststore_password)
+    if truststore_path:
+        cmd += KEYTOOL_KEYSTORE.format(truststore_path)
+    return cmd
+
+
+def import_cert_and_key(security_server_keys_dir):
+  import_cert_path = get_validated_filepath_input( \
+      "Enter path to Certificate: ", \
+      "Certificate not found")
+  import_key_path  =  get_validated_filepath_input( \
+      "Enter path to Private Key: ", "Private Key not found")
+  pem_password = get_validated_string_input("Please enter password for Private Key: ", "", None, None, True)
+
+  certInfoDict = get_cert_info(import_cert_path)
+
+  if not certInfoDict:
+    print_warning_msg('Unable to get Certificate information')
+  else:
+    #Validate common name of certificate
+    if not is_valid_cert_host(certInfoDict):
+      print_warning_msg('Unable to validate Certificate hostname')
+
+    #Validate issue and expirations dates of certificate
+    if not is_valid_cert_exp(certInfoDict):
+      print_warning_msg('Unable to validate Certificate issue and expiration dates')
+
+  #jetty requires private key files with non-empty key passwords
+  retcode = 0
+  err = ''
+  if not pem_password:
+    print 'Generating random password for HTTPS keystore...done.'
+    pem_password = generate_random_string()
+    retcode, out, err = run_os_command(CHANGE_KEY_PWD_CND.format(
+        import_key_path, pem_password))
+    import_key_path += '.secured'
+
+  if retcode == 0:
+    keystoreFilePath = os.path.join(security_server_keys_dir, \
+                                    SSL_KEYSTORE_FILE_NAME)
+    keystoreFilePathTmp = os.path.join(tempfile.gettempdir(), \
+                                       SSL_KEYSTORE_FILE_NAME)
+    passFilePath = os.path.join(security_server_keys_dir, \
+                                SSL_KEY_PASSWORD_FILE_NAME)
+    passFilePathTmp = os.path.join(tempfile.gettempdir(), \
+                                   SSL_KEY_PASSWORD_FILE_NAME)
+    passinFilePath = os.path.join(tempfile.gettempdir(), \
+                                  SSL_PASSIN_FILE)
+    passwordFilePath = os.path.join(tempfile.gettempdir(), \
+                                    SSL_PASSWORD_FILE)
+
+    with open(passFilePathTmp, 'w+') as passFile:
+      passFile.write(pem_password)
+      passFile.close
+      pass
+
+    set_file_permissions(passFilePath, "660", read_ambari_user(), False)
+
+    copy_file(passFilePathTmp, passinFilePath)
+    copy_file(passFilePathTmp, passwordFilePath)
+
+    retcode, out, err = run_os_command(EXPRT_KSTR_CMD.format(import_cert_path, \
+                                                             import_key_path, passwordFilePath, passinFilePath, keystoreFilePathTmp))
+  if retcode == 0:
+    print 'Importing and saving Certificate...done.'
+    import_file_to_keystore(keystoreFilePathTmp, keystoreFilePath)
+    import_file_to_keystore(passFilePathTmp, passFilePath)
+
+    import_file_to_keystore(import_cert_path, os.path.join( \
+        security_server_keys_dir, SSL_CERT_FILE_NAME))
+    import_file_to_keystore(import_key_path, os.path.join( \
+        security_server_keys_dir, SSL_KEY_FILE_NAME))
+
+    #Validate keystore
+    retcode, out, err = run_os_command(VALIDATE_KEYSTORE_CMD.format(keystoreFilePath, \
+                                                                    passwordFilePath, passinFilePath))
+
+    remove_file(passinFilePath)
+    remove_file(passwordFilePath)
+
+    if not retcode == 0:
+      print 'Error during keystore validation occured!:'
+      print err
+      return False
+
+    return True
+  else:
+    print_error_msg('Could not import Certificate and Private Key.')
+    print 'SSL error on exporting keystore: ' + err.rstrip() + \
+        '.\nPlease ensure that provided Private Key password is correct and ' + \
+        're-import Certificate.'
+
+    return False
+
+
+def import_file_to_keystore(source, destination):
+  shutil.copy(source, destination)
+  set_file_permissions(destination, "660", read_ambari_user(), False)
+
+
+def generate_random_string(length=SSL_KEY_PASSWORD_LENGTH):
+  chars = string.digits + string.ascii_letters
+  return ''.join(random.choice(chars) for x in range(length))
+
+
+def get_cert_info(path):
+  retcode, out, err = run_os_command(GET_CRT_INFO_CMD.format(path))
+
+  if retcode != 0:
+    print 'Error getting Certificate info'
+    print err
+    return None
+
+  if out:
+    certInfolist = out.split(os.linesep)
+  else:
+    print 'Empty Certificate info'
+    return None
+
+  notBefore = None
+  notAfter = None
+  subject = None
+
+  for item in range(len(certInfolist)):
+    if certInfolist[item].startswith('notAfter='):
+      notAfter = certInfolist[item].split('=')[1]
+
+    if certInfolist[item].startswith('notBefore='):
+      notBefore = certInfolist[item].split('=')[1]
+
+    if certInfolist[item].startswith('subject='):
+      subject = certInfolist[item].split('=', 1)[1]
+
+    #Convert subj to dict
+  pattern = re.compile(r"[A-Z]{1,2}=[\w.-]{1,}")
+  if subject:
+    subjList = pattern.findall(subject)
+    keys = [item.split('=')[0] for item in subjList]
+    values = [item.split('=')[1] for item in subjList]
+    subjDict = dict(zip(keys, values))
+
+    result = subjDict
+    result['notBefore'] = notBefore
+    result['notAfter'] = notAfter
+    result['subject'] = subject
+
+    return result
+  else:
+    return {}
+
+
+def is_valid_cert_exp(certInfoDict):
+  if certInfoDict.has_key(NOT_BEFORE_ATTR):
+    notBefore = certInfoDict[NOT_BEFORE_ATTR]
+  else:
+    print_warning_msg('There is no Not Before value in Certificate')
+    return False
+
+  if certInfoDict.has_key(NOT_AFTER_ATTR):
+    notAfter = certInfoDict['notAfter']
+  else:
+    print_warning_msg('There is no Not After value in Certificate')
+    return False
+
+  notBeforeDate = datetime.datetime.strptime(notBefore, SSL_DATE_FORMAT)
+  notAfterDate = datetime.datetime.strptime(notAfter, SSL_DATE_FORMAT)
+
+  currentDate = datetime.datetime.now()
+
+  if currentDate > notAfterDate:
+    print_warning_msg('Certificate expired on: ' + str(notAfterDate))
+    return False
+
+  if currentDate < notBeforeDate:
+    print_warning_msg('Certificate will be active from: ' + str(notBeforeDate))
+    return False
+
+  return True
+
+
+def is_valid_cert_host(certInfoDict):
+  if certInfoDict.has_key(COMMON_NAME_ATTR):
+    commonName = certInfoDict[COMMON_NAME_ATTR]
+  else:
+    print_warning_msg('There is no Common Name in Certificate')
+    return False
+
+  fqdn = get_fqdn()
+
+  if not fqdn:
+    print_warning_msg('Failed to get server FQDN')
+    return False
+
+  if commonName != fqdn:
+    print_warning_msg('Common Name in Certificate: ' + commonName + ' does not match the server FQDN: ' + fqdn)
+    return False
+
+  return True
+
+
+def get_fqdn(timeout=2):
+  properties = get_ambari_properties()
+  if properties == -1:
+    print "Error reading ambari properties"
+    return None
+
+  get_fqdn_service_url = properties[GET_FQDN_SERVICE_URL]
+  try:
+    handle = urllib2.urlopen(get_fqdn_service_url, '', timeout)
+    str = handle.read()
+    handle.close()
+    return str
+  except Exception:
+    return socket.getfqdn().lower()
+
+
+def is_valid_https_port(port):
+  properties = get_ambari_properties()
+  if properties == -1:
+    print "Error getting ambari properties"
+    return False
+
+  one_way_port = properties[SRVR_ONE_WAY_SSL_PORT_PROPERTY]
+  if not one_way_port:
+    one_way_port = SRVR_ONE_WAY_SSL_PORT
+
+  two_way_port = properties[SRVR_TWO_WAY_SSL_PORT_PROPERTY]
+  if not two_way_port:
+    two_way_port = SRVR_TWO_WAY_SSL_PORT
+
+  if port.strip() == one_way_port.strip():
+    print "Port for https can't match the port for one way authentication port(" + one_way_port + ")"
+    return False
+
+  if port.strip() == two_way_port.strip():
+    print "Port for https can't match the port for two way authentication port(" + two_way_port + ")"
+    return False
+
+  return True
+
+
+def import_cert_and_key_action(security_server_keys_dir, properties):
+  if import_cert_and_key(security_server_keys_dir):
+    properties.process_pair(SSL_SERVER_CERT_NAME, SSL_CERT_FILE_NAME)
+    properties.process_pair(SSL_SERVER_KEY_NAME, SSL_KEY_FILE_NAME)
+    properties.process_pair(SSL_API, "true")
+    return True
+  else:
+    return False
+
+def run_component_https_cmd(cmd):
+  retcode, out, err = run_os_command(cmd)
+
+  if not retcode == 0:
+    err = 'Error occured during truststore setup ! :' + out + " : " + err
+    raise FatalException(1, err)
+
+
+def setup_https(args):
+  if not is_root():
+        err = 'ambari-server setup-https should be run with ' \
+              'root-level privileges'
+        raise FatalException(4, err)
+  args.exit_message = None
+  if not get_silent():
+    properties = get_ambari_properties()
+    try:
+      security_server_keys_dir = properties.get_property(SSL_KEY_DIR)
+      client_api_ssl_port = DEFAULT_SSL_API_PORT if properties.get_property(SSL_API_PORT) in ("") \
+            else properties.get_property(SSL_API_PORT)
+      api_ssl = properties.get_property(SSL_API) in ['true']
+      cert_was_imported = False
+      cert_must_import = True
+      if api_ssl:
+        if get_YN_input("Do you want to disable HTTPS [y/n] (n)? ", False):
+          properties.process_pair(SSL_API, "false")
+          cert_must_import=False
+        else:
+          properties.process_pair(SSL_API_PORT, \
+                                  get_validated_string_input( \
+                                      "SSL port ["+str(client_api_ssl_port)+"] ? ", \
+                                      str(client_api_ssl_port), \
+                                      "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
+          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
+      else:
+        if get_YN_input("Do you want to configure HTTPS [y/n] (y)? ", True):
+          properties.process_pair(SSL_API_PORT, \
+                                  get_validated_string_input("SSL port ["+str(client_api_ssl_port)+"] ? ", \
+                                                               str(client_api_ssl_port), "^[0-9]{1,5}$", "Invalid port.", False, validatorFunction = is_valid_https_port))
+          cert_was_imported = import_cert_and_key_action(security_server_keys_dir, properties)
+        else:
+          return False
+
+      if cert_must_import and not cert_was_imported:
+        print 'Setup of HTTPS failed. Exiting.'
+        return False
+
+      conf_file = find_properties_file()
+      f = open(conf_file, 'w')
+      properties.store(f, "Changed by 'ambari-server setup-https' command")
+
+      ambari_user = read_ambari_user()
+      if ambari_user:
+        adjust_directory_permissions(ambari_user)
+      return True
+    except (KeyError), e:
+        err = 'Property ' + str(e) + ' is not defined'
+        raise FatalException(1, err)
+  else:
+    warning = "setup-https is not enabled in silent mode."
+    raise NonFatalException(warning)
+
+
+def setup_component_https(component, command, property, alias):
+  if not get_silent():
+    jdk_path = find_jdk()
+    if jdk_path is None:
+      err = "No JDK found, please run the \"ambari-server setup\" " \
+            "command to install a JDK automatically or install any " \
+            "JDK manually to " + configDefaults.JDK_INSTALL_DIR
+      raise FatalException(1, err)
+
+    properties = get_ambari_properties()
+
+    use_https = properties.get_property(property) in ['true']
+
+    if use_https:
+      if get_YN_input("Do you want to disable HTTPS for " + component + " [y/n] (n)? ", False):
+        truststore_path = get_truststore_path(properties)
+        truststore_password = get_truststore_password(properties)
+
+        run_component_https_cmd(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
+
+        properties.process_pair(property, "false")
+      else:
+        return
+    else:
+      if get_YN_input("Do you want to configure HTTPS for " + component + " [y/n] (y)? ", True):
+        truststore_type = get_truststore_type(properties)
+        truststore_path = get_truststore_path(properties)
+        truststore_password = get_truststore_password(properties)
+
+        run_os_command(get_delete_cert_command(jdk_path, alias, truststore_path, truststore_password))
+
+        import_cert_path = get_validated_filepath_input( \
+            "Enter path to " + component + " Certificate: ", \
+            "Certificate not found")
+
+        run_component_https_cmd(get_import_cert_command(jdk_path, alias, truststore_type, import_cert_path, truststore_path, truststore_password))
+
+        properties.process_pair(property, "true")
+      else:
+        return
+
+    conf_file = find_properties_file()
+    f = open(conf_file, 'w')
+    properties.store(f, "Changed by 'ambari-server " + command + "' command")
+  else:
+    print command + " is not enabled in silent mode."
+
+def setup_ganglia_https():
+  setup_component_https("Ganglia", "setup-ganglia-https", GANGLIA_HTTPS,
+                        "ganglia_cert")

ambari-server/src/main/python/ambari_server/userInput.py

@@ -119,3 +119,36 @@ def get_prompt_default(defaultStr=None):
     return ""
   else:
     return '(' + defaultStr + ')'
+
+
+def read_password(passwordDefault,
+                  passwordPattern,
+                  passwordPrompt=None,
+                  passwordDescr=None):
+
+  input = True
+  while(input):
+    # setup password
+    if passwordPrompt is None:
+      passwordPrompt = 'Password (' + passwordDefault + '): '
+
+    if passwordDescr is None:
+      passwordDescr = "Invalid characters in password. Use only alphanumeric or " \
+                      "_ or - characters"
+
+    password = get_validated_string_input(passwordPrompt, passwordDefault,
+                                          passwordPattern, passwordDescr, True)
+    if not password:
+      print 'Password cannot be blank.'
+      continue
+
+    if password != passwordDefault:
+      password1 = get_validated_string_input("Re-enter password: ",
+                                             passwordDefault, passwordPattern, passwordDescr, True)
+      if password != password1:
+        print "Passwords do not match"
+        continue
+
+    input = False
+
+  return password

ambari-server/src/main/python/ambari_server_main.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import getpass
+
+import os
+import stat
+import subprocess
+import tempfile
+import sys
+
+from ambari_commons.exceptions import FatalException
+from ambari_commons.logging_utils import get_debug_mode, get_verbose, print_warning_msg, print_info_msg, \
+  set_debug_mode_from_options
+from ambari_commons.os_check import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons.os_utils import is_root
+from ambari_server.dbConfiguration import ensure_dbms_is_running, ensure_jdbc_driver_is_installed
+from ambari_server.serverConfiguration import configDefaults, find_jdk, get_ambari_classpath, get_ambari_properties, \
+  get_conf_dir, get_is_persisted, get_is_secure, get_java_exe_path, get_original_master_key, read_ambari_user, \
+  PID_NAME, RESOURCES_DIR_DEFAULT, RESOURCES_DIR_PROPERTY, SECURITY_KEY_ENV_VAR_NAME, SECURITY_MASTER_KEY_LOCATION, \
+  SETUP_OR_UPGRADE_MSG, check_database_name_property, parse_properties_file
+from ambari_server.serverUtils import is_server_runing, refresh_stack_hash
+from ambari_server.setupHttps import get_fqdn
+from ambari_server.setupSecurity import save_master_key
+from ambari_server.utils import check_reverse_lookup, save_pid, locate_file, looking_for_pid, wait_for_pid, \
+  save_main_pid_ex, check_exitcode
+
+
+# debug settings
+SERVER_START_DEBUG = False
+SUSPEND_START_MODE = False
+
+# server commands
+ambari_provider_module_option = ""
+ambari_provider_module = os.environ.get('AMBARI_PROVIDER_MODULE')
+if ambari_provider_module is not None:
+  ambari_provider_module_option = "-Dprovider.module.class=" + \
+                                  ambari_provider_module + " "
+
+jvm_args = os.getenv('AMBARI_JVM_ARGS', '-Xms512m -Xmx2048m')
+
+SERVER_START_CMD = "{0} " \
+    "-server -XX:NewRatio=3 " \
+    "-XX:+UseConcMarkSweepGC " + \
+    "-XX:-UseGCOverheadLimit -XX:CMSInitiatingOccupancyFraction=60 " + \
+    "{1} {2} " \
+    "-cp {3} "\
+    "org.apache.ambari.server.controller.AmbariServer " \
+    "> {4} 2>&1 || echo $? > {5} &"
+SERVER_START_CMD_DEBUG = "{0} " \
+    "-server -XX:NewRatio=2 " \
+    "-XX:+UseConcMarkSweepGC " + \
+    "{1} {2} " \
+    " -Xdebug -Xrunjdwp:transport=dt_socket,address=5005," \
+    "server=y,suspend={6} " \
+    "-cp {3} " + \
+    "org.apache.ambari.server.controller.AmbariServer " \
+    "> {4} 2>&1 || echo $? > {5} &"
+
+SERVER_START_CMD_WINDOWS = "{0} " \
+    "-server -XX:NewRatio=3 " \
+    "-XX:+UseConcMarkSweepGC " + \
+    "-XX:-UseGCOverheadLimit -XX:CMSInitiatingOccupancyFraction=60 " \
+    "{1} {2} " \
+    "-cp {3} " \
+    "org.apache.ambari.server.controller.AmbariServer"
+SERVER_START_CMD_DEBUG_WINDOWS = "{0} " \
+    "-server -XX:NewRatio=2 " \
+    "-XX:+UseConcMarkSweepGC " \
+    "{1} {2} " \
+    "-Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend={4} " \
+    "-cp {3}" \
+    "org.apache.ambari.server.controller.AmbariServer"
+
+SERVER_INIT_TIMEOUT = 5
+SERVER_START_TIMEOUT = 10
+
+SERVER_PING_TIMEOUT_WINDOWS = 5
+SERVER_PING_ATTEMPTS_WINDOWS = 4
+
+SERVER_SEARCH_PATTERN = "org.apache.ambari.server.controller.AmbariServer"
+
+EXITCODE_NAME = "ambari-server.exitcode"
+
+AMBARI_SERVER_DIE_MSG = "Ambari Server java process died with exitcode {0}. Check {1} for more information."
+
+# linux open-file limit
+ULIMIT_OPEN_FILES_KEY = 'ulimit.open.files'
+ULIMIT_OPEN_FILES_DEFAULT = 10000
+
+
+def get_resources_location(properties):
+  res_location = properties[RESOURCES_DIR_PROPERTY]
+  if res_location is None:
+    res_location = RESOURCES_DIR_DEFAULT
+  return res_location
+
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def ensure_can_start_under_current_user(ambari_user):
+  #Ignore the requirement to run as root. In Windows, by default the child process inherits the security context
+  # and the environment from the parent process.
+  return ""
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def ensure_can_start_under_current_user(ambari_user):
+  current_user = getpass.getuser()
+  if ambari_user is None:
+    err = "Unable to detect a system user for Ambari Server.\n" + SETUP_OR_UPGRADE_MSG
+    raise FatalException(1, err)
+  if current_user != ambari_user and not is_root():
+    err = "Unable to start Ambari Server as user {0}. Please either run \"ambari-server start\" " \
+          "command as root, as sudo or as user \"{1}\"".format(current_user, ambari_user)
+    raise FatalException(1, err)
+  return current_user
+
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def ensure_server_security_is_configured():
+  pass
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def ensure_server_security_is_configured():
+  if not is_root():
+    print "Unable to check iptables status when starting without root privileges."
+    print "Please do not forget to disable or adjust iptables if needed"
+
+
+def get_ulimit_open_files(properties):
+  open_files_val = properties[ULIMIT_OPEN_FILES_KEY]
+  open_files = int(open_files_val) if (open_files_val and int(open_files_val) > 0) else ULIMIT_OPEN_FILES_DEFAULT
+  return open_files
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode):
+  if class_path.find(' ') != -1:
+    conf_dir = '"' + class_path + '"'
+  command_base = SERVER_START_CMD_DEBUG_WINDOWS if debug_start else SERVER_START_CMD_WINDOWS
+  command = command_base.format(
+      java_exe,
+      ambari_provider_module_option,
+      jvm_args,
+      conf_dir,
+      suspend_mode)
+  return command
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def generate_child_process_param_list(ambari_user, current_user, java_exe, class_path, debug_start, suspend_mode):
+  from ambari_commons.os_linux import ULIMIT_CMD
+
+  properties = get_ambari_properties()
+
+  isSecure = get_is_secure(properties)
+  (isPersisted, masterKeyFile) = get_is_persisted(properties)
+  environ = os.environ.copy()
+  # Need to handle master key not persisted scenario
+  if isSecure and not masterKeyFile:
+    prompt = False
+    masterKey = environ.get(SECURITY_KEY_ENV_VAR_NAME)
+
+    if masterKey is not None and masterKey != "":
+      pass
+    else:
+      keyLocation = environ.get(SECURITY_MASTER_KEY_LOCATION)
+
+      if keyLocation is not None:
+        try:
+          # Verify master key can be read by the java process
+          with open(keyLocation, 'r'):
+            pass
+        except IOError:
+          print_warning_msg("Cannot read Master key from path specified in "
+                            "environemnt.")
+          prompt = True
+      else:
+        # Key not provided in the environment
+        prompt = True
+
+    if prompt:
+      import pwd
+
+      masterKey = get_original_master_key(properties)
+      tempDir = tempfile.gettempdir()
+      tempFilePath = tempDir + os.sep + "masterkey"
+      save_master_key(masterKey, tempFilePath, True)
+      if ambari_user != current_user:
+        uid = pwd.getpwnam(ambari_user).pw_uid
+        gid = pwd.getpwnam(ambari_user).pw_gid
+        os.chown(tempFilePath, uid, gid)
+      else:
+        os.chmod(tempFilePath, stat.S_IREAD | stat.S_IWRITE)
+
+      if tempFilePath is not None:
+        environ[SECURITY_MASTER_KEY_LOCATION] = tempFilePath
+
+  command_base = SERVER_START_CMD_DEBUG if debug_start else SERVER_START_CMD
+
+  command = "%s %s; %s" % (ULIMIT_CMD, str(get_ulimit_open_files(properties)),
+      command_base.format(java_exe,
+          ambari_provider_module_option,
+          jvm_args,
+          class_path,
+          configDefaults.SERVER_OUT_FILE,
+          os.path.join(configDefaults.PID_DIR, EXITCODE_NAME),
+          suspend_mode)
+  )
+
+  # required to start properly server instance
+  os.chdir(configDefaults.ROOT_FS_PATH)
+
+  #For properly daemonization server should be started using shell as parent
+  if is_root() and ambari_user != "root":
+    # To inherit exported environment variables (especially AMBARI_PASSPHRASE),
+    # from subprocess, we have to skip --login option of su command. That's why
+    # we change dir to / (otherwise subprocess can face with 'permission denied'
+    # errors while trying to list current directory
+    param_list = [locate_file('su', '/bin'), ambari_user, "-s", locate_file('sh', '/bin'), "-c", command]
+  else:
+    param_list = [locate_file('sh', '/bin'), "-c", command]
+  return param_list
+
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def wait_for_server_start(pidFile, scmStatus):
+  # Wait for the HTTP port to be open
+  iter_start = 0
+  while iter_start < SERVER_PING_ATTEMPTS_WINDOWS and not get_fqdn(SERVER_PING_TIMEOUT_WINDOWS):
+    if scmStatus is not None:
+      scmStatus.reportStartPending()
+    iter_start += 1
+
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
+def wait_for_server_start(pidFile, scmStatus):
+  #wait for server process for SERVER_START_TIMEOUT seconds
+  sys.stdout.write('Waiting for server start...')
+  sys.stdout.flush()
+
+  pids = looking_for_pid(SERVER_SEARCH_PATTERN, SERVER_INIT_TIMEOUT)
+  found_pids = wait_for_pid(pids, SERVER_START_TIMEOUT)
+
+  sys.stdout.write('\n')
+  sys.stdout.flush()
+
+  if found_pids <= 0:
+    exitcode = check_exitcode(os.path.join(configDefaults.PID_DIR, EXITCODE_NAME))
+    raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
+  else:
+    save_main_pid_ex(pids, pidFile, [locate_file('sh', '/bin'),
+                                     locate_file('bash', '/bin')], True)
+
+
+def server_process_main(options, scmStatus=None):
+  # debug mode, including stop Java process at startup
+  try:
+    set_debug_mode_from_options(options)
+  except AttributeError:
+    pass
+
+  if not check_reverse_lookup():
+    print_warning_msg("The hostname was not found in the reverse DNS lookup. "
+                      "This may result in incorrect behavior. "
+                      "Please check the DNS setup and fix the issue.")
+
+  check_database_name_property()
+  parse_properties_file(options)
+
+  ambari_user = read_ambari_user()
+  current_user = ensure_can_start_under_current_user(ambari_user)
+
+  print_info_msg("Ambari Server is not running...")
+
+  jdk_path = find_jdk()
+  if jdk_path is None:
+    err = "No JDK found, please run the \"ambari-server setup\" " \
+          "command to install a JDK automatically or install any " \
+          "JDK manually to " + configDefaults.JDK_INSTALL_DIR
+    raise FatalException(1, err)
+
+  properties = get_ambari_properties()
+
+  # Preparations
+  if is_root():
+    print configDefaults.MESSAGE_SERVER_RUNNING_AS_ROOT
+
+  ensure_jdbc_driver_is_installed(options, properties)
+
+  ensure_dbms_is_running(options, properties, scmStatus)
+
+  if scmStatus is not None:
+    scmStatus.reportStartPending()
+
+  refresh_stack_hash(properties)
+
+  if scmStatus is not None:
+    scmStatus.reportStartPending()
+
+  ensure_server_security_is_configured()
+
+  if scmStatus is not None:
+    scmStatus.reportStartPending()
+
+  java_exe = get_java_exe_path()
+
+  class_path = get_conf_dir()
+  class_path = os.path.abspath(class_path) + os.pathsep + get_ambari_classpath()
+
+  debug_mode = get_debug_mode()
+  debug_start = (debug_mode & 1) or SERVER_START_DEBUG
+  suspend_start = (debug_mode & 2) or SUSPEND_START_MODE
+  suspend_mode = 'y' if suspend_start else 'n'
+
+  param_list = generate_child_process_param_list(ambari_user, current_user,
+                                                 java_exe, class_path, debug_start, suspend_mode)
+
+  if not os.path.exists(configDefaults.PID_DIR):
+    os.makedirs(configDefaults.PID_DIR, 0755)
+
+  environ = os.environ.copy()
+  print_info_msg("Running server: " + str(param_list))
+  procJava = subprocess.Popen(param_list, env=environ)
+
+  pidJava = procJava.pid
+  if pidJava <= 0:
+    procJava.terminate()
+    exitcode = procJava.returncode
+    exitfile = os.path.join(configDefaults.PID_DIR, EXITCODE_NAME)
+    save_pid(exitcode, exitfile)
+
+    if scmStatus is not None:
+      scmStatus.reportStopPending()
+
+    raise FatalException(-1, AMBARI_SERVER_DIE_MSG.format(exitcode, configDefaults.SERVER_OUT_FILE))
+  else:
+    pidfile = os.path.join(configDefaults.PID_DIR, PID_NAME)
+    save_pid(pidJava, pidfile)
+    print "Server PID at: "+pidfile
+    print "Server out at: "+configDefaults.SERVER_OUT_FILE
+    print "Server log at: "+configDefaults.SERVER_LOG_FILE
+
+    wait_for_server_start(pidfile, scmStatus)
+
+  if scmStatus is not None:
+    scmStatus.reportStarted()
+
+  return procJava

ambari-server/src/main/python/ambari_windows_service.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import optparse
+
+import os
+import sys
+
+from ambari_commons.ambari_service import AmbariService
+from ambari_commons.logging_utils import set_silent, set_verbose, print_info_msg
+from ambari_commons.os_utils import remove_file
+from ambari_commons.os_windows import SvcStatusCallback
+from ambari_server.serverConfiguration import configDefaults, get_ambari_properties, get_value_from_properties, \
+  DEBUG_MODE_KEY, PID_NAME, SERVER_OUT_FILE_KEY, SERVICE_PASSWORD_KEY, SERVICE_USERNAME_KEY, SUSPEND_START_MODE_KEY, \
+  VERBOSE_OUTPUT_KEY
+
+
+class AmbariServerService(AmbariService):
+  AmbariService._svc_name_ = "Ambari Server"
+  AmbariService._svc_display_name_ = "Ambari Server"
+  AmbariService._svc_description_ = "Ambari Server"
+
+  AmbariService._AdjustServiceVersion()
+
+  # Adds the necessary script dir to the Python's modules path
+  def _adjustPythonPath(self, current_dir):
+    python_path = os.path.join(current_dir, "sbin")
+    sys.path.insert(0, python_path)
+    print_info_msg("sys.path=" + str(sys.path))
+
+  def SvcDoRun(self):
+    from ambari_server_main import server_process_main
+
+    scmStatus = SvcStatusCallback(self)
+
+    properties = get_ambari_properties()
+    self.options.verbose = get_value_from_properties(properties, VERBOSE_OUTPUT_KEY, False)
+    self.options.debug = get_value_from_properties(properties, DEBUG_MODE_KEY, False)
+    self.options.suspend_start = get_value_from_properties(properties, SUSPEND_START_MODE_KEY, False)
+
+    # set verbose
+    set_verbose(self.options.verbose)
+
+    self.redirect_output_streams()
+
+    childProc = server_process_main(self.options, scmStatus)
+
+    if not self._StopOrWaitForChildProcessToFinish(childProc):
+      return
+
+    pid_file_path = os.path.join(configDefaults.PID_DIR, PID_NAME)
+    remove_file(pid_file_path)
+    pass
+
+  def _InitOptionsParser(self):
+    # No command-line options needed when starting as a service
+    return optparse.OptionParser()
+
+  def redirect_output_streams(self):
+    properties = get_ambari_properties()
+
+    outFilePath = properties[SERVER_OUT_FILE_KEY]
+    if (outFilePath is None or outFilePath == ""):
+      outFilePath = configDefaults.SERVER_OUT_FILE
+
+    self._RedirectOutputStreamsToFile(outFilePath)
+    pass
+
+def ctrlHandler(ctrlType):
+  AmbariServerService.DefCtrlCHandler()
+  return True
+
+def svcsetup():
+  AmbariServerService.set_ctrl_c_handler(ctrlHandler)
+
+  scriptFile, ext = os.path.splitext(__file__.replace('/', os.sep))
+  classPath = scriptFile + "." + AmbariServerService.__name__
+
+  # we don't save password between 'setup' runs, so we can't run Install every time. We run 'setup' only if user and
+  # password provided or if service not installed
+  if (SERVICE_USERNAME_KEY in os.environ and SERVICE_PASSWORD_KEY in os.environ):
+    AmbariServerService.Install(classPath=classPath, username=os.environ[SERVICE_USERNAME_KEY], password=os.environ[SERVICE_PASSWORD_KEY])
+  elif AmbariServerService.QueryStatus() == "not installed":
+    AmbariServerService.Install(classPath)

ambari-server/src/main/windows/ambari-server.ps1

@@ -42,7 +42,7 @@ $AMBARI_LOG_DIR="\var\log\ambari-server"
 $OUTFILE_STDOUT=Join-Path -path $AMBARI_LOG_DIR -childpath "ambari-server.stdout"
 $OUTFILE_STDERR=Join-Path -path $AMBARI_LOG_DIR -childpath "ambari-server.stderr"
 $LOGFILE=Join-Path -path $AMBARI_LOG_DIR -childpath "ambari-server.log"
-$AMBARI_SERVER_PY_SCRIPT=Join-Path -path $PSScriptRoot -childpath "sbin\ambari-server-windows.py"
+$AMBARI_SERVER_PY_SCRIPT=Join-Path -path $PSScriptRoot -childpath "sbin\ambari-server.py"
 if($AMBARI_SERVER_PY_SCRIPT.Contains(' '))
 {
   $AMBARI_SERVER_PY_SCRIPT = """" + $AMBARI_SERVER_PY_SCRIPT + """"