AMBARI-2349. Enhance processing of ojdbc.jar before starting ambari server

(Oleksandr via mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1492504 13f79535-47bb-0310-9956-ffa450edef68

ambari-server/src/main/python/ambari-server.py

@@ -78,6 +78,9 @@ JAVA_SHARE_PATH="/usr/share/java"
 BOLD_ON='\033[1m'
 BOLD_OFF='\033[0m'
 
+#Common messages
+PRESS_ENTER_MSG="Press <enter> to continue."
+
 if ambari_provider_module is not None:
   ambari_provider_module_option = "-Dprovider.module.class=" +\
                                   ambari_provider_module + " "
@@ -160,6 +163,11 @@ DATABASE_PORTS =["5432", "1521", "3306"]
 DATABASE_DRIVER_NAMES = ["org.postgresql.Driver", "oracle.jdbc.driver.OracleDriver", "com.mysql.jdbc.Driver"]
 DATABASE_CONNECTION_STRINGS = ["jdbc:postgresql://{0}:{1}/{2}", "jdbc:oracle:thin:@{0}:{1}/{2}", "jdbc:mysql://{0}:{1}/{2}"]
 DATABASE_CLI_TOOLS = [["psql"], ["sqlplus", "sqlplus64"], ["mysql"]]
+DATABASE_CLI_TOOLS_DESC = ["psql", "sqlplus", "mysql"]
+DATABASE_CLI_TOOLS_USAGE = ['su -postgres --command=psql -f {0} -v username=\'"{1}"\' -v password="\'{2}\'"',
+                            'sqlplus {1}/{2} < {0} ',
+                            'mysql --user={1} --password={2} {3}<{0}']
+
 DATABASE_INIT_SCRIPTS = ['/var/lib/ambari-server/resources/Ambari-DDL-Postgres-REMOTE-CREATE.sql',
                          '/var/lib/ambari-server/resources/Ambari-DDL-Oracle-CREATE.sql',
                          '/var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql']
@@ -551,13 +559,6 @@ def track_jdk(base_name, url, local_name):
 # Downloads the JDK
 #
 def download_jdk(args):
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  if conf_file is None:
-    print 'File %s not found in search path $%s: %s' %\
-          (AMBARI_PROPERTIES_FILE, AMBARI_CONF_VAR, get_conf_dir())
-    return -1
-  print_info_msg('Loading properties from ' + conf_file)
-
   if get_JAVA_HOME():
     return 0
 
@@ -571,14 +572,11 @@ def download_jdk(args):
     write_property(JAVA_HOME_PROPERTY, args.java_home)
     return 0
 
-  properties = None
-  try:
-    properties = Properties()
-    properties.load(open(conf_file))
-  except (Exception), e:
-    print 'Could not read "%s": %s' % (conf_file, e)
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
     return -1
-
+  conf_file = properties.fileName
   try:
     jdk_url = properties['jdk.url']
     resources_dir = properties['resources.dir']  
@@ -828,19 +826,11 @@ def update_ambari_properties():
 # Configures the OS settings in ambari properties.
 #
 def configure_os_settings():
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  if conf_file is None:
-    print_error_msg ('File %s not found in search path $%s: %s'
-                   % (AMBARI_PROPERTIES_FILE, AMBARI_CONF_VAR, get_conf_dir()))
-    return -1
-  print_info_msg ('Loading properties from ' + conf_file)
-  properties = None
-  try:
-    properties = Properties()
-    properties.load(open(conf_file))
-  except (Exception), e:
-    print_error_msg ('Could not read "%s": %s' % (conf_file, e))
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
     return -1
+    
   try:
     conf_os_type = properties[OS_TYPE_PROPERTY]
     if conf_os_type != '':
@@ -861,30 +851,23 @@ def configure_os_settings():
   if os_name == 'suse':
     os_name = 'sles'
   os_version = os_info[1].split('.', 1)[0]
-  master_os_type = os_name + os_version
-  with open(conf_file, "a") as ambariConf:
-    ambariConf.write(OS_TYPE_PROPERTY + "=" + master_os_type)
-    ambariConf.write("\n")
-    ambariConf.closed
+  master_os_type = os_name + os_version    
+  write_property(OS_TYPE_PROPERTY, master_os_type)
   return 0
 
 
 
 def get_JAVA_HOME():
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  properties = Properties()
-  
-  try:
-    properties.load(open(conf_file))
-    java_home = properties[JAVA_HOME_PROPERTY]
-    if (not 0 == len(java_home)) and (os.path.exists(java_home)):
-      return java_home
-  except (Exception), e:
-    print 'Could not read "%s": %s' % (conf_file, e)
-  
-  return None
-
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
+    return None
+    
+  java_home = properties[JAVA_HOME_PROPERTY]
+  if (not 0 == len(java_home)) and (os.path.exists(java_home)):
+    return java_home
 
+  return None
 
 #
 # Finds the available JDKs.
@@ -982,9 +965,6 @@ def setup(args):
     print_error_msg ('Failed to stop iptables. Exiting.')
     sys.exit(retcode)
 
-  print 'Configuring database...'
-  prompt_db_properties(args)
-
   print 'Checking JDK...'
   retcode = download_jdk(args)
   if not retcode == 0:
@@ -997,6 +977,9 @@ def setup(args):
     print_error_msg ('Configure of OS settings in '
                    'ambari.properties failed. Exiting.')
     sys.exit(retcode)
+    
+  print 'Configuring database...'
+  prompt_db_properties(args)
   
   #DB setup should be done last after doing any setup.
   
@@ -1080,30 +1063,36 @@ def reset(args):
   print "Resetting the Server database..."
 
   parse_properties_file(args)
-
   # configure_database_username_password(args)
   if args.persistence_type=="remote":
+    client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]
+    client_usage_cmd_drop = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_DROP_SCRIPTS[DATABASE_INDEX], args.database_username,
+                                                     args.database_password, args.database_name)
+    client_usage_cmd_init = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX], args.database_username,
+                                                     args.database_password, args.database_name)
     if get_db_cli_tool(args) != -1:
       retcode, out, err = execute_remote_script(args, DATABASE_DROP_SCRIPTS[DATABASE_INDEX])
       if not retcode == 0:
         if retcode == -1:
-          print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
-          DATABASE_DROP_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+          print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+          'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+          ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
         print err
         return retcode
 
       retcode, out, err = execute_remote_script(args, DATABASE_INIT_SCRIPTS[DATABASE_INDEX])
       if not retcode == 0:
         if retcode == -1:
-          print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
-          DATABASE_DROP_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+          print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+          'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+          ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
         print err
         return retcode
 
     else:
-      print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + ' client in the path, for reset please run the following DDL against the DB: ' +
-      DATABASE_INIT_SCRIPTS[DATABASE_INDEX] + ", then: " + DATABASE_DROP_SCRIPTS[DATABASE_INDEX], True)
-      print_error_msg(DATABASE_CLI_TOOLS[DATABASE_INDEX] + " not found. Unable to perform automatic reset.")
+      print_warning_msg('Cannot find ' + client_desc + ' client in the path to reset the Ambari Server schema. To reset Ambari Server schema ' +
+      'you must run the following DDL against the database to drop the schema:' + os.linesep + client_usage_cmd_drop + os.linesep +
+      ', then you must run the following DDL against the database to create the schema ' + os.linesep + client_usage_cmd_init + os.linesep )
       return -1
 
   else:
@@ -1446,13 +1435,9 @@ def prompt_db_properties(args):
 
 # Store set of properties for remote database connection
 def store_remote_properties(args):
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  properties = Properties()
-
-  try:
-    properties.load(open(conf_file))
-  except Exception, e:
-    print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
     return -1
 
   properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "remote")
@@ -1478,6 +1463,9 @@ def store_remote_properties(args):
   properties.process_pair(JDBC_RCA_USER_NAME_PROPERTY, args.database_username)
   properties.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
 
+
+  conf_file = properties.fileName
+
   try:
     properties.store(open(conf_file, "w"))
   except Exception, e:
@@ -1488,14 +1476,21 @@ def store_remote_properties(args):
 
 # Initialize remote database schema
 def setup_remote_db(args):
+    
+  not_found_msg = "Cannot find {0} {1} client in the path to load the Ambari Server schema.\
+ Before starting Ambari Server, you must run the following DDL against the database to create \
+the schema ".format(DATABASE_NAMES[DATABASE_INDEX], str(DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]))
 
+  client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INIT_SCRIPTS[DATABASE_INDEX], args.database_username,
+                                                     args.database_password, args.database_name)
+  
   retcode, out, err = execute_remote_script(args, DATABASE_INIT_SCRIPTS[DATABASE_INDEX])
   if retcode != 0:
 
     if retcode == -1:
-      print_warning_msg('Cannot find ' + DATABASE_NAMES[DATABASE_INDEX] + 
-                        ' client in the path, for setup please run the following DDL against the DB: ' +
-                        DATABASE_INIT_SCRIPTS[DATABASE_INDEX], True)
+      print_warning_msg(not_found_msg + os.linesep + client_usage_cmd)
+      if not SILENT:
+        raw_input(PRESS_ENTER_MSG)
       return retcode
 
     print err
@@ -1540,7 +1535,8 @@ def execute_remote_script(args, scriptPath):
   tool = get_db_cli_tool(args)
   if not tool:
     args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
-    print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+    if VERBOSE:
+      print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
     return -1, "Client wasn't found", "Client wasn't found"
 
   if args.database == "postgres":
@@ -1578,13 +1574,9 @@ def execute_remote_script(args, scriptPath):
   return -2, "Wrong database", "Wrong database"
 
 def configure_database_username_password(args):
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  properties = Properties()
-
-  try:
-    properties.load(open(conf_file))
-  except Exception, e:
-    print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
     return -1
 
   username = properties[JDBC_USER_NAME_PROPERTY]
@@ -1601,14 +1593,11 @@ def configure_database_username_password(args):
 
 # Store local database connection properties
 def store_local_properties(args):
-  conf_file = search_file(AMBARI_PROPERTIES_FILE, get_conf_dir())
-  properties = Properties()
-
-  try:
-    properties.load(open(conf_file))
-  except Exception, e:
-    print 'Could not read ambari config file "%s": %s' % (conf_file, e)
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
     return -1
+
   properties.removeOldProp(JDBC_SCHEMA_PROPERTY)
   properties.removeOldProp(JDBC_HOSTNAME_PROPERTY)
   properties.removeOldProp(JDBC_DATABASE_PROPERTY)
@@ -1622,6 +1611,8 @@ def store_local_properties(args):
   properties.process_pair(PERSISTENCE_TYPE_PROPERTY, "local")
   properties.process_pair(JDBC_USER_NAME_PROPERTY, args.database_username)
   properties.process_pair(JDBC_PASSWORD_FILE_PROPERTY, store_password_file(args.database_password, JDBC_PASSWORD_FILENAME))
+  
+  conf_file = properties.fileName
 
   try:
     properties.store(open(conf_file, "w"))
@@ -1921,6 +1912,7 @@ class Properties(object):
     if stream.mode != 'r':
       raise ValueError, 'Stream should be opened in read-only mode!'
     try:
+      self.fileName = os.path.abspath(stream.name)
       lines = stream.readlines()
       self.__parse(lines)
     except IOError, e:

ambari-server/src/test/python/TestAmbaryServer.py

@@ -630,21 +630,17 @@ class TestAmbariServer(TestCase):
   @patch("os.stat")
   @patch("os.path.isfile")
   @patch("os.path.exists")
-  @patch("__builtin__.open")
   @patch.object(ambari_server, "track_jdk")
   @patch.object(ambari_server, "get_YN_input")
   @patch.object(ambari_server, "run_os_command")
-  @patch.object(ambari_server, "Properties")
   @patch.object(ambari_server, "write_property")
   @patch.object(ambari_server, "print_info_msg")
   @patch.object(ambari_server, "get_JAVA_HOME")
-  @patch.object(ambari_server, "get_conf_dir")
-  @patch.object(ambari_server, "search_file")
-  def test_download_jdk(self, search_file_mock, get_conf_dir_mock,
+  @patch.object(ambari_server, "get_ambari_properties")
+  def test_download_jdk(self, get_ambari_properties_mock,
                         get_JAVA_HOME_mock, print_info_msg_mock,
-                        write_property_mock, Properties_mock,
-                        run_os_command_mock, get_YN_input_mock, track_jdk_mock,
-                        openMock, path_existsMock,
+                        write_property_mock, run_os_command_mock,
+                        get_YN_input_mock, track_jdk_mock, path_existsMock,
                         path_isfileMock, statMock):
 
     out = StringIO.StringIO()
@@ -652,15 +648,16 @@ class TestAmbariServer(TestCase):
 
     args = MagicMock()
     args.java_home = "somewhere"
-    search_file_mock.return_value = None
+    path_existsMock.return_value = False
+    get_JAVA_HOME_mock.return_value = False
+    get_ambari_properties_mock.return_value = -1
 
     rcode = ambari_server.download_jdk(args)
 
     self.assertEqual(-1, rcode)
-    self.assertTrue(search_file_mock.called)
-    self.assertTrue(get_conf_dir_mock.called)
+    self.assertTrue(get_ambari_properties_mock.called)
 
-    search_file_mock.return_value = "something"
+    #search_file_mock.return_value = "something"
     get_JAVA_HOME_mock.return_value = True
     path_existsMock.return_value = True
     rcode = ambari_server.download_jdk(args)
@@ -671,14 +668,9 @@ class TestAmbariServer(TestCase):
     self.assertEqual(0, rcode)
     self.assertTrue(write_property_mock.called)
 
-    p = MagicMock()
-    Properties_mock.return_value = p
-    openMock.side_effect = Exception("test exception")
     path_existsMock.return_value = False
-    rcode = ambari_server.download_jdk(args)
-    self.assertEqual(-1, rcode)
-
-    openMock.side_effect = None
+    p = MagicMock()
+    get_ambari_properties_mock.return_value = p
     p.__getitem__.side_effect = KeyError("test exception")
     rcode = ambari_server.download_jdk(args)
     self.assertEqual(-1, rcode)
@@ -740,30 +732,21 @@ class TestAmbariServer(TestCase):
 
   @patch("platform.linux_distribution")
   @patch("platform.system")
-  @patch("__builtin__.open")
-  @patch.object(ambari_server, "Properties")
   @patch.object(ambari_server, "print_info_msg")
   @patch.object(ambari_server, "print_error_msg")
-  @patch.object(ambari_server, "search_file")
+  @patch.object(ambari_server, "get_ambari_properties")
+  @patch.object(ambari_server, "write_property")
   @patch.object(ambari_server, "get_conf_dir")
-  def test_configure_os_settings(self, get_conf_dir_mock, search_file_mock,
-                                 print_error_msg_mock, print_info_msg_mock,
-                                 Properties_mock, openMock, systemMock,
-                                 distMock):
+  def test_configure_os_settings(self, get_conf_dir_mock, write_property_mock, get_ambari_properties_mock, print_error_msg_mock,
+                                 print_info_msg_mock, systemMock, distMock):
 
-    search_file_mock.return_value = None
+    get_ambari_properties_mock.return_value = -1
     rcode = ambari_server.configure_os_settings()
     self.assertEqual(-1, rcode)
-
-    search_file_mock.return_value = "something"
+    
     p = MagicMock()
-    Properties_mock.return_value = p
-    openMock.side_effect = Exception("exception")
-    rcode = ambari_server.configure_os_settings()
-    self.assertEqual(-1, rcode)
-
-    p.__getitem__.return_value = "something"
-    openMock.side_effect = None
+    p[ambari_server.OS_TYPE_PROPERTY] = 'somevalue'
+    get_ambari_properties_mock.return_value = p
     rcode = ambari_server.configure_os_settings()
     self.assertEqual(0, rcode)
 
@@ -774,10 +757,9 @@ class TestAmbariServer(TestCase):
 
     systemMock.return_value = "Linux"
     distMock.return_value = ("CentOS", "6.3", None)
-    f = MagicMock()
-    openMock.return_value = f
     rcode = ambari_server.configure_os_settings()
     self.assertEqual(0, rcode)
+    self.assertTrue(write_property_mock.called)
 
 
 
@@ -1102,13 +1084,14 @@ class TestAmbariServer(TestCase):
 
 
   @patch("sys.exit")
+  @patch.object(ambari_server, "download_jdk")
   @patch.object(ambari_server, "get_db_cli_tool")
   @patch.object(ambari_server, "store_remote_properties")
   @patch.object(ambari_server, "is_local_database")
   @patch.object(ambari_server, "check_iptables")
   @patch.object(ambari_server, "check_jdbc_drivers")
   def test_setup_remote_db_wo_client(self, check_jdbc_drivers_mock, check_iptables_mock, is_local_db_mock,
-                                     store_remote_properties_mock, get_db_cli_tool_mock, exit_mock):
+                                     store_remote_properties_mock, get_db_cli_tool_mock, download_jdk_mock, exit_mock):
 
     out = StringIO.StringIO()
     sys.stdout = out
@@ -1217,6 +1200,101 @@ class TestAmbariServer(TestCase):
     
     sys.stdout = sys.__stdout__
 
+  @patch.object(ambari_server, "get_ambari_properties")
+  @patch.object(ambari_server, "find_jdbc_driver")
+  @patch.object(ambari_server, "copy_files")
+  @patch.object(ambari_server, "print_error_msg")
+  @patch.object(ambari_server, "print_warning_msg")
+  @patch('__builtin__.raw_input')
+  @patch("sys.exit")
+  def test_check_jdbc_drivers(self, exit_mock, raw_input_mock, print_warning_msg, print_error_msg_mock, copy_files_mock,
+                              find_jdbc_driver_mock, get_ambari_properties_mock):
+
+    out = StringIO.StringIO()
+    sys.stdout = out
+
+    args = MagicMock()
+    
+    # Check positive scenario
+    drivers_list = ['driver_file']
+    resources_dir = '/tmp'
+    
+    get_ambari_properties_mock.return_value = {ambari_server.RESOURCES_DIR_KEY : resources_dir}
+    find_jdbc_driver_mock.return_value = drivers_list
+    
+    args.database = "oracle"
+    
+    rcode = ambari_server.check_jdbc_drivers(args)
+    
+    self.assertEqual(0, rcode)
+    copy_files_mock.assert_called_with(drivers_list, resources_dir)
+    
+    # Check negative scenarios
+    # Silent option, no drivers
+    ambari_server.SILENT = True
+    
+    find_jdbc_driver_mock.return_value = -1
+    
+    rcode = ambari_server.check_jdbc_drivers(args)
+    
+    self.assertTrue(print_error_msg_mock.called)
+    self.assertTrue(exit_mock.called)
+    
+    # Non-Silent option, no drivers
+    ambari_server.SILENT = False
+    
+    find_jdbc_driver_mock.return_value = -1
+    
+    rcode = ambari_server.check_jdbc_drivers(args)
+    
+    self.assertTrue(exit_mock.called)
+    self.assertTrue(print_error_msg_mock.called)
+    
+    # Non-Silent option, no drivers at first ask, present drivers after that
+    
+    find_jdbc_driver_mock.side_effect = [-1, drivers_list]
+    
+    rcode = ambari_server.check_jdbc_drivers(args)
+    
+    self.assertEqual(0, rcode)
+    copy_files_mock.assert_called_with(drivers_list, resources_dir)
+    
+    # Non-Silent option, no drivers at first ask, present drivers after that
+    find_jdbc_driver_mock.reset()
+    find_jdbc_driver_mock.side_effect = [-1, -1]
+    
+    rcode = ambari_server.check_jdbc_drivers(args)
+    
+    self.assertTrue(exit_mock.called)
+    self.assertTrue(print_error_msg_mock.called)
+    
+    
+    sys.stdout = sys.__stdout__
+    
+    
+  @patch.object(ambari_server, "search_file")
+  def test_get_ambari_properties(self, search_file_mock):
+
+    search_file_mock.return_value = None
+    rcode = ambari_server.get_ambari_properties()
+    self.assertEqual(rcode, -1)
+  
+    tf1 = tempfile.NamedTemporaryFile()
+    search_file_mock.return_value = tf1.name
+    prop_name='name'
+    prop_value='val'
+    
+    with open(tf1.name, 'w') as fout:
+      fout.write(prop_name + '=' + prop_value)
+    fout.close()
+
+    properties = ambari_server.get_ambari_properties()
+    
+    self.assertEqual(properties[prop_name], prop_value)
+    self.assertEqual(properties.fileName, os.path.abspath(tf1.name))
+    
+    sys.stdout = sys.__stdout__
+
   @patch.object(ambari_server, "search_file")
   def test_parse_properties_file(self, search_file_mock):