AMBARI-16912. Oozie and Hive Server start fail during EU with missing DB class exception.(vbrodetskyi)

ambari-server/src/main/python/ambari_server/dbConfiguration.py

@@ -40,6 +40,7 @@ SETUP_DB_CONNECT_ATTEMPTS = 3
 
 USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
 PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
+CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere"]
 DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere"]
 DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "mssql": "Microsoft SQL Server", "postgres":
   "PostgreSQL", "sqlanywhere": "SQL Anywhere"}
@@ -47,6 +48,15 @@ LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sq
 AMBARI_DATABASE_NAME = "ambari"
 AMBARI_DATABASE_TITLE = "ambari"
 
+TAR_GZ_ARCHIVE_TYPE = ".tar.gz"
+
+default_connectors_map = { "mssql":"sqljdbc4.jar",
+                           "mysql":"mysql-connector-java.jar",
+                           "postgres":"postgresql-jdbc.jar",
+                           "oracle":"ojdbc.jar",
+                           "sqlanywhere":"sajdbc4.jar",
+                           "hsqldb":"hsqldb.jar"}
+
 STORAGE_TYPE_LOCAL = 'local'
 STORAGE_TYPE_REMOTE = 'remote'
 

ambari-server/src/main/python/ambari_server/serverSetup.py

@@ -34,7 +34,7 @@ from ambari_commons.os_check import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons.os_utils import copy_files, run_os_command, is_root
 from ambari_commons.str_utils import compress_backslashes
-from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers
+from ambari_server.dbConfiguration import DBMSConfigFactory, TAR_GZ_ARCHIVE_TYPE, default_connectors_map, check_jdbc_drivers
 from ambari_server.serverConfiguration import configDefaults, JDKRelease, \
   get_ambari_properties, get_is_secure, get_is_persisted, get_java_exe_path, get_JAVA_HOME, \
   get_resources_location, get_value_from_properties, read_ambari_user, update_properties, validate_jdk, write_property, \
@@ -60,7 +60,6 @@ SE_MODE_PERMISSIVE = "permissive"
 
 PERSISTENCE_TYPE_PROPERTY = "server.persistence.type"
 
-TAR_GZ_ARCHIVE_TYPE = ".tar.gz"
 
 # Non-root user setup commands
 NR_USER_COMMENT = "Ambari user"
@@ -77,11 +76,7 @@ UNTAR_JDK_ARCHIVE = "tar --no-same-owner -xvf {0}"
 JDK_PROMPT = "[{0}] {1}\n"
 JDK_VALID_CHOICES = "^[{0}{1:d}]$"
 
-default_connectors_map = { "mssql":"sqljdbc4.jar",
-                           "mysql":"mysql-connector-java.jar",
-                           "postgres":"postgresql-jdbc.jar",
-                           "oracle":"ojdbc.jar",
-                           "sqlanywhere":"sajdbc4.jar"}
+
 
 def get_supported_jdbc_drivers():
   factory = DBMSConfigFactory()

ambari-server/src/main/python/ambari_server/serverUpgrade.py

@@ -31,10 +31,10 @@ import optparse
 from ambari_commons.exceptions import FatalException
 from ambari_commons.logging_utils import print_info_msg, print_warning_msg, print_error_msg, get_verbose
 from ambari_commons.os_utils import is_root, run_os_command
-from ambari_server.dbConfiguration import DBMSConfigFactory, check_jdbc_drivers, \
-  get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST
+from ambari_server.dbConfiguration import DBMSConfigFactory, CUSTOM_JDBC_DB_NAMES, TAR_GZ_ARCHIVE_TYPE,  check_jdbc_drivers, \
+  get_jdbc_driver_path, ensure_jdbc_driver_is_installed, LINUX_DBMS_KEYS_LIST, default_connectors_map
 from ambari_server.properties import Properties
-from ambari_server.serverConfiguration import configDefaults, \
+from ambari_server.serverConfiguration import configDefaults, get_resources_location, update_properties, \
   check_database_name_property, get_ambari_properties, get_ambari_version, \
   get_java_exe_path, get_stack_location, parse_properties_file, read_ambari_user, update_ambari_properties, \
   update_database_name_property, get_admin_views_dir, get_views_dir, \
@@ -413,6 +413,26 @@ def upgrade(args):
   if properties.get_property(LDAP_PRIMARY_URL_PROPERTY) and not properties.get_property(IS_LDAP_CONFIGURED):
     args.warnings.append("Existing LDAP configuration is detected. You must run the \"ambari-server setup-ldap\" command to adjust existing LDAP configuration.")
 
+  # adding custom jdbc name and previous custom jdbc properties
+  # we need that to support new dynamic jdbc names for upgraded ambari
+  add_jdbc_properties(properties)
+
+
+def add_jdbc_properties(properties):
+  for db_name in CUSTOM_JDBC_DB_NAMES:
+    if db_name == "sqlanywhere":
+      symlink_name = db_name + "-jdbc-driver" + TAR_GZ_ARCHIVE_TYPE
+    else:
+      symlink_name = db_name + "-jdbc-driver.jar"
+
+    resources_dir = get_resources_location(properties)
+    custom_db_jdbc_property_name = "custom." + db_name + ".jdbc.name"
+
+    if os.path.lexists(os.path.join(resources_dir, symlink_name)):
+      properties.process_pair(custom_db_jdbc_property_name, symlink_name)
+      properties.process_pair("previous." + custom_db_jdbc_property_name, default_connectors_map[db_name])
+      update_properties(properties)
+
 
 #
 # Set current cluster version (run Finalize during manual RU)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py

@@ -104,8 +104,10 @@ def hive_service(name, action='start', upgrade_type=None):
         path_to_jdbc = format("{hive_lib}/") + params.default_connectors_map[params.hive_jdbc_driver]
         if not os.path.isfile(path_to_jdbc):
           path_to_jdbc = format("{hive_lib}/") + "*"
-          print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
-                " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+          error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
+                          " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+          print error_message
+          Logger.error(error_message)
 
       db_connection_check_command = format(
         "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service_interactive.py

@@ -73,8 +73,10 @@ def hive_service_interactive(name, action='start', upgrade_type=None):
         path_to_jdbc = format("{hive_interactive_lib}/") + params.default_connectors_map[params.hive_jdbc_driver]
         if not os.path.isfile(path_to_jdbc):
           path_to_jdbc = format("{hive_interactive_lib}/") + "*"
-          print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
+          error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
                 " in hive lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+          print error_message
+          Logger.error(error_message)
 
       db_connection_check_command = format(
         "{java64_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{hive_jdbc_connection_url}' {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")

ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py

@@ -20,6 +20,7 @@ limitations under the License.
 import os
 from resource_management import *
 from resource_management.core.shell import as_user
+from resource_management.core.logger import Logger
 from resource_management.libraries.functions.show_logs import show_logs
 from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -73,8 +74,10 @@ def oozie_service(action = 'start', upgrade_type=None):
         path_to_jdbc = format("{oozie_libext_dir}/") + params.default_connectors_map[params.jdbc_driver_name]
         if not os.path.isfile(path_to_jdbc):
           path_to_jdbc = format("{oozie_libext_dir}/") + "*"
-          print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
+          error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.jdbc_driver_name] + \
                 " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+          print error_message
+          Logger.error(error_message)
 
       db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}")
     else:

ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/package/scripts/kms.py

@@ -156,8 +156,10 @@ def kms(upgrade_type=None):
         path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + params.default_connectors_map[params.db_flavor]
         if not os.path.isfile(path_to_jdbc):
           path_to_jdbc = format("{kms_home}/ews/webapp/lib/") + "*"
-          print "Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.hive_jdbc_driver] + \
+          error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.db_flavor] + \
                 " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
+          print error_message
+          Logger.error(error_message)
 
       cp = cp + os.pathsep + path_to_jdbc