AMBARI-17955. Enabled download of berkeley DB jar file to be provisioned during install (Venkat Ranganathan via alejandro)

ambari-server/src/main/python/ambari_server/dbConfiguration.py

@@ -40,11 +40,11 @@ SETUP_DB_CONNECT_ATTEMPTS = 3
 
 USERNAME_PATTERN = "^[a-zA-Z_][a-zA-Z0-9_\-]*$"
 PASSWORD_PATTERN = "^[a-zA-Z0-9_-]*$"
-CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere"]
-DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere"]
+CUSTOM_JDBC_DB_NAMES = ["postgres", "mysql", "mssql", "oracle", "hsqldb", "sqlanywhere", "bdb"]
+DATABASE_NAMES = ["postgres", "oracle", "mysql", "mssql", "sqlanywhere", "bdb"]
 DATABASE_FULL_NAMES = {"oracle": "Oracle", "mysql": "MySQL", "mssql": "Microsoft SQL Server", "postgres":
-  "PostgreSQL", "sqlanywhere": "SQL Anywhere"}
-LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sqlanywhere']
+  "PostgreSQL", "sqlanywhere": "SQL Anywhere", "bdb" : "Berkeley DB"}
+LINUX_DBMS_KEYS_LIST = [ 'embedded', 'oracle', 'mysql', 'postgres', 'mssql', 'sqlanywhere', 'bdb']
 AMBARI_DATABASE_NAME = "ambari"
 AMBARI_DATABASE_TITLE = "ambari"
 
@@ -55,7 +55,8 @@ default_connectors_map = { "mssql":"sqljdbc4.jar",
                            "postgres":"postgresql-jdbc.jar",
                            "oracle":"ojdbc.jar",
                            "sqlanywhere":"sajdbc4.jar",
-                           "hsqldb":"hsqldb.jar"}
+                           "hsqldb":"hsqldb.jar",
+                           "bdb": 'je-5.0.73.jar'}
 
 STORAGE_TYPE_LOCAL = 'local'
 STORAGE_TYPE_REMOTE = 'remote'
@@ -343,7 +344,7 @@ class DBMSConfigFactoryWindows(DBMSConfigFactory):
 class DBMSConfigFactoryLinux(DBMSConfigFactory):
   def __init__(self):
     from ambari_server.dbConfiguration_linux import createPGConfig, createOracleConfig, createMySQLConfig, \
-      createMSSQLConfig, createSQLAConfig
+      createMSSQLConfig, createSQLAConfig, createBDBConfig
 
     self.DBMS_KEYS_LIST = LINUX_DBMS_KEYS_LIST
 
@@ -353,7 +354,8 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
       'postgres',
       'mssql',
       'hsqldb',
-      'sqlanywhere'
+      'sqlanywhere',
+      'bdb'
     ]
 
     self.DBMS_LIST = [
@@ -362,7 +364,9 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
       DBMSDesc(self.DBMS_KEYS_LIST[2], STORAGE_TYPE_REMOTE, 'MySQL / MariaDB', '', createMySQLConfig),
       DBMSDesc(self.DBMS_KEYS_LIST[3], STORAGE_TYPE_REMOTE, 'PostgreSQL', '', createPGConfig),
       DBMSDesc(self.DBMS_KEYS_LIST[4], STORAGE_TYPE_REMOTE, 'Microsoft SQL Server', 'Tech Preview', createMSSQLConfig),
-      DBMSDesc(self.DBMS_KEYS_LIST[5], STORAGE_TYPE_REMOTE, 'SQL Anywhere', '', createSQLAConfig)
+      DBMSDesc(self.DBMS_KEYS_LIST[5], STORAGE_TYPE_REMOTE, 'SQL Anywhere', '', createSQLAConfig),
+      DBMSDesc(self.DBMS_KEYS_LIST[6], STORAGE_TYPE_REMOTE, 'BDB', '', createBDBConfig)
+
     ]
 
     self.DBMS_DICT = \
@@ -381,6 +385,8 @@ class DBMSConfigFactoryLinux(DBMSConfigFactory):
       self.DBMS_KEYS_LIST[3] + '-' + STORAGE_TYPE_REMOTE: 3,
       self.DBMS_KEYS_LIST[5] + '-': 5,
       self.DBMS_KEYS_LIST[5] + '-' + STORAGE_TYPE_REMOTE: 5,
+      self.DBMS_KEYS_LIST[6] + '-': 6,
+      self.DBMS_KEYS_LIST[6] + '-' + STORAGE_TYPE_LOCAL: 6,
     }
 
     self.DBMS_PROMPT_PATTERN = "[{0}] - {1}{2}\n"

ambari-server/src/main/python/ambari_server/dbConfiguration_linux.py

@@ -1119,3 +1119,37 @@ class SQLAConfig(LinuxDBMSConfig):
 
 def createSQLAConfig(options, properties, storage_type, dbId):
   return SQLAConfig(options, properties, storage_type)
+
+class BDBConfig(LinuxDBMSConfig):
+  def __init__(self, options, properties, storage_type):
+    super(BDBConfig, self).__init__(options, properties, storage_type)
+
+    #Init the database configuration data here, if any
+    self.dbms = "bdb"
+    self.dbms_full_name = "Berkeley DB Jar file"
+    self.driver_class_name = "com.berkeleydb.Driver"
+    self.driver_file_name = "je-5.0.73.jar"
+    self.driver_symlink_name = "bdb-jdbc-driver.jar"
+
+    self.database_storage_name = "Database"
+    self.client_tool_usage_pattern = ''
+
+  #
+  # Private implementation
+  #
+
+
+  def _is_jdbc_driver_installed(self, properties):
+    return LinuxDBMSConfig._find_jdbc_driver("*je-*.jar")
+
+  def _get_jdbc_driver_path(self, properties):
+    super(BDBConfig, self)._get_jdbc_driver_path(properties)
+
+
+  def _configure_database_name(self):
+    self.database_name = LinuxDBMSConfig._get_validated_db_name(self.database_storage_name, self.database_name)
+    return True
+
+
+def createBDBConfig(options, properties, storage_type, dbId):
+  return BDBConfig(options, properties, storage_type)

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py

@@ -18,11 +18,13 @@ limitations under the License.
 """
 
 import os.path
+import traceback
 
 # Local Imports
 from resource_management.core.environment import Environment
 from resource_management.core.source import InlineTemplate
 from resource_management.core.source import Template
+from resource_management.core.source import  DownloadSource
 from resource_management.core.resources import Execute
 from resource_management.core.resources.service import Service
 from resource_management.core.resources.service import ServiceConfig
@@ -196,6 +198,24 @@ def falcon(type, action = None, upgrade_type=None):
     process_exists = format("ls {server_pid_file} && ps -p {pid}")
 
     if action == 'start':
+      if not os.path.exists(params.target_jar_file):
+        try :
+          File(params.target_jar_file,
+           content = DownloadSource(params.bdb_resource_name))
+        except :
+           exc_msg = traceback.format_exc()
+           exception_message = format("Caught Exception while downloading {bdb_resource_name}:\n{exc_msg}")
+           Logger.error(exception_message)
+
+        if not os.path.isfile(params.target_jar_file) :
+          error_message = """
+If you are using bdb as the Falcon graph db store, please run
+ambari-server setup --jdbc-db=bdb --jdbc-driver=<path to je5.0.73.jar
+on the ambari server host.  Otherwise falcon startup will fail.
+Otherwise please configure Falcon to use HBase as the backend as described
+in the Falcon documentation.
+"""
+          Logger.error(error_message)
       try:
         Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'),
           user = params.falcon_user,

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py

@@ -45,6 +45,7 @@ version = default("/commandParams/version", None)
 stack_version_unformatted = status_params.stack_version_unformatted
 stack_version_formatted = status_params.stack_version_formatted
 upgrade_direction = default("/commandParams/upgrade_direction", None)
+jdk_location = config['hostLevelParams']['jdk_location']
 
 # current host stack version
 current_version = default("/hostLevelParams/current_version", None)
@@ -134,6 +135,7 @@ supports_falcon_extensions = (stack_version_formatted and check_stack_feature(St
 local_data_mirroring_dir = format('{stack_root}/current/falcon-server/data-mirroring')
 dfs_data_mirroring_dir = "/apps/data-mirroring"
 
+
 ########################################################
 ############# Atlas related params #####################
 ########################################################
@@ -169,6 +171,11 @@ default_fs = config['configurations']['core-site']['fs.defaultFS']
 
 dfs_type = default("/commandParams/dfs_type", "")
 
+bdb_jar_name = "je-5.0.73.jar"
+bdb_resource_name = format("{jdk_location}/{bdb_jar_name}")
+target_jar_file = os.path.join(falcon_webinf_lib, bdb_jar_name)
+
+
 import functools
 #create partial functions with common arguments for every HdfsResource call
 #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code

ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py

@@ -86,6 +86,12 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
     self.__atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
     super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
 
+  def recommendFalconConfigurations(self, configurations, clusterData, services, hosts):
+    # Set the classname for the Falcon hook to 2.5 classname = all other 2.3
+    # stack advisor changes are needed.
+    self.atlasFalconHookClassName = "org.apache.atlas.falcon.service.AtlasService"
+    super(HDP25StackAdvisor,self).recommendFalconConfigurations(configurations, clusterData, services, hosts)
+
   def createComponentLayoutRecommendations(self, services, hosts):
     parentComponentLayoutRecommendations = super(HDP25StackAdvisor, self).createComponentLayoutRecommendations(
       services, hosts)
@@ -396,6 +402,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
       "ATLAS": self.recommendAtlasConfigurations,
       "RANGER_KMS": self.recommendRangerKMSConfigurations,
       "STORM": self.recommendStormConfigurations,
+      "FALCON": self.recommendFalconConfigurations,
       "OOZIE": self.recommendOozieConfigurations,
       "SPARK2": self.recommendSpark2Configurations
     }

ambari-server/src/test/python/TestAmbariServer.py

@@ -5331,7 +5331,7 @@ class TestAmbariServer(TestCase):
     get_ambari_properties_mock.return_value = properties
     get_ambari_properties_3_mock.side_effect = get_ambari_properties_2_mock.side_effect = [properties, properties2, properties2]
 
-    isfile_mock.side_effect = [False, True, False, False]
+    isfile_mock.side_effect = [False, True, False, False, False]
 
     try:
       upgrade(args)
@@ -6550,7 +6550,7 @@ class TestAmbariServer(TestCase):
 
     isdir_mock.return_value = True
 
-    isfile_mock.side_effect = [True, False, False, False]
+    isfile_mock.side_effect = [True, False, False, False, False]
 
     del args.database_index
     del args.persistence_type
@@ -6572,7 +6572,7 @@ class TestAmbariServer(TestCase):
     get_ambari_properties_mock.reset_mock()
     os_symlink_mock.reset_mock()
 
-    isfile_mock.side_effect = [False, False, False, False]
+    isfile_mock.side_effect = [False, False, False, False, False]
 
     check_jdbc_drivers(args)
 

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -43,6 +43,10 @@ class TestFalconServer(RMFTestCase):
 
     self.assert_configure_default()
 
+    self.assertResourceCalled('File', '/usr/lib/falcon/server/webapp/falcon/WEB-INF/lib/je-5.0.73.jar',
+      content=DownloadSource('http://c6401.ambari.apache.org:8080/resources//je-5.0.73.jar')
+    )
+
     self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000',
       path = ['/usr/bin'],
       user = 'falcon',
@@ -368,6 +372,7 @@ class TestFalconServer(RMFTestCase):
         owner = 'falcon',
         create_parents = True,
     )
+   
     self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-start -port 15000',
         environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'},
         path = ['/usr/hdp/current/hadoop-client/bin'],
@@ -375,7 +380,7 @@ class TestFalconServer(RMFTestCase):
         not_if = 'ls /var/run/falcon/falcon.pid && ps -p ',
     )
     self.assertNoMoreResources()
-    
+
   @patch("resource_management.libraries.functions.security_commons.build_expectations")
   @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
   @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")