AMBARI-11630 - Falcon Server Seems To Have Hadoop 2.2 Paths After Upgrade (jonathanhurley)

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py

@@ -17,177 +17,195 @@ limitations under the License.
 
 """
 
-from resource_management import *
 import os.path
+
+from resource_management.core.environment import Environment
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.core.resources import Execute
+from resource_management.core.resources.service import Service
+from resource_management.core.resources.service import ServiceConfig
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.libraries.script import Script
+from resource_management.libraries.resources import PropertiesFile
+from resource_management.libraries.functions import format
+
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
-@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+@OsFamilyFuncImpl(os_family = OsFamilyImpl.DEFAULT)
 def falcon(type, action = None):
   import params
+
   if action == 'config':
     Directory(params.falcon_pid_dir,
-              owner=params.falcon_user,
-              recursive=True
-    )
+      owner = params.falcon_user,
+      recursive = True)
+
     Directory(params.falcon_log_dir,
-              owner=params.falcon_user,
-              recursive=True
-    )
+      owner = params.falcon_user,
+      recursive = True)
+
     Directory(params.falcon_webapp_dir,
-              owner=params.falcon_user,
-              recursive=True
-    )
+      owner = params.falcon_user,
+      recursive = True)
+
     Directory(params.falcon_home,
-              owner=params.falcon_user,
-              recursive=True
-    )
+      owner = params.falcon_user,
+      recursive = True)
+
     Directory(params.etc_prefix_dir,
-              mode=0755,
-              recursive=True
-    )
+      mode = 0755,
+      recursive = True)
+
     Directory(params.falcon_conf_dir,
-              owner=params.falcon_user,
-              recursive=True
-    )
+      owner = params.falcon_user,
+      recursive = True)
+
     File(params.falcon_conf_dir + '/falcon-env.sh',
-         content=InlineTemplate(params.falcon_env_sh_template),
-         owner=params.falcon_user
-    )
+      content = InlineTemplate(params.falcon_env_sh_template),
+      owner = params.falcon_user)
+
     File(params.falcon_conf_dir + '/client.properties',
-         content=Template('client.properties.j2'),
-         mode=0644,
-         owner=params.falcon_user
-    )
+      content = Template('client.properties.j2'),
+      mode = 0644,
+      owner = params.falcon_user)
+
     PropertiesFile(params.falcon_conf_dir + '/runtime.properties',
-                   properties=params.falcon_runtime_properties,
-                   mode=0644,
-                   owner=params.falcon_user
-    )
+      properties = params.falcon_runtime_properties,
+      mode = 0644,
+      owner = params.falcon_user)
+
     PropertiesFile(params.falcon_conf_dir + '/startup.properties',
-                   properties=params.falcon_startup_properties,
-                   mode=0644,
-                   owner=params.falcon_user
-    )
+      properties = params.falcon_startup_properties,
+      mode = 0644,
+      owner = params.falcon_user)
 
     if params.falcon_graph_storage_directory:
       Directory(params.falcon_graph_storage_directory,
-                owner=params.falcon_user,
-                group=params.user_group,
-                mode=0775,
-                recursive=True,
-                cd_access="a"
-      )
+        owner = params.falcon_user,
+        group = params.user_group,
+        mode = 0775,
+        recursive = True,
+        cd_access = "a")
 
     if params.falcon_graph_serialize_path:
       Directory(params.falcon_graph_serialize_path,
-                owner=params.falcon_user,
-                group=params.user_group,
-                mode=0775,
-                recursive=True,
-                cd_access="a"
-      )
+        owner = params.falcon_user,
+        group = params.user_group,
+        mode = 0775,
+        recursive = True,
+        cd_access = "a")
 
   if type == 'server':
     if action == 'config':
       if params.store_uri[0:4] == "hdfs":
         params.HdfsResource(params.store_uri,
-                             type="directory",
-                             action="create_on_execute",
-                             owner=params.falcon_user,
-                             mode=0755
-        )
+          type = "directory",
+          action = "create_on_execute",
+          owner = params.falcon_user,
+          mode = 0755)
       elif params.store_uri[0:4] == "file":
         Directory(params.store_uri[7:],
-                  owner=params.falcon_user,
-                  recursive=True
-        )
+          owner = params.falcon_user,
+          recursive = True)
+
+      # TODO change to proper mode
       params.HdfsResource(params.flacon_apps_dir,
-                           type="directory",
-                           action="create_on_execute",
-                           owner=params.falcon_user,
-                           mode=0777 #TODO change to proper mode
-      )
+        type = "directory",
+        action = "create_on_execute",
+        owner = params.falcon_user,
+        mode = 0777)
+
       if params.falcon_store_uri[0:4] == "hdfs":
         params.HdfsResource(params.falcon_store_uri,
-                             type="directory",
-                             action="create_on_execute",
-                             owner=params.falcon_user,
-                             mode=0755
-        )
+          type = "directory",
+          action = "create_on_execute",
+          owner = params.falcon_user,
+          mode = 0755)
       elif params.falcon_store_uri[0:4] == "file":
         Directory(params.falcon_store_uri[7:],
-                  owner=params.falcon_user,
-                  recursive=True
-        )
+          owner = params.falcon_user,
+          recursive = True)
+
       if params.supports_hive_dr:
         params.HdfsResource(params.dfs_data_mirroring_dir,
-                            type="directory",
-                            action="create_on_execute",
-                            owner=params.falcon_user,
-                            group=params.user_group,
-                            recursive_chown = True,
-                            recursive_chmod = True,
-                            mode=0770,
-                            source=params.local_data_mirroring_dir
-        )
-      params.HdfsResource(None, action="execute")
+          type = "directory",
+          action = "create_on_execute",
+          owner = params.falcon_user,
+          group = params.user_group,
+          recursive_chown = True,
+          recursive_chmod = True,
+          mode = 0770,
+          source = params.local_data_mirroring_dir)
+
+      params.HdfsResource(None, action = "execute")
       Directory(params.falcon_local_dir,
-                owner=params.falcon_user,
-                recursive=True,
-                cd_access="a"
-      )
+        owner = params.falcon_user,
+        recursive = True,
+        cd_access = "a")
+
       if params.falcon_embeddedmq_enabled == True:
-        Directory(os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")),
-                  owner=params.falcon_user,
-                  recursive=True
-        )
+        Directory(
+          os.path.abspath(os.path.join(params.falcon_embeddedmq_data, "..")),
+          owner = params.falcon_user,
+          recursive = True)
+
         Directory(params.falcon_embeddedmq_data,
-                  owner=params.falcon_user,
-                  recursive=True
-        )
+          owner = params.falcon_user,
+          recursive = True)
+
+    # although Falcon's falcon-config.sh will use 'which hadoop' to figure
+    # this out, in an upgraded cluster, it's possible that 'which hadoop'
+    # still points to older binaries; it's safer to just pass in the
+    # hadoop home directory to use
+    environment_dictionary = { "HADOOP_HOME" : params.hadoop_home_dir }
 
     if action == 'start':
       Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'),
-              user=params.falcon_user,
-              path=params.hadoop_bin_dir
-      )
+        user = params.falcon_user,
+        path = params.hadoop_bin_dir,
+        environment=environment_dictionary)
+
     if action == 'stop':
       Execute(format('{falcon_home}/bin/falcon-stop'),
-              user=params.falcon_user,
-              path=params.hadoop_bin_dir
-      )
-      File(params.server_pid_file,
-           action='delete'
-      )
-
-@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+        user = params.falcon_user,
+        path = params.hadoop_bin_dir,
+        environment=environment_dictionary)
+
+      File(params.server_pid_file, action = 'delete')
+
+
+@OsFamilyFuncImpl(os_family = OSConst.WINSRV_FAMILY)
 def falcon(type, action = None):
   import params
+
   if action == 'config':
     env = Environment.get_instance()
     # These 2 parameters are used in ../templates/client.properties.j2
     env.config.params["falcon_host"] = params.falcon_host
     env.config.params["falcon_port"] = params.falcon_port
     File(os.path.join(params.falcon_conf_dir, 'falcon-env.sh'),
-         content=InlineTemplate(params.falcon_env_sh_template)
-    )
+      content = InlineTemplate(params.falcon_env_sh_template))
+
     File(os.path.join(params.falcon_conf_dir, 'client.properties'),
-         content=Template('client.properties.j2')
-    )
+      content = Template('client.properties.j2'))
+
     PropertiesFile(os.path.join(params.falcon_conf_dir, 'runtime.properties'),
-                   properties=params.falcon_runtime_properties
-    )
+      properties = params.falcon_runtime_properties)
+
     PropertiesFile(os.path.join(params.falcon_conf_dir, 'startup.properties'),
-                   properties=params.falcon_startup_properties
-    )
+      properties = params.falcon_startup_properties)
 
   if type == 'server':
     ServiceConfig(params.falcon_win_service_name,
-                  action="change_user",
-                  username=params.falcon_user,
-                  password = Script.get_password(params.falcon_user))
+      action = "change_user",
+      username = params.falcon_user,
+      password = Script.get_password(params.falcon_user))
+
     if action == 'start':
-      Service(params.falcon_win_service_name, action="start")
+      Service(params.falcon_win_service_name, action = "start")
+
     if action == 'stop':
-      Service(params.falcon_win_service_name, action="stop")
+      Service(params.falcon_win_service_name, action = "stop")

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py

@@ -19,13 +19,17 @@ limitations under the License.
 
 import falcon_server_upgrade
 
-from resource_management import *
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import hdp_select
-from resource_management.libraries.functions.version import *
-from resource_management.libraries.functions.security_commons import build_expectations, \
-  cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
-  FILE_TYPE_PROPERTIES
+from resource_management.libraries.functions import check_process_status
+from resource_management.libraries.functions.security_commons import build_expectations
+from resource_management.libraries.functions.security_commons import cached_kinit_executor
+from resource_management.libraries.functions.security_commons import get_params_from_filesystem
+from resource_management.libraries.functions.security_commons import validate_security_config_properties
+from resource_management.libraries.functions.security_commons import FILE_TYPE_PROPERTIES
+
 from falcon import falcon
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -46,6 +50,7 @@ class FalconServer(Script):
     import params
     env.set_params(params)
     falcon('server', action='stop')
+
     # if performing an upgrade, backup some directories after stopping falcon
     if rolling_restart:
       falcon_server_upgrade.post_stop_backup()
@@ -71,7 +76,7 @@ class FalconServerLinux(FalconServer):
 
     # this function should not execute if the version can't be determined or
     # is not at least HDP 2.2.0.0
-    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
+    if Script.is_hdp_stack_less_than("2.2"):
       return
 
     Logger.info("Executing Falcon Server Rolling Upgrade pre-restart")
@@ -144,11 +149,15 @@ class FalconServerWindows(FalconServer):
 
   def install(self, env):
     import params
+    from resource_management.libraries.functions.windows_service_utils import check_windows_service_exists
+
     if not check_windows_service_exists(params.falcon_win_service_name):
       self.install_packages(env)
 
   def status(self, env):
     import status_params
+    from resource_management.libraries.functions.windows_service_utils import check_windows_service_status
+
     check_windows_service_status(status_params.falcon_win_service_name)
 
 if __name__ == "__main__":

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py

@@ -38,6 +38,7 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 etc_prefix_dir = "/etc/falcon"
 
 # hadoop params
+hadoop_home_dir = hdp_select.get_hadoop_dir("home")
 hadoop_bin_dir = hdp_select.get_hadoop_dir("bin")
 
 if Script.is_hdp_stack_greater_or_equal("2.2"):

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -33,34 +33,37 @@ class TestFalconServer(RMFTestCase):
 
   def test_start_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
-                       classname="FalconServer",
-                       command="start",
-                       config_file="default.json",
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES
-    )
+      classname="FalconServer",
+      command="start",
+      config_file="default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
     self.assert_configure_default()
+
     self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000',
-                              path = ['/usr/bin'],
-                              user = 'falcon',
-                              )
+      path = ['/usr/bin'],
+      user = 'falcon',
+      environment = {'HADOOP_HOME': '/usr/lib/hadoop'})
+
     self.assertNoMoreResources()
 
   def test_stop_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
-                       classname="FalconServer",
-                       command="stop",
-                       config_file="default.json",
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES
-    )
+      classname="FalconServer",
+      command="stop",
+      config_file="default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
     self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-stop',
-                              path = ['/usr/bin'],
-                              user = 'falcon',
-                              )
+      path = ['/usr/bin'],
+      user = 'falcon',
+      environment = {'HADOOP_HOME': '/usr/lib/hadoop'})
+
     self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
-                              action = ['delete'],
-                              )
+      action = ['delete'])
+
     self.assertNoMoreResources()
 
   def test_configure_default(self):
@@ -198,7 +201,8 @@ class TestFalconServer(RMFTestCase):
 
     self.assertResourceCalled('Execute',
       '/usr/hdp/current/falcon-server/bin/falcon-stop',
-      path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon')
+      path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon',
+      environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'})
 
     self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
       action = ['delete'])
@@ -322,10 +326,12 @@ class TestFalconServer(RMFTestCase):
   @patch.object(tarfile, 'open')
   @patch.object(shutil, 'rmtree')
   def test_pre_rolling_restart(self, tarfile_open_mock, rmtree_mock):
-    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
     tarfile_open_mock.return_value = MagicMock()
+
     with open(config_file, "r") as f:
       json_content = json.load(f)
+
     version = '2.2.1.0-3242'
     json_content['commandParams']['version'] = version
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
@@ -345,9 +351,11 @@ class TestFalconServer(RMFTestCase):
   @patch.object(shutil, 'rmtree')
   @patch("resource_management.core.shell.call")
   def test_pre_rolling_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock):
-    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json"
+
     with open(config_file, "r") as f:
       json_content = json.load(f)
+
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version