#!/usr/bin/env python ''' Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ''' import json from mock.mock import MagicMock, patch import shutil from stacks.utils.RMFTestCase import * import tarfile import tempfile @patch("platform.linux_distribution", new = MagicMock(return_value="Linux")) @patch.object(tempfile, "gettempdir", new=MagicMock(return_value="/tmp")) class TestFalconServer(RMFTestCase): COMMON_SERVICES_PACKAGE_DIR = "FALCON/0.5.0.2.1/package" STACK_VERSION = "2.1" UPGRADE_STACK_VERSION = "2.2" def test_start_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="start", config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) self.assert_configure_default() self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000', path = ['/usr/bin'], user = 'falcon', environment = {'HADOOP_HOME': '/usr/lib/hadoop'}) self.assertNoMoreResources() def test_stop_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="stop", config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-stop', path = ['/usr/bin'], user = 'falcon', environment = {'HADOOP_HOME': '/usr/lib/hadoop'}) self.assertResourceCalled('File', '/var/run/falcon/falcon.pid', action = ['delete']) self.assertNoMoreResources() def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="configure", config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assert_configure_default() self.assertNoMoreResources() def assert_configure_default(self): self.assertResourceCalled('Directory', '/var/run/falcon', owner = 'falcon', recursive = True ) self.assertResourceCalled('Directory', '/var/log/falcon', owner = 'falcon', recursive = True ) self.assertResourceCalled('Directory', '/var/lib/falcon/webapp', owner = 'falcon', recursive = True ) self.assertResourceCalled('Directory', '/usr/lib/falcon', owner = 'falcon', recursive = True ) self.assertResourceCalled('Directory', '/etc/falcon', mode = 0755, recursive = True ) self.assertResourceCalled('Directory', '/etc/falcon/conf', owner = 'falcon', recursive = True ) self.assertResourceCalled('File', '/etc/falcon/conf/falcon-env.sh', content = InlineTemplate(self.getConfig()['configurations']['falcon-env']['content']), owner = 'falcon', group = 'hadoop' ) self.assertResourceCalled('File', '/etc/falcon/conf/client.properties', content = Template('client.properties.j2'), mode = 0644, owner = 'falcon' ) self.assertResourceCalled('PropertiesFile', '/etc/falcon/conf/runtime.properties', mode = 0644, properties = self.getConfig()['configurations']['falcon-runtime.properties'], owner = 'falcon' ) self.assertResourceCalled('PropertiesFile', '/etc/falcon/conf/startup.properties', mode = 0644, properties = self.getConfig()['configurations']['falcon-startup.properties'], owner = 'falcon' ) self.assertResourceCalled('Directory', '/hadoop/falcon/store', owner = 'falcon', recursive = True ) self.assertResourceCalled('HdfsResource', '/apps/falcon', security_enabled = False, hadoop_bin_dir = '/usr/bin', keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', dfs_type = '', owner = 'falcon', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, ) self.assertResourceCalled('Directory', '/hadoop/falcon/store', owner = 'falcon', recursive = True, ) self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', security_enabled = False, hadoop_bin_dir = '/usr/bin', keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', dfs_type = '', owner = 'falcon', group='users', hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', recursive_chown = True, recursive_chmod = True, action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0770, source='/usr/hdp/current/falcon-server/data-mirroring' ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, hadoop_bin_dir = '/usr/bin', keytab = UnknownConfigurationMock(), kinit_path_local = '/usr/bin/kinit', user = 'hdfs', dfs_type = '', action = ['execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020', hadoop_conf_dir = '/etc/hadoop/conf', ) self.assertResourceCalled('Directory', '/hadoop/falcon', owner = 'falcon', recursive = True, cd_access='a' ) self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq', owner = 'falcon', recursive = True ) self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq/data', owner = 'falcon', recursive = True ) @patch("os.path.isdir") @patch("os.path.exists") @patch("os.path.isfile") def test_upgrade(self, isfile_mock, exists_mock, isdir_mock): isdir_mock.return_value = True exists_mock.side_effect = [False,False,True, True] isfile_mock.return_value = True self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname = "FalconServer", command = "restart", config_file = "falcon-upgrade.json", hdp_stack_version = self.UPGRADE_STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-stop', path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'}) self.assertResourceCalled('File', '/var/run/falcon/falcon.pid', action = ['delete']) self.assertResourceCalled('Execute', ('tar', '-zcvhf', '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', '/usr/hdp/current/falcon-server/conf'), sudo = True, ) self.assertResourceCalled('Execute', ('tar', '-zcvhf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', u'/hadoop/falcon'), sudo = True, ) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', u'2.2.1.0-2135'), sudo = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', '-C', '/usr/hdp/current/falcon-server/conf/'), sudo = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', '-C', u'/hadoop/falcon/'), sudo = True, ) self.assertResourceCalled('Directory', '/tmp/falcon-upgrade-backup', action = ['delete'], ) self.assertResourceCalled('Directory', '/var/run/falcon', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Directory', '/var/log/falcon', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/webapp', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Directory', '/etc/falcon', recursive = True, mode = 0755, ) self.assertResourceCalled('Directory', '/usr/hdp/current/falcon-server/conf', owner = 'falcon', recursive = True, ) self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/falcon-env.sh', owner = 'falcon', content = InlineTemplate(self.getConfig()['configurations']['falcon-env']['content']), group = 'hadoop' ) self.assertResourceCalled('File', '/usr/hdp/current/falcon-server/conf/client.properties', owner = 'falcon', content = Template('client.properties.j2'), mode = 0644, ) self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/runtime.properties', owner = 'falcon', mode = 0644, properties = {u'*.domain': u'${falcon.app.type}', u'*.log.cleanup.frequency.days.retention': u'days(7)', u'*.log.cleanup.frequency.hours.retention': u'minutes(1)', u'*.log.cleanup.frequency.minutes.retention': u'hours(6)', u'*.log.cleanup.frequency.months.retention': u'months(3)'}, ) self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/falcon-server/conf/startup.properties', owner = 'falcon', mode = 0644, properties = self.getConfig()['configurations']['falcon-startup.properties'], ) self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage/graphdb', owner = 'falcon', recursive = True, group = 'hadoop', mode = 0775, cd_access = 'a', ) self.assertResourceCalled('Directory', '/hadoop/falcon/data/lineage', owner = 'falcon', recursive = True, group = 'hadoop', mode = 0775, cd_access = 'a', ) self.assertResourceCalled('Directory', '/hadoop/falcon/store', owner = 'falcon', recursive = True, ) self.assertResourceCalled('HdfsResource', '/apps/falcon', security_enabled = False, hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', keytab = UnknownConfigurationMock(), default_fs = 'hdfs://c6401.ambari.apache.org:8020', hdfs_site = self.getConfig()['configurations']['hdfs-site'], kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', dfs_type = '', owner = 'falcon', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', type = 'directory', action = ['create_on_execute'], mode = 0777, ) self.assertResourceCalled('Directory', '/hadoop/falcon/store', owner = 'falcon', recursive = True, ) self.assertResourceCalled('HdfsResource', '/apps/data-mirroring', security_enabled = False, hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', keytab = UnknownConfigurationMock(), source = '/usr/hdp/current/falcon-server/data-mirroring', default_fs = 'hdfs://c6401.ambari.apache.org:8020', user = 'hdfs', dfs_type = '', hdfs_site = self.getConfig()['configurations']['hdfs-site'], kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), recursive_chmod = True, recursive_chown = True, owner = 'falcon', group = 'users', hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', type = 'directory', action = ['create_on_execute'], mode = 0770, ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin', keytab = UnknownConfigurationMock(), default_fs = 'hdfs://c6401.ambari.apache.org:8020', hdfs_site = self.getConfig()['configurations']['hdfs-site'], kinit_path_local = '/usr/bin/kinit', principal_name = UnknownConfigurationMock(), user = 'hdfs', dfs_type = '', action = ['execute'], hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf', ) self.assertResourceCalled('Directory', '/hadoop/falcon', owner = 'falcon', recursive = True, cd_access = 'a', ) self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq/data', owner = 'falcon', recursive = True, ) self.assertResourceCalled('Execute', '/usr/hdp/current/falcon-server/bin/falcon-start -port 15000', environment = {'HADOOP_HOME': '/usr/hdp/current/hadoop-client'}, path = ['/usr/hdp/current/hadoop-client/bin'], user = 'falcon', ) self.assertNoMoreResources() @patch("resource_management.libraries.functions.security_commons.build_expectations") @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem") @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties") @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor") @patch("resource_management.libraries.script.Script.put_structured_out") def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock): # Test that function works when is called with correct parameters security_params = { 'startup': { '*.falcon.service.authentication.kerberos.keytab': 'path/to/falcon/service/keytab', '*.falcon.service.authentication.kerberos.principal': 'falcon_service_keytab', '*.falcon.http.authentication.kerberos.keytab': 'path/to/falcon/http/keytab', '*.falcon.http.authentication.kerberos.principal': 'falcon_http_principal' } } result_issues = [] props_value_check = {"*.falcon.authentication.type": "kerberos", "*.falcon.http.authentication.type": "kerberos"} props_empty_check = ["*.falcon.service.authentication.kerberos.principal", "*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.principal", "*.falcon.http.authentication.kerberos.keytab"] props_read_check = ["*.falcon.service.authentication.kerberos.keytab", "*.falcon.http.authentication.kerberos.keytab"] get_params_mock.return_value = security_params validate_security_config_mock.return_value = result_issues self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="security_status", config_file="secured.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) get_params_mock.assert_called_with('/etc/falcon/conf', {'startup.properties': 'PROPERTIES'}) build_exp_mock.assert_called_with('startup', props_value_check, props_empty_check, props_read_check) put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"}) self.assertTrue(cached_kinit_executor_mock.call_count, 2) cached_kinit_executor_mock.assert_called_with('/usr/bin/kinit', self.config_dict['configurations']['falcon-env']['falcon_user'], security_params['startup']['*.falcon.http.authentication.kerberos.keytab'], security_params['startup']['*.falcon.http.authentication.kerberos.principal'], self.config_dict['hostname'], '/tmp') # Testing that the exception throw by cached_executor is caught cached_kinit_executor_mock.reset_mock() cached_kinit_executor_mock.side_effect = Exception("Invalid command") try: self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="security_status", config_file="secured.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) except: self.assertTrue(True) # Testing with a security_params which doesn't contains startup empty_security_params = {} cached_kinit_executor_mock.reset_mock() get_params_mock.reset_mock() put_structured_out_mock.reset_mock() get_params_mock.return_value = empty_security_params self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="security_status", config_file="secured.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."}) # Testing with not empty result_issues result_issues_with_params = { 'startup': "Something bad happened" } validate_security_config_mock.reset_mock() get_params_mock.reset_mock() validate_security_config_mock.return_value = result_issues_with_params get_params_mock.return_value = security_params self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="security_status", config_file="secured.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) # Testing with security_enable = false self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname="FalconServer", command="security_status", config_file="default.json", hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"}) @patch('os.path.isfile', new=MagicMock(return_value=True)) def test_pre_rolling_restart(self): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.2.1.0-3242' json_content['commandParams']['version'] = version self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname = "FalconServer", command = "pre_rolling_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', version), sudo=True,) self.printResources() @patch('os.path.isfile', new=MagicMock(return_value=True)) @patch.object(tarfile, 'open') @patch.object(shutil, 'rmtree') @patch("resource_management.core.shell.call") def test_pre_rolling_restart_23(self, tarfile_open_mock, rmtree_mock, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/falcon-upgrade.json" with open(config_file, "r") as f: json_content = json.load(f) version = '2.3.0.0-1234' json_content['commandParams']['version'] = version mocks_dict = {} self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py", classname = "FalconServer", command = "pre_rolling_restart", config_dict = json_content, hdp_stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, call_mocks = [(0, None), (0, None)], mocks_dict = mocks_dict) self.assertResourceCalled('Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'falcon-server', version), sudo=True,) self.assertResourceCalled('Execute', ('tar', '-xvf', '/tmp/falcon-upgrade-backup/falcon-conf-backup.tar', '-C', '/usr/hdp/current/falcon-server/conf/'), sudo = True, ) self.assertResourceCalled('Execute', ('tar', '-xvf', '/tmp/falcon-upgrade-backup/falcon-local-backup.tar', '-C', u'/hadoop/falcon/'), sudo = True, ) self.assertResourceCalled('Directory', '/tmp/falcon-upgrade-backup', action = ['delete'], ) self.assertNoMoreResources() self.assertEquals(1, mocks_dict['call'].call_count) self.assertEquals(1, mocks_dict['checked_call'].call_count) self.assertEquals( ('conf-select', 'set-conf-dir', '--package', 'falcon', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'), mocks_dict['checked_call'].call_args_list[0][0][0]) self.assertEquals( ('conf-select', 'create-conf-dir', '--package', 'falcon', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'), mocks_dict['call'].call_args_list[0][0][0])