Verify if restricting acls on /var/lib/ambari-agent/data will be OK (aonishuk)

ambari-agent/conf/unix/ambari-agent.ini

@@ -19,7 +19,6 @@ secured_url_port=8441
 
 [agent]
 prefix=/var/lib/ambari-agent/data
-tmp_dir=/var/lib/ambari-agent/data/tmp
 ;loglevel=(DEBUG/INFO)
 loglevel=INFO
 data_cleanup_interval=86400

ambari-agent/conf/unix/install-helper.sh

@@ -58,8 +58,9 @@ do_install(){
   # on nano Ubuntu, when umask=027 those folders are created without 'x' bit for 'others'.
   # which causes failures when hadoop users try to access tmp_dir
   chmod a+x /var/lib/ambari-agent
-  chmod a+x /var/lib/ambari-agent/data
-  chmod 777 /var/lib/ambari-agent/data/tmp
+  
+  chmod 777 /var/lib/ambari-agent/tmp
+  chmod 700 /var/lib/ambari-agent/data
 }
 
 do_remove(){

ambari-agent/pom.xml

@@ -391,7 +391,7 @@
               <groupname>root</groupname>
             </mapping>
             <mapping>
-              <directory>/var/lib/${project.artifactId}/data/tmp</directory>
+              <directory>/var/lib/${project.artifactId}/tmp</directory>
               <filemode>777</filemode>
               <username>root</username>
               <groupname>root</groupname>
@@ -596,7 +596,7 @@
               <paths>
                 <path>${package.pid.dir}</path>
                 <path>/var/lib/${project.artifactId}/data</path>
-                <path>/var/lib/${project.artifactId}/data/tmp</path>
+                <path>/var/lib/${project.artifactId}/tmp</path>
                 <path>/var/lib/${project.artifactId}/keys</path>
                 <path>${package.log.dir}</path>
                 <path>/var/lib/${project.artifactId}/lib</path>

ambari-agent/src/main/python/ambari_agent/Constants.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+AGENT_TMP_DIR = "/var/lib/ambari-agent/tmp"

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -29,6 +29,7 @@ from FileCache import FileCache
 from AgentException import AgentException
 from PythonExecutor import PythonExecutor
 from PythonReflectiveExecutor import PythonReflectiveExecutor
+import Constants
 import hostname
 
 
@@ -61,7 +62,7 @@ class CustomServiceOrchestrator():
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
-    self.exec_tmp_dir = config.get('agent', 'tmp_dir')
+    self.exec_tmp_dir = Constants.AGENT_TMP_DIR
     self.file_cache = FileCache(config)
     self.status_commands_stdout = os.path.join(self.tmp_dir,
                                                'status_command_stdout.txt')

ambari-agent/src/main/python/ambari_agent/alerts/metric_alert.py

@@ -30,6 +30,7 @@ from alerts.base_alert import BaseAlert
 from ambari_commons.urllib_handlers import RefreshHeaderProcessor
 from resource_management.libraries.functions.get_port_from_url import get_port_from_url
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from ambari_agent import Constants
 
 logger = logging.getLogger()
 
@@ -199,7 +200,7 @@ class MetricAlert(BaseAlert):
       content = ''
       try:
         if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
-          tmp_dir = self.config.get('agent', 'tmp_dir')
+          tmp_dir = Constants.AGENT_TMP_DIR
           if tmp_dir is None:
             tmp_dir = gettempdir()
 

ambari-agent/src/main/python/ambari_agent/alerts/script_alert.py

@@ -25,6 +25,7 @@ import re
 from alerts.base_alert import BaseAlert
 from resource_management.core.environment import Environment
 from resource_management.core.logger import Logger
+from ambari_agent import Constants
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +100,7 @@ class ScriptAlert(BaseAlert):
       matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
       if matchObj:
         basedir = matchObj.group(1)
-        with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env:
+        with Environment(basedir, tmp_dir=Constants.AGENT_TMP_DIR) as env:
           return cmd_module.execute(configurations, self.parameters, self.host_name)
       else:
         return cmd_module.execute(configurations, self.parameters, self.host_name)

ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py

@@ -31,6 +31,7 @@ from resource_management.libraries.functions.get_port_from_url import get_port_f
 from resource_management.libraries.functions.curl_krb_request import curl_krb_request
 from ambari_commons import OSCheck
 from ambari_commons.inet_utils import resolve_address
+from ambari_agent import Constants
 
 # hashlib is supplied as of Python 2.5 as the replacement interface for md5
 # and other secure hashes.  In 2.6, md5 is deprecated.  Import hashlib if
@@ -171,7 +172,7 @@ class WebAlert(BaseAlert):
         # Create the kerberos credentials cache (ccache) file and set it in the environment to use
         # when executing curl. Use the md5 hash of the combination of the principal and keytab file
         # to generate a (relatively) unique cache filename so that we can use it as needed.
-        tmp_dir = self.config.get('agent', 'tmp_dir')
+        tmp_dir = Constants.AGENT_TMP_DIR
         if tmp_dir is None:
           tmp_dir = gettempdir()
 

ambari-agent/src/main/python/ambari_agent/security.py

@@ -35,6 +35,7 @@ logger = logging.getLogger(__name__)
 GEN_AGENT_KEY = 'openssl req -new -newkey rsa:1024 -nodes -keyout "%(keysdir)s' \
                 + os.sep + '%(hostname)s.key" -subj /OU=%(hostname)s/ ' \
                 '-out "%(keysdir)s' + os.sep + '%(hostname)s.csr"'
+KEY_FILENAME = '%(hostname)s.key'
 
 
 class VerifiedHTTPSConnection(httplib.HTTPSConnection):
@@ -190,7 +191,7 @@ class CertificateManager():
 
     if not agent_key_exists:
       logger.info("Agent key not exists, generating request")
-      self.genAgentCrtReq()
+      self.genAgentCrtReq(self.getAgentKeyName())
     else:
       logger.info("Agent key exists, ok")
 
@@ -252,10 +253,12 @@ class CertificateManager():
                    '\nExiting..')
       raise ssl.SSLError
 
-  def genAgentCrtReq(self):
+  def genAgentCrtReq(self, keyname):
+    keysdir = os.path.abspath(self.config.get('security', 'keysdir'))
     generate_script = GEN_AGENT_KEY % {
       'hostname': hostname.hostname(self.config),
-      'keysdir': os.path.abspath(self.config.get('security', 'keysdir'))}
+      'keysdir': keysdir}
+    
     logger.info(generate_script)
     if platform.system() == 'Windows':
       p = subprocess.Popen(generate_script, stdout=subprocess.PIPE)
@@ -264,6 +267,8 @@ class CertificateManager():
       p = subprocess.Popen([generate_script], shell=True,
                            stdout=subprocess.PIPE)
       p.communicate()
+    # this is required to be 600 for security concerns.
+    os.chmod(keyname, 0600)
 
   def initSecurity(self):
     self.checkCertExists()

ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py

@@ -41,8 +41,10 @@ class TestCertGeneration(TestCase):
     config.set('security', 'server_crt', 'ca.crt')
     self.certMan = CertificateManager(config)
 
-  def test_generation(self):
-    self.certMan.genAgentCrtReq()
+  @patch.object(os, "chmod")
+  def test_generation(self, chmod_mock):
+    self.certMan.genAgentCrtReq('/dummy_dir/hostname.key')
+    self.assertTrue(chmod_mock.called)
     self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
     self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
   def tearDown(self):

ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py.orig

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import os
+import tempfile
+import shutil
+from unittest import TestCase
+
+from ambari_agent.security import CertificateManager
+from ambari_agent import AmbariConfig
+from mock.mock import patch, MagicMock
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+class TestCertGeneration(TestCase):
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  def setUp(self):
+    self.tmpdir = tempfile.mkdtemp()
+    config = AmbariConfig.AmbariConfig()
+    #config.add_section('server')
+    config.set('server', 'hostname', 'example.com')
+    config.set('server', 'url_port', '777')
+    #config.add_section('security')
+    config.set('security', 'keysdir', self.tmpdir)
+    config.set('security', 'server_crt', 'ca.crt')
+    self.certMan = CertificateManager(config)
+
+  def test_generation(self):
+    self.certMan.genAgentCrtReq()
+    self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
+    self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
+  def tearDown(self):
+    shutil.rmtree(self.tmpdir)
+

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py

@@ -56,7 +56,6 @@ class TestCustomServiceOrchestrator(TestCase):
     self.config = ConfigParser.RawConfigParser()
     self.config.add_section('agent')
     self.config.set('agent', 'prefix', tmpdir)
-    self.config.set('agent', 'tmp_dir', exec_tmp_dir)
     self.config.set('agent', 'cache_dir', "/cachedir")
     self.config.add_section('python')
     self.config.set('python', 'custom_actions_dir', tmpdir)

ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py.orig

@@ -0,0 +1,643 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import ConfigParser
+from multiprocessing.pool import ThreadPool
+import os
+
+import pprint
+from ambari_commons import shell
+
+from unittest import TestCase
+import threading
+import tempfile
+import time
+from threading import Thread
+
+from mock.mock import MagicMock, patch
+import StringIO
+import sys
+
+from ambari_agent.ActionQueue import ActionQueue
+from ambari_agent.AgentException import AgentException
+from ambari_agent.AmbariConfig import AmbariConfig
+from ambari_agent.BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle
+from ambari_agent.CustomServiceOrchestrator import CustomServiceOrchestrator
+from ambari_agent.FileCache import FileCache
+from ambari_agent.PythonExecutor import PythonExecutor
+from ambari_commons import OSCheck
+from only_for_platform import get_platform, os_distro_value, PLATFORM_WINDOWS
+
+class TestCustomServiceOrchestrator(TestCase):
+
+  def setUp(self):
+    # disable stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+    # generate sample config
+    tmpdir = tempfile.gettempdir()
+    exec_tmp_dir = os.path.join(tmpdir, 'tmp')
+    self.config = ConfigParser.RawConfigParser()
+    self.config.add_section('agent')
+    self.config.set('agent', 'prefix', tmpdir)
+    self.config.set('agent', 'tmp_dir', exec_tmp_dir)
+    self.config.set('agent', 'cache_dir', "/cachedir")
+    self.config.add_section('python')
+    self.config.set('python', 'custom_actions_dir', tmpdir)
+
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  @patch.object(FileCache, "__init__")
+  def test_add_reg_listener_to_controller(self, FileCache_mock):
+    FileCache_mock.return_value = None
+    dummy_controller = MagicMock()
+    config = AmbariConfig().getConfig()
+    tempdir = tempfile.gettempdir()
+    config.set('agent', 'prefix', tempdir)
+    CustomServiceOrchestrator(config, dummy_controller)
+    self.assertTrue(dummy_controller.registration_listeners.append.called)
+
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  @patch.object(CustomServiceOrchestrator, 'decompressClusterHostInfo')
+  @patch("ambari_agent.hostname.public_hostname")
+  @patch("os.path.isfile")
+  @patch("os.unlink")
+  @patch.object(FileCache, "__init__")
+  def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
+                                isfile_mock, hostname_mock,
+                                decompress_cluster_host_info_mock):
+    FileCache_mock.return_value = None
+    hostname_mock.return_value = "test.hst"
+    command = {
+      'commandType': 'EXECUTION_COMMAND',
+      'role': u'DATANODE',
+      'roleCommand': u'INSTALL',
+      'commandId': '1-1',
+      'taskId': 3,
+      'clusterName': u'cc',
+      'serviceName': u'HDFS',
+      'configurations':{'global' : {}},
+      'configurationTags':{'global' : { 'tag': 'v1' }},
+      'clusterHostInfo':{'namenode_host' : ['1'],
+                         'slave_hosts'   : ['0', '1'],
+                         'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+                         'all_ping_ports': ['8670:0,1']},
+      'hostLevelParams':{}
+    }
+    
+    decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
+                         'slave_hosts'   : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+                         'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+                         'all_ping_ports': ['8670', '8670']}
+    
+    config = AmbariConfig().getConfig()
+    tempdir = tempfile.gettempdir()
+    config.set('agent', 'prefix', tempdir)
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+    isfile_mock.return_value = True
+    # Test dumping EXECUTION_COMMAND
+    json_file = orchestrator.dump_command_to_json(command)
+    self.assertTrue(os.path.exists(json_file))
+    self.assertTrue(os.path.getsize(json_file) > 0)
+    if get_platform() != PLATFORM_WINDOWS:
+      self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+    self.assertTrue(json_file.endswith("command-3.json"))
+    self.assertTrue(decompress_cluster_host_info_mock.called)
+    os.unlink(json_file)
+    # Test dumping STATUS_COMMAND
+    command['commandType']='STATUS_COMMAND'
+    decompress_cluster_host_info_mock.reset_mock()
+    json_file = orchestrator.dump_command_to_json(command)
+    self.assertTrue(os.path.exists(json_file))
+    self.assertTrue(os.path.getsize(json_file) > 0)
+    if get_platform() != PLATFORM_WINDOWS:
+      self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+    self.assertTrue(json_file.endswith("status_command.json"))
+    self.assertFalse(decompress_cluster_host_info_mock.called)
+    os.unlink(json_file)
+    # Testing side effect of dump_command_to_json
+    self.assertEquals(command['public_hostname'], "test.hst")
+    self.assertTrue(unlink_mock.called)
+
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  @patch("ambari_agent.hostname.public_hostname")
+  @patch("os.path.isfile")
+  @patch("os.unlink")
+  @patch.object(FileCache, "__init__")
+  def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock,
+                                isfile_mock, hostname_mock):
+    FileCache_mock.return_value = None
+    hostname_mock.return_value = "test.hst"
+    command = {
+      'commandType': 'EXECUTION_COMMAND',
+      'role': u'DATANODE',
+      'roleCommand': u'INSTALL',
+      'commandId': '1-1',
+      'taskId': 3,
+      'clusterName': u'cc',
+      'serviceName': u'HDFS',
+      'configurations':{'global' : {}},
+      'configurationTags':{'global' : { 'tag': 'v1' }},
+      'clusterHostInfo':{'namenode_host' : ['1'],
+                         'slave_hosts'   : ['0', '1'],
+                         'all_racks'   : [u'/default-rack:0'],
+                         'ambari_server_host' : 'a.b.c',
+                         'all_ipv4_ips'   : [u'192.168.12.101:0'],
+                         'all_hosts'     : ['h1.hortonworks.com', 'h2.hortonworks.com'],
+                         'all_ping_ports': ['8670:0,1']},
+      'hostLevelParams':{}
+    }
+
+    config = AmbariConfig().getConfig()
+    tempdir = tempfile.gettempdir()
+    config.set('agent', 'prefix', tempdir)
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+    isfile_mock.return_value = True
+    # Test dumping EXECUTION_COMMAND
+    json_file = orchestrator.dump_command_to_json(command)
+    self.assertTrue(os.path.exists(json_file))
+    self.assertTrue(os.path.getsize(json_file) > 0)
+    if get_platform() != PLATFORM_WINDOWS:
+      self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+    self.assertTrue(json_file.endswith("command-3.json"))
+    os.unlink(json_file)
+    # Test dumping STATUS_COMMAND
+    json_file = orchestrator.dump_command_to_json(command, True)
+    self.assertTrue(os.path.exists(json_file))
+    self.assertTrue(os.path.getsize(json_file) > 0)
+    if get_platform() != PLATFORM_WINDOWS:
+      self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
+    self.assertTrue(json_file.endswith("command-3.json"))
+    os.unlink(json_file)
+    # Testing side effect of dump_command_to_json
+    self.assertEquals(command['public_hostname'], "test.hst")
+    self.assertTrue(unlink_mock.called)
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  @patch("os.path.exists")
+  @patch.object(FileCache, "__init__")
+  def test_resolve_script_path(self, FileCache_mock, exists_mock):
+    FileCache_mock.return_value = None
+    dummy_controller = MagicMock()
+    config = AmbariConfig().getConfig()
+    orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+    # Testing existing path
+    exists_mock.return_value = True
+    path = orchestrator.\
+      resolve_script_path(os.path.join("HBASE", "package"), os.path.join("scripts", "hbase_master.py"))
+    self.assertEqual(os.path.join("HBASE", "package", "scripts", "hbase_master.py"), path)
+    # Testing not existing path
+    exists_mock.return_value = False
+    try:
+      orchestrator.resolve_script_path("/HBASE",
+                                       os.path.join("scripts", "hbase_master.py"))
+      self.fail('ExpectedException not thrown')
+    except AgentException:
+      pass # Expected
+
+
+  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+  @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+  @patch.object(FileCache, "get_host_scripts_base_dir")
+  @patch.object(FileCache, "get_service_base_dir")
+  @patch.object(FileCache, "get_hook_base_dir")
+  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+  @patch.object(PythonExecutor, "run_file")
+  @patch.object(FileCache, "__init__")
+  def test_runCommand(self, FileCache_mock,
+                      run_file_mock, dump_command_to_json_mock,
+                      get_hook_base_dir_mock, get_service_base_dir_mock, 
+                      get_host_scripts_base_dir_mock, 
+                      resolve_hook_script_path_mock, 
+                      resolve_script_path_mock):
+    
+    FileCache_mock.return_value = None
+    command = {
+      'role' : 'REGION_SERVER',
+      'hostLevelParams' : {
+        'stack_name' : 'HDP',
+        'stack_version' : '2.0.7',
+        'jdk_location' : 'some_location'
+      },
+      'commandParams': {
+        'script_type': 'PYTHON',
+        'script': 'scripts/hbase_regionserver.py',
+        'command_timeout': '600',
+        'service_package_folder' : 'HBASE'
+      },
+      'taskId' : '3',
+      'roleCommand': 'INSTALL'
+    }
+    
+    get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+    get_service_base_dir_mock.return_value = "/basedir/"
+    resolve_script_path_mock.return_value = "/basedir/scriptpath"
+    resolve_hook_script_path_mock.return_value = \
+      ('/hooks_dir/prefix-command/scripts/hook.py',
+       '/hooks_dir/prefix-command')
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    unix_process_id = 111
+    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+    get_hook_base_dir_mock.return_value = "/hooks/"
+    # normal run case
+    run_file_mock.return_value = {
+        'stdout' : 'sss',
+        'stderr' : 'eee',
+        'exitcode': 0,
+      }
+    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+    self.assertEqual(ret['exitcode'], 0)
+    self.assertTrue(run_file_mock.called)
+    self.assertEqual(run_file_mock.call_count, 3)
+
+    run_file_mock.reset_mock()
+
+    # Case when we force another command
+    run_file_mock.return_value = {
+        'stdout' : 'sss',
+        'stderr' : 'eee',
+        'exitcode': 0,
+      }
+    ret = orchestrator.runCommand(command, "out.txt", "err.txt",
+              forced_command_name=CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+    ## Check that override_output_files was true only during first call
+    print run_file_mock
+    self.assertEquals(run_file_mock.call_args_list[0][0][8], True)
+    self.assertEquals(run_file_mock.call_args_list[1][0][8], False)
+    self.assertEquals(run_file_mock.call_args_list[2][0][8], False)
+    ## Check that forced_command_name was taken into account
+    self.assertEqual(run_file_mock.call_args_list[0][0][1][0],
+                                  CustomServiceOrchestrator.SCRIPT_TYPE_PYTHON)
+
+    run_file_mock.reset_mock()
+
+    # unknown script type case
+    command['commandParams']['script_type'] = "SOME_TYPE"
+    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+    self.assertEqual(ret['exitcode'], 1)
+    self.assertFalse(run_file_mock.called)
+    self.assertTrue("Unknown script type" in ret['stdout'])
+
+    #By default returns empty dictionary
+    self.assertEqual(ret['structuredOut'], '{}')
+
+    pass
+
+  @patch("ambari_commons.shell.kill_process_with_children")
+  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+  @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+  @patch.object(FileCache, "get_host_scripts_base_dir")
+  @patch.object(FileCache, "get_service_base_dir")
+  @patch.object(FileCache, "get_hook_base_dir")
+  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+  @patch.object(PythonExecutor, "run_file")
+  @patch.object(FileCache, "__init__")
+  def test_cancel_command(self, FileCache_mock,
+                      run_file_mock, dump_command_to_json_mock,
+                      get_hook_base_dir_mock, get_service_base_dir_mock,
+                      get_host_scripts_base_dir_mock,
+                      resolve_hook_script_path_mock, resolve_script_path_mock,
+                      kill_process_with_children_mock):
+    FileCache_mock.return_value = None
+    command = {
+      'role' : 'REGION_SERVER',
+      'hostLevelParams' : {
+        'stack_name' : 'HDP',
+        'stack_version' : '2.0.7',
+        'jdk_location' : 'some_location'
+      },
+      'commandParams': {
+        'script_type': 'PYTHON',
+        'script': 'scripts/hbase_regionserver.py',
+        'command_timeout': '600',
+        'service_package_folder' : 'HBASE'
+      },
+      'taskId' : '3',
+      'roleCommand': 'INSTALL'
+    }
+    
+    get_host_scripts_base_dir_mock.return_value = "/host_scripts"
+    get_service_base_dir_mock.return_value = "/basedir/"
+    resolve_script_path_mock.return_value = "/basedir/scriptpath"
+    resolve_hook_script_path_mock.return_value = \
+      ('/hooks_dir/prefix-command/scripts/hook.py',
+       '/hooks_dir/prefix-command')
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    unix_process_id = 111
+    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+    get_hook_base_dir_mock.return_value = "/hooks/"
+    run_file_mock_return_value = {
+      'stdout' : 'killed',
+      'stderr' : 'killed',
+      'exitcode': 1,
+      }
+    def side_effect(*args, **kwargs):
+      time.sleep(0.2)
+      return run_file_mock_return_value
+    run_file_mock.side_effect = side_effect
+
+    _, out = tempfile.mkstemp()
+    _, err = tempfile.mkstemp()
+    pool = ThreadPool(processes=1)
+    async_result = pool.apply_async(orchestrator.runCommand, (command, out, err))
+
+    time.sleep(0.1)
+    orchestrator.cancel_command(command['taskId'], 'reason')
+
+    ret = async_result.get()
+
+    self.assertEqual(ret['exitcode'], 1)
+    self.assertEquals(ret['stdout'], 'killed\nCommand aborted. reason')
+    self.assertEquals(ret['stderr'], 'killed\nCommand aborted. reason')
+
+    self.assertTrue(kill_process_with_children_mock.called)
+    self.assertFalse(command['taskId'] in orchestrator.commands_in_progress.keys())
+    self.assertTrue(os.path.exists(out))
+    self.assertTrue(os.path.exists(err))
+    try:
+      os.remove(out)
+      os.remove(err)
+    except:
+      pass
+
+  from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  @patch.object(CustomServiceOrchestrator, "get_py_executor")
+  @patch("ambari_commons.shell.kill_process_with_children")
+  @patch.object(FileCache, "__init__")
+  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
+  @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
+  @patch.object(StackVersionsFileHandler, "read_stack_version")
+  def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock,
+                                    resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock,
+                                    get_py_executor_mock):
+    FileCache_mock.return_value = None
+    FileCache_mock.cache_dir = MagicMock()
+    resolve_hook_script_path_mock.return_value = None
+#     shell.kill_process_with_children = MagicMock()
+    dummy_controller = MagicMock()
+    cfg = AmbariConfig()
+    cfg.set('agent', 'tolerate_download_failures', 'true')
+    cfg.set('agent', 'prefix', '.')
+    cfg.set('agent', 'cache_dir', 'background_tasks')
+
+    actionQueue = ActionQueue(cfg, dummy_controller)
+
+    dummy_controller.actionQueue = actionQueue
+    orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
+    orchestrator.file_cache = MagicMock()
+    def f (a, b):
+      return ""
+    orchestrator.file_cache.get_service_base_dir = f
+    actionQueue.customServiceOrchestrator = orchestrator
+
+    import TestActionQueue
+    import copy
+
+    pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
+    TestActionQueue.patch_output_file(pyex)
+    pyex.prepare_process_result = MagicMock()
+    get_py_executor_mock.return_value = pyex
+    orchestrator.dump_command_to_json = MagicMock()
+
+    lock = threading.RLock()
+    complete_done = threading.Condition(lock)
+
+    complete_was_called = {}
+    def command_complete_w(process_condenced_result, handle):
+      with lock:
+        complete_was_called['visited']= ''
+        complete_done.wait(3)
+
+    actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
+    execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
+    actionQueue.put([execute_command])
+    actionQueue.processBackgroundQueueSafeEmpty()
+
+    time.sleep(.1)
+
+    orchestrator.cancel_command(19,'')
+    self.assertTrue(kill_process_with_children_mock.called)
+    kill_process_with_children_mock.assert_called_with(33)
+
+    with lock:
+      complete_done.notifyAll()
+
+    with lock:
+      self.assertTrue(complete_was_called.has_key('visited'))
+
+    time.sleep(.1)
+
+    runningCommand = actionQueue.commandStatuses.get_command_status(19)
+    self.assertTrue(runningCommand is not None)
+    self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
+
+
+  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+  @patch.object(PythonExecutor, "run_file")
+  @patch.object(FileCache, "__init__")
+  @patch.object(FileCache, "get_custom_actions_base_dir")
+  def test_runCommand_custom_action(self, get_custom_actions_base_dir_mock,
+                                    FileCache_mock,
+                                    run_file_mock, dump_command_to_json_mock):
+    FileCache_mock.return_value = None
+    get_custom_actions_base_dir_mock.return_value = "some path"
+    _, script = tempfile.mkstemp()
+    command = {
+      'role' : 'any',
+      'commandParams': {
+        'script_type': 'PYTHON',
+        'script': 'some_custom_action.py',
+        'command_timeout': '600',
+        'jdk_location' : 'some_location'
+      },
+      'taskId' : '3',
+      'roleCommand': 'ACTIONEXECUTE'
+    }
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    unix_process_id = 111
+    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
+    # normal run case
+    run_file_mock.return_value = {
+      'stdout' : 'sss',
+      'stderr' : 'eee',
+      'exitcode': 0,
+      }
+    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+    self.assertEqual(ret['exitcode'], 0)
+    self.assertTrue(run_file_mock.called)
+    # Hoooks are not supported for custom actions,
+    # that's why run_file() should be called only once
+    self.assertEqual(run_file_mock.call_count, 1)
+
+
+  @patch("os.path.isfile")
+  @patch.object(FileCache, "__init__")
+  def test_resolve_hook_script_path(self, FileCache_mock, isfile_mock):
+    FileCache_mock.return_value = None
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    # Testing None param
+    res1 = orchestrator.resolve_hook_script_path(None, "prefix", "command",
+                                            "script_type")
+    self.assertEqual(res1, None)
+    # Testing existing hook script
+    isfile_mock.return_value = True
+    res2 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+                                            "script_type")
+    self.assertEqual(res2, (os.path.join('hooks_dir', 'prefix-command', 'scripts', 'hook.py'),
+                            os.path.join('hooks_dir', 'prefix-command')))
+    # Testing not existing hook script
+    isfile_mock.return_value = False
+    res3 = orchestrator.resolve_hook_script_path("hooks_dir", "prefix", "command",
+                                                 "script_type")
+    self.assertEqual(res3, None)
+
+
+  @patch.object(CustomServiceOrchestrator, "runCommand")
+  @patch.object(FileCache, "__init__")
+  def test_requestComponentStatus(self, FileCache_mock, runCommand_mock):
+    FileCache_mock.return_value = None
+    status_command = {
+      "serviceName" : 'HDFS',
+      "commandType" : "STATUS_COMMAND",
+      "clusterName" : "",
+      "componentName" : "DATANODE",
+      'configurations':{}
+    }
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    # Test alive case
+    runCommand_mock.return_value = {
+      "exitcode" : 0
+    }
+
+    status = orchestrator.requestComponentStatus(status_command)
+    self.assertEqual(runCommand_mock.return_value, status)
+
+    # Test dead case
+    runCommand_mock.return_value = {
+      "exitcode" : 1
+    }
+    status = orchestrator.requestComponentStatus(status_command)
+    self.assertEqual(runCommand_mock.return_value, status)
+
+  @patch.object(CustomServiceOrchestrator, "runCommand")
+  @patch.object(FileCache, "__init__")
+  def test_requestComponentSecurityState(self, FileCache_mock, runCommand_mock):
+    FileCache_mock.return_value = None
+    status_command = {
+      "serviceName" : 'HDFS',
+      "commandType" : "STATUS_COMMAND",
+      "clusterName" : "",
+      "componentName" : "DATANODE",
+      'configurations':{}
+    }
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+    # Test securityState
+    runCommand_mock.return_value = {
+      'exitcode' : 0,
+      'structuredOut' : {'securityState': 'UNSECURED'}
+    }
+
+    status = orchestrator.requestComponentSecurityState(status_command)
+    self.assertEqual('UNSECURED', status)
+
+    # Test case where exit code indicates failure
+    runCommand_mock.return_value = {
+      "exitcode" : 1
+    }
+    status = orchestrator.requestComponentSecurityState(status_command)
+    self.assertEqual('UNKNOWN', status)
+
+  @patch.object(FileCache, "__init__")
+  def test_requestComponentSecurityState_realFailure(self, FileCache_mock):
+    '''
+    Tests the case where the CustomServiceOrchestrator attempts to call a service's security_status
+    method, but fails to do so because the script or method was not found.
+    :param FileCache_mock:
+    :return:
+    '''
+    FileCache_mock.return_value = None
+    status_command = {
+      "serviceName" : 'BOGUS_SERVICE',
+      "commandType" : "STATUS_COMMAND",
+      "clusterName" : "",
+      "componentName" : "DATANODE",
+      'configurations':{}
+    }
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+    status = orchestrator.requestComponentSecurityState(status_command)
+    self.assertEqual('UNKNOWN', status)
+
+
+  @patch.object(CustomServiceOrchestrator, "get_py_executor")
+  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
+  @patch.object(FileCache, "__init__")
+  @patch.object(FileCache, "get_custom_actions_base_dir")
+  def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
+                                    FileCache_mock,
+                                    dump_command_to_json_mock,
+                                    get_py_executor_mock):
+    FileCache_mock.return_value = None
+    get_custom_actions_base_dir_mock.return_value = "some path"
+    _, script = tempfile.mkstemp()
+    command = {
+      'role' : 'any',
+      'commandParams': {
+        'script_type': 'PYTHON',
+        'script': 'some_custom_action.py',
+        'command_timeout': '600',
+        'jdk_location' : 'some_location'
+      },
+      'taskId' : '13',
+      'roleCommand': 'ACTIONEXECUTE',
+      'commandType': 'BACKGROUND_EXECUTION_COMMAND',
+      '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
+                                                   MagicMock(), MagicMock())
+    }
+    dummy_controller = MagicMock()
+    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
+
+    import TestActionQueue
+    pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
+    TestActionQueue.patch_output_file(pyex)
+    pyex.condenseOutput = MagicMock()
+    get_py_executor_mock.return_value = pyex
+    orchestrator.dump_command_to_json = MagicMock()
+
+    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
+    self.assertEqual(ret['exitcode'], 777)
+
+  def tearDown(self):
+    # enable stdout
+    sys.stdout = sys.__stdout__
+
+

ambari-agent/src/test/python/ambari_agent/TestSecurity.py

@@ -343,12 +343,14 @@ class TestSecurity(unittest.TestCase):
 
   @patch("subprocess.Popen")
   @patch("subprocess.Popen.communicate")
-  def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+  @patch.object(os, "chmod")
+  def test_genAgentCrtReq(self, chmod_mock, communicate_mock, popen_mock):
     man = CertificateManager(self.config)
     p = MagicMock(spec=subprocess.Popen)
     p.communicate = communicate_mock
     popen_mock.return_value = p
-    man.genAgentCrtReq()
+    man.genAgentCrtReq('/dummy-keysdir/hostname.key')
+    self.assertTrue(chmod_mock.called)
     self.assertTrue(popen_mock.called)
     self.assertTrue(communicate_mock.called)
 

ambari-agent/src/test/python/ambari_agent/TestSecurity.py.orig

@@ -0,0 +1,389 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import StringIO
+import sys, subprocess
+from mock.mock import MagicMock, patch, ANY
+import mock.mock
+import unittest
+import logging
+import signal
+import ConfigParser
+import ssl
+import os
+import tempfile
+
+from ambari_commons import OSCheck
+from only_for_platform import os_distro_value
+
+with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
+  from ambari_agent import NetUtil
+  from ambari_agent.security import CertificateManager
+  from ambari_agent import ProcessHelper, main
+  from ambari_agent.AmbariConfig import AmbariConfig
+  from ambari_agent.Controller import Controller
+  from ambari_agent import security
+
+aa = mock.mock.mock_open()
+class TestSecurity(unittest.TestCase):
+
+  @patch.object(OSCheck, "os_distribution", new = MagicMock(return_value = os_distro_value))
+  def setUp(self):
+    # disable stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+    # Create config
+    self.config = AmbariConfig()
+    # Instantiate CachedHTTPSConnection (skip connect() call)
+    with patch.object(security.VerifiedHTTPSConnection, "connect"):
+      self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config)
+
+
+  def tearDown(self):
+    # enable stdout
+    sys.stdout = sys.__stdout__
+
+
+  ### VerifiedHTTPSConnection ###
+
+  @patch.object(security.CertificateManager, "initSecurity")
+  @patch("socket.create_connection")
+  @patch("ssl.wrap_socket")
+  def test_VerifiedHTTPSConnection_connect(self, wrap_socket_mock,
+                                           create_connection_mock,
+                                            init_security_mock):
+    init_security_mock.return_value = None
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    connection = security.VerifiedHTTPSConnection("example.com",
+      self.config.get('server', 'secured_url_port'), self.config)
+    connection._tunnel_host = False
+    connection.sock = None
+    connection.connect()
+    self.assertTrue(wrap_socket_mock.called)
+
+  ### VerifiedHTTPSConnection with no certificates creation
+  @patch.object(security.CertificateManager, "initSecurity")
+  @patch("socket.create_connection")
+  @patch("ssl.wrap_socket")
+  def test_Verified_HTTPSConnection_non_secure_connect(self, wrap_socket_mock,
+                                                    create_connection_mock,
+                                                    init_security_mock):
+    connection = security.VerifiedHTTPSConnection("example.com",
+      self.config.get('server', 'secured_url_port'), self.config)
+    connection._tunnel_host = False
+    connection.sock = None
+    connection.connect()
+    self.assertFalse(init_security_mock.called)
+
+  ### VerifiedHTTPSConnection with two-way SSL authentication enabled
+  @patch.object(security.CertificateManager, "initSecurity")
+  @patch("socket.create_connection")
+  @patch("ssl.wrap_socket")
+  def test_Verified_HTTPSConnection_two_way_ssl_connect(self, wrap_socket_mock,
+                                                    create_connection_mock,
+                                                    init_security_mock):
+    wrap_socket_mock.side_effect=ssl.SSLError()
+    connection = security.VerifiedHTTPSConnection("example.com",
+      self.config.get('server', 'secured_url_port'), self.config)
+    connection._tunnel_host = False
+    connection.sock = None
+    try:
+      connection.connect()
+    except ssl.SSLError:
+      pass
+    self.assertTrue(init_security_mock.called)
+
+  ### CachedHTTPSConnection ###
+
+  @patch.object(security.VerifiedHTTPSConnection, "connect")
+  def test_CachedHTTPSConnection_connect(self, vhc_connect_mock):
+    self.config.set('server', 'hostname', 'dummy.server.hostname')
+    self.config.set('server', 'secured_url_port', '443')
+    # Testing not connected case
+    self.cachedHTTPSConnection.connected = False
+    self.cachedHTTPSConnection.connect()
+    self.assertTrue(vhc_connect_mock.called)
+    vhc_connect_mock.reset_mock()
+    # Testing already connected case
+    self.cachedHTTPSConnection.connect()
+    self.assertFalse(vhc_connect_mock.called)
+
+
+  @patch.object(security.CachedHTTPSConnection, "connect")
+  def test_forceClear(self, connect_mock):
+    # Testing if httpsconn instance changed
+    old = self.cachedHTTPSConnection.httpsconn
+    self.cachedHTTPSConnection.forceClear()
+    self.assertNotEqual(old, self.cachedHTTPSConnection.httpsconn)
+
+
+  @patch.object(security.CachedHTTPSConnection, "connect")
+  def test_request(self, connect_mock):
+    httpsconn_mock = MagicMock(create = True)
+    self.cachedHTTPSConnection.httpsconn = httpsconn_mock
+
+    dummy_request = MagicMock(create = True)
+    dummy_request.get_method.return_value = "dummy_get_method"
+    dummy_request.get_full_url.return_value = "dummy_full_url"
+    dummy_request.get_data.return_value = "dummy_get_data"
+    dummy_request.headers = "dummy_headers"
+
+    responce_mock = MagicMock(create = True)
+    responce_mock.read.return_value = "dummy responce"
+    httpsconn_mock.getresponse.return_value = responce_mock
+
+    # Testing normal case
+    responce = self.cachedHTTPSConnection.request(dummy_request)
+
+    self.assertEqual(responce, responce_mock.read.return_value)
+    httpsconn_mock.request.assert_called_once_with(
+      dummy_request.get_method.return_value,
+      dummy_request.get_full_url.return_value,
+      dummy_request.get_data.return_value,
+      dummy_request.headers)
+
+    # Testing case of exception
+    try:
+      def side_eff():
+        raise Exception("Dummy exception")
+      httpsconn_mock.read.side_effect = side_eff
+      responce = self.cachedHTTPSConnection.request(dummy_request)
+      self.fail("Should raise IOError")
+    except Exception, err:
+      # Expected
+      pass
+
+
+  ### CertificateManager ###
+
+
+  @patch("ambari_agent.hostname.hostname")
+  def test_getAgentKeyName(self, hostname_mock):
+    hostname_mock.return_value = "dummy.hostname"
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    man = CertificateManager(self.config)
+    res = man.getAgentKeyName()
+    self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.key"))
+
+
+  @patch("ambari_agent.hostname.hostname")
+  def test_getAgentCrtName(self, hostname_mock):
+    hostname_mock.return_value = "dummy.hostname"
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    man = CertificateManager(self.config)
+    res = man.getAgentCrtName()
+    self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.crt"))
+
+
+  @patch("ambari_agent.hostname.hostname")
+  def test_getAgentCrtReqName(self, hostname_mock):
+    hostname_mock.return_value = "dummy.hostname"
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    man = CertificateManager(self.config)
+    res = man.getAgentCrtReqName()
+    self.assertEquals(res, os.path.abspath("/dummy-keysdir/dummy.hostname.csr"))
+
+
+  def test_getSrvrCrtName(self):
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    man = CertificateManager(self.config)
+    res = man.getSrvrCrtName()
+    self.assertEquals(res, os.path.abspath("/dummy-keysdir/ca.crt"))
+
+
+  @patch("os.path.exists")
+  @patch.object(security.CertificateManager, "loadSrvrCrt")
+  @patch.object(security.CertificateManager, "getAgentKeyName")
+  @patch.object(security.CertificateManager, "genAgentCrtReq")
+  @patch.object(security.CertificateManager, "getAgentCrtName")
+  @patch.object(security.CertificateManager, "reqSignCrt")
+  def test_checkCertExists(self, reqSignCrt_mock, getAgentCrtName_mock,
+                           genAgentCrtReq_mock, getAgentKeyName_mock,
+                           loadSrvrCrt_mock, exists_mock):
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    getAgentKeyName_mock.return_value = "dummy AgentKeyName"
+    getAgentCrtName_mock.return_value = "dummy AgentCrtName"
+    man = CertificateManager(self.config)
+
+    # Case when all files exist
+    exists_mock.side_effect = [True, True, True]
+    man.checkCertExists()
+    self.assertFalse(loadSrvrCrt_mock.called)
+    self.assertFalse(genAgentCrtReq_mock.called)
+    self.assertFalse(reqSignCrt_mock.called)
+
+    # Absent server cert
+    exists_mock.side_effect = [False, True, True]
+    man.checkCertExists()
+    self.assertTrue(loadSrvrCrt_mock.called)
+    self.assertFalse(genAgentCrtReq_mock.called)
+    self.assertFalse(reqSignCrt_mock.called)
+    loadSrvrCrt_mock.reset_mock()
+
+    # Absent agent key
+    exists_mock.side_effect = [True, False, True]
+    man.checkCertExists()
+    self.assertFalse(loadSrvrCrt_mock.called)
+    self.assertTrue(genAgentCrtReq_mock.called)
+    self.assertFalse(reqSignCrt_mock.called)
+    genAgentCrtReq_mock.reset_mock()
+
+    # Absent agent cert
+    exists_mock.side_effect = [True, True, False]
+    man.checkCertExists()
+    self.assertFalse(loadSrvrCrt_mock.called)
+    self.assertFalse(genAgentCrtReq_mock.called)
+    self.assertTrue(reqSignCrt_mock.called)
+    reqSignCrt_mock.reset_mock()
+
+
+
+  @patch("urllib2.OpenerDirector.open")
+  @patch.object(security.CertificateManager, "getSrvrCrtName")
+  def test_loadSrvrCrt(self, getSrvrCrtName_mock, urlopen_mock):
+    read_mock = MagicMock(create=True)
+    read_mock.read.return_value = "dummy_cert"
+    urlopen_mock.return_value = read_mock
+    _, tmpoutfile = tempfile.mkstemp()
+    getSrvrCrtName_mock.return_value = tmpoutfile
+
+    man = CertificateManager(self.config)
+    man.loadSrvrCrt()
+
+    # Checking file contents
+    saved = open(tmpoutfile, 'r').read()
+    self.assertEqual(saved, read_mock.read.return_value)
+    try:
+      os.unlink(tmpoutfile)
+    except:
+      pass
+
+
+  @patch("ambari_agent.hostname.hostname")
+  @patch('__builtin__.open', create=True, autospec=True)
+  @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+  @patch('ambari_simplejson.dumps')
+  @patch('urllib2.Request')
+  @patch("urllib2.OpenerDirector.open")
+  @patch('ambari_simplejson.loads')
+  def test_reqSignCrt(self, loads_mock, urlopen_mock, request_mock, dumps_mock, open_mock, hostname_mock):
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+    man = CertificateManager(self.config)
+    hostname_mock.return_value = "dummy-hostname"
+
+    open_mock.return_value.read.return_value = "dummy_request"
+    urlopen_mock.return_value.read.return_value = "dummy_server_request"
+    loads_mock.return_value = {
+      'result': 'OK',
+      'signedCa': 'dummy-crt'
+    }
+
+    # Test normal server interaction
+    man.reqSignCrt()
+
+    self.assertEqual(dumps_mock.call_args[0][0], {
+      'csr'       : 'dummy_request',
+      'passphrase' : 'dummy-passphrase'
+    })
+    self.assertEqual(open_mock.return_value.write.call_args[0][0], 'dummy-crt')
+
+    # Test negative server reply
+    dumps_mock.reset_mock()
+    open_mock.return_value.write.reset_mock()
+    loads_mock.return_value = {
+      'result': 'FAIL',
+      'signedCa': 'fail-crt'
+    }
+
+    # If certificate signing failed, then exception must be raised
+    try:
+      man.reqSignCrt()
+      self.fail()
+    except ssl.SSLError:
+      pass
+    self.assertFalse(open_mock.return_value.write.called)
+
+    # Test connection fail
+    dumps_mock.reset_mock()
+    open_mock.return_value.write.reset_mock()
+
+    try:
+      man.reqSignCrt()
+      self.fail("Expected exception here")
+    except Exception, err:
+      # expected
+      pass
+
+    # Test malformed JSON response
+    open_mock.return_value.write.reset_mock()
+    loads_mock.side_effect = Exception()
+    try:
+      man.reqSignCrt()
+      self.fail("Expected exception here")
+    except ssl.SSLError:
+      pass
+    self.assertFalse(open_mock.return_value.write.called)
+
+  @patch("subprocess.Popen")
+  @patch("subprocess.Popen.communicate")
+  def test_genAgentCrtReq(self, communicate_mock, popen_mock):
+    man = CertificateManager(self.config)
+    p = MagicMock(spec=subprocess.Popen)
+    p.communicate = communicate_mock
+    popen_mock.return_value = p
+    man.genAgentCrtReq()
+    self.assertTrue(popen_mock.called)
+    self.assertTrue(communicate_mock.called)
+
+  @patch("ambari_agent.hostname.hostname")
+  @patch('__builtin__.open', create=True, autospec=True)
+  @patch("urllib2.OpenerDirector.open")
+  @patch.dict('os.environ', {'DUMMY_PASSPHRASE': 'dummy-passphrase'})
+  def test_reqSignCrt_malformedJson(self, urlopen_mock, open_mock, hostname_mock):
+    hostname_mock.return_value = "dummy-hostname"
+    open_mock.return_value.read.return_value = "dummy_request"
+    self.config.set('security', 'keysdir', '/dummy-keysdir')
+    self.config.set('security', 'passphrase_env_var_name', 'DUMMY_PASSPHRASE')
+    man = CertificateManager(self.config)
+
+    # test valid JSON response
+    urlopen_mock.return_value.read.return_value = '{"result": "OK", "signedCa":"dummy"}'
+    try:
+      man.reqSignCrt()
+    except ssl.SSLError:
+      self.fail("Unexpected exception!")
+    open_mock.return_value.write.assert_called_with(u'dummy')
+
+    # test malformed JSON response
+    open_mock.return_value.write.reset_mock()
+    urlopen_mock.return_value.read.return_value = '{malformed_object}'
+    try:
+      man.reqSignCrt()
+      self.fail("Expected exception!")
+    except ssl.SSLError:
+      pass
+    self.assertFalse(open_mock.return_value.write.called)
+
+  @patch.object(security.CertificateManager, "checkCertExists")
+  def test_initSecurity(self, checkCertExists_method):
+    man = CertificateManager(self.config)
+    man.initSecurity()
+    self.assertTrue(checkCertExists_method.called)
+

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -61,7 +61,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
 <BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package
 <STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
 <LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
-<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp
+<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/tmp
 """
 
 _PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}

ambari-server/src/main/python/bootstrap.py

@@ -47,7 +47,7 @@ MAX_PARALLEL_BOOTSTRAPS = 20
 # How many seconds to wait between polling parallel bootstraps
 POLL_INTERVAL_SEC = 1
 DEBUG = False
-DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp"
+DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/tmp"
 DEFAULT_AGENT_DATA_FOLDER = "/var/lib/ambari-agent/data"
 DEFAULT_AGENT_LIB_FOLDER = "/var/lib/ambari-agent"
 PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:" + DEFAULT_AGENT_TEMP_FOLDER

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py

@@ -202,7 +202,7 @@ class NameNodeDefault(NameNode):
       # Create the kerberos credentials cache (ccache) file and set it in the environment to use
       # when executing HDFS rebalance command. Use the md5 hash of the combination of the principal and keytab file
       # to generate a (relatively) unique cache filename so that we can use it as needed.
-      # TODO: params.tmp_dir=/var/lib/ambari-agent/data/tmp. However hdfs user doesn't have access to this path.
+      # TODO: params.tmp_dir=/var/lib/ambari-agent/tmp. However hdfs user doesn't have access to this path.
       # TODO: Hence using /tmp
       ccache_file_name = "hdfs_rebalance_cc_" + _md5(format("{hdfs_principal_name}|{hdfs_user_keytab}")).hexdigest()
       ccache_file_path = os.path.join(tempfile.gettempdir(), ccache_file_name)

ambari-server/src/test/python/TestBootstrap.py

@@ -112,8 +112,8 @@ class TestBootstrap(TestCase):
     utime = 1234
     bootstrap_obj.getUtime = MagicMock(return_value=utime)
     ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
-    expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
-               "ambariServer root  8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+    expected = "sudo -S python /var/lib/ambari-agent/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+               "ambariServer root  8440 < /var/lib/ambari-agent/tmp/host_pass{0}".format(utime)
     self.assertEquals(ret, expected)
 
 
@@ -353,11 +353,11 @@ class TestBootstrap(TestCase):
     self.assertEquals(res, expected)
     command = str(init_mock.call_args[0][3])
     self.assertEqual(command,
-                     "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
-                     "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+                     "sudo mkdir -p /var/lib/ambari-agent/tmp ; "
+                     "sudo chown -R root /var/lib/ambari-agent/tmp ; "
                      "sudo chmod 755 /var/lib/ambari-agent ; "
                      "sudo chmod 755 /var/lib/ambari-agent/data ; "
-                     "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+                     "sudo chmod 777 /var/lib/ambari-agent/tmp")
 
   @patch.object(BootstrapDefault, "getOsCheckScript")
   @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
@@ -520,7 +520,7 @@ class TestBootstrap(TestCase):
     command = str(init_mock.call_args[0][3])
     self.assertEqual(command,
                      "chmod a+x OsCheckScriptRemoteLocation && "
-                     "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+                     "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/tmp OsCheckScriptRemoteLocation centos6")
 
 
   @patch.object(SSH, "__init__")

ambari-server/src/test/python/TestBootstrap.py.orig

@@ -0,0 +1,879 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import *
+import bootstrap
+import time
+import subprocess
+import os
+import logging
+import tempfile
+import pprint
+
+from ambari_commons.os_check import OSCheck
+from bootstrap import PBootstrap, Bootstrap, BootstrapDefault, SharedState, HostLog, SCP, SSH
+from unittest import TestCase
+from subprocess import Popen
+from bootstrap import AMBARI_PASSPHRASE_VAR_NAME
+from mock.mock import MagicMock, call
+from mock.mock import patch
+from mock.mock import create_autospec
+from only_for_platform import not_for_platform, os_distro_value, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+class TestBootstrap(TestCase):
+
+  def setUp(self):
+    logging.basicConfig(level=logging.ERROR)
+
+
+  def test_getRemoteName(self):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                      "setupAgentFile", "ambariServer", "centos6", None, "8440", "root")
+    res = bootstrap_obj = Bootstrap("hostname", shared_state)
+    utime1 = 1234
+    utime2 = 12345
+    bootstrap_obj.getUtime = MagicMock(return_value=utime1)
+    remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh")
+    self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1))
+
+    bootstrap_obj.getUtime.return_value=utime2
+    remote1 = bootstrap_obj.getRemoteName("/tmp/setupAgent.sh")
+    self.assertEquals(remote1, "/tmp/setupAgent{0}.sh".format(utime1))
+
+    remote2 = bootstrap_obj.getRemoteName("/tmp/host_pass")
+    self.assertEquals(remote2, "/tmp/host_pass{0}".format(utime2))
+
+
+  # TODO: Test bootstrap timeout
+
+  # TODO: test_return_error_message_for_missing_sudo_package
+
+  def test_getAmbariPort(self):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    self.assertEquals(bootstrap_obj.getAmbariPort(),"8440")
+    shared_state.server_port = None
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    self.assertEquals(bootstrap_obj.getAmbariPort(),"null")
+
+
+  @patch.object(subprocess, "Popen")
+  @patch("sys.stderr")
+  @patch("sys.exit")
+  @patch.object(PBootstrap, "run")
+  @patch("os.path.dirname")
+  @patch("os.path.realpath")
+  def test_bootstrap_main(self, dirname_mock, realpath_mock, run_mock, exit_mock, stderr_mock, subprocess_Popen_mock):
+    bootstrap.main(["bootstrap.py", "hostname,hostname2", "/tmp/bootstrap", "root", "sshkey_file", "setupAgent.py", "ambariServer", \
+                    "centos6", "1.1.1", "8440", "root", "passwordfile"])
+    self.assertTrue(run_mock.called)
+    run_mock.reset_mock()
+    bootstrap.main(["bootstrap.py", "hostname,hostname2", "/tmp/bootstrap", "root", "sshkey_file", "setupAgent.py", "ambariServer", \
+                    "centos6", "1.1.1", "8440", "root", None])
+    self.assertTrue(run_mock.called)
+    run_mock.reset_mock()
+    def side_effect(retcode):
+      raise Exception(retcode, "sys.exit")
+    exit_mock.side_effect = side_effect
+    try:
+      bootstrap.main(["bootstrap.py","hostname,hostname2", "/tmp/bootstrap"])
+      self.fail("sys.exit(2)")
+    except Exception:
+    # Expected
+      pass
+    self.assertTrue(exit_mock.called)
+
+
+  @patch("os.environ")
+  def test_getRunSetupWithPasswordCommand(self, environ_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    environ_mock.__getitem__.return_value = "TEST_PASSPHRASE"
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    utime = 1234
+    bootstrap_obj.getUtime = MagicMock(return_value=utime)
+    ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname")
+    expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \
+               "ambariServer root  8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime)
+    self.assertEquals(ret, expected)
+
+
+  def test_generateRandomFileName(self):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    self.assertTrue(bootstrap_obj.generateRandomFileName(None) == bootstrap_obj.getUtime())
+
+
+
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(OSCheck, "is_suse_family")
+  def test_getRepoDir(self, is_suse_family, is_redhat_family):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    # Suse
+    is_redhat_family.return_value = False
+    is_suse_family.return_value = True
+    res = bootstrap_obj.getRepoDir()
+    self.assertEquals(res, "/etc/zypp/repos.d")
+    # non-Suse
+    is_suse_family.return_value = False
+    is_redhat_family.return_value = True
+    res = bootstrap_obj.getRepoDir()
+    self.assertEquals(res, "/etc/yum.repos.d")
+
+  def test_getSetupScript(self):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    self.assertEquals(bootstrap_obj.shared_state.script_dir, "scriptDir")
+
+
+  def test_run_setup_agent_command_ends_with_project_version(self):
+    os.environ[AMBARI_PASSPHRASE_VAR_NAME] = ""
+    version = "1.1.1"
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               version, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname")
+    self.assertTrue(runSetupCommand.endswith(version + " 8440"))
+
+
+  def test_agent_setup_command_without_project_version(self):
+    os.environ[AMBARI_PASSPHRASE_VAR_NAME] = ""
+    version = None
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               version, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    runSetupCommand = bootstrap_obj.getRunSetupCommand("hostname")
+    self.assertTrue(runSetupCommand.endswith(" 8440"))
+
+
+  # TODO: test_os_check_fail_fails_bootstrap_execution
+
+
+  def test_host_log(self):
+    tmp_file, tmp_filename  = tempfile.mkstemp()
+    dummy_log = HostLog(tmp_filename)
+    # First write to log
+    dummy_log.write("a\nb\nc")
+    # Read it
+    with open(tmp_filename) as f:
+      s = f.read()
+      etalon = "a\nb\nc\n"
+      self.assertEquals(s, etalon)
+    # Next write
+    dummy_log.write("Yet another string")
+    # Read it
+    with open(tmp_filename) as f:
+      s = f.read()
+      etalon = "a\nb\nc\nYet another string\n"
+      self.assertEquals(s, etalon)
+    # Should not append line end if it already exists
+    dummy_log.write("line break->\n")
+    # Read it
+    with open(tmp_filename) as f:
+      s = f.read()
+      etalon = "a\nb\nc\nYet another string\nline break->\n"
+      self.assertEquals(s, etalon)
+    # Cleanup
+    os.unlink(tmp_filename)
+
+
+  @patch("subprocess.Popen")
+  def test_SCP(self, popenMock):
+    params = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                                  "setupAgentFile", "ambariServer", "centos6",
+                                  "1.2.1", "8440", "root")
+    host_log_mock = MagicMock()
+    log = {'text': ""}
+    def write_side_effect(text):
+      log['text'] = log['text'] + text
+
+    host_log_mock.write.side_effect = write_side_effect
+    scp = SCP(params.user, params.sshkey_file, "dummy-host", "src/file",
+              "dst/file", params.bootdir, host_log_mock)
+    log_sample = "log_sample"
+    error_sample = "error_sample"
+    # Successful run
+    process = MagicMock()
+    popenMock.return_value = process
+    process.communicate.return_value = (log_sample, error_sample)
+    process.returncode = 0
+
+    retcode = scp.run()
+
+    self.assertTrue(popenMock.called)
+    self.assertTrue(log_sample in log['text'])
+    self.assertTrue(error_sample in log['text'])
+    command_str = str(popenMock.call_args[0][0])
+    self.assertEquals(command_str, "['scp', '-r', '-o', 'ConnectTimeout=60', '-o', "
+        "'BatchMode=yes', '-o', 'StrictHostKeyChecking=no', '-i', 'sshkey_file',"
+        " 'src/file', 'root@dummy-host:dst/file']")
+    self.assertEqual(retcode["exitstatus"], 0)
+
+    log['text'] = ""
+    #unsuccessfull run
+    process.returncode = 1
+
+    retcode = scp.run()
+
+    self.assertTrue(log_sample in log['text'])
+    self.assertTrue(error_sample in log['text'])
+    self.assertEqual(retcode["exitstatus"], 1)
+
+
+  @patch("subprocess.Popen")
+  def test_SSH(self, popenMock):
+    params = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                                  "setupAgentFile", "ambariServer", "centos6",
+                                  "1.2.1", "8440", "root")
+    host_log_mock = MagicMock()
+    log = {'text': ""}
+    def write_side_effect(text):
+      log['text'] = log['text'] + text
+
+    host_log_mock.write.side_effect = write_side_effect
+    ssh = SSH(params.user, params.sshkey_file, "dummy-host", "dummy-command",
+              params.bootdir, host_log_mock)
+    log_sample = "log_sample"
+    error_sample = "error_sample"
+    # Successful run
+    process = MagicMock()
+    popenMock.return_value = process
+    process.communicate.return_value = (log_sample, error_sample)
+    process.returncode = 0
+
+    retcode = ssh.run()
+
+    self.assertTrue(popenMock.called)
+    self.assertTrue(log_sample in log['text'])
+    self.assertTrue(error_sample in log['text'])
+    command_str = str(popenMock.call_args[0][0])
+    self.assertEquals(command_str, "['ssh', '-o', 'ConnectTimeOut=60', '-o', "
+            "'StrictHostKeyChecking=no', '-o', 'BatchMode=yes', '-tt', '-i', "
+            "'sshkey_file', 'root@dummy-host', 'dummy-command']")
+    self.assertEqual(retcode["exitstatus"], 0)
+
+    log['text'] = ""
+    #unsuccessfull run
+    process.returncode = 1
+
+    retcode = ssh.run()
+
+    self.assertTrue(log_sample in log['text'])
+    self.assertTrue(error_sample in log['text'])
+    self.assertEqual(retcode["exitstatus"], 1)
+
+    log['text'] = ""
+    # unsuccessful run with error message
+    process.returncode = 1
+
+    dummy_error_message = "dummy_error_message"
+    ssh = SSH(params.user, params.sshkey_file, "dummy-host", "dummy-command",
+              params.bootdir, host_log_mock, errorMessage= dummy_error_message)
+    retcode = ssh.run()
+
+    self.assertTrue(log_sample in log['text'])
+    self.assertTrue(error_sample in log['text'])
+    self.assertTrue(dummy_error_message in log['text'])
+    self.assertEqual(retcode["exitstatus"], 1)
+
+
+  def test_getOsCheckScript(self):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    ocs = bootstrap_obj.getOsCheckScript()
+    self.assertEquals(ocs, "scriptDir/os_check_type.py")
+
+
+  @patch.object(BootstrapDefault, "getRemoteName")
+  def test_getOsCheckScriptRemoteLocation(self, getRemoteName_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    v = "/tmp/os_check_type1374259902.py"
+    getRemoteName_mock.return_value = v
+    ocs = bootstrap_obj.getOsCheckScriptRemoteLocation()
+    self.assertEquals(ocs, v)
+
+
+  @patch.object(BootstrapDefault, "is_suse")
+  def test_getRepoFile(self, is_suse_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    is_suse_mock.return_value = False
+    rf = bootstrap_obj.getRepoFile()
+    self.assertEquals(rf, "/etc/yum.repos.d/ambari.repo")
+
+
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_createTargetDir(self, write_mock, run_mock,
+                            init_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.createTargetDir()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command,
+                     "sudo mkdir -p /var/lib/ambari-agent/data/tmp ; "
+                     "sudo chown -R root /var/lib/ambari-agent/data/tmp ; "
+                     "sudo chmod 755 /var/lib/ambari-agent ; "
+                     "sudo chmod 755 /var/lib/ambari-agent/data ; "
+                     "sudo chmod 777 /var/lib/ambari-agent/data/tmp")
+
+  @patch.object(BootstrapDefault, "getOsCheckScript")
+  @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
+  @patch.object(SCP, "__init__")
+  @patch.object(SCP, "run")
+  @patch.object(HostLog, "write")
+  def test_copyOsCheckScript(self, write_mock, run_mock, init_mock,
+                    getOsCheckScriptRemoteLocation_mock, getOsCheckScript_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    getOsCheckScript_mock.return_value = "OsCheckScript"
+    getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation"
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.copyOsCheckScript()
+    self.assertEquals(res, expected)
+    input_file = str(init_mock.call_args[0][3])
+    remote_file = str(init_mock.call_args[0][4])
+    self.assertEqual(input_file, "OsCheckScript")
+    self.assertEqual(remote_file, "OsCheckScriptRemoteLocation")
+
+
+  @patch.object(BootstrapDefault, "getRemoteName")
+  @patch.object(BootstrapDefault, "hasPassword")
+  @patch.object(OSCheck, "is_suse_family")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  def test_getRepoFile(self, is_redhat_family, is_ubuntu_family, is_suse_family, hasPassword_mock, getRemoteName_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    is_redhat_family.return_value = True
+    is_ubuntu_family.return_value = False
+    is_suse_family.return_value = False
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    # Without password
+    hasPassword_mock.return_value = False
+    getRemoteName_mock.return_value = "RemoteName"
+    rf = bootstrap_obj.getMoveRepoFileCommand("target")
+    self.assertEquals(rf, "sudo mv RemoteName target/ambari.repo")
+    # With password
+    hasPassword_mock.return_value = True
+    getRemoteName_mock.return_value = "RemoteName"
+    rf = bootstrap_obj.getMoveRepoFileCommand("target")
+    self.assertEquals(rf, "sudo -S mv RemoteName target/ambari.repo < RemoteName")
+
+  @patch("os.path.exists")
+  @patch.object(OSCheck, "is_suse_family")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(BootstrapDefault, "getMoveRepoFileCommand")
+  @patch.object(BootstrapDefault, "getRepoDir")
+  @patch.object(BootstrapDefault, "getRepoFile")
+  @patch.object(BootstrapDefault, "getRemoteName")
+  @patch.object(SCP, "__init__")
+  @patch.object(SCP, "run")
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_copyNeededFiles(self, write_mock, ssh_run_mock, ssh_init_mock,
+                           scp_run_mock, scp_init_mock,
+                           getRemoteName_mock, getRepoFile_mock, getRepoDir,
+                           getMoveRepoFileCommand, is_redhat_family, is_ubuntu_family, is_suse_family,
+                           os_path_exists_mock):
+    #
+    # Ambari repo file exists
+    #
+    def os_path_exists_side_effect(*args, **kwargs):
+      if args[0] == getRepoFile_mock():
+        return True
+      else:
+        return False
+
+    os_path_exists_mock.side_effect = os_path_exists_side_effect
+    os_path_exists_mock.return_value = None
+
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    is_redhat_family.return_value = True
+    is_ubuntu_family.return_value = False
+    is_suse_family.return_value = False
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    getMoveRepoFileCommand.return_value = "MoveRepoFileCommand"
+    getRepoDir.return_value  = "RepoDir"
+    getRemoteName_mock.return_value = "RemoteName"
+    getRepoFile_mock.return_value = "RepoFile"
+    expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+    expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"}
+    expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    scp_init_mock.return_value = None
+    ssh_init_mock.return_value = None
+    # Testing max retcode return
+    scp_run_mock.side_effect = [expected1, expected3]
+    ssh_run_mock.side_effect = [expected2, expected4]
+    res = bootstrap_obj.copyNeededFiles()
+    self.assertEquals(res, expected1["exitstatus"])
+    input_file = str(scp_init_mock.call_args[0][3])
+    remote_file = str(scp_init_mock.call_args[0][4])
+    self.assertEqual(input_file, "setupAgentFile")
+    self.assertEqual(remote_file, "RemoteName")
+    command = str(ssh_init_mock.call_args[0][3])
+    self.assertEqual(command, "sudo chmod 644 RepoFile")
+    # Another order
+    expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"}
+    expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    expected3 = {"exitstatus": 1, "log": "log1", "errormsg": "errorMsg"}
+    expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    scp_run_mock.side_effect = [expected1, expected3]
+    ssh_run_mock.side_effect = [expected2, expected4]
+    res = bootstrap_obj.copyNeededFiles()
+    self.assertEquals(res, expected2["exitstatus"])
+    # yet another order
+    expected1 = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"}
+    expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    expected3 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+    expected4 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    scp_run_mock.side_effect = [expected1, expected3]
+    ssh_run_mock.side_effect = [expected2, expected4]
+    res = bootstrap_obj.copyNeededFiles()
+    self.assertEquals(res, expected3["exitstatus"])
+
+    #
+    #Ambari repo file does not exist
+    #
+    os_path_exists_mock.side_effect = None
+    os_path_exists_mock.return_value = False
+
+    #Expectations:
+    # SSH will not be called at all
+    # SCP will be called once for copying the setup script file
+    scp_run_mock.reset_mock()
+    ssh_run_mock.reset_mock()
+    expectedResult = {"exitstatus": 33, "log": "log33", "errormsg": "errorMsg"}
+    scp_run_mock.side_effect = [expectedResult]
+    res = bootstrap_obj.copyNeededFiles()
+    self.assertFalse(ssh_run_mock.called)
+    self.assertEquals(res, expectedResult["exitstatus"])
+
+  @patch.object(BootstrapDefault, "getOsCheckScriptRemoteLocation")
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_runOsCheckScript(self, write_mock, run_mock,
+                            init_mock, getOsCheckScriptRemoteLocation_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    getOsCheckScriptRemoteLocation_mock.return_value = "OsCheckScriptRemoteLocation"
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.runOsCheckScript()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command,
+                     "chmod a+x OsCheckScriptRemoteLocation && "
+                     "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6")
+
+
+  @patch.object(SSH, "__init__")
+  @patch.object(BootstrapDefault, "getRunSetupCommand")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_runSetupAgent(self, write_mock, run_mock,
+                         getRunSetupCommand_mock, init_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    getRunSetupCommand_mock.return_value = "RunSetupCommand"
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.runSetupAgent()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command, "RunSetupCommand")
+
+
+  @patch.object(BootstrapDefault, "hasPassword")
+  @patch.object(BootstrapDefault, "getRunSetupWithPasswordCommand")
+  @patch.object(BootstrapDefault, "getRunSetupWithoutPasswordCommand")
+  def test_getRunSetupCommand(self, getRunSetupWithoutPasswordCommand_mock,
+                              getRunSetupWithPasswordCommand_mock,
+                              hasPassword_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    # With password
+    hasPassword_mock.return_value = True
+    getRunSetupWithPasswordCommand_mock.return_value = "RunSetupWithPasswordCommand"
+    getRunSetupWithoutPasswordCommand_mock.return_value = "RunSetupWithoutPasswordCommand"
+    res = bootstrap_obj.getRunSetupCommand("dummy-host")
+    self.assertEqual(res, "RunSetupWithPasswordCommand")
+    # Without password
+    hasPassword_mock.return_value = False
+    res = bootstrap_obj.getRunSetupCommand("dummy-host")
+    self.assertEqual(res, "RunSetupWithoutPasswordCommand")
+
+
+  @patch.object(HostLog, "write")
+  def test_createDoneFile(self, write_mock):
+    tmp_dir = tempfile.gettempdir()
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", tmp_dir,
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    done_file = os.path.join(tmp_dir, "hostname.done")
+    expected = 42
+    bootstrap_obj.createDoneFile(expected)
+    with open(done_file) as df:
+      res = df.read()
+      self.assertEqual(res, str(expected))
+    os.unlink(done_file)
+
+  @patch.object(OSCheck, "is_suse_family")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_checkSudoPackage(self, write_mock, run_mock, init_mock, is_redhat_family, is_ubuntu_family, is_suse_family):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    is_redhat_family.return_value = True
+    is_ubuntu_family.return_value = False
+    is_suse_family.return_value = False
+    res = bootstrap_obj.checkSudoPackage()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command, "rpm -qa | grep -e '^sudo\-'")
+
+  @patch.object(OSCheck, "is_suse_family")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_checkSudoPackageUbuntu(self, write_mock, run_mock, init_mock,
+                                  is_redhat_family, is_ubuntu_family, is_suse_family):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "ubuntu12",
+                               None, "8440", "root")
+    is_redhat_family.return_value = False
+    is_ubuntu_family.return_value = True
+    is_suse_family.return_value = False
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    expected = 42
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.checkSudoPackage()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command, "dpkg --get-selections|grep -e '^sudo\s*install'")
+
+
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  @patch.object(BootstrapDefault, "getPasswordFile")
+  def test_deletePasswordFile(self, getPasswordFile_mock, write_mock, run_mock,
+                              init_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    expected = 42
+    getPasswordFile_mock.return_value = "PasswordFile"
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.deletePasswordFile()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command, "rm PasswordFile")
+
+
+  @patch.object(BootstrapDefault, "getPasswordFile")
+  @patch.object(SCP, "__init__")
+  @patch.object(SCP, "run")
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  def test_copyPasswordFile(self, write_mock, ssh_run_mock,
+                            ssh_init_mock, scp_run_mock,
+                            scp_init_mock, getPasswordFile_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root", password_file="PasswordFile")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    getPasswordFile_mock.return_value = "PasswordFile"
+     # Testing max retcode return
+    expected1 = {"exitstatus": 42, "log": "log42", "errormsg": "errorMsg"}
+    expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    scp_init_mock.return_value = None
+    scp_run_mock.return_value = expected1
+    ssh_init_mock.return_value = None
+    ssh_run_mock.return_value = expected2
+    res = bootstrap_obj.copyPasswordFile()
+    self.assertEquals(res, expected1["exitstatus"])
+    input_file = str(scp_init_mock.call_args[0][3])
+    remote_file = str(scp_init_mock.call_args[0][4])
+    self.assertEqual(input_file, "PasswordFile")
+    self.assertEqual(remote_file, "PasswordFile")
+    command = str(ssh_init_mock.call_args[0][3])
+    self.assertEqual(command, "chmod 600 PasswordFile")
+    # Another order
+    expected1 = {"exitstatus": 0, "log": "log0", "errormsg": "errorMsg"}
+    expected2 = {"exitstatus": 17, "log": "log17", "errormsg": "errorMsg"}
+    scp_run_mock.return_value = expected1
+    ssh_run_mock.return_value = expected2
+
+
+  @patch.object(SSH, "__init__")
+  @patch.object(SSH, "run")
+  @patch.object(HostLog, "write")
+  @patch.object(BootstrapDefault, "getPasswordFile")
+  def test_changePasswordFileModeOnHost(self, getPasswordFile_mock, write_mock,
+                                        run_mock, init_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    expected = 42
+    getPasswordFile_mock.return_value = "PasswordFile"
+    init_mock.return_value = None
+    run_mock.return_value = expected
+    res = bootstrap_obj.changePasswordFileModeOnHost()
+    self.assertEquals(res, expected)
+    command = str(init_mock.call_args[0][3])
+    self.assertEqual(command, "chmod 600 PasswordFile")
+
+
+  @patch.object(HostLog, "write")
+  def test_try_to_execute(self, write_mock):
+    expected = 43
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    # Normal case
+    def act_normal_return_int():
+      return 43
+    ret = bootstrap_obj.try_to_execute(act_normal_return_int)
+    self.assertEqual(ret["exitstatus"], expected)
+    self.assertFalse(write_mock.called)
+    write_mock.reset_mock()
+    def act_normal_return():
+        return {"exitstatus": 43}
+    ret = bootstrap_obj.try_to_execute(act_normal_return)
+    self.assertEqual(ret["exitstatus"], expected)
+    self.assertFalse(write_mock.called)
+    write_mock.reset_mock()
+    # Exception scenario
+    def act():
+      raise IOError()
+    ret = bootstrap_obj.try_to_execute(act)
+    self.assertEqual(ret["exitstatus"], 177)
+    self.assertTrue(write_mock.called)
+
+
+  @patch.object(BootstrapDefault, "try_to_execute")
+  @patch.object(BootstrapDefault, "hasPassword")
+  @patch.object(BootstrapDefault, "createDoneFile")
+  @patch.object(HostLog, "write")
+  @patch("logging.warn")
+  @patch("logging.error")
+  def test_run(self, error_mock, warn_mock, write_mock, createDoneFile_mock,
+               hasPassword_mock, try_to_execute_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    # Testing workflow without password
+    bootstrap_obj.copied_password_file = False
+    hasPassword_mock.return_value = False
+    try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}
+    bootstrap_obj.run()
+    self.assertEqual(try_to_execute_mock.call_count, 7) # <- Adjust if changed
+    self.assertTrue(createDoneFile_mock.called)
+    self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0)
+
+    try_to_execute_mock.reset_mock()
+    createDoneFile_mock.reset_mock()
+    # Testing workflow with password
+    bootstrap_obj.copied_password_file = True
+    hasPassword_mock.return_value = True
+    try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}
+    bootstrap_obj.run()
+    self.assertEqual(try_to_execute_mock.call_count, 10) # <- Adjust if changed
+    self.assertTrue(createDoneFile_mock.called)
+    self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0)
+
+    error_mock.reset_mock()
+    write_mock.reset_mock()
+    try_to_execute_mock.reset_mock()
+    createDoneFile_mock.reset_mock()
+    # Testing workflow when some action failed before copying password
+    bootstrap_obj.copied_password_file = False
+    hasPassword_mock.return_value = False
+    try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 1, "log":"log1", "errormsg":"errormsg1"}]
+    bootstrap_obj.run()
+    self.assertEqual(try_to_execute_mock.call_count, 2) # <- Adjust if changed
+    self.assertTrue("ERROR" in error_mock.call_args[0][0])
+    self.assertTrue("ERROR" in write_mock.call_args[0][0])
+    self.assertTrue(createDoneFile_mock.called)
+    self.assertEqual(bootstrap_obj.getStatus()["return_code"], 1)
+
+    try_to_execute_mock.reset_mock()
+    createDoneFile_mock.reset_mock()
+    # Testing workflow when some action failed after copying password
+    bootstrap_obj.copied_password_file = True
+    hasPassword_mock.return_value = True
+    try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 42, "log":"log42", "errormsg":"errormsg42"}, {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}]
+    bootstrap_obj.run()
+    self.assertEqual(try_to_execute_mock.call_count, 3) # <- Adjust if changed
+    self.assertTrue(createDoneFile_mock.called)
+    self.assertEqual(bootstrap_obj.getStatus()["return_code"], 42)
+
+    error_mock.reset_mock()
+    write_mock.reset_mock()
+    try_to_execute_mock.reset_mock()
+    createDoneFile_mock.reset_mock()
+    # Testing workflow when some action failed after copying password and
+    # removing password failed too
+    bootstrap_obj.copied_password_file = True
+    hasPassword_mock.return_value = True
+    try_to_execute_mock.side_effect = [{"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"}, {"exitstatus": 17, "log":"log17", "errormsg":"errormsg17"}, {"exitstatus": 19, "log":"log19", "errormsg":"errormsg19"}]
+    bootstrap_obj.run()
+    self.assertEqual(try_to_execute_mock.call_count, 3) # <- Adjust if changed
+    self.assertTrue("ERROR" in write_mock.call_args_list[0][0][0])
+    self.assertTrue("ERROR" in error_mock.call_args[0][0])
+    self.assertTrue("WARNING" in write_mock.call_args_list[1][0][0])
+    self.assertTrue("WARNING" in warn_mock.call_args[0][0])
+    self.assertTrue(createDoneFile_mock.called)
+    self.assertEqual(bootstrap_obj.getStatus()["return_code"], 17)
+
+
+  @patch.object(BootstrapDefault, "createDoneFile")
+  @patch.object(HostLog, "write")
+  def test_interruptBootstrap(self, write_mock, createDoneFile_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    bootstrap_obj = Bootstrap("hostname", shared_state)
+    bootstrap_obj.interruptBootstrap()
+    self.assertTrue(createDoneFile_mock.called)
+
+
+  @patch("time.sleep")
+  @patch("time.time")
+  @patch("logging.warn")
+  @patch("logging.info")
+  @patch.object(BootstrapDefault, "start")
+  @patch.object(BootstrapDefault, "interruptBootstrap")
+  @patch.object(BootstrapDefault, "getStatus")
+  def test_PBootstrap(self, getStatus_mock, interruptBootstrap_mock, start_mock,
+                      info_mock, warn_mock, time_mock, sleep_mock):
+    shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir",
+                               "setupAgentFile", "ambariServer", "centos6",
+                               None, "8440", "root")
+    n = 180
+    time = 100500
+    time_mock.return_value = time
+    hosts = []
+    for i in range(0, n):
+      hosts.append("host" + str(i))
+    # Testing normal case
+    getStatus_mock.return_value = {"return_code": 0,
+                                   "start_time": time + 999}
+    pbootstrap_obj = PBootstrap(hosts, shared_state)
+    pbootstrap_obj.run()
+    self.assertEqual(start_mock.call_count, n)
+    self.assertEqual(interruptBootstrap_mock.call_count, 0)
+
+    start_mock.reset_mock()
+    getStatus_mock.reset_mock()
+    # Testing case of timeout
+    def fake_return_code_generator():
+      call_number = 0
+      while True:
+        call_number += 1
+        if call_number % 5 != 0:   # ~80% of hosts finish successfully
+          yield 0
+        else:
+          yield None
+
+    def fake_start_time_generator():
+      while True:
+        yield time - bootstrap.HOST_BOOTSTRAP_TIMEOUT - 1
+
+    return_code_generator = fake_return_code_generator()
+    start_time_generator = fake_start_time_generator()
+
+    def status_get_item_mock(item):
+      if item == "return_code":
+        return return_code_generator.next()
+      elif item == "start_time":
+        return start_time_generator.next()
+
+    dict_mock = MagicMock()
+    dict_mock.__getitem__.side_effect = status_get_item_mock
+    getStatus_mock.return_value = dict_mock
+
+    pbootstrap_obj.run()
+    self.assertEqual(start_mock.call_count, n)
+    self.assertEqual(interruptBootstrap_mock.call_count, n / 5)
+