AMBARI-8298. Distribute Repositories/Install Components - custom action script (dlysnichenko)

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -76,7 +76,7 @@ class CustomServiceOrchestrator():
 
   def map_task_to_process(self, task_id, processId):
     with self.commands_in_progress_lock:
-      logger.debug('Maps taskId=%s to pid=%s'%(task_id, processId))
+      logger.debug('Maps taskId=%s to pid=%s' % (task_id, processId))
       self.commands_in_progress[task_id] = processId
 
   def cancel_command(self, task_id, reason):
@@ -86,12 +86,12 @@ class CustomServiceOrchestrator():
         self.commands_in_progress[task_id] = reason
         logger.info("Canceling command with task_id - {tid}, " \
                     "reason - {reason} . Killing process {pid}"
-        .format(tid = str(task_id), reason = reason, pid = pid))
+                    .format(tid=str(task_id), reason=reason, pid=pid))
         shell.kill_process_with_children(pid)
-      else:
-        logger.warn("Unable to find pid by taskId = %s"%task_id)
+      else: 
+        logger.warn("Unable to find pid by taskId = %s" % task_id)
 
-  def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name = None,
+  def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None,
                  override_output_files = True):
     """
     forced_command_name may be specified manually. In this case, value, defined at
@@ -113,14 +113,14 @@ class CustomServiceOrchestrator():
         task_id = command['taskId']
         command_name = command['roleCommand']
       except KeyError:
-        pass # Status commands have no taskId
+        pass  # Status commands have no taskId
 
-      if forced_command_name is not None: # If not supplied as an argument
+      if forced_command_name is not None:  # If not supplied as an argument
         command_name = forced_command_name
 
       if command_name == self.CUSTOM_ACTION_COMMAND:
         base_dir = self.file_cache.get_custom_actions_base_dir(server_url_prefix)
-        script_tuple = (os.path.join(base_dir, script) , base_dir)
+        script_tuple = (os.path.join(base_dir, 'scripts', script), base_dir)
         hook_dir = None
       else:
         if command_name == self.CUSTOM_COMMAND_COMMAND:
@@ -145,7 +145,7 @@ class CustomServiceOrchestrator():
 
       # Execute command using proper interpreter
       handle = None
-      if(command.has_key('__handle')):
+      if command.has_key('__handle'):
         handle = command['__handle']
         handle.on_background_command_started = self.map_task_to_process
         del command['__handle']
@@ -164,7 +164,7 @@ class CustomServiceOrchestrator():
       # Executing hooks and script
       ret = None
       from ActionQueue import ActionQueue
-      if(command.has_key('commandType') and command['commandType'] == ActionQueue.BACKGROUND_EXECUTION_COMMAND and len(filtered_py_file_list) > 1):
+      if command.has_key('commandType') and command['commandType'] == ActionQueue.BACKGROUND_EXECUTION_COMMAND and len(filtered_py_file_list) > 1:
         raise AgentException("Background commands are supported without hooks only")
 
       for py_file, current_base_dir in filtered_py_file_list:
@@ -205,6 +205,7 @@ class CustomServiceOrchestrator():
         'exitcode': 1,
       }
     return ret
+
   def command_canceled_reason(self, task_id):
     with self.commands_in_progress_lock:
       if self.commands_in_progress.has_key(task_id):#Background command do not push in this collection (TODO)

ambari-agent/src/test/python/resource_management/TestListAmbariManagedRepos.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from mock.mock import patch
+from mock.mock import MagicMock
+from mock.mock import patch, MagicMock
+import glob
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions.list_ambari_managed_repos import *
+from resource_management.core.exceptions import Fail
+from unittest import TestCase
+
+
+class TestListAmbariManagedRepos(TestCase):
+
+  @patch("glob.glob")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(OSCheck, "is_suse_family")
+  def test_normal_flow_ubuntu(self, is_suse_family_mock,
+                       is_redhat_family_mock, is_ubuntu_family_mock, glob_mock):
+    is_ubuntu_family_mock.return_value = True
+    is_redhat_family_mock.return_value = False
+    is_suse_family_mock.return_value = False
+    glob_mock.side_effect = \
+    [
+      [
+        "/etc/apt/sources.list.d/HDP-1.1.1.repo",
+        "/etc/apt/sources.list.d/HDP-1.1.2.repo",
+        "/etc/apt/sources.list.d/HDP-1.1.3.repo",
+        "/etc/apt/sources.list.d/HDP-UTILS-1.1.3.repo",
+      ],
+      [
+        "/etc/apt/sources.list.d/HDP-UTILS-1.1.3.repo",
+      ],
+      []
+    ]
+    res = list_ambari_managed_repos()
+    self.assertEquals(glob_mock.call_args_list[0][0][0], "/etc/apt/sources.list.d/HDP*")
+    self.assertEquals(res, ['HDP-1.1.1', 'HDP-1.1.2', 'HDP-1.1.3', 'HDP-UTILS-1.1.3'])
+    self.assertTrue(glob_mock.call_count > 1)
+
+  @patch("glob.glob")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(OSCheck, "is_suse_family")
+  def test_normal_flow_rhel(self, is_suse_family_mock,
+                              is_redhat_family_mock, is_ubuntu_family_mock, glob_mock):
+    is_ubuntu_family_mock.return_value = False
+    is_redhat_family_mock.return_value = True
+    is_suse_family_mock.return_value = False
+    glob_mock.side_effect = \
+      [
+        [
+          "/etc/yum.repos.d/HDP-1.1.1.repo",
+          "/etc/yum.repos.d/HDP-1.1.2.repo",
+          "/etc/yum.repos.d/HDP-1.1.3.repo",
+          "/etc/yum.repos.d/HDP-UTILS-1.1.3.repo",
+          ],
+        [
+          "/etc/yum.repos.d/HDP-UTILS-1.1.3.repo",
+          ],
+        []
+      ]
+    res = list_ambari_managed_repos()
+    self.assertEquals(glob_mock.call_args_list[0][0][0], "/etc/yum.repos.d/HDP*")
+    self.assertEquals(res, ['HDP-1.1.1', 'HDP-1.1.2', 'HDP-1.1.3', 'HDP-UTILS-1.1.3'])
+    self.assertTrue(glob_mock.call_count > 1)
+
+
+  @patch("glob.glob")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(OSCheck, "is_suse_family")
+  def test_normal_flow_sles(self, is_suse_family_mock,
+                              is_redhat_family_mock, is_ubuntu_family_mock, glob_mock):
+    is_ubuntu_family_mock.return_value = False
+    is_redhat_family_mock.return_value = False
+    is_suse_family_mock.return_value = True
+    glob_mock.side_effect = \
+      [
+        [
+          "/etc/zypp/repos.d/HDP-1.1.1.repo",
+          "/etc/zypp/repos.d/HDP-1.1.2.repo",
+          "/etc/zypp/repos.d/HDP-1.1.3.repo",
+          "/etc/zypp/repos.d/HDP-UTILS-1.1.3.repo",
+          ],
+        [
+          "/etc/zypp/repos.d/HDP-UTILS-1.1.3.repo",
+          ],
+        []
+      ]
+    res = list_ambari_managed_repos()
+    self.assertEquals(glob_mock.call_args_list[0][0][0], "/etc/zypp/repos.d/HDP*")
+    self.assertEquals(res, ['HDP-1.1.1', 'HDP-1.1.2', 'HDP-1.1.3', 'HDP-UTILS-1.1.3'])
+    self.assertTrue(glob_mock.call_count > 1)
+
+
+  @patch.object(OSCheck, "is_ubuntu_family")
+  @patch.object(OSCheck, "is_redhat_family")
+  @patch.object(OSCheck, "is_suse_family")
+  def test_normal_flow_unknown_os(self, is_suse_family_mock,
+                            is_redhat_family_mock, is_ubuntu_family_mock):
+    is_ubuntu_family_mock.return_value = False
+    is_redhat_family_mock.return_value = False
+    is_suse_family_mock.return_value = False
+    try:
+      list_ambari_managed_repos()
+      self.fail("Should throw a Fail")
+    except Fail:
+      pass  # Expected

ambari-common/src/main/python/resource_management/libraries/functions/list_ambari_managed_repos.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+__all__ = ["list_ambari_managed_repos"]
+import os
+import glob
+from ambari_commons.os_check import OSCheck
+from resource_management.core.exceptions import Fail
+
+# TODO : get it dynamically from the server
+repository_names = ["HDP", "HDP-UTILS"]
+
+
+def list_ambari_managed_repos():
+  """
+  Lists all repositories that are present at host
+  """
+  if OSCheck.is_ubuntu_family():
+    repo_dir = '/etc/apt/sources.list.d/'
+  elif OSCheck.is_redhat_family():  # Centos/RHEL 5/6
+    repo_dir = '/etc/yum.repos.d/'
+  elif OSCheck.is_suse_family():
+    repo_dir = '/etc/zypp/repos.d/'
+  else:
+    raise Fail('Can not dermine repo dir')
+  repos = []
+  for name in repository_names:
+    # List all files that match pattern
+    files = glob.glob(os.path.join(repo_dir, name) + '*')
+    for f in files:
+      filename = os.path.basename(f)
+      # leave out extension
+      reponame = os.path.splitext(filename)[0]
+      repos.append(reponame)
+  # get uniq strings
+  seen = set()
+  uniq = [s for s in repos if not (s in seen or seen.add(s))]
+  return uniq

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java

@@ -78,9 +78,6 @@ public class AmbariActionExecutionHelper {
    * @throws AmbariException
    */
   public void validateAction(ExecuteActionRequest actionRequest) throws AmbariException {
-
-
-
     if (actionRequest.getActionName() == null || actionRequest.getActionName().isEmpty()) {
       throw new AmbariException("Action name must be specified");
     }

ambari-server/src/main/resources/custom_action_definitions/system_action_definitions.xml

@@ -22,9 +22,9 @@
   <actionDefinition>
     <actionName>check_host</actionName>
     <actionType>SYSTEM</actionType>
-    <inputs></inputs>
-    <targetService></targetService>
-    <targetComponent></targetComponent>
+    <inputs/>
+    <targetService/>
+    <targetComponent/>
     <defaultTimeout>60</defaultTimeout>
     <description>General check for host</description>
     <targetType>ANY</targetType>
@@ -32,11 +32,21 @@
   <actionDefinition>
     <actionName>validate_configs</actionName>
     <actionType>SYSTEM</actionType>
-    <inputs></inputs>
-    <targetService></targetService>
-    <targetComponent></targetComponent>
+    <inputs/>
+    <targetService/>
+    <targetComponent/>
     <defaultTimeout>60</defaultTimeout>
     <description>Validate if provided service config can be applied to specified hosts</description>
     <targetType>ALL</targetType>
   </actionDefinition>
+  <actionDefinition>
+    <actionName>install_packages</actionName>
+    <actionType>SYSTEM</actionType>
+    <inputs>base_urls, package_list</inputs>
+    <targetService/>
+    <targetComponent/>
+    <defaultTimeout>60</defaultTimeout>
+    <description>Distribute repositories and install packages</description>
+    <targetType>ALL</targetType>
+  </actionDefinition>
 </actionDefinitions>

ambari-server/src/main/resources/custom_actions/scripts/install_packages.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import json
+import sys
+import traceback
+from resource_management import *
+from resource_management.libraries.functions.list_ambari_managed_repos import *
+from ambari_commons.os_check import OSCheck, OSConst
+
+
+class InstallPackages(Script):
+  """
+  This script is a part of Rolling Upgrade workflow and is described at
+  appropriate design doc.
+  It installs repositories to the node and then installs packages.
+  For now, repositories are installed into individual files.
+  """
+
+  UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+
+  def actionexecute(self, env):
+    delayed_fail = False
+    package_install_result = False
+
+    # Parse parameters
+    config = Script.get_config()
+    base_urls = json.loads(config['commandParams']['base_urls'])
+    package_list = json.loads(config['commandParams']['package_list'])
+
+    # Install/update repositories
+    installed_repositories = []
+    try:
+      for url_info in base_urls:
+        self.install_repository(url_info)
+
+      installed_repositories = list_ambari_managed_repos()
+    except Exception, err:
+      print "Can not distribute repositories."
+      print traceback.format_exc()
+      delayed_fail = True
+
+    # Install packages
+    if not delayed_fail:
+      try:
+        for package in package_list:
+          Package(package)
+        package_install_result = True
+      except Exception, err:
+        print "Can not install packages."
+        print traceback.format_exc()
+        delayed_fail = True
+        # TODO : remove already installed packages in case of fail
+
+    # Build structured output
+    structured_output = {
+      'ambari_repositories': installed_repositories,
+      'package_installation_result': 'SUCCESS' if package_install_result else 'FAIL'
+    }
+    self.put_structured_out(structured_output)
+
+    # Provide correct exit code
+    if delayed_fail:
+      raise Fail("Failed to distribute repositories/install packages")
+
+
+  def install_repository(self, url_info):
+    template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else "repo_ubuntu.j2"
+
+    repo = {
+      'repoName': url_info['id']
+    }
+
+    if not 'baseurl' in url_info:
+      repo['baseurl'] = None
+    else:
+      repo['baseurl'] = url_info['baseurl']
+
+    if not 'mirrorsList' in url_info:
+      repo['mirrorsList'] = None
+    else:
+      repo['mirrorsList'] = url_info['mirrorslist']
+
+    ubuntu_components = [repo['repoName']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX
+
+    Repository(repo['repoName'],
+      action = "create",
+      base_url = repo['baseurl'],
+      mirror_list = repo['mirrorsList'],
+      repo_file_name = repo['repoName'],
+      repo_template = template,
+      components = ubuntu_components,  # ubuntu specific
+    )
+
+
+if __name__ == "__main__":
+  InstallPackages().execute()

ambari-server/src/main/resources/custom_actions/templates/repo_suse_rhel.j2

@@ -0,0 +1,7 @@
+[{{repo_id}}]
+name={{repo_file_name}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+
+path=/
+enabled=1
+gpgcheck=0

ambari-server/src/main/resources/custom_actions/templates/repo_ubuntu.j2

@@ -0,0 +1 @@
+{{package_type}} {{base_url}} {{components}}

ambari-server/src/test/python/TestCheckHost.py → ambari-server/src/test/python/custom_actions/TestCheckHost.py

@@ -29,7 +29,6 @@ from mock.mock import patch
 from mock.mock import MagicMock
 from unittest import TestCase
 
-check_host = __import__('check_host')
 from check_host import CheckHost
 
 class TestCheckHost(TestCase):

ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+import os
+import socket
+from resource_management import Script,ConfigDictionary
+from mock.mock import patch
+from mock.mock import MagicMock
+from stacks.utils.RMFTestCase import *
+from install_packages import InstallPackages
+from mock.mock import patch, MagicMock
+
+
+class TestInstallPackages(RMFTestCase):
+
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_normal_flow(self, put_structured_out):
+    self.executeScript("scripts/install_packages.py",
+                       classname="InstallPackages",
+                       command="actionexecute",
+                       config_file="install_packages_config.json",
+                       target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+                       os_type=('Suse', '11', 'Final'),
+    )
+    self.assertTrue(put_structured_out.called)
+    self.assertEquals(put_structured_out.call_args[0][0],
+                      {'package_installation_result': 'SUCCESS',
+                       'ambari_repositories': []})
+    self.assertResourceCalled('Repository', 'HDP-2.2.0.0-885',
+                              base_url=u'http://host1/hdp',
+                              action=['create'],
+                              components=[u'HDP-2.2.0.0-885', 'main'],
+                              repo_template='repo_suse_rhel.j2',
+                              repo_file_name=u'HDP-2.2.0.0-885',
+                              mirror_list=None,
+    )
+    self.assertResourceCalled('Repository', 'HDP-UTILS-1.0.0.20',
+                              base_url=u'http://host1/hdp-utils',
+                              action=['create'],
+                              components=[u'HDP-UTILS-1.0.0.20', 'main'],
+                              repo_template='repo_suse_rhel.j2',
+                              repo_file_name=u'HDP-UTILS-1.0.0.20',
+                              mirror_list=None,
+    )
+    self.assertResourceCalled('Package', 'python-rrdtool-1.4.5', )
+    self.assertResourceCalled('Package', 'libganglia-3.5.0-99', )
+    self.assertResourceCalled('Package', 'ganglia-*', )
+    self.assertNoMoreResources()
+
+
+  @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos",
+         new=MagicMock(return_value=["HDP-UTILS-1.0.0.20"]))
+  @patch("resource_management.libraries.script.Script.put_structured_out")
+  def test_exclude_existing_repo(self, put_structured_out):
+    self.executeScript("scripts/install_packages.py",
+                       classname="InstallPackages",
+                       command="actionexecute",
+                       config_file="install_packages_config.json",
+                       target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
+                       os_type=('Suse', '11', 'Final'),
+    )
+    self.assertTrue(put_structured_out.called)
+    self.assertEquals(put_structured_out.call_args[0][0],
+                      {'package_installation_result': 'SUCCESS',
+                       'ambari_repositories': ["HDP-UTILS-1.0.0.20"]})
+    self.assertResourceCalled('Repository', 'HDP-2.2.0.0-885',
+                              base_url=u'http://host1/hdp',
+                              action=['create'],
+                              components=[u'HDP-2.2.0.0-885', 'main'],
+                              repo_template='repo_suse_rhel.j2',
+                              repo_file_name=u'HDP-2.2.0.0-885',
+                              mirror_list=None,
+    )
+    self.assertResourceCalled('Repository', 'HDP-UTILS-1.0.0.20',
+                              base_url=u'http://host1/hdp-utils',
+                              action=['create'],
+                              components=[u'HDP-UTILS-1.0.0.20', 'main'],
+                              repo_template='repo_suse_rhel.j2',
+                              repo_file_name=u'HDP-UTILS-1.0.0.20',
+                              mirror_list=None,
+    )
+    self.assertResourceCalled('Package', 'python-rrdtool-1.4.5', )
+    self.assertResourceCalled('Package', 'libganglia-3.5.0-99', )
+    self.assertResourceCalled('Package', 'ganglia-*', )
+    self.assertNoMoreResources()
+
+

ambari-server/src/test/python/custom_actions/configs/install_packages_config.json

@@ -0,0 +1,77 @@
+{
+    "configuration_attributes": {}, 
+    "roleCommand": "ACTIONEXECUTE", 
+    "clusterName": "cc", 
+    "hostname": "0b3.vm", 
+    "passiveInfo": [], 
+    "hostLevelParams": {
+        "jdk_location": "http://0b3.vm:8080/resources/", 
+        "ambari_db_rca_password": "mapred", 
+        "java_home": "/usr/jdk64/jdk1.7.0_67", 
+        "ambari_db_rca_url": "jdbc:postgresql://0b3.vm/ambarirca", 
+        "jce_name": "UnlimitedJCEPolicyJDK7.zip", 
+        "oracle_jdbc_url": "http://0b3.vm:8080/resources//ojdbc6.jar", 
+        "stack_version": "2.1", 
+        "stack_name": "HDP", 
+        "db_name": "ambari", 
+        "ambari_db_rca_driver": "org.postgresql.Driver", 
+        "jdk_name": "jdk-7u67-linux-x64.tar.gz", 
+        "ambari_db_rca_username": "mapred", 
+        "db_driver_filename": "mysql-connector-java.jar", 
+        "mysql_jdbc_url": "http://0b3.vm:8080/resources//mysql-connector-java.jar"
+    }, 
+    "commandType": "EXECUTION_COMMAND", 
+    "roleParams": {
+        "base_urls": "[{\"id\": \"HDP-2.2.0.0-885\", \"type\": \"HDP\",\"baseurl\": \"http://host1/hdp\"}, {\"id\": \"HDP-UTILS-1.0.0.20\", \"type\": \"HDP-UTILS\", \"baseurl\": \"http://host1/hdp-utils\"}]", 
+        "package_list": "[\"python-rrdtool-1.4.5\", \"libganglia-3.5.0-99\", \"ganglia-*\"]"
+    }, 
+    "serviceName": "null", 
+    "role": "install_packages", 
+    "forceRefreshConfigTags": [], 
+    "taskId": 61, 
+    "public_hostname": "0b3.vm", 
+    "configurations": {}, 
+    "commandParams": {
+        "command_timeout": "60", 
+        "script_type": "PYTHON", 
+        "base_urls": "[{\"id\": \"HDP-2.2.0.0-885\", \"type\": \"HDP\",\"baseurl\": \"http://host1/hdp\"}, {\"id\": \"HDP-UTILS-1.0.0.20\", \"type\": \"HDP-UTILS\", \"baseurl\": \"http://host1/hdp-utils\"}]", 
+        "package_list": "[\"python-rrdtool-1.4.5\", \"libganglia-3.5.0-99\", \"ganglia-*\"]", 
+        "script": "install_packages.py"
+    }, 
+    "commandId": "14-1", 
+    "clusterHostInfo": {
+        "snamenode_host": [
+            "0b3.vm"
+        ], 
+        "nm_hosts": [
+            "0b3.vm"
+        ], 
+        "app_timeline_server_hosts": [
+            "0b3.vm"
+        ], 
+        "all_ping_ports": [
+            "8670"
+        ], 
+        "rm_host": [
+            "0b3.vm"
+        ], 
+        "all_hosts": [
+            "0b3.vm"
+        ], 
+        "slave_hosts": [
+            "0b3.vm"
+        ], 
+        "namenode_host": [
+            "0b3.vm"
+        ], 
+        "ambari_server_host": [
+            "0b3.vm"
+        ], 
+        "zookeeper_hosts": [
+            "0b3.vm"
+        ], 
+        "hs_host": [
+            "0b3.vm"
+        ]
+    }
+}

ambari-server/src/test/python/stacks/utils/RMFTestCase.py

@@ -34,24 +34,41 @@ with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
   from resource_management.libraries.script.script import Script
   from resource_management.libraries.script.config_dictionary import UnknownConfiguration
 
+PATH_TO_STACKS = "main/resources/stacks/HDP"
+PATH_TO_STACK_TESTS = "test/python/stacks/"
+
+PATH_TO_CUSTOM_ACTIONS = "main/resources/custom_actions"
+PATH_TO_CUSTOM_ACTION_TESTS = "test/python/custom_actions"
 
-PATH_TO_STACKS = os.path.normpath("main/resources/stacks/HDP")
-PATH_TO_STACK_TESTS = os.path.normpath("test/python/stacks/")
 
 class RMFTestCase(TestCase):
+
+  # (default) build all paths to test stack scripts
+  TARGET_STACKS = 'TARGET_STACKS'
+
+  # (default) build all paths to test custom action scripts
+  TARGET_CUSTOM_ACTIONS = 'TARGET_CUSTOM_ACTIONS'
+
   def executeScript(self, path, classname=None, command=None, config_file=None,
                     config_dict=None,
                     # common mocks for all the scripts
                     config_overrides = None,
                     shell_mock_value = (0, "OK."), 
                     os_type=('Suse','11','Final'),
-                    kinit_path_local="/usr/bin/kinit"
+                    kinit_path_local="/usr/bin/kinit",
+                    target=TARGET_STACKS
                     ):
     norm_path = os.path.normpath(path)
     src_dir = RMFTestCase._getSrcFolder()
-    stack_version = norm_path.split(os.sep)[0]
-    stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
-    configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
+    if target == self.TARGET_STACKS:
+      stack_version = norm_path.split(os.sep)[0]
+      stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
+      configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
+    elif target == self.TARGET_CUSTOM_ACTIONS:
+      stacks_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTIONS)
+      configs_path = os.path.join(src_dir, PATH_TO_CUSTOM_ACTION_TESTS, "configs")
+    else:
+      raise RuntimeError("Wrong target value %s", target)
     script_path = os.path.join(stacks_path, norm_path)
     if config_file is not None and config_dict is None:
       config_file_path = os.path.join(configs_path, config_file)
@@ -80,8 +97,8 @@ class RMFTestCase(TestCase):
     try:
       with patch.object(platform, 'linux_distribution', return_value=os_type):
         script_module = imp.load_source(classname, script_path)
-    except IOError:
-      raise RuntimeError("Cannot load class %s from %s",classname, norm_path)
+    except IOError, err:
+      raise RuntimeError("Cannot load class %s from %s: %s" % (classname, norm_path, err.message))
     
     script_class_inst = RMFTestCase._get_attr(script_module, classname)()
     method = RMFTestCase._get_attr(script_class_inst, command)

ambari-server/src/test/python/unitTests.py

@@ -30,6 +30,7 @@ SERVICE_EXCLUDE = ["configs"]
 
 TEST_MASK = '[Tt]est*.py'
 CUSTOM_TEST_MASK = '_[Tt]est*.py'
+
 def get_parent_path(base, directory_name):
   """
   Returns absolute path for directory_name, if directory_name present in base.
@@ -43,7 +44,8 @@ def get_parent_path(base, directory_name):
     done = True if os.path.split(base)[-1] == directory_name else False
   return base
 
-def get_test_files(path, mask = None, recursive=True):
+
+def get_test_files(path, mask=None, recursive=True):
   """
   Returns test files for path recursively
   """
@@ -52,13 +54,14 @@ def get_test_files(path, mask = None, recursive=True):
 
   for item in directory_items:
     add_to_pythonpath = False
-    if os.path.isfile(path + "/" + item):
+    p = os.path.join(path, item)
+    if os.path.isfile(p):
       if fnmatch.fnmatch(item, mask):
         add_to_pythonpath = True
         current.append(item)
-    elif os.path.isdir(path + "/" + item):
+    elif os.path.isdir(p):
       if recursive:
-        current.extend(get_test_files(path + "/" + item, mask = mask))
+        current.extend(get_test_files(p, mask = mask))
     if add_to_pythonpath:
       sys.path.append(path)
   return current
@@ -102,10 +105,12 @@ def stack_test_executor(base_folder, service, stack, custom_tests, executor_resu
   sys.stdout.flush()
   sys.stderr.flush()
   exit_code = 0 if textRunner.wasSuccessful() else 1
-  executor_result.put({'exit_code':exit_code,
-                  'tests_run':textRunner.testsRun,
-                  'errors':[(str(item[0]),str(item[1]),"ERROR") for item in textRunner.errors],
-                  'failures':[(str(item[0]),str(item[1]),"FAIL") for item in textRunner.failures]})
+  executor_result.put({'exit_code': exit_code,
+                       'tests_run': textRunner.testsRun,
+                       'errors': [(str(item[0]), str(item[1]), "ERROR") for item
+                                  in textRunner.errors],
+                       'failures': [(str(item[0]), str(item[1]), "FAIL") for
+                                    item in textRunner.failures]})
   executor_result.put(0) if textRunner.wasSuccessful() else executor_result.put(1)
 
 def main():
@@ -115,37 +120,36 @@ def main():
       custom_tests = True
   pwd = os.path.abspath(os.path.dirname(__file__))
 
-  ambari_server_folder = get_parent_path(pwd,'ambari-server')
-  ambari_agent_folder = os.path.join(ambari_server_folder,"../ambari-agent")
-  ambari_common_folder = os.path.join(ambari_server_folder,"../ambari-common")
-  sys.path.append(ambari_common_folder + "/src/main/python")
-  sys.path.append(ambari_common_folder + "/src/main/python/ambari_jinja2")
-  sys.path.append(ambari_common_folder + "/src/main/python")
-  sys.path.append(ambari_common_folder + "/src/test/python")
-  sys.path.append(ambari_agent_folder + "/src/main/python")
-  sys.path.append(ambari_server_folder + "/src/test/python")
-  sys.path.append(ambari_server_folder + "/src/main/python")
-  sys.path.append(ambari_server_folder + "/src/main/resources/scripts")
-  sys.path.append(ambari_server_folder + "/src/main/resources/custom_actions")
+  ambari_server_folder = get_parent_path(pwd, 'ambari-server')
+  ambari_agent_folder = os.path.join(ambari_server_folder, "../ambari-agent")
+  ambari_common_folder = os.path.join(ambari_server_folder, "../ambari-common")
+  sys.path.append(os.path.join(ambari_common_folder, "src/main/python"))
+  sys.path.append(os.path.join(ambari_common_folder, "src/main/python/ambari_jinja2"))
+  sys.path.append(os.path.join(ambari_common_folder, "src/test/python"))
+  sys.path.append(os.path.join(ambari_agent_folder, "src/main/python"))
+  sys.path.append(os.path.join(ambari_server_folder, "src/test/python"))
+  sys.path.append(os.path.join(ambari_server_folder, "src/main/python"))
+  sys.path.append(os.path.join(ambari_server_folder, "src/main/resources/scripts"))
+  sys.path.append(os.path.join(ambari_server_folder, "src/main/resources/custom_actions/scripts"))
   
-  stacks_folder = pwd+'/stacks'
+  stacks_folder = os.path.join(pwd, 'stacks')
   #generate test variants(path, service, stack)
   test_variants = []
   for stack in os.listdir(stacks_folder):
-    current_stack_dir = stacks_folder+"/"+stack
+    current_stack_dir = os.path.join(stacks_folder, stack)
     if os.path.isdir(current_stack_dir) and stack not in STACK_EXCLUDE:
       for service in os.listdir(current_stack_dir):
-        current_service_dir = current_stack_dir+"/"+service
+        current_service_dir = os.path.join(current_stack_dir, service)
         if os.path.isdir(current_service_dir) and service not in SERVICE_EXCLUDE:
           if service == 'hooks':
             for hook in os.listdir(current_service_dir):
-              test_variants.append({'directory':current_service_dir + "/" + hook,
-                                    'service':hook,
-                                    'stack':stack})
+              test_variants.append({'directory': os.path.join(current_service_dir, hook),
+                                    'service': hook,
+                                    'stack': stack})
           else:
-            test_variants.append({'directory':current_service_dir,
-                                  'service':service,
-                                  'stack':stack})
+            test_variants.append({'directory': current_service_dir,
+                                  'service': service,
+                                  'stack': stack})
 
   #run tests for every service in every stack in separate process
   has_failures = False
@@ -155,7 +159,7 @@ def main():
   for variant in test_variants:
     executor_result = multiprocessing.Queue()
     sys.stderr.write( "Running tests for stack:{0} service:{1}\n"
-                      .format(variant['stack'],variant['service']))
+                      .format(variant['stack'], variant['service']))
     process = multiprocessing.Process(target=stack_test_executor,
                                       args=(variant['directory'],
                                             variant['service'],
@@ -183,18 +187,27 @@ def main():
   else:
     test_mask = TEST_MASK
 
-  tests = get_test_files(pwd, mask=test_mask, recursive=False)
-  #TODO Add an option to randomize the tests' execution
-  #shuffle(tests)
-  modules = [os.path.basename(s)[:-3] for s in tests]
-  suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in
-    modules]
-  testSuite = unittest.TestSuite(suites)
-  textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite)
-  test_runs += textRunner.testsRun
-  test_errors.extend([(str(item[0]),str(item[1]),"ERROR") for item in textRunner.errors])
-  test_failures.extend([(str(item[0]),str(item[1]),"FAIL") for item in textRunner.failures])
-  tests_status = textRunner.wasSuccessful() and not has_failures
+  test_dirs = [
+    (os.path.join(pwd, 'custom_actions'), "\nRunning tests for custom actions\n"),
+    (pwd, "\nRunning tests for ambari-server\n"),
+  ]
+
+  for test_dir, msg in test_dirs:
+    sys.stderr.write(msg)
+    tests = get_test_files(test_dir, mask=test_mask, recursive=False)
+    #TODO Add an option to randomize the tests' execution
+    #shuffle(tests)
+    modules = [os.path.basename(s)[:-3] for s in tests]
+    suites = [unittest.defaultTestLoader.loadTestsFromName(name) for name in
+      modules]
+    testSuite = unittest.TestSuite(suites)
+    textRunner = unittest.TextTestRunner(verbosity=2).run(testSuite)
+    test_runs += textRunner.testsRun
+    test_errors.extend(
+      [(str(item[0]), str(item[1]), "ERROR") for item in textRunner.errors])
+    test_failures.extend(
+      [(str(item[0]), str(item[1]), "FAIL") for item in textRunner.failures])
+    tests_status = textRunner.wasSuccessful() and not has_failures
 
   if not tests_status:
     sys.stderr.write("----------------------------------------------------------------------\n")