AMBARI-8694 - Upgrade Pack definition for Core Slaves Components (jonathanhurley)

ambari-server/src/main/java/org/apache/ambari/server/orm/dao/AlertsDAO.java

@@ -341,10 +341,9 @@ public class AlertsDAO {
 
   /**
    * Retrieve the summary alert information for all hosts. This is different
-   * from {@link #findCurrentCounts(long, String, String)} since this will not
-   * return alerts that are not related to a particular host, such as aggregate
-   * alerts. In general, {@link #findCurrentCounts(long, String, String)} and
-   * this method will return very similar counts.
+   * from {@link #findCurrentCounts(long, String, String)} since this will
+   * return only alerts related to hosts and those values will be the total
+   * number of hosts affected, not the total number of alerts.
    *
    * @param clusterId
    *          the cluster id

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/datanode.py

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 """
-
+import datanode_upgrade
 from hdfs_datanode import datanode
 from resource_management import *
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
@@ -30,14 +30,25 @@ class DataNode(Script):
     self.install_packages(env, params.exclude_packages)
     env.set_params(params)
 
+
   def pre_rolling_restart(self, env):
-    Logger.info("Executing Rolling Upgrade pre-restart")
+    Logger.info("Executing DataNode Rolling Upgrade pre-restart")
     import params
     env.set_params(params)
 
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
       Execute(format("hdp-select set hadoop-hdfs-datanode {version}"))
 
+
+  def post_rolling_restart(self, env):
+    Logger.info("Executing DataNode Rolling Upgrade post-restart")
+    import params
+    env.set_params(params)
+
+    # ensure the DataNode has started and rejoined the cluster
+    datanode_upgrade.post_upgrade_check()
+
+
   def start(self, env, rolling_restart=False):
     import params
 
@@ -45,11 +56,19 @@ class DataNode(Script):
     self.configure(env)
     datanode(action="start")
 
+
   def stop(self, env, rolling_restart=False):
     import params
 
     env.set_params(params)
-    datanode(action="stop")
+
+    # pre-upgrade steps shutdown the datanode, so there's no need to call
+    # action=stop
+    if rolling_restart:
+      datanode_upgrade.pre_upgrade_shutdown()
+    else:
+      datanode(action="stop")
+
 
   def configure(self, env):
     import params
@@ -57,6 +76,7 @@ class DataNode(Script):
     hdfs()
     datanode(action="configure")
 
+
   def status(self, env):
     import status_params
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/datanode_upgrade.py

@@ -0,0 +1,114 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.core.shell import call
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.decorator import retry
+
+
+def pre_upgrade_shutdown():
+  """
+  Runs the "shutdownDatanode {ipc_address} upgrade" command to shutdown the
+  DataNode in preparation for an upgrade. This will then periodically check
+  "getDatanodeInfo" to ensure the DataNode has shutdown correctly.
+  This function will obtain the Kerberos ticket if security is enabled.
+  :return:
+  """
+  import params
+
+  Logger.info('DataNode executing "shutdownDatanode" command in preparation for upgrade...')
+  if params.security_enabled:
+    Execute(params.dn_kinit_cmd, user = params.hdfs_user)
+
+  command = format('hdfs dfsadmin -shutdownDatanode {dfs_dn_ipc_address} upgrade')
+  Execute(command, user=params.hdfs_user, tries=1 )
+
+  # verify that the datanode is down
+  _check_datanode_shutdown()
+
+
+def post_upgrade_check():
+  """
+  Verifies that the DataNode has rejoined the cluster. This function will
+  obtain the Kerberos ticket if security is enabled.
+  :return:
+  """
+  import params
+
+  Logger.info("Checking that the DataNode has rejoined the cluster after upgrade...")
+  if params.security_enabled:
+    Execute(params.dn_kinit_cmd,user = params.hdfs_user)
+
+  # verify that the datanode has started and rejoined the HDFS cluster
+  _check_datanode_startup()
+
+
+@retry(times=12, sleep_time=10, err_class=Fail)
+def _check_datanode_shutdown():
+  """
+  Checks that a DataNode is down by running "hdfs dfsamin getDatanodeInfo"
+  several times, pausing in between runs. Once the DataNode stops responding
+  this method will return, otherwise it will raise a Fail(...) and retry
+  automatically.
+  :return:
+  """
+  import params
+
+  command = format('hdfs dfsadmin -getDatanodeInfo {dfs_dn_ipc_address}')
+
+  try:
+    Execute(command, user=params.hdfs_user, tries=1)
+  except:
+    Logger.info("DataNode has successfully shutdown for upgrade.")
+    return
+
+  Logger.info("DataNode has not shutdown.")
+  raise Fail('DataNode has not shutdown.')
+
+
+@retry(times=12, sleep_time=10, err_class=Fail)
+def _check_datanode_startup():
+  """
+  Checks that a DataNode is reported as being alive via the
+  "hdfs dfsadmin -report -live" command. Once the DataNode is found to be
+  alive this method will return, otherwise it will raise a Fail(...) and retry
+  automatically.
+  :return:
+  """
+  import params
+
+  try:
+    # 'su - hdfs -c "hdfs dfsadmin -report -live"'
+    command = 'hdfs dfsadmin -report -live'
+    return_code, hdfs_output = call(command, user=params.hdfs_user)
+  except:
+    raise Fail('Unable to determine if the DataNode has started after upgrade.')
+
+  if return_code == 0:
+    if params.hostname.lower() in hdfs_output.lower():
+      Logger.info("DataNode {0} reports that it has rejoined the cluster.".format(params.hostname))
+      return
+    else:
+      raise Fail("DataNode {0} was not found in the list of live DataNodes".format(params.hostname))
+
+  # return_code is not 0, fail
+  raise Fail("Unable to determine if the DataNode has started after upgrade (result code {0})".format(str(return_code)))

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py

@@ -43,6 +43,7 @@ dfs_dn_addr = default('/configurations/hdfs-site/dfs.datanode.address', None)
 dfs_dn_http_addr = default('/configurations/hdfs-site/dfs.datanode.http.address', None)
 dfs_dn_https_addr = default('/configurations/hdfs-site/dfs.datanode.https.address', None)
 dfs_http_policy = default('/configurations/hdfs-site/dfs.http.policy', None)
+dfs_dn_ipc_address = config['configurations']['hdfs-site']['dfs.datanode.ipc.address']
 secure_dn_ports_are_in_use = False
 
 #hadoop params

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager.py

@@ -18,32 +18,16 @@ limitations under the License.
 Ambari Agent
 
 """
-import re
+
+import nodemanager_upgrade
 
 from resource_management import *
-from resource_management.libraries.functions.decorator import retry
 from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions.format import format
-from resource_management.core.shell import call
 
 from yarn import yarn
 from service import service
 
-
-@retry(times=10, sleep_time=2, err_class=Fail)
-def call_and_match_output(command, regex_expression, err_message):
-  """
-  Call the command and performs a regex match on the output for the specified expression.
-  :param command: Command to call
-  :param regex_expression: Regex expression to search in the output
-  """
-  # TODO Rolling Upgrade, does this work in Ubuntu? If it doesn't see dynamic_variable_interpretation.py to see how stdout was redirected
-  # to a temporary file, which was then read.
-  code, out = call(command, verbose=True)
-  if not (out and re.search(regex_expression, out, re.IGNORECASE)):
-    raise Fail(err_message)
-
-
 class Nodemanager(Script):
   def install(self, env):
     self.install_packages(env)
@@ -54,7 +38,7 @@ class Nodemanager(Script):
     yarn(name="nodemanager")
 
   def pre_rolling_restart(self, env):
-    Logger.info("Executing Rolling Upgrade post-restart")
+    Logger.info("Executing NodeManager Rolling Upgrade pre-restart")
     import params
     env.set_params(params)
 
@@ -65,25 +49,20 @@ class Nodemanager(Script):
     import params
     env.set_params(params)
     self.configure(env) # FOR SECURITY
-    service('nodemanager',
-            action='start'
-    )
+    service('nodemanager',action='start')
 
   def post_rolling_restart(self, env):
-    Logger.info("Executing Rolling Upgrade post-restart")
+    Logger.info("Executing NodeManager Rolling Upgrade post-restart")
     import params
     env.set_params(params)
 
-    nm_status_command = format("yarn node -status {nm_address}")
-    call_and_match_output(nm_status_command, 'Node-State : RUNNING',  "Failed to check NodeManager status")
+    nodemanager_upgrade.post_upgrade_check()
 
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)
 
-    service('nodemanager',
-            action='stop'
-    )
+    service('nodemanager',action='stop')
 
   def status(self, env):
     import status_params

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/nodemanager_upgrade.py

@@ -0,0 +1,74 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import subprocess
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.system import Execute
+from resource_management.core.shell import call
+from resource_management.libraries.functions.decorator import retry
+
+
+def post_upgrade_check():
+  '''
+  Checks that the NodeManager has rejoined the cluster.
+  This function will obtain the Kerberos ticket if security is enabled.
+  :return:
+  '''
+  import params
+
+  Logger.info('NodeManager executing "yarn node -list -states=RUNNING" to verify the node has rejoined the cluster...')
+  if params.security_enabled and params.nodemanager_kinit_cmd:
+    Execute(params.nodemanager_kinit_cmd, user = params.yarn_user)
+
+  _check_nodemanager_startup()
+
+
+@retry(times=12, sleep_time=10, err_class=Fail)
+def _check_nodemanager_startup():
+  '''
+  Checks that a NodeManager is in a RUNNING state in the cluster via
+  "yarn node -list -states=RUNNING" command. Once the NodeManager is found to be
+  alive this method will return, otherwise it will raise a Fail(...) and retry
+  automatically.
+  :return:
+  '''
+  import params
+
+  command = 'yarn node -list -states=RUNNING'
+
+  try:
+    # 'su - yarn -c "yarn node -status c6401.ambari.apache.org:45454"'
+    return_code, yarn_output = call(command, user=params.hdfs_user)
+  except:
+    raise Fail('Unable to determine if the NodeManager has started after upgrade.')
+
+  if return_code == 0:
+    hostname = params.hostname.lower()
+    nodemanager_address = params.nm_address.lower()
+    yarn_output = yarn_output.lower()
+
+    if hostname in yarn_output or nodemanager_address in yarn_output:
+      Logger.info('NodeManager with ID {0} has rejoined the cluster.'.format(nodemanager_address))
+      return
+    else:
+      raise Fail('NodeManager with ID {0} was not found in the list of running NodeManagers'.format(nodemanager_address))
+
+  raise Fail('Unable to determine if the NodeManager has started after upgrade (result code {0})'.format(str(return_code)))

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/params.py

@@ -131,11 +131,15 @@ exclude_file_path = default("/configurations/yarn-site/yarn.resourcemanager.node
 ats_host = set(default("/clusterHostInfo/app_timeline_server_hosts", []))
 has_ats = not len(ats_host) == 0
 
+# default kinit commands
+rm_kinit_cmd = ""
+yarn_timelineservice_kinit_cmd = ""
+nodemanager_kinit_cmd = ""
+
 if security_enabled:
   _rm_principal_name = config['configurations']['yarn-site']['yarn.resourcemanager.principal']
-  _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab']
   _rm_principal_name = _rm_principal_name.replace('_HOST',hostname.lower())
-  
+  _rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab']
   rm_kinit_cmd = format("{kinit_path_local} -kt {_rm_keytab} {_rm_principal_name};")
 
   # YARN timeline security options are only available in HDP Champlain
@@ -144,9 +148,12 @@ if security_enabled:
     _yarn_timelineservice_principal_name = _yarn_timelineservice_principal_name.replace('_HOST', hostname.lower())
     _yarn_timelineservice_keytab = config['configurations']['yarn-site']['yarn.timeline-service.keytab']
     yarn_timelineservice_kinit_cmd = format("{kinit_path_local} -kt {_yarn_timelineservice_keytab} {_yarn_timelineservice_principal_name};")
-else:
-  rm_kinit_cmd = ""
-  yarn_timelineservice_kinit_cmd = ""
+
+  if 'yarn.nodemanager.principal' in config['configurations']['yarn-site']:
+    _nodemanager_principal_name = config['configurations']['yarn-site']['yarn.nodemanager.principal']
+    _nodemanager_keytab = config['configurations']['yarn-site']['yarn.nodemanager.keytab']
+    nodemanager_kinit_cmd = format("{kinit_path_local} -kt {_nodemanager_keytab} {_nodemanager_principal_name};")
+
 
 yarn_log_aggregation_enabled = config['configurations']['yarn-site']['yarn.log-aggregation-enable']
 yarn_nm_app_log_dir =  config['configurations']['yarn-site']['yarn.nodemanager.remote-app-log-dir']
@@ -155,7 +162,6 @@ mapreduce_jobhistory_done_dir = config['configurations']['mapred-site']['mapredu
 jobhistory_heapsize = default("/configurations/mapred-env/jobhistory_heapsize", "900")
 
 #for create_hdfs_directory
-hostname = config["hostname"]
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
 import functools

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml

@@ -97,7 +97,6 @@
   
   
   <processing>
-  
     <service name="ZOOKEEPER">
       <component name="ZOOKEEPER_SERVER">
         <!-- TODO, optimization
@@ -130,7 +129,7 @@
 
     <service name="HDFS">
       <component name="NAMENODE">
-        
+
         <pre-upgrade>
           <!-- Backup the image,
           Enter Safemode if not already in it,
@@ -183,10 +182,10 @@
             <every>1</every>
           </task>
         </pre-upgrade>
-        
+
         <upgrade>
           <task xsi:type="restart" />
-        </upgrade>        
+        </upgrade>
 
         <!-- This step should be done once the user clicks on the "Finalize" button. So the name post-upgrade is misleading. -->
         <post-upgrade>
@@ -200,42 +199,6 @@
       </component>
 
       <component name="DATANODE">
-        <pre-upgrade>
-          <!-- Shutdown the datanode,
-
-          Will retry 50 times.
-          Property dfs.datanode.ipc.address = 0.0.0.0:8010 needs to evaluate to current host.
-          $ su hdfs -c 'hdfs dfsadmin -shutdownDatanode <DATANODE_HOST:IPC_PORT> upgrade'
-          E.g.,
-          $ su hdfs -c 'hdfs dfsadmin -shutdownDatanode c6407.ambari.apache.org:8010 upgrade'
-
-          Will retry 50 times.
-          $ su hdfs -c 'hdfs dfsadmin -getDatanodeInfo c6407.ambari.apache.org:8010'
-          Datanode unreachable.
-
-          Change the version,
-          $ hdp-select set hadoop-hdfs-datanode 2.2.0.1-885
-
-          Start the datanode,
-          $ su - hdfs -c '/usr/hdp/current/hadoop-hdfs-datanode/../hadoop/sbin/hadoop-daemon.sh start datanode'
-          starting datanode, logging to /var/log/hadoop/hdfs/hadoop-hdfs-datanode-c6407.ambari.apache.org.out
-
-          Verify it is live,
-          $ su - hdfs -c 'hdfs dfsadmin -report -live'
-          Live datanodes (1):
-          -->
-          <task xsi:type="execute">
-            <first>su {{hadoop-env/hdfs_user}} -c 'hdfs dfsadmin -getDatanodeInfo {{hdfs-site/dfs.datanode.ipc.address}}'</first>
-            <command>su {{hadoop-env/hdfs_user}} -c 'hdfs dfsadmin -shutdownDatanode {{hdfs-site/dfs.datanode.ipc.address}} upgrade'</command>
-          </task>
-
-          <!-- After shutting down the datanode, this command is expected to fail with 255, so ignore only that return code. -->
-          <task xsi:type="execute">
-            <command>su {{hadoop-env/hdfs_user}} -c 'hdfs dfsadmin -getDatanodeInfo {{hdfs-site/dfs.datanode.ipc.address}}'</command>
-            <ignore>255</ignore>
-          </task>
-        </pre-upgrade>
-
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -18,9 +18,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 from stacks.utils.RMFTestCase import *
-from ambari_commons import OSCheck
 import json
 from mock.mock import MagicMock, patch
+from resource_management.core.exceptions import Fail
 
 class TestDatanode(RMFTestCase):
 
@@ -416,3 +416,75 @@ class TestDatanode(RMFTestCase):
                               recursive = True,
                               recursive_permission = True
                               )
+
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart(self, process_mock, time_mock):
+    process_output = """
+      Live datanodes (2):
+
+      Name: 192.168.64.102:50010 (c6401.ambari.apache.org)
+      Hostname: c6401.ambari.apache.org
+      Decommission Status : Normal
+      Configured Capacity: 524208947200 (488.21 GB)
+      DFS Used: 193069056 (184.13 MB)
+      Non DFS Used: 29264986112 (27.26 GB)
+      DFS Remaining: 494750892032 (460.77 GB)
+      DFS Used%: 0.04%
+      DFS Remaining%: 94.38%
+      Configured Cache Capacity: 0 (0 B)
+      Cache Used: 0 (0 B)
+      Cache Remaining: 0 (0 B)
+      Cache Used%: 100.00%
+      Cache Remaining%: 0.00%
+      Xceivers: 2
+      Last contact: Fri Dec 12 20:47:21 UTC 2014
+    """
+
+    process = MagicMock()
+    process.communicate.return_value = [process_output]
+    process.returncode = 0
+    process_mock.return_value = process
+
+    self.executeScript("2.0.6/services/HDFS/package/scripts/datanode.py",
+      classname = "DataNode", command = "post_rolling_restart", config_file="default.json",  )
+
+    self.assertTrue(process_mock.called)
+    self.assertEqual(process_mock.call_count,1)
+
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart_datanode_not_ready(self, process_mock, time_mock):
+    process = MagicMock()
+    process.communicate.return_value = ['There are no DataNodes here!']
+    process.returncode = 0
+    process_mock.return_value = process
+
+    try:
+      self.executeScript("2.0.6/services/HDFS/package/scripts/datanode.py",
+        classname = "DataNode", command = "post_rolling_restart", config_file="default.json",  )
+      self.fail('Missing DataNode should have caused a failure')
+    except Fail,fail:
+      self.assertTrue(process_mock.called)
+      self.assertEqual(process_mock.call_count,12)
+
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart_bad_returncode(self, process_mock, time_mock):
+    process = MagicMock()
+    process.communicate.return_value = ['some']
+    process.returncode = 999
+    process_mock.return_value = process
+
+    try:
+      self.executeScript("2.0.6/services/HDFS/package/scripts/datanode.py",
+        classname = "DataNode", command = "post_rolling_restart", config_file="default.json",  )
+      self.fail('Invalid return code should cause a failure')
+    except Fail,fail:
+      self.assertTrue(process_mock.called)
+      self.assertEqual(process_mock.call_count,12)
+
+

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -19,6 +19,7 @@ limitations under the License.
 '''
 from mock.mock import MagicMock, call, patch
 from stacks.utils.RMFTestCase import *
+from resource_management.core.exceptions import Fail
 import os
 
 origin_exists = os.path.exists
@@ -556,3 +557,63 @@ class TestNodeManager(RMFTestCase):
                               owner = 'mapred',
                               group = 'hadoop',
                               )
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart(self, process_mock, time_mock):
+    process_output = """
+      c6401.ambari.apache.org:45454  RUNNING  c6401.ambari.apache.org:8042  0
+    """
+
+    process = MagicMock()
+    process.communicate.return_value = [process_output]
+    process.returncode = 0
+    process_mock.return_value = process
+
+    self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+      classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+
+    self.assertTrue(process_mock.called)
+    self.assertEqual(process_mock.call_count,1)
+
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart_nodemanager_not_ready(self, process_mock, time_mock):
+    process_output = """
+      c9999.ambari.apache.org:45454  RUNNING  c9999.ambari.apache.org:8042  0
+    """
+
+    process = MagicMock()
+    process.communicate.return_value = [process_output]
+    process.returncode = 0
+    process_mock.return_value = process
+
+    try:
+      self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+        classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+      self.fail('Missing NodeManager should have caused a failure')
+    except Fail,fail:
+      self.assertTrue(process_mock.called)
+      self.assertEqual(process_mock.call_count,12)
+
+
+  @patch('time.sleep')
+  @patch("subprocess.Popen")
+  def test_post_rolling_restart_nodemanager_not_ready(self, process_mock, time_mock):
+    process_output = """
+      c6401.ambari.apache.org:45454  RUNNING  c6401.ambari.apache.org:8042  0
+    """
+
+    process = MagicMock()
+    process.communicate.return_value = [process_output]
+    process.returncode = 999
+    process_mock.return_value = process
+
+    try:
+      self.executeScript("2.0.6/services/YARN/package/scripts/nodemanager.py",
+        classname="Nodemanager", command = "post_rolling_restart", config_file="default.json")
+      self.fail('Invalid return code should cause a failure')
+    except Fail,fail:
+      self.assertTrue(process_mock.called)
+      self.assertEqual(process_mock.call_count,12)