AMBARI-11892 - Finalize Cluster Fails Because Some Components Are Not Included In The Upgrade Pack (jonathanhurley)

ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py

@@ -70,7 +70,8 @@ SERVICE_CHECK_DIRECTORY_MAP = {
   "HDFS_SERVICE_CHECK" : "hadoop-client",
   "TEZ_SERVICE_CHECK" : "hadoop-client",
   "PIG_SERVICE_CHECK" : "hadoop-client",
-  "HIVE_SERVICE_CHECK" : "hadoop-client"
+  "HIVE_SERVICE_CHECK" : "hadoop-client",
+  "MAHOUT_SERVICE_CHECK" : "mahout-client"
 }
 
 # /usr/hdp/current/hadoop-client/[bin|sbin|libexec|lib]

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/phoenix_queryserver.py

@@ -17,7 +17,8 @@ limitations under the License.
 
 """
 
-import sys
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.script import Script
 from phoenix_service import phoenix_service
 from hbase import hbase
@@ -28,30 +29,45 @@ class PhoenixQueryServer(Script):
   def install(self, env):
     self.install_packages(env)
 
+
   def get_stack_to_component(self):
     return {"HDP": "phoenix-server"}
 
+
   def configure(self, env):
     import params
     env.set_params(params)
     hbase(name='queryserver')
 
+
   def start(self, env, rolling_restart=False):
     import params
     env.set_params(params)
     self.configure(env)
     phoenix_service('start')
 
+
   def stop(self, env, rolling_restart=False):
     import params
     env.set_params(params)
     phoenix_service('stop')
 
+
+  def pre_rolling_restart(self, env):
+    import params
+    env.set_params(params)
+
+    if Script.is_hdp_stack_greater_or_equal("2.3"):
+      conf_select.select(params.stack_name, "phoenix", params.version)
+      hdp_select.select("phoenix-server", params.version)
+
+
   def status(self, env):
     import status_params
     env.set_params(status_params)
     phoenix_service('status')
 
+
   def security_status(self, env):
     self.put_structured_out({"securityState": "UNSECURED"})
 

ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/metainfo.xml

@@ -16,61 +16,63 @@
    limitations under the License.
 -->
 <metainfo>
-    <schemaVersion>2.0</schemaVersion>
-    <services>
-        <service>
-            <name>MAHOUT</name>
-            <displayName>Mahout</displayName>
-            <comment>Project of the Apache Software Foundation to produce free implementations of distributed or
-                otherwise scalable machine learning algorithms focused primarily in the areas of collaborative
-                filtering, clustering and classification</comment>
-            <version>1.0.0.2.3</version>
-            <components>
-                <component>
-                    <name>MAHOUT</name>
-                    <displayName>Mahout</displayName>
-                    <category>CLIENT</category>
-                    <cardinality>0+</cardinality>
-                    <versionAdvertised>true</versionAdvertised>
-                    <commandScript>
-                        <script>scripts/mahout_client.py</script>
-                        <scriptType>PYTHON</scriptType>
-                        <timeout>1200</timeout>
-                    </commandScript>
-                    <configFiles>
-                        <configFile>
-                            <type>env</type>
-                            <fileName>log4j.properties</fileName>
-                            <dictionaryName>mahout-log4j</dictionaryName>
-                        </configFile>
-                    </configFiles>
-                </component>
-            </components>
-            <osSpecifics>
-                <osSpecific>
-                    <osFamily>any</osFamily>
-                    <packages>
-                        <package>
-                            <name>mahout</name>
-                        </package>
-                    </packages>
-                </osSpecific>
-            </osSpecifics>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>MAHOUT</name>
+      <displayName>Mahout</displayName>
+      <comment>
+        Project of the Apache Software Foundation to produce free implementations of distributed or
+        otherwise scalable machine learning algorithms focused primarily in the areas of collaborative
+        filtering, clustering and classification
+      </comment>
+      <version>1.0.0.2.3</version>
+      <components>
+        <component>
+          <name>MAHOUT</name>
+          <displayName>Mahout</displayName>
+          <category>CLIENT</category>
+          <cardinality>0+</cardinality>
+          <versionAdvertised>true</versionAdvertised>
+          <commandScript>
+            <script>scripts/mahout_client.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>1200</timeout>
+          </commandScript>
+          <configFiles>
+            <configFile>
+              <type>env</type>
+              <fileName>log4j.properties</fileName>
+              <dictionaryName>mahout-log4j</dictionaryName>
+            </configFile>
+          </configFiles>
+        </component>
+      </components>
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>mahout</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
 
-            <commandScript>
-                <script>scripts/service_check.py</script>
-                <scriptType>PYTHON</scriptType>
-                <timeout>300</timeout>
-            </commandScript>
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
 
-            <requiredServices>
-                <service>YARN</service>
-            </requiredServices>
+      <requiredServices>
+        <service>YARN</service>
+      </requiredServices>
 
-            <configuration-dependencies>
-                <config-type>mahout-log4j</config-type>
-            </configuration-dependencies>
+      <configuration-dependencies>
+        <config-type>mahout-log4j</config-type>
+      </configuration-dependencies>
 
-        </service>
-    </services>
+    </service>
+  </services>
 </metainfo>

ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout_client.py

@@ -18,10 +18,10 @@ limitations under the License.
 Ambari Agent
 
 """
-
-import sys
-from resource_management import *
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.libraries.functions import hdp_select
 from resource_management.libraries.functions import conf_select
+from resource_management.libraries.script import Script
 from mahout import mahout
 
 
@@ -30,23 +30,26 @@ class MahoutClient(Script):
   def get_stack_to_component(self):
     return {"HDP": "mahout-client"}
 
+
   def pre_rolling_restart(self, env):
     import params
     env.set_params(params)
 
     conf_select.select(params.stack_name, "mahout", params.version)
-    Execute(('hdp-select', 'set', 'mahout-client', params.version),
-            sudo = True)
+    hdp_select.select("mahout-client", params.version )
+
 
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
 
+
   def configure(self, env):
     import params
     env.set_params(params)
     mahout()
 
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml

@@ -133,6 +133,7 @@
       
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
+        <component>PHOENIX_QUERY_SERVER</component>
       </service>
 
       <service name="YARN">
@@ -225,6 +226,10 @@
         <component>SQOOP</component>
       </service>
 
+      <service name="MAHOUT">
+        <component>MAHOUT</component>
+      </service>
+
       <service name="HIVE">
         <component>HIVE_CLIENT</component>
         <component>HCAT</component>
@@ -283,6 +288,17 @@
       </service>
     </group>
 
+    <group name="ACCUMULO" title="Accumulo">
+      <skippable>true</skippable>
+      <service name="ACCUMULO">
+        <component>ACCUMULO_MASTER</component>
+        <component>ACCUMULO_TSERVER</component>
+        <component>ACCUMULO_MONITOR</component>
+        <component>ACCUMULO_GC</component>
+        <component>ACCUMULO_TRACER</component>
+        <component>ACCUMULO_CLIENT</component>
+      </service>
+    </group>
 
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
@@ -414,6 +430,7 @@
         </upgrade>
       </component>
     </service>
+
     <service name="HBASE">
       <component name="HBASE_MASTER">
         <upgrade>
@@ -432,6 +449,12 @@
           <task xsi:type="restart" />
         </upgrade>
       </component>
+
+      <component name="PHOENIX_QUERY_SERVER">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
     </service>
 
     <service name="TEZ">
@@ -458,6 +481,14 @@
       </component>
     </service>
 
+    <service name="MAHOUT">
+      <component name="MAHOUT">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+    </service>
+
     <service name="HIVE">
       <component name="HIVE_METASTORE">
         <upgrade>
@@ -628,5 +659,38 @@
         </upgrade>
       </component>
     </service>
+
+    <service name="ACCUMULO">
+      <component name="ACCUMULO_MASTER">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="ACCUMULO_MONITOR">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="ACCUMULO_GC">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="ACCUMULO_TRACER">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="ACCUMULO_TSERVER">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="ACCUMULO_CLIENT">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+    </service>
   </processing>
 </upgrade>

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_phoenix_queryserver.py

@@ -17,124 +17,145 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import json
+
 from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 from unittest import skip
 
-@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
-@patch("os.path.exists", new = MagicMock(return_value=True))
+
+@patch("platform.linux_distribution", new = MagicMock(return_value = "Linux"))
+@patch("os.path.exists", new = MagicMock(return_value = True))
 class TestPhoenixQueryServer(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HBASE/0.96.0.2.0/package"
   STACK_VERSION = "2.3"
   TMP_PATH = "/tmp/hbase-hbase"
 
   def test_configure_default(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "configure",
-                   config_file="hbase_default.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "configure",
+      config_file = "hbase_default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
     )
 
     self.assert_configure_default()
     self.assertNoMoreResources()
-    
+
   def test_start_default(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "start",
-                   config_file="hbase_default.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "start",
+      config_file = "hbase_default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     self.assert_configure_default()
-    self.assertResourceCalled('Execute', '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
-                            environment = {'JAVA_HOME': '/usr/jdk64/jdk1.8.0_40', 'HBASE_CONF_DIR': '/usr/hdp/current/hbase-regionserver/conf'},
-                            user = 'hbase'
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
+      environment = {'JAVA_HOME':'/usr/jdk64/jdk1.8.0_40',
+      'HBASE_CONF_DIR':'/usr/hdp/current/hbase-regionserver/conf'},
+      user = 'hbase'
     )
     self.assertNoMoreResources()
 
   def test_stop_default(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "stop",
-                   config_file="hbase_default.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
-    )
-    
-    self.assertResourceCalled('Execute', '/usr/hdp/current/phoenix-server/bin/queryserver.py stop',
-        on_timeout = '! ( ls /var/run/hbase/phoenix-hbase-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hbase/phoenix-hbase-server.pid` >/dev/null 2>&1 ) || ambari-sudo.sh -H -E kill -9 `cat /var/run/hbase/phoenix-hbase-server.pid`',
-        timeout = 30,
-        environment = {'JAVA_HOME': '/usr/jdk64/jdk1.8.0_40', 'HBASE_CONF_DIR': '/usr/hdp/current/hbase-regionserver/conf'},
-        user = 'hbase'
-    )
-    
-    self.assertResourceCalled('Execute', 'rm -f /var/run/hbase/phoenix-hbase-server.pid',
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "stop",
+      config_file = "hbase_default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/phoenix-server/bin/queryserver.py stop',
+      on_timeout = '! ( ls /var/run/hbase/phoenix-hbase-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hbase/phoenix-hbase-server.pid` >/dev/null 2>&1 ) || ambari-sudo.sh -H -E kill -9 `cat /var/run/hbase/phoenix-hbase-server.pid`',
+      timeout = 30,
+      environment = {'JAVA_HOME':'/usr/jdk64/jdk1.8.0_40',
+      'HBASE_CONF_DIR':'/usr/hdp/current/hbase-regionserver/conf'},
+      user = 'hbase'
+    )
+
+    self.assertResourceCalled('Execute',
+      'rm -f /var/run/hbase/phoenix-hbase-server.pid',
     )
     self.assertNoMoreResources()
-    
+
   def test_configure_secured(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "configure",
-                   config_file="hbase_secure.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "configure",
+      config_file = "hbase_secure.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
     )
 
     self.assert_configure_secured()
     self.assertNoMoreResources()
-    
+
   def test_start_secured(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "start",
-                   config_file="hbase_secure.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "start",
+      config_file = "hbase_secure.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     self.assert_configure_secured()
-    self.assertResourceCalled('Execute', '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
-                          environment = {'JAVA_HOME': '/usr/jdk64/jdk1.8.0_40', 'HBASE_CONF_DIR': '/usr/hdp/current/hbase-regionserver/conf'},
-                          user = 'hbase'
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
+      environment = {'JAVA_HOME':'/usr/jdk64/jdk1.8.0_40',
+      'HBASE_CONF_DIR':'/usr/hdp/current/hbase-regionserver/conf'},
+      user = 'hbase'
     )
     self.assertNoMoreResources()
 
   def test_stop_secured(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "stop",
-                   config_file="hbase_secure.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "stop",
+      config_file = "hbase_secure.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES
     )
 
-    self.assertResourceCalled('Execute', '/usr/hdp/current/phoenix-server/bin/queryserver.py stop',
-        on_timeout = '! ( ls /var/run/hbase/phoenix-hbase-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hbase/phoenix-hbase-server.pid` >/dev/null 2>&1 ) || ambari-sudo.sh -H -E kill -9 `cat /var/run/hbase/phoenix-hbase-server.pid`',
-        timeout = 30,
-        environment = {'JAVA_HOME': '/usr/jdk64/jdk1.8.0_40', 'HBASE_CONF_DIR': '/usr/hdp/current/hbase-regionserver/conf'},
-        user = 'hbase'
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/phoenix-server/bin/queryserver.py stop',
+      on_timeout = '! ( ls /var/run/hbase/phoenix-hbase-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hbase/phoenix-hbase-server.pid` >/dev/null 2>&1 ) || ambari-sudo.sh -H -E kill -9 `cat /var/run/hbase/phoenix-hbase-server.pid`',
+      timeout = 30,
+      environment = {'JAVA_HOME':'/usr/jdk64/jdk1.8.0_40',
+      'HBASE_CONF_DIR':'/usr/hdp/current/hbase-regionserver/conf'},
+      user = 'hbase'
     )
-    
-    self.assertResourceCalled('Execute', 'rm -f /var/run/hbase/phoenix-hbase-server.pid',
+
+    self.assertResourceCalled('Execute',
+      'rm -f /var/run/hbase/phoenix-hbase-server.pid',
     )
     self.assertNoMoreResources()
 
   @skip("there's nothing to upgrade to yet")
   def test_start_default_24(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
-                   classname = "PhoenixQueryServer",
-                   command = "start",
-                   config_file="hbase-rs-2.4.json",
-                   hdp_stack_version = self.STACK_VERSION,
-                   target = RMFTestCase.TARGET_COMMON_SERVICES)
-    
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "start",
+      config_file = "hbase-rs-2.4.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
     self.assertResourceCalled('Directory', '/etc/hbase',
       mode = 0755)
 
-    self.assertResourceCalled('Directory', '/usr/hdp/current/hbase-regionserver/conf',
+    self.assertResourceCalled('Directory',
+      '/usr/hdp/current/hbase-regionserver/conf',
       owner = 'hbase',
       group = 'hadoop',
       recursive = True)
@@ -144,17 +165,21 @@ class TestPhoenixQueryServer(RMFTestCase):
       group = 'hadoop',
       conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
       configurations = self.getConfig()['configurations']['hbase-site'],
-      configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site'])
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hbase-site'])
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['core-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['core-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'core-site']
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
+    self.assertResourceCalled('File',
+      '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
       owner = 'hbase',
-      content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']))
+      content = InlineTemplate(
+        self.getConfig()['configurations']['hbase-env']['content']))
 
     self.assertResourceCalled('Directory', '/var/run/hbase',
       owner = 'hbase',
@@ -165,14 +190,14 @@ class TestPhoenixQueryServer(RMFTestCase):
       recursive = True)
 
     self.assertResourceCalled('File',
-                              '/usr/lib/phoenix/bin/log4j.properties',
-                              mode=0644,
-                              group='hadoop',
-                              owner='hbase',
-                              content='log4jproperties\nline2')
-
+      '/usr/lib/phoenix/bin/log4j.properties',
+      mode = 0644,
+      group = 'hadoop',
+      owner = 'hbase',
+      content = 'log4jproperties\nline2')
 
-    self.assertResourceCalled('Execute', '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/phoenix-server/bin/queryserver.py start',
       not_if = 'ls /var/run/hbase/phoenix-hbase-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hbase/phoenix-hbase-server.pid` >/dev/null 2>&1',
       user = 'hbase')
 
@@ -180,184 +205,224 @@ class TestPhoenixQueryServer(RMFTestCase):
 
   def assert_configure_default(self):
     self.assertResourceCalled('Directory', '/etc/hbase',
-                              mode = 0755
+      mode = 0755
+    )
+    self.assertResourceCalled('Directory',
+      '/usr/hdp/current/hbase-regionserver/conf',
+      owner = 'hbase',
+      group = 'hadoop',
+      recursive = True,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/hbase-regionserver/conf',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              recursive = True,
-                              )
     self.assertResourceCalled('Directory', self.TMP_PATH,
-                              owner = 'hbase',
-                              mode=0775,
-                              recursive = True,
-                              cd_access='a'
+      owner = 'hbase',
+      mode = 0775,
+      recursive = True,
+      cd_access = 'a'
     )
     self.assertResourceCalled('Directory', self.TMP_PATH + '/local',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              mode=0775,
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      mode = 0775,
+      recursive = True,
+    )
     self.assertResourceCalled('Directory', self.TMP_PATH + '/local/jars',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              mode=0775,
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      mode = 0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hbase-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hbase-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hbase-site']
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['core-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['core-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'core-site']
     )
     self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hdfs-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hdfs-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hdfs-site']
     )
     self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
-                              owner = 'hdfs',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hadoop-client/conf',
-                              configurations = self.getConfig()['configurations']['hdfs-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+      owner = 'hdfs',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hadoop-client/conf',
+      configurations = self.getConfig()['configurations']['hdfs-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hdfs-site']
     )
     self.assertResourceCalled('XmlConfig', 'hbase-policy.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hbase-policy'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hbase-policy']
-                              )
-    self.assertResourceCalled('File', '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
-                              owner = 'hbase',
-                              content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
-                              )
-    self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
-                              owner = 'hbase',
-                              template_tag = 'GANGLIA-RS',
-                              )
-    self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/regionservers',
-                              owner = 'hbase',
-                              template_tag = None,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hbase-policy'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hbase-policy']
+    )
+    self.assertResourceCalled('File',
+      '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
+      owner = 'hbase',
+      content = InlineTemplate(
+        self.getConfig()['configurations']['hbase-env']['content']),
+    )
+    self.assertResourceCalled('TemplateConfig',
+      '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
+      owner = 'hbase',
+      template_tag = 'GANGLIA-RS',
+    )
+    self.assertResourceCalled('TemplateConfig',
+      '/usr/hdp/current/hbase-regionserver/conf/regionservers',
+      owner = 'hbase',
+      template_tag = None,
+    )
     self.assertResourceCalled('Directory', '/var/run/hbase',
-                              owner = 'hbase',
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      recursive = True,
+    )
     self.assertResourceCalled('Directory', '/var/log/hbase',
-                              owner = 'hbase',
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      recursive = True,
+    )
     self.assertResourceCalled('File',
-                              '/usr/hdp/current/hbase-regionserver/conf/log4j.properties',
-                              mode=0644,
-                              group='hadoop',
-                              owner='hbase',
-                              content='log4jproperties\nline2'
+      '/usr/hdp/current/hbase-regionserver/conf/log4j.properties',
+      mode = 0644,
+      group = 'hadoop',
+      owner = 'hbase',
+      content = 'log4jproperties\nline2'
     )
 
   def assert_configure_secured(self):
     self.assertResourceCalled('Directory', '/etc/hbase',
-                              mode = 0755
+      mode = 0755
+    )
+    self.assertResourceCalled('Directory',
+      '/usr/hdp/current/hbase-regionserver/conf',
+      owner = 'hbase',
+      group = 'hadoop',
+      recursive = True,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/hbase-regionserver/conf',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              recursive = True,
-                              )
     self.assertResourceCalled('Directory', self.TMP_PATH,
-                              owner = 'hbase',
-                              mode=0775,
-                              recursive = True,
-                              cd_access='a'
+      owner = 'hbase',
+      mode = 0775,
+      recursive = True,
+      cd_access = 'a'
     )
     self.assertResourceCalled('Directory', self.TMP_PATH + '/local',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              mode=0775,
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      mode = 0775,
+      recursive = True,
+    )
     self.assertResourceCalled('Directory', self.TMP_PATH + '/local/jars',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              mode=0775,
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      mode = 0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hbase-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hbase-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hbase-site']
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['core-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['core-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'core-site']
     )
     self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hdfs-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hdfs-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hdfs-site']
     )
     self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
-                              owner = 'hdfs',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hadoop-client/conf',
-                              configurations = self.getConfig()['configurations']['hdfs-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
+      owner = 'hdfs',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hadoop-client/conf',
+      configurations = self.getConfig()['configurations']['hdfs-site'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hdfs-site']
     )
     self.assertResourceCalled('XmlConfig', 'hbase-policy.xml',
-                              owner = 'hbase',
-                              group = 'hadoop',
-                              conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
-                              configurations = self.getConfig()['configurations']['hbase-policy'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['hbase-policy']
-    )
-    self.assertResourceCalled('File', '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
-                              owner = 'hbase',
-                              content = InlineTemplate(self.getConfig()['configurations']['hbase-env']['content']),
-                              )
-    self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
-                              owner = 'hbase',
-                              template_tag = 'GANGLIA-RS',
-                              )
-    self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/regionservers',
-                              owner = 'hbase',
-                              template_tag = None,
-                              )
-    self.assertResourceCalled('TemplateConfig', '/usr/hdp/current/hbase-regionserver/conf/hbase_queryserver_jaas.conf',
-                              owner = 'hbase',
-                              template_tag = None,
-                              )
+      owner = 'hbase',
+      group = 'hadoop',
+      conf_dir = '/usr/hdp/current/hbase-regionserver/conf',
+      configurations = self.getConfig()['configurations']['hbase-policy'],
+      configuration_attributes = self.getConfig()['configuration_attributes'][
+        'hbase-policy']
+    )
+    self.assertResourceCalled('File',
+      '/usr/hdp/current/hbase-regionserver/conf/hbase-env.sh',
+      owner = 'hbase',
+      content = InlineTemplate(
+        self.getConfig()['configurations']['hbase-env']['content']),
+    )
+    self.assertResourceCalled('TemplateConfig',
+      '/usr/hdp/current/hbase-regionserver/conf/hadoop-metrics2-hbase.properties',
+      owner = 'hbase',
+      template_tag = 'GANGLIA-RS',
+    )
+    self.assertResourceCalled('TemplateConfig',
+      '/usr/hdp/current/hbase-regionserver/conf/regionservers',
+      owner = 'hbase',
+      template_tag = None,
+    )
+    self.assertResourceCalled('TemplateConfig',
+      '/usr/hdp/current/hbase-regionserver/conf/hbase_queryserver_jaas.conf',
+      owner = 'hbase',
+      template_tag = None,
+    )
     self.assertResourceCalled('Directory', '/var/run/hbase',
-                              owner = 'hbase',
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      recursive = True,
+    )
     self.assertResourceCalled('Directory', '/var/log/hbase',
-                              owner = 'hbase',
-                              recursive = True,
-                              )
+      owner = 'hbase',
+      recursive = True,
+    )
     self.assertResourceCalled('File',
-                              '/usr/hdp/current/hbase-regionserver/conf/log4j.properties',
-                              mode=0644,
-                              group='hadoop',
-                              owner='hbase',
-                              content='log4jproperties\nline2'
+      '/usr/hdp/current/hbase-regionserver/conf/log4j.properties',
+      mode = 0644,
+      group = 'hadoop',
+      owner = 'hbase',
+      content = 'log4jproperties\nline2'
     )
+
+  def test_upgrade_restart(self):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.3/configs/hbase_default.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+
+    json_content['commandParams']['version'] = '2.3.0.0-1234'
+
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/phoenix_queryserver.py",
+      classname = "PhoenixQueryServer",
+      command = "pre_rolling_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Execute', 'hdp-select set phoenix-server 2.3.0.0-1234')
+
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.3/MAHOUT/test_mahout_client.py

@@ -17,92 +17,102 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
+import itertools
+import json
 
-from mock.mock import patch
 from stacks.utils.RMFTestCase import *
-import json
 
 class TestMahoutClient(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "MAHOUT/1.0.0.2.3/package"
   STACK_VERSION = "2.3"
 
   def test_configure_default(self):
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
-                       classname = "MahoutClient",
-                       command = "configure",
-                       config_file="default.json",
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES
-    )
-
-    self.assertResourceCalled('Directory', '/usr/hdp/current/mahout-client/conf',
-                              owner = 'mahout',
-                              group = 'hadoop',
-                              recursive = True,
-                              )
-    self.assertResourceCalled('File', '/usr/hdp/current/mahout-client/conf/log4j.properties',
-                              content = self.getConfig()['configurations']['mahout-log4j']['content'],
-                              owner = 'mahout',
-                              group = 'hadoop',
-                              mode = 0644,
-                              )
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
+      classname = "MahoutClient",
+      command = "configure",
+      config_file = "default.json",
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Directory',
+      '/usr/hdp/current/mahout-client/conf',
+      owner = 'mahout',
+      group = 'hadoop',
+      recursive = True)
+
+    self.assertResourceCalled('File',
+      '/usr/hdp/current/mahout-client/conf/log4j.properties',
+      content = self.getConfig()['configurations']['mahout-log4j']['content'],
+      owner = 'mahout',
+      group = 'hadoop',
+      mode = 0644 )
+
     self.assertNoMoreResources()
 
   def test_pre_rolling_restart(self):
-    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
     version = '2.2.1.0-3242'
     json_content['commandParams']['version'] = version
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
-                       classname = "MahoutClient",
-                       command = "pre_rolling_restart",
-                       config_dict = json_content,
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES)
-    self.assertResourceCalled('Execute',
-                              ('hdp-select', 'set', 'mahout-client', version), sudo = True)
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
+      classname = "MahoutClient",
+      command = "pre_rolling_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+    self.assertResourceCalled('Execute', "hdp-select set mahout-client 2.2.1.0-3242")
     self.assertNoMoreResources()
 
-  @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock):
-    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+  def test_pre_rolling_restart_23(self):
+    config_file = self.get_src_folder() + "/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
+
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version
+
     # test to make sure conf_select is working correctly
     json_content['commandParams']['upgrade_direction'] = 'upgrade'
     json_content['hostLevelParams']['stack_name'] = 'HDP'
     json_content['hostLevelParams']['stack_version'] = '2.3'
 
     mocks_dict = {}
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
-                       classname = "MahoutClient",
-                       command = "pre_rolling_restart",
-                       config_dict = json_content,
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES,
-                       call_mocks = [(0, None), (0, None), (0, None), (0, None)],
-                       mocks_dict = mocks_dict)
-
-    self.assertResourceCalled('Execute',
-                              ('hdp-select', 'set', 'mahout-client', version), sudo = True)
+    self.executeScript(
+      self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mahout_client.py",
+      classname = "MahoutClient",
+      command = "pre_rolling_restart",
+      config_dict = json_content,
+      hdp_stack_version = self.STACK_VERSION,
+      target = RMFTestCase.TARGET_COMMON_SERVICES,
+      call_mocks = itertools.cycle([(0, None)]),
+      mocks_dict = mocks_dict)
+
+    self.assertResourceCalled('Execute', "hdp-select set mahout-client 2.3.0.0-1234")
     self.assertNoMoreResources()
 
     import sys
-    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/conf", sys.modules["params"].hadoop_conf_dir)
 
-    self.assertEquals(4, mocks_dict['call'].call_count)
+    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/conf",
+      sys.modules["params"].hadoop_conf_dir)
+
+    self.assertEquals(6, mocks_dict['call'].call_count)
+
     self.assertEquals(
       "conf-select create-conf-dir --package hadoop --stack-version 2.3.0.0-1234 --conf-version 0",
-       mocks_dict['call'].call_args_list[0][0][0])
+      mocks_dict['call'].call_args_list[0][0][0])
+
     self.assertEquals(
       "conf-select set-conf-dir --package hadoop --stack-version 2.3.0.0-1234 --conf-version 0",
-       mocks_dict['call'].call_args_list[1][0][0])
+      mocks_dict['call'].call_args_list[1][0][0])
+
     self.assertEquals(
       "conf-select create-conf-dir --package mahout --stack-version 2.3.0.0-1234 --conf-version 0",
-       mocks_dict['call'].call_args_list[2][0][0])
+      mocks_dict['call'].call_args_list[2][0][0])
+
     self.assertEquals(
       "conf-select set-conf-dir --package mahout --stack-version 2.3.0.0-1234 --conf-version 0",
-       mocks_dict['call'].call_args_list[3][0][0])
+      mocks_dict['call'].call_args_list[3][0][0])