AMBARI-11009. Storm service check fails with multiple nimbus hosts, remove messages from Storm RU now that it is rolling, allow nimbus.seeds to be edited (alejandro)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py

@@ -81,8 +81,12 @@ user_group = config['configurations']['cluster-env']['user_group']
 java64_home = config['hostLevelParams']['java_home']
 jps_binary = format("{java64_home}/bin/jps")
 nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
-nimbus_seeds_supported = config['configurations']['storm-env']['nimbus_seeds_supported']
-nimbus_host = config['configurations']['storm-site']['nimbus.seeds'] if nimbus_seeds_supported else config['configurations']['storm-site']['nimbus.host']
+
+# nimbus.seeds is supported in HDP 2.3.0.0 and higher
+nimbus_seeds_supported = default('/configurations/storm-env/nimbus_seeds_supported', False)
+nimbus_host = default('/configurations/storm-site/nimbus.host', None)
+nimbus_seeds = default('/configurations/storm-site/nimbus.seeds', None)
+
 rest_api_port = "8745"
 rest_api_admin_port = "8746"
 rest_api_conf_file = format("{conf_dir}/config.yaml")

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/service_check.py

@@ -53,7 +53,13 @@ class ServiceCheckDefault(ServiceCheck):
          content=StaticFile("wordCount.jar")
     )
 
-    cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}")
+    cmd = ""
+    if params.nimbus_seeds_supported:
+      # Because this command is guaranteed to run on one of the hosts with storm client, there is no need
+      # to specify "-c nimbus.seeds={nimbus_seeds}"
+      cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique}")
+    elif params.nimbus_host is not None:
+      cmd = format("storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}")
 
     Execute(cmd,
             logoutput=True,

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/yaml_utils.py

@@ -24,7 +24,7 @@ import resource_management
 from resource_management.core.source import InlineTemplate
 from resource_management.core.resources.system import File
 
-def escape_yaml_propetry(value):
+def escape_yaml_property(value):
   unquouted = False
   unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
   if value in unquouted_values:
@@ -61,18 +61,18 @@ def replace_jaas_placeholder(name, security_enabled, conf_dir):
   else:
     return name
 
-storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_propetry(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }}
+storm_yaml_template = """{% for key, value in configurations|dictsort if not key.startswith('_') %}{{key}} : {{ escape_yaml_property(replace_jaas_placeholder(resource_management.core.source.InlineTemplate(value).get_content().strip(), security_enabled, conf_dir)) }}
 {% endfor %}"""
 
 def yaml_config_template(configurations):
   return InlineTemplate(storm_yaml_template, configurations=configurations,
-                        extra_imports=[escape_yaml_propetry, replace_jaas_placeholder, resource_management,
+                        extra_imports=[escape_yaml_property, replace_jaas_placeholder, resource_management,
                                        resource_management.core, resource_management.core.source])
 
 def yaml_config(filename, configurations = None, conf_dir = None, owner = None, group = None):
   import params
-  config_content = InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
-{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+  config_content = InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_property(value) }}
+{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property])
 
   File (os.path.join(params.conf_dir, filename),
         content = config_content,

ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/STORM/package/scripts/yaml_config.py

@@ -21,7 +21,7 @@ limitations under the License.
 import re
 from resource_management import *
 
-def escape_yaml_propetry(value):
+def escape_yaml_property(value):
   unquouted = False
   unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
   
@@ -58,8 +58,8 @@ def yaml_config(
   owner = None,
   group = None
 ):
-    config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_propetry(value) }}
-{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_propetry])
+    config_content = source.InlineTemplate('''{% for key, value in configurations_dict.items() %}{{ key }}: {{ escape_yaml_property(value) }}
+{% endfor %}''', configurations_dict=configurations, extra_imports=[escape_yaml_property])
 
     File (format("{conf_dir}/{filename}"),
       content = config_content,

ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-site.xml

@@ -22,7 +22,7 @@
   <property>
     <name>nimbus.seeds</name>
     <value>localhost</value>
-    <description>The host that the nimbus server is running on.</description>
+    <description>Comma-delimited list of the hosts running nimbus server.</description>
   </property>
   <property>
     <name>topology.min.replication.count</name>

ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml

@@ -569,11 +569,6 @@
 
     <service name="STORM">
       <component name="NIMBUS">
-        <pre-upgrade>
-          <task xsi:type="manual">
-            <message>Before continuing, please deactivate and kill any currently running topologies.</message>
-          </task>
-        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>
@@ -597,11 +592,6 @@
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>
-        <post-upgrade>
-          <task xsi:type="manual">
-            <message>Please rebuild your topology using the new Storm version dependencies and resubmit it using the newly created jar.</message>
-          </task>
-        </post-upgrade>
       </component>
     </service>
 

ambari-server/src/test/python/stacks/2.3/STORM/test_service_check.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+import resource_management.libraries.functions
+from test_storm_base import TestStormBase
+
+@patch.object(resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value=''))
+
+class TestStormServiceCheck(TestStormBase):
+
+  def test_service_check(self):
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
+                       classname="ServiceCheck",
+                       command="service_check",
+                       config_file="storm_default.json",
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES
+    )
+
+    self.assertResourceCalled('File', '/tmp/wordCount.jar',
+      content = StaticFile('wordCount.jar'),
+    )
+    self.assertResourceCalled('Execute', 'storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount',
+      logoutput = True,
+      path = ['/usr/hdp/current/storm-client/bin'],
+      user = 'storm'
+    )
+    self.assertResourceCalled('Execute', 'storm kill WordCount',
+      path = ['/usr/hdp/current/storm-client/bin'],
+      user = 'storm'
+    )

ambari-server/src/test/python/stacks/2.3/STORM/test_storm_base.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from stacks.utils.RMFTestCase import *
+
+
+class TestStormBase(RMFTestCase):
+  COMMON_SERVICES_PACKAGE_DIR = "STORM/0.9.1.2.1/package"
+  STACK_VERSION = "2.3"
+
+  def assert_configure_default(self, confDir="/etc/storm/conf"):
+    import params
+    self.assertResourceCalled('Directory', '/var/log/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      mode = 0777,
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/run/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('Directory', '/hadoop/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('Directory', confDir,
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('File', confDir + '/config.yaml',
+      owner = 'storm',
+      content = Template('config.yaml.j2'),
+      group = 'hadoop',
+    )
+    
+    storm_yarn_content = self.call_storm_template_and_assert(confDir=confDir)
+    
+    self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted')
+
+    self.assertResourceCalled('File', confDir + '/storm-env.sh',
+                              owner = 'storm',
+                              content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content'])
+                              )
+    return storm_yarn_content
+
+  def assert_configure_secured(self, confDir='/etc/storm/conf'):
+    import params
+    self.assertResourceCalled('Directory', '/var/log/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      mode = 0777,
+      recursive = True,
+    )
+    self.assertResourceCalled('Directory', '/var/run/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('Directory', '/hadoop/storm',
+      owner = 'storm',
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('Directory', confDir,
+      group = 'hadoop',
+      recursive = True,
+      cd_access='a'
+    )
+    self.assertResourceCalled('File', confDir + '/config.yaml',
+      owner = 'storm',
+      content = Template('config.yaml.j2'),
+      group = 'hadoop',
+    )
+    storm_yarn_content = self.call_storm_template_and_assert(confDir=confDir)
+    
+    self.assertTrue(storm_yarn_content.find('_JAAS_PLACEHOLDER') == -1, 'Placeholder have to be substituted')
+    
+    self.assertResourceCalled('File', confDir + '/storm-env.sh',
+                              owner = 'storm',
+                              content = InlineTemplate(self.getConfig()['configurations']['storm-env']['content'])
+                              )
+    self.assertResourceCalled('TemplateConfig', confDir + '/storm_jaas.conf',
+      owner = 'storm',
+    )
+    return storm_yarn_content
+
+  def call_storm_template_and_assert(self, confDir="/etc/storm/conf"):
+    import yaml_utils
+
+    with RMFTestCase.env as env:
+      storm_yarn_temlate = yaml_utils.yaml_config_template(self.getConfig()['configurations']['storm-site'])
+
+      self.assertResourceCalled('File', confDir + '/storm.yaml',
+        owner = 'storm',
+        content= storm_yarn_temlate,
+        group = 'hadoop'
+      )
+
+      return storm_yarn_temlate.get_content()

ambari-web/app/data/HDP2/site_properties.js

@@ -1035,7 +1035,6 @@ var hdp2properties = [
     "name": "nimbus.seeds",
     "displayName": "nimbus.seeds",
     "displayType": "masterHosts",
-    "isOverridable": false,
     "serviceName": "STORM",
     "filename": "storm-site.xml",
     "category": "NIMBUS",