AMBARI-15527: Stack featurize STORM service (jluniya)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -48,4 +48,5 @@ class StackFeature:
   TEZ_FOR_SPARK = "tez_for_spark"
   SPARK_16PLUS = "spark_16plus"
   SPARK_THRIFTSERVER = "spark_thriftserver"
-    
+  STORM_KERBEROS = "storm_kerberos"
+  STORM_AMS = "storm_ams"

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -72,6 +72,16 @@ _DEFAULT_STACK_FEATURES = {
       "name": "spark_thriftserver",
       "description": "Spark Thrift Server",
       "min_version": "2.3.2.0"
+    },
+    {
+      "name": "storm_kerberos",
+      "description": "Storm Kerberos support (AMBARI-7570)",
+      "min_version": "2.2.0.0"
+    },
+    {
+      "name": "storm_ams",
+      "description": "Storm AMS integration (AMBARI-10710)",
+      "min_version": "2.2.0.0"
     }
   ]
 }

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/drpc_server.py

@@ -25,7 +25,8 @@ from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from storm import storm
 from service import service
 from service_check import ServiceCheck
@@ -36,7 +37,8 @@ from resource_management.libraries.functions.security_commons import build_expec
 class DrpcServer(Script):
 
   def get_stack_to_component(self):
-    return {"HDP": "storm-client"}
+    import params
+    return {params.stack_name: "storm-client"}
 
   def install(self, env):
     self.install_packages(env)
@@ -52,7 +54,7 @@ class DrpcServer(Script):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
 

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/nimbus.py

@@ -25,7 +25,8 @@ from resource_management.libraries.functions import format
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from storm import storm
 from service import service
 from resource_management.libraries.functions.security_commons import build_expectations, \
@@ -38,7 +39,8 @@ from resource_management.core.resources.service import Service
 
 class Nimbus(Script):
   def get_stack_to_component(self):
-    return {"HDP": "storm-nimbus"}
+    import params
+    return {params.stack_name : "storm-nimbus"}
 
   def install(self, env):
     self.install_packages(env)
@@ -56,7 +58,7 @@ class NimbusDefault(Nimbus):
   def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
       stack_select.select("storm-nimbus", params.version)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/nimbus_prod.py

@@ -26,12 +26,14 @@ from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 class Nimbus(Script):
 
   def get_stack_to_component(self):
-    return {"HDP": "storm-nimbus"}
+    import params
+    return {params.stack_name : "storm-nimbus"}
 
   def install(self, env):
     self.install_packages(env)
@@ -47,7 +49,7 @@ class Nimbus(Script):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
       stack_select.select("storm-nimbus", params.version)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py

@@ -34,10 +34,13 @@ from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 # server configurations
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
+stack_root = status_params.stack_root
 sudo = AMBARI_SUDO_BINARY
 
 stack_name = default("/hostLevelParams/stack_name", None)
@@ -47,9 +50,11 @@ version = default("/commandParams/version", None)
 storm_component_home_dir = status_params.storm_component_home_dir
 conf_dir = status_params.conf_dir
 
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-stack_version_formatted = format_stack_version(stack_version_unformatted)
-stack_is_hdp22_or_further = Script.is_stack_greater_or_equal("2.2")
+stack_version_unformatted = status_params.stack_version_unformatted
+stack_version_formatted = status_params.stack_version_formatted
+stack_supports_ru = stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted)
+stack_supports_storm_kerberos = stack_version_formatted and check_stack_feature(StackFeature.STORM_KERBEROS, stack_version_formatted)
+stack_supports_storm_ams = stack_version_formatted and check_stack_feature(StackFeature.STORM_AMS, stack_version_formatted)
 
 # default hadoop params
 rest_lib_dir = "/usr/lib/storm/contrib/storm-rest"
@@ -57,7 +62,7 @@ storm_bin_dir = "/usr/bin"
 storm_lib_dir = "/usr/lib/storm/lib/"
 
 # hadoop parameters for 2.2+
-if stack_is_hdp22_or_further:
+if stack_supports_ru:
   rest_lib_dir = format("{storm_component_home_dir}/contrib/storm-rest")
   storm_bin_dir = format("{storm_component_home_dir}/bin")
   storm_lib_dir = format("{storm_component_home_dir}/lib")
@@ -122,30 +127,26 @@ if security_enabled:
   storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase)
   storm_keytab_path = config['configurations']['storm-env']['storm_keytab']
 
-  if stack_is_hdp22_or_further:
+  if stack_supports_storm_kerberos:
     storm_ui_keytab_path = config['configurations']['storm-env']['storm_ui_keytab']
     _storm_ui_jaas_principal_name = config['configurations']['storm-env']['storm_ui_principal_name']
     storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',_hostname_lowercase)
-
     storm_bare_jaas_principal = get_bare_principal(_storm_principal_name)
-
     _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name']
     nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST', _hostname_lowercase)
     nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name)
     nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab']
 
 kafka_bare_jaas_principal = None
-if stack_is_hdp22_or_further:
+if stack_supports_storm_kerberos:
   if security_enabled:
     storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport']
     # generate KafkaClient jaas config if kafka is kerberoized
     _kafka_principal_name = default("/configurations/kafka-env/kafka_principal_name", None)
     kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name)
-
   else:
     storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.nonsecure.transport']
 
-
 ams_collector_hosts = default("/clusterHostInfo/metrics_collector_hosts", [])
 has_metric_collector = not len(ams_collector_hosts) == 0
 if has_metric_collector:

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/rest_api.py

@@ -25,7 +25,6 @@ from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
 
 from storm import storm
 from service import service
@@ -40,7 +39,8 @@ class StormRestApi(Script):
   """
 
   def get_stack_to_component(self):
-    return {"HDP": "storm-client"}
+    import params
+    return {params.stack_name : "storm-client"}
 
   def install(self, env):
     self.install_packages(env)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/status_params.py

@@ -20,6 +20,9 @@ limitations under the License.
 from resource_management.libraries.script import Script
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.functions import default, format
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from ambari_commons import OSCheck
 
 # a map of the Ambari role to the component name
@@ -35,6 +38,9 @@ SERVER_ROLE_DIRECTORY_MAP = {
 component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "STORM_SERVICE_CHECK")
 
 config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+stack_version_formatted = format_stack_version(stack_version_unformatted)
 
 if OSCheck.is_windows_family():
   nimbus_win_service_name = "nimbus"
@@ -66,9 +72,9 @@ else:
 
   storm_component_home_dir = "/usr/lib/storm"
   conf_dir = "/etc/storm/conf"
-  if Script.is_stack_greater_or_equal("2.2"):
-    storm_component_home_dir = format("/usr/hdp/current/{component_directory}")
-    conf_dir = format("/usr/hdp/current/{component_directory}/conf")
+  if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+    storm_component_home_dir = format("{stack_root}/current/{component_directory}")
+    conf_dir = format("{stack_root}/current/{component_directory}/conf")
 
   storm_user = config['configurations']['storm-env']['storm_user']
   storm_ui_principal = default('/configurations/storm-env/storm_ui_principal_name', None)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py

@@ -26,7 +26,8 @@ from resource_management.libraries.resources.template_config import TemplateConf
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.script.script import Script
 from resource_management.core.source import Template
-from resource_management.libraries.functions import compare_versions
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from storm_yaml_utils import yaml_config_template, yaml_config
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
@@ -131,7 +132,7 @@ def storm(name=None):
     TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
                    owner=params.storm_user
     )
-    if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
+    if params.stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.stack_version_formatted):
       TemplateConfig(format("{conf_dir}/client_jaas.conf"),
                      owner=params.storm_user
       )

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py

@@ -27,6 +27,7 @@ from resource_management.core.resources.system import File
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
 
 class StormUpgrade(Script):
   """
@@ -68,9 +69,9 @@ class StormUpgrade(Script):
     zookeeper_data_cleared = False
     for storm_zookeeper_server in storm_zookeeper_server_list:
       # Determine where the zkCli.sh shell script is
-      zk_command_location = "/usr/hdp/current/zookeeper-client/bin/zkCli.sh"
+      zk_command_location = os.path.join(params.stack_root, "current", "zookeeper-client", "bin", "zkCli.sh")
       if params.version is not None:
-        zk_command_location = "/usr/hdp/{0}/zookeeper/bin/zkCli.sh".format(params.version)
+        zk_command_location = os.path.join(params.stack_root, params.version, "zookeeper", "bin", "zkCli.sh")
 
       # create the ZooKeeper delete command
       command = "{0} -server {1}:{2} rmr /storm".format(

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor.py

@@ -25,7 +25,8 @@ from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from storm import storm
 from service import service
 from ambari_commons import OSConst
@@ -35,7 +36,8 @@ from resource_management.core.resources.service import Service
 
 class Supervisor(Script):
   def get_stack_to_component(self):
-    return {"HDP": "storm-supervisor"}
+    import params
+    return {params.stack_name : "storm-supervisor"}
 
   def install(self, env):
     self.install_packages(env)
@@ -74,7 +76,7 @@ class SupervisorDefault(Supervisor):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
       stack_select.select("storm-supervisor", params.version)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/supervisor_prod.py

@@ -27,13 +27,15 @@ from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 
 class Supervisor(Script):
 
   def get_stack_to_component(self):
-    return {"HDP": "storm-supervisor"}
+    import params
+    return {params.stack_name : "storm-supervisor"}
 
   def install(self, env):
     self.install_packages(env)
@@ -48,7 +50,7 @@ class Supervisor(Script):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
       stack_select.select("storm-supervisor", params.version)

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/ui_server.py

@@ -29,7 +29,8 @@ from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import format
 from resource_management.core.resources.system import Link
 from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_JAAS_CONF
@@ -42,7 +43,8 @@ from resource_management.core.resources.service import Service
 class UiServer(Script):
 
   def get_stack_to_component(self):
-    return {"HDP": "storm-client"}
+    import params
+    return {params.stack_name : "storm-client"}
 
   def install(self, env):
     self.install_packages(env)
@@ -79,7 +81,7 @@ class UiServerDefault(UiServer):
   def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       conf_select.select(params.stack_name, "storm", params.version)
       stack_select.select("storm-client", params.version)
 

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/config.yaml.j2

@@ -46,7 +46,7 @@ ganglia:
   #spoof: "192.168.1.1:storm"
 {% endif %}
 
-{% if has_metric_collector and stack_is_hdp22_or_further %}
+{% if has_metric_collector and stack_supports_storm_ams %}
 enableGanglia: False
 
 ganglia:

ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/templates/storm_jaas.conf.j2

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #}
-{% if stack_is_hdp22_or_further %}
+{% if stack_supports_storm_kerberos %}
 StormServer {
    com.sun.security.auth.module.Krb5LoginModule required
    useKeyTab=true

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -46,6 +46,16 @@
       "name": "spark_thriftserver",
       "description": "Spark Thrift Server",
       "min_version": "2.3.2.0"
+    },
+    {
+      "name": "storm_kerberos",
+      "description": "Storm Kerberos support (AMBARI-7570)",
+      "min_version": "2.2.0.0"
+    },
+    {
+      "name": "storm_ams",
+      "description": "Storm AMS integration (AMBARI-10710)",
+      "min_version": "2.2.0.0"
     }
   ]
 }