AMBARI-14435: Stack Featurization for ZOOKEEPER service (Juanjo Marron via jluniya)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -18,7 +18,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-__all__ = ["Direction", "SafeMode"]
+__all__ = ["Direction", "SafeMode", "StackFeature"]
 
 class Direction:
   """
@@ -34,3 +34,15 @@ class SafeMode:
   ON = "ON"
   OFF = "OFF"
   UNKNOWN = "UNKNOWN"
+  
+class StackFeature:
+  """
+  Stack Feature supported
+  """
+  SNAPPY = "snappy"
+  EXPRESS_UPGRADE = "express_upgrade"
+  ROLLING_UPGRADE = "rolling_upgrade"
+  CONFIG_VERSIONING = "config_versioning"
+  RANGER = "ranger"
+  NFS = "nfs"
+    

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+# simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+import ambari_simplejson as json
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.default import default
+  
+_DEFAULT_STACK_FEATURES = {
+  "stack_features": [
+    { "name": "snappy", "description" : "Snappy compressor/decompressor support" , "min_version" : "2.0.0.0" , "max_version" : "2.2.0.0" }
+    ,
+    { "name": "express_upgrade", "description" : "Express upgrade support", "min_version" : "2.1.0.0" }
+    ,
+    { "name": "rolling_upgrade", "description" : "Rolling upgrade support", "min_version" : "2.2.0.0" }
+    ,
+    { "name": "config_versioning", "description" : "Configurable versions support", "min_version" : "2.3.0.0" }
+    ,
+    { "name": "ranger", "description" : "Ranger Service support", "min_version" : "2.2.0.0" }
+    ,
+    { "name": "nfs", "description" : "NFS support", "min_version" : "2.3.0.0" }
+  ]
+}
+
+def check_stack_feature(stack_feature, stack_version):
+  """
+  Given a stack_feature and a specific stack_version, it validates that the feature is supported by the stack_version.
+  :param stack_feature: Feature name to check if it is supported by the stack. For example: "rolling_upgrade"
+  :param stack_version: Version of the stack
+  :return: Will return True if successful, otherwise, False. 
+  """
+  stack_features_config = default("/configurations/cluster-env/stack_features", None)
+  data = _DEFAULT_STACK_FEATURES
+  
+  if stack_features_config:
+    data = json.loads(stack_features_config)
+  
+  for feature in data["stack_features"]:
+    if feature["name"] == stack_feature:
+        if "min_version" in feature:
+            min_version = feature["min_version"] 
+            if Script.is_stack_less_than(min_version):
+                return False
+        if "max_version" in feature:
+            max_version = feature["max_version"]
+            if Script.is_stack_greater_or_equal(max_version):
+                return False
+        return True 
+        
+  return False    

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/params_linux.py

@@ -24,15 +24,17 @@ import os
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.script.script import Script
 
 # server configurations
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-stack_version_formatted = format_stack_version(stack_version_unformatted)
+stack_version_formatted = status_params.stack_version_formatted
+stack_root = status_params.stack_root
 
 stack_name = default("/hostLevelParams/stack_name", None)
 current_version = default("/hostLevelParams/current_version", None)
@@ -48,11 +50,11 @@ zk_cli_shell = "/usr/lib/zookeeper/bin/zkCli.sh"
 config_dir = "/etc/zookeeper/conf"
 zk_smoke_out = os.path.join(tmp_dir, "zkSmoke.out")
 
-# hadoop parameters for 2.2+
-if Script.is_stack_greater_or_equal("2.2"):
-  zk_home = format("/usr/hdp/current/{component_directory}")
-  zk_bin = format("/usr/hdp/current/{component_directory}/bin")
-  zk_cli_shell = format("/usr/hdp/current/{component_directory}/bin/zkCli.sh")
+# hadoop parameters for stacks that support rolling_upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+  zk_home = format("{stack_root}/current/{component_directory}")
+  zk_bin = format("{stack_root}/current/{component_directory}/bin")
+  zk_cli_shell = format("{stack_root}/current/{component_directory}/bin/zkCli.sh")
   config_dir = status_params.config_dir
 
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/status_params.py

@@ -20,6 +20,9 @@ limitations under the License.
 from ambari_commons import OSCheck
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.script.script import Script
 
@@ -46,7 +49,11 @@ else:
   kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
   tmp_dir = Script.get_tmp_dir()
   zk_user =  config['configurations']['zookeeper-env']['zk_user']
+  
+  stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+  stack_version_formatted = format_stack_version(stack_version_unformatted)
+  stack_root = Script.get_stack_root()
 
   config_dir = "/etc/zookeeper/conf"
-  if Script.is_stack_greater_or_equal("2.2"):
-    config_dir = format("/usr/hdp/current/{component_directory}/conf")
+  if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+    config_dir = format("{stack_root}/current/{component_directory}/conf")

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper.py

@@ -24,7 +24,9 @@ import sys
 from resource_management import *
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
@@ -35,8 +37,8 @@ def zookeeper(type = None, upgrade_type=None):
   if type == 'server':
     # This path may be missing after Ambari upgrade. We need to create it. We need to do this before any configs will
     # be applied.
-    if upgrade_type is None and not os.path.exists("/usr/hdp/current/zookeeper-server") and params.current_version\
-      and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if upgrade_type is None and not os.path.exists(os.path.join(params.stack_root,"/current/zookeeper-server")) and params.current_version\
+      and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
       conf_select.select(params.stack_name, "zookeeper", params.current_version)
       stack_select.select("zookeeper-server", params.version)
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_client.py

@@ -23,8 +23,10 @@ import sys
 from resource_management import *
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.version import compare_versions, format_stack_version
 from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.stack_features import check_stack_feature 
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
@@ -65,7 +67,7 @@ class ZookeeperClientLinux(ZookeeperClient):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
       conf_select.select(params.stack_name, "zookeeper", params.version)
       stack_select.select("zookeeper-client", params.version)
 

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_server.py

@@ -25,7 +25,9 @@ from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions import get_unique_id_and_date
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \
   FILE_TYPE_JAAS_CONF
@@ -73,8 +75,8 @@ class ZookeeperServerLinux(ZookeeperServer):
     Logger.info("Executing Stack Upgrade pre-restart")
     import params
     env.set_params(params)
-
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+    
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
       conf_select.select(params.stack_name, "zookeeper", params.version)
       stack_select.select("zookeeper-server", params.version)
 
@@ -114,7 +116,7 @@ class ZookeeperServerLinux(ZookeeperServer):
     env.set_params(status_params)
 
     if status_params.security_enabled:
-      # Expect the following files to be available in status_params.config_dir:
+      # Expect the following files to be available in params.config_dir:
       #   zookeeper_jaas.conf
       #   zookeeper_client_jaas.conf
       try:

ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5.2.0/package/scripts/zookeeper_service.py

@@ -25,15 +25,17 @@ from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions.version import compare_versions, format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature 
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def zookeeper_service(action='start', upgrade_type=None):
   import params
 
   # This path may be missing after Ambari upgrade. We need to create it.
-  if upgrade_type is None and not os.path.exists("/usr/hdp/current/zookeeper-server") and params.current_version \
-    and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
+  if upgrade_type is None and not os.path.exists(os.path.join(params.stack_root,"/current/zookeeper-server")) and params.current_version \
+    and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
     conf_select.select(params.stack_name, "zookeeper", params.current_version)
     stack_select.select("zookeeper-server", params.version)
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/configuration/cluster-env.xml

@@ -174,6 +174,17 @@ gpgcheck=0</value>
         </value-attributes>
     </property>
 
+    <property>
+        <name>stack_features</name>
+        <value></value>
+        <description>List of features supported by the stack</description>
+        <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+        <value-attributes>
+            <property-file-name>stack_features.json</property-file-name>
+            <property-file-type>json</property-file-type>
+        </value-attributes>
+    </property>
+
     <property>
         <name>stack_root</name>
         <value>/usr/hdp</value>

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -0,0 +1,15 @@
+{
+  "stack_features": [
+    { "name": "snappy", "description" : "Snappy compressor/decompressor support" , "min_version" : "2.0.0.0" , "max_version" : "2.2.0.0" }
+    ,
+    { "name": "express_upgrade", "description" : "Express upgrade support", "min_version" : "2.1.0.0" }
+    ,
+    { "name": "rolling_upgrade", "description" : "Rolling upgrade support", "min_version" : "2.2.0.0" }
+    ,
+    { "name": "config_versioning", "description" : "Configurable versions support", "min_version" : "2.3.0.0" }
+    ,
+    { "name": "ranger", "description" : "Ranger Service support", "min_version" : "2.2.0.0" }
+    ,
+    { "name": "nfs", "description" : "NFS support", "min_version" : "2.3.0.0" }
+  ]
+}