AMBARI-15575: Stack Featurize Knox Service (jluniya)

ambari-common/src/main/python/resource_management/libraries/functions/constants.py

@@ -61,3 +61,5 @@ class StackFeature:
   PIG_ON_TEZ = "pig_on_tez"
   RANGER_USERSYNC_NON_ROOT = "ranger_usersync_non_root"
   ACCUMULO_KERBEROS_USER_AUTH = "accumulo_kerberos_user_auth"
+  KNOX_VERSIONED_DATA_DIR = "knox_versioned_data_dir"
+  KNOX_SSO_TOPOLOGY = "knox_sso_topology"

ambari-common/src/main/python/resource_management/libraries/functions/stack_features.py

@@ -138,6 +138,16 @@ _DEFAULT_STACK_FEATURES = {
       "name": "accumulo_kerberos_user_auth",
       "description": "Accumulo Kerberos User Auth (AMBARI-10163)",
       "min_version": "2.3.0.0"
+    },
+    {
+      "name": "knox_versioned_data_dir",
+      "description": "Use versioned data dir for Knox (AMBARI-13164)",
+      "min_version": "2.3.2.0"
+    },
+    {
+      "name": "knox_sso_topology",
+      "description": "Knox SSO Topology support (AMBARI-13975)",
+      "min_version": "2.3.8.0"
     }
   ]
 }
@@ -151,7 +161,10 @@ def check_stack_feature(stack_feature, stack_version):
   """
   stack_features_config = default("/configurations/cluster-env/stack_features", None)
   data = _DEFAULT_STACK_FEATURES
-  
+
+  if not stack_version:
+    return False
+
   if stack_features_config:
     data = json.loads(stack_features_config)
   

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py

@@ -31,6 +31,8 @@ from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
 from resource_management.core.logger import Logger
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def knox():
@@ -66,7 +68,7 @@ def knox():
      content=InlineTemplate(params.admin_topology_template)
   )
 
-  if Script.is_stack_greater_or_equal_to(params.version_formatted, "2.3.8.0"):
+  if params.version_formatted and check_stack_feature(StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted):
       File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"),
          group=params.knox_group,
          owner=params.knox_user,
@@ -123,7 +125,7 @@ def knox():
          content=InlineTemplate(params.admin_topology_template)
     )
 
-    if Script.is_stack_greater_or_equal_to(params.version_formatted, "2.3.8.0"):
+    if params.version_formatted and check_stack_feature(StackFeature.KNOX_SSO_TOPOLOGY, params.version_formatted):
         File(os.path.join(params.knox_conf_dir, "topologies", "knoxsso.xml"),
             group=params.knox_group,
             owner=params.knox_user,

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py

@@ -25,7 +25,6 @@ from resource_management.libraries.functions import conf_select, tar_archive
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions.check_process_status import check_process_status
 from resource_management.libraries.functions import format
-from resource_management.libraries.functions.version import compare_versions, format_stack_version
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import Direction
@@ -47,11 +46,14 @@ import upgrade
 from knox import knox, update_knox_logfolder_permissions
 from knox_ldap import ldap
 from setup_ranger_knox import setup_ranger_knox
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 
 class KnoxGateway(Script):
   def get_stack_to_component(self):
-    return {"HDP": "knox-server"}
+    import params
+    return {params.stack_name: "knox-server"}
 
   def install(self, env):
     import params
@@ -113,8 +115,7 @@ class KnoxGatewayDefault(KnoxGateway):
   def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
-    if params.version and compare_versions(format_stack_version(params.version), '2.2.0.0') >= 0:
-
+    if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
       absolute_backup_dir = None
       if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
         Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir))

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py

@@ -33,9 +33,12 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 # server configurations
 config = Script.get_config()
+stack_root = Script.get_stack_root()
 
 tmp_dir = Script.get_tmp_dir()
 stack_name = default("/hostLevelParams/stack_name", None)
@@ -54,18 +57,16 @@ upgrade_from_version = default("/hostLevelParams/current_version", None)
 
 # server configurations
 # Default value used in HDP 2.3.0.0 and earlier.
-
 knox_data_dir = '/var/lib/knox/data'
 
 # Important, it has to be strictly greater than 2.3.0.0!!!
-if stack_name and stack_name.upper() == "HDP":
-  Logger.info(format("HDP version to use is {version_formatted}"))
-  if Script.is_stack_greater(version_formatted, "2.3.0.0"):
-    # This is the current version. In the case of a Rolling Upgrade, it will be the newer version.
-    # In the case of a Downgrade, it will be the version downgrading to.
-    # This is always going to be a symlink to /var/lib/knox/data_${version}
-    knox_data_dir = format('/usr/hdp/{version}/knox/data')
-    Logger.info(format("Detected HDP with stack version {version}, will use knox_data_dir = {knox_data_dir}"))
+Logger.info(format("Stack version to use is {version_formatted}"))
+if version_formatted and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, version_formatted):
+  # This is the current version. In the case of a Rolling Upgrade, it will be the newer version.
+  # In the case of a Downgrade, it will be the version downgrading to.
+  # This is always going to be a symlink to /var/lib/knox/data_${version}
+  knox_data_dir = format('{stack_root}/{version}/knox/data')
+  Logger.info(format("Detected stack with version {version}, will use knox_data_dir = {knox_data_dir}"))
 
 
 knox_master_secret_path = format('{knox_data_dir}/security/master')
@@ -83,15 +84,14 @@ ldap_bin = '/usr/lib/knox/bin/ldap.sh'
 knox_client_bin = '/usr/lib/knox/bin/knoxcli.sh'
 
 # HDP 2.2+ parameters
-if Script.is_stack_greater_or_equal("2.2"):
-  knox_bin = '/usr/hdp/current/knox-server/bin/gateway.sh'
-  knox_conf_dir = '/usr/hdp/current/knox-server/conf'
-  ldap_bin = '/usr/hdp/current/knox-server/bin/ldap.sh'
-  knox_client_bin = '/usr/hdp/current/knox-server/bin/knoxcli.sh'
-
-  knox_master_secret_path = '/usr/hdp/current/knox-server/data/security/master'
-  knox_cert_store_path = '/usr/hdp/current/knox-server/data/security/keystores/gateway.jks'
-  knox_data_dir = '/usr/hdp/current/knox-server/data/'
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+  knox_bin = format('{stack_root}/current/knox-server/bin/gateway.sh')
+  knox_conf_dir = format('{stack_root}/current/knox-server/conf')
+  ldap_bin = format('{stack_root}/current/knox-server/bin/ldap.sh')
+  knox_client_bin = format('{stack_root}/current/knox-server/bin/knoxcli.sh')
+  knox_master_secret_path = format('{stack_root}/current/knox-server/data/security/master')
+  knox_cert_store_path = format('{stack_root}/current/knox-server/data/security/keystores/gateway.jks')
+  knox_data_dir = format('{stack_root}/current/knox-server/data/')
 
 knox_group = default("/configurations/knox-env/knox_group", "knox")
 mode = 0644
@@ -211,7 +211,7 @@ if has_oozie:
   oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url'])
 
 # Knox managed properties
-knox_managed_pid_symlink= "/usr/hdp/current/knox-server/pids"
+knox_managed_pid_symlink= format('{stack_root}/current/knox-server/pids')
 
 # server configurations
 knox_master_secret = config['configurations']['knox-env']['knox_master_secret']
@@ -301,7 +301,7 @@ if has_ranger_admin:
   downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
 
   driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}")
-  driver_curl_target = format("/usr/hdp/current/knox-server/ext/{jdbc_jar_name}")
+  driver_curl_target = format("{stack_root}/current/knox-server/ext/{jdbc_jar_name}")
 
   knox_ranger_plugin_config = {
     'username': repo_config_username,

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/status_params.py

@@ -22,16 +22,23 @@ from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.script.script import Script
 from ambari_commons import OSCheck
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
+
 
 config = Script.get_config()
+stack_root = Script.get_stack_root()
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
 
 if OSCheck.is_windows_family():
   knox_gateway_win_service_name = "gateway"
   knox_ldap_win_service_name = "ldap"
 else:
   knox_conf_dir = '/etc/knox/conf'
-  if Script.is_stack_greater_or_equal("2.2"):
-    knox_conf_dir = '/usr/hdp/current/knox-server/conf'
+  if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+    knox_conf_dir = format('{stack_root}/current/knox-server/conf')
   knox_pid_dir = config['configurations']['knox-env']['knox_pid_dir']
   knox_pid_file = format("{knox_pid_dir}/gateway.pid")
   ldap_pid_file = format("{knox_pid_dir}/ldap.pid")

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py

@@ -27,7 +27,8 @@ from resource_management.core.exceptions import Fail
 from resource_management.libraries.functions import tar_archive
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import Direction
-from resource_management.libraries.functions.version import compare_versions,format_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions import StackFeature
 
 
 BACKUP_TEMP_DIR = "knox-upgrade-backup"
@@ -81,10 +82,9 @@ def _get_directory_mappings_during_upgrade():
   # By default, use this for all stacks.
   knox_data_dir = '/var/lib/knox/data'
 
-  if params.stack_name and params.stack_name.upper() == "HDP" and \
-          compare_versions(format_stack_version(params.upgrade_from_version), "2.3.0.0") > 0:
+  if params.upgrade_from_version and check_stack_feature(StackFeature.KNOX_VERSIONED_DATA_DIR, params.upgrade_from_version):
     # Use the version that is being upgraded from.
-    knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')
+    knox_data_dir = format('{stack_root}/{upgrade_from_version}/knox/data')
 
 
   directories = {knox_data_dir: BACKUP_DATA_ARCHIVE,

ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json

@@ -112,6 +112,16 @@
       "name": "accumulo_kerberos_user_auth",
       "description": "Accumulo Kerberos User Auth (AMBARI-10163)",
       "min_version": "2.3.0.0"
+    },
+    {
+      "name": "knox_versioned_data_dir",
+      "description": "Use versioned data dir for Knox (AMBARI-13164)",
+      "min_version": "2.3.2.0"
+    },
+    {
+      "name": "knox_sso_topology",
+      "description": "Knox SSO Topology support (AMBARI-13975)",
+      "min_version": "2.3.8.0"
     }
   ]
 }