AMBARI-7892. WebHCat to support versioned rpms in Ambari (alejandro)

ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py

@@ -36,6 +36,7 @@ variables cannot be determined ahead of time, but rather, depends on what files
 
 It assumes that {{ hdp_stack_version }} is constructed as ${major.minor.patch.rev}-${build_number}
 E.g., 998.2.2.1.0-998
+Please note that "-${build_number}" is optional.
 Whereas {{ component_version }} is up to the Component to define, may be 3.0.1 or 301.
 """
 
@@ -81,10 +82,10 @@ def __get_tar_source_and_dest_folder(tarball_prefix):
   return component_tar_source_file, component_tar_destination_folder
 
 
-def __create_regex_pattern(file_path, rpm_version):
+def __create_regex_pattern(file_path, hdp_stack_version):
   """
   :param file_path: Input file path
-  :param rpm_version: HDP rpm version, such as 2.2.0.0
+  :param hdp_stack_version: Stack version, such as 2.2.0.0
   :return: Returns an expression that uses file system regex that can be used with ls and hadoop fs -ls
   """
   # Perform the variable interpretation
@@ -94,16 +95,16 @@ def __create_regex_pattern(file_path, rpm_version):
 
   # IMPORTANT, the build version was used in HDP 2.2, but may not be needed in future versions.
   if "{{ hdp_stack_version }}" in file_path_pattern:
-    file_path_pattern = file_path_pattern.replace("{{ hdp_stack_version }}", rpm_version + "-*")   # the trailing "-*" is the build number
+    file_path_pattern = file_path_pattern.replace("{{ hdp_stack_version }}", hdp_stack_version + "*")   # the trailing "*" is the optional build number
   return file_path_pattern
 
 
-def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, rpm_version):
+def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, hdp_stack_version):
   """
   :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
   :param source_file_pattern: Regex pattern of the source file from the local file system
   :param component_tar_destination_folder: Destination folder to copy the file to in HDFS
-  :param rpm_version: Stack version number without the build version. E.g., 2.2.0.0
+  :param hdp_stack_version: Stack version number without the build version. E.g., 2.2.0.0
   :return: Returns a list of tuples (x, y), where x is the source file in the local file system,
   and y is the destination file path in HDFS
   """
@@ -115,12 +116,12 @@ def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_t
     hdp_build_version = None
 
     # Attempt to retrieve the hdp_build_version and component_version.
-    # In case the build number has dots, attempt to match as many as possible.
-    pattern = "%s-(.*)\\.%s-([0-9\\.]*)\\..*" % (tarball_prefix, str(rpm_version).replace(".", "\\."))
+    # In case the build number (which is optional) has dots, attempt to match as many as possible.
+    pattern = "%s-(.*)\\.%s-?([0-9\\.]*)\\..*" % (tarball_prefix, str(hdp_stack_version).replace(".", "\\."))
     m = re.search(pattern, file_base_name)
     if m and len(m.groups()) == 2:
       component_version = str(m.group(1))
-      hdp_build_version = str(m.group(2))
+      hdp_build_version = str(m.group(2))   # optional, so may be empty.
 
     missing_a_variable = False
     # The destination_file_path will be interpreted as well.
@@ -133,13 +134,14 @@ def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_t
         missing_a_variable = True
 
     if "{{ hdp_stack_version }}" in destination_file_path:
-      if hdp_build_version :
-        destination_file_path = destination_file_path.replace("{{ hdp_stack_version }}", "%s-%s" % (rpm_version, hdp_build_version))
+      if hdp_build_version and hdp_build_version.strip() != "":
+        destination_file_path = destination_file_path.replace("{{ hdp_stack_version }}", "%s-%s" %
+                                                              (hdp_stack_version, hdp_build_version))
       else:
-        missing_a_variable = True
+        destination_file_path = destination_file_path.replace("{{ hdp_stack_version }}", "%s" % hdp_stack_version)
 
     if missing_a_variable:
-      print("WARNING. Could not identify HDP stack version or Component version in file %s , "
+      print("WARNING. Could not identify Component version in file %s , "
             "so will not copy to HDFS." % str(file))
     else:
       source_and_dest_pairs.append((file, destination_file_path))
@@ -164,8 +166,6 @@ def __copy_files(source_and_dest_pairs, file_owner, kinit_if_needed):
     for (source, destination) in source_and_dest_pairs:
       try:
         destination_dir = os.path.dirname(destination)
-        create_dir_cmd = "dfs -mkdir -p %s" % destination_dir
-        test_dir_exists = "dfs -test -e %s" % destination_dir
 
         params.HdfsDirectory(destination_dir,
                              action="create",
@@ -195,25 +195,25 @@ def copy_tarballs_to_hdfs(tarball_prefix, component_user, file_owner):
   :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
 
   In order to call this function, params.py must have all of the following,
-  rpm_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
+  hdp_stack_version, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
   hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
   """
   import params
 
-  if not hasattr(params, "rpm_version") or params.rpm_version is None:
-    Logger.warning("cluster-env.xml does not have rpm_version")
+  if not hasattr(params, "hdp_stack_version") or params.hdp_stack_version is None:
+    Logger.warning("Could not find hdp_stack_version")
     return 1
 
   component_tar_source_file, component_tar_destination_folder = __get_tar_source_and_dest_folder(tarball_prefix)
   if not component_tar_source_file or not component_tar_destination_folder:
     return 1
 
-  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.rpm_version)
+  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.hdp_stack_version)
   # This is just the last segment
   file_name_pattern = source_file_pattern.split('/')[-1:][0]
-  tar_destination_folder_pattern = __create_regex_pattern(component_tar_destination_folder, params.rpm_version)
+  tar_destination_folder_pattern = __create_regex_pattern(component_tar_destination_folder, params.hdp_stack_version)
 
-  # Pattern for searching the file in HDFS. E.g. value, hdfs:///hdp/apps/2.2.0.0-*/tez/tez-*.2.2.0.0-*.tar.gz
+  # Pattern for searching the file in HDFS. E.g. value, hdfs:///hdp/apps/2.2.0.0*/tez/tez-*.2.2.0.0*.tar.gz
   hdfs_file_pattern = os.path.join(tar_destination_folder_pattern, file_name_pattern)
   does_hdfs_file_exist_cmd = "fs -ls %s" % hdfs_file_pattern
 
@@ -241,7 +241,7 @@ def copy_tarballs_to_hdfs(tarball_prefix, component_user, file_owner):
 
   if not does_hdfs_file_exist:
     source_and_dest_pairs = __populate_source_and_dests(tarball_prefix, source_file_pattern,
-                                                        component_tar_destination_folder, params.rpm_version)
+                                                        component_tar_destination_folder, params.hdp_stack_version)
     return __copy_files(source_and_dest_pairs, file_owner, kinit_if_needed)
   return 1
 
@@ -256,16 +256,16 @@ def __map_local_file_to_hdfs_file(tarball_prefix):
   """
   import params
 
-  if not hasattr(params, "rpm_version") or params.rpm_version is None:
-    Logger.warning("cluster-env.xml does not have rpm_version")
+  if not hasattr(params, "hdp_stack_version") or params.hdp_stack_version is None:
+    Logger.warning("Could not find hdp_stack_version")
     return ""
 
   component_tar_source_file, component_tar_destination_folder = __get_tar_source_and_dest_folder(tarball_prefix)
   if not component_tar_source_file or not component_tar_destination_folder:
     return ""
 
-  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.rpm_version)
-  source_and_dest_pairs = __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, params.rpm_version)
+  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.hdp_stack_version)
+  source_and_dest_pairs = __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, params.hdp_stack_version)
   if source_and_dest_pairs and len(source_and_dest_pairs) == 1:
     return source_and_dest_pairs[0][1]
 
@@ -280,7 +280,7 @@ def interpret_dynamic_version_property(property_value, tarball_prefix, delimiter
   :return: Returns a tuple of (x, y), where x is a bool indicating if at least one variable was substituted, and y
   is the interpretation of the property value if an interpretation could be done, otherwise it remains unchanged.
 
-  Notice that params must have the rpm_version attribute.
+  Notice that params must have the hdp_stack_version attribute.
   """
   import params
 
@@ -298,7 +298,7 @@ def interpret_dynamic_version_property(property_value, tarball_prefix, delimiter
         if __contains_dynamic_variable(elem):
           # Need to do dynamic interpretation, and slight regex escaping. Must not escape " " since it is used in
           # the dynamic variable string.
-          elem_pattern = __create_regex_pattern(elem, params.rpm_version).replace(".", "\\.").replace("*", ".*")
+          elem_pattern = __create_regex_pattern(elem, params.hdp_stack_version).replace(".", "\\.").replace("*", ".*")
           p = re.compile(elem_pattern)
           m = p.match(versioned_tarball)
           if m:

ambari-common/src/main/python/resource_management/libraries/functions/version.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+
+def _normalize(v, desired_segments=0):
+  """
+  :param v: Input string of the form "#.#.#" or "#.#.#.#"
+  :param desired_segments: If greater than 0, and if v has fewer segments this parameter, will pad v with segments
+  containing "0" until the desired segments is reached.
+  :return: Returns a list of integers representing the segments of the version
+  """
+  v_list = v.split(".")
+  if desired_segments > 0 and len(v_list) < desired_segments:
+    v_list = v_list + ((desired_segments - len(v_list)) * ["0", ])
+  return [int(x) for x in v_list]
+
+
+def format_hdp_stack_version(input):
+  """
+  :param input: Input string, e.g. "2.2"
+  :return: Returns a well-formatted HDP stack version of the form #.#.#.# as a string.
+  """
+  if input:
+    normalized = _normalize(str(input))
+    if len(normalized) == 2:
+      normalized = normalized + [0, 0]
+    elif len(normalized) == 3:
+      normalized = normalized + [0, ]
+    normalized = [str(x) for x in normalized]   # need to convert each number into a string
+    return ".".join(normalized)
+  return ""
+
+
+def compare_versions(version1, version2):
+  """
+  Used to compare either Ambari Versions, or Stack versions
+  E.g., Ambari version 1.6.1 vs 1.7.0,
+  Stack Version 2.0.6.0 vs 2.2.0.0
+  :param version1: First parameter for version
+  :param version2: Second parameter for version
+  :return: Returns -1 if version1 is before version2, 0 if they are equal, and 1 if version1 is after version2
+  """
+  max_segments = max(len(version1.split(".")), len(version2.split(".")))
+  return cmp(_normalize(version1, desired_segments=max_segments), _normalize(version2, desired_segments=max_segments))

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/webhcat-site.xml

@@ -25,7 +25,7 @@ limitations under the License.
 
   <property>
     <name>templeton.port</name>
-      <value>50111</value>
+    <value>50111</value>
     <description>The HTTP port for the main server.</description>
   </property>
 
@@ -47,7 +47,6 @@ limitations under the License.
     <description>Jars to add the the classpath.</description>
   </property>
 
-
   <property>
     <name>templeton.hadoop</name>
     <value>/usr/bin/hadoop</value>
@@ -116,14 +115,12 @@ limitations under the License.
   </property>
 
   <property>
-   <name>templeton.override.enabled</name>
-   <value>false</value>
-   <description>
-     Enable the override path in templeton.override.jars
-   </description>
- </property>
+    <name>templeton.override.enabled</name>
+    <value>false</value>
+    <description>Enable the override path in templeton.override.jars</description>
+  </property>
 
- <property>
+  <property>
     <name>templeton.streaming.jar</name>
     <value>hdfs:///apps/webhcat/hadoop-streaming.jar</value>
     <description>The hdfs path to the Hadoop streaming jar file.</description>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from resource_management import *
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 import status_params
 import os
 
@@ -26,22 +27,18 @@ import os
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
-
+# This is expected to be of the form #.#.#.#
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
 stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
 
 # Hadoop params
-# TODO, this logic assumes that the existence of HDP version is 2.2 or greater.
-# Instead, it should initialize these parameters in a file inside the HDP 2.2 stack.
-if stack_is_hdp22_or_further:
+# TODO, this logic should initialize these parameters in a file inside the HDP 2.2 stack.
+if compare_versions(hdp_stack_version, "2.2.0.0") >= 0:
   hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   hadoop_home = '/usr/hdp/current/hadoop-client'
-  hadoop_streeming_jars = "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming-*.jar"
   hive_bin = '/usr/hdp/current/hive-client/bin'
   hive_lib = '/usr/hdp/current/hive-client/lib'
-  pig_tar_file = '/usr/hdp/current/pig-client/pig.tar.gz'
-  hive_tar_file = '/usr/hdp/current/hive-client/hive.tar.gz'
-  sqoop_tar_file = '/usr/hdp/current/sqoop-client/sqoop*.tar.gz'
 
   hcat_lib = '/usr/hdp/current/hive-webhcat/share/hcatalog'
   webhcat_bin_dir = '/usr/hdp/current/hive-webhcat/sbin'
@@ -56,7 +53,7 @@ else:
   hive_tar_file = '/usr/share/HDP-webhcat/hive.tar.gz'
   sqoop_tar_file = '/usr/share/HDP-webhcat/sqoop*.tar.gz'
 
-  if str(hdp_stack_version).startswith('2.0'):
+  if compare_versions(hdp_stack_version, "2.1.0.0") < 0:
     hcat_lib = '/usr/lib/hcatalog/share/hcatalog'
     webhcat_bin_dir = '/usr/lib/hcatalog/sbin'
   # for newer versions
@@ -70,8 +67,7 @@ hive_client_conf_dir = "/etc/hive/conf"
 hive_server_conf_dir = '/etc/hive/conf.server'
 
 
-
-if str(hdp_stack_version).startswith('2.0'):
+if compare_versions(hdp_stack_version, "2.1.0.0") < 0:
   hcat_conf_dir = '/etc/hcatalog/conf'
   config_dir = '/etc/hcatalog/conf'
 # for newer versions
@@ -147,7 +143,7 @@ hive_metastore_pid = status_params.hive_metastore_pid
 java_share_dir = '/usr/share/java'
 driver_curl_target = format("{java_share_dir}/{jdbc_jar_name}")
 
-hdfs_user =  config['configurations']['hadoop-env']['hdfs_user']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 user_group = config['configurations']['cluster-env']['user_group']
 artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
 
@@ -173,7 +169,7 @@ mysql_host = config['clusterHostInfo']['hive_mysql_host']
 mysql_adduser_path = format("{tmp_dir}/addMysqlUser.sh")
 
 ######## Metastore Schema
-if str(hdp_stack_version).startswith('2.0'):
+if compare_versions(hdp_stack_version, "2.1.0.0") < 0:
   init_metastore_schema = False
 else:
   init_metastore_schema = True
@@ -261,8 +257,8 @@ import functools
 #to create hdfs directory we need to call params.HdfsDirectory in code
 HdfsDirectory = functools.partial(
   HdfsDirectory,
-  conf_dir=hadoop_conf_dir,
-  hdfs_user=hdfs_user,
+  conf_dir = hadoop_conf_dir,
+  hdfs_user = hdfs_principal_name if security_enabled else hdfs_user,
   security_enabled = security_enabled,
   keytab = hdfs_user_keytab,
   kinit_path_local = kinit_path_local,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py

@@ -18,20 +18,48 @@ limitations under the License.
 Ambari Agent
 
 """
-from resource_management import *
 import sys
 import os.path
 import glob
 
+from resource_management import *
+from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from resource_management.libraries.script.config_dictionary import MutableConfigDictionary
+from resource_management.libraries.functions.dynamic_variable_interpretation import interpret_dynamic_version_property
+
+
+def __inject_config_variables(mutable_configs):
+  """
+  :param mutable_configs: Mutable Configuration Dictionary
+  :return: Returns the mutable configuration dictionary where each of the dynamic properties have been injected
+  with the value of the versioned tarball or jar in HDFS.
+  """
+  if mutable_configs is not None and "configurations" in mutable_configs and \
+          mutable_configs["configurations"] is not None and "webhcat-site" in mutable_configs["configurations"]:
+    webhcat_config = mutable_configs['configurations']['webhcat-site']
+
+    properties_and_prefix_tuple_list = [('pig', 'templeton.pig.archive'), ('hive', 'templeton.hive.archive'),
+                                        ('sqoop', 'templeton.sqoop.archive'), ('hadoop-streaming', 'templeton.streaming.jar')]
+    for (prefix, prop_name) in properties_and_prefix_tuple_list:
+      prop_value = webhcat_config[prop_name]
+      if prop_value:
+        found_at_least_one_replacement, new_value = interpret_dynamic_version_property(prop_value, prefix, ",")
+        if found_at_least_one_replacement:
+          webhcat_config[prop_name] = new_value
+  return mutable_configs
+
 
 def webhcat():
   import params
 
-  params.HdfsDirectory(params.webhcat_apps_dir,
-                       action="create_delayed",
-                       owner=params.webhcat_user,
-                       mode=0755
-  )
+  if compare_versions(params.hdp_stack_version, "2.2.0.0") < 0:
+    params.HdfsDirectory(params.webhcat_apps_dir,
+                         action="create_delayed",
+                         owner=params.webhcat_user,
+                         mode=0755
+    )
+  
   if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
     params.HdfsDirectory(params.hcat_hdfs_user_dir,
                          action="create_delayed",
@@ -62,20 +90,6 @@ def webhcat():
             owner=params.webhcat_user,
             group=params.user_group)
 
-  XmlConfig("webhcat-site.xml",
-            conf_dir=params.config_dir,
-            configurations=params.config['configurations']['webhcat-site'],
-            configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
-            owner=params.webhcat_user,
-            group=params.user_group,
-  )
-
-  File(format("{config_dir}/webhcat-env.sh"),
-       owner=params.webhcat_user,
-       group=params.user_group,
-       content=InlineTemplate(params.webhcat_env_sh_template)
-  )
-
   if params.security_enabled:
     kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
   else:
@@ -87,22 +101,14 @@ def webhcat():
             path='/bin'
     )
 
-  # TODO, fix this in AMBARI-7842.
-  # The source of these tarballs will no longer be deterministic, since the build version is not known until HDP 2.2
-  # is deployed.
-  # Also, this logic should be specific to HDP 2.2 stack.
-  CopyFromLocal(params.hadoop_streeming_jars,
-                owner=params.webhcat_user,
-                mode=0755,
-                dest_dir=params.webhcat_apps_dir,
-                kinnit_if_needed=kinit_if_needed,
-                hdfs_user=params.hdfs_user,
-                hadoop_bin_dir=params.hadoop_bin_dir,
-                hadoop_conf_dir=params.hadoop_conf_dir
-  )
-
-  if (os.path.isfile(params.pig_tar_file)):
-    CopyFromLocal(params.pig_tar_file,
+  # TODO, these checks that are specific to HDP 2.2 and greater should really be in a script specific to that stack.
+  if compare_versions(params.hdp_stack_version, "2.2.0.0") >= 0:
+    copy_tarballs_to_hdfs('hive', params.webhcat_user, params.hdfs_user)
+    copy_tarballs_to_hdfs('pig', params.webhcat_user, params.hdfs_user)
+    copy_tarballs_to_hdfs('hadoop-streaming', params.webhcat_user, params.hdfs_user)
+    copy_tarballs_to_hdfs('sqoop', params.webhcat_user, params.hdfs_user)
+  else:
+    CopyFromLocal(params.hadoop_streeming_jars,
                   owner=params.webhcat_user,
                   mode=0755,
                   dest_dir=params.webhcat_apps_dir,
@@ -112,18 +118,18 @@ def webhcat():
                   hadoop_conf_dir=params.hadoop_conf_dir
     )
 
-  CopyFromLocal(params.hive_tar_file,
-                owner=params.webhcat_user,
-                mode=0755,
-                dest_dir=params.webhcat_apps_dir,
-                kinnit_if_needed=kinit_if_needed,
-                hdfs_user=params.hdfs_user,
-                hadoop_bin_dir=params.hadoop_bin_dir,
-                hadoop_conf_dir=params.hadoop_conf_dir
-  )
-
-  if (len(glob.glob(params.sqoop_tar_file)) > 0):
-    CopyFromLocal(params.sqoop_tar_file,
+    if (os.path.isfile(params.pig_tar_file)):
+      CopyFromLocal(params.pig_tar_file,
+                    owner=params.webhcat_user,
+                    mode=0755,
+                    dest_dir=params.webhcat_apps_dir,
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user,
+                    hadoop_bin_dir=params.hadoop_bin_dir,
+                    hadoop_conf_dir=params.hadoop_conf_dir
+      )
+
+    CopyFromLocal(params.hive_tar_file,
                   owner=params.webhcat_user,
                   mode=0755,
                   dest_dir=params.webhcat_apps_dir,
@@ -132,3 +138,33 @@ def webhcat():
                   hadoop_bin_dir=params.hadoop_bin_dir,
                   hadoop_conf_dir=params.hadoop_conf_dir
     )
+
+    if (len(glob.glob(params.sqoop_tar_file)) > 0):
+      CopyFromLocal(params.sqoop_tar_file,
+                    owner=params.webhcat_user,
+                    mode=0755,
+                    dest_dir=params.webhcat_apps_dir,
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user,
+                    hadoop_bin_dir=params.hadoop_bin_dir,
+                    hadoop_conf_dir=params.hadoop_conf_dir
+      )
+
+  mutable_configs = MutableConfigDictionary(params.config)
+  # TODO, this is specific to HDP 2.2, but it is safe to call in earlier versions.
+  # It should eventually be moved to scripts specific to the HDP 2.2 stack.
+  mutable_configs = __inject_config_variables(mutable_configs)
+
+  XmlConfig("webhcat-site.xml",
+            conf_dir=params.config_dir,
+            configurations=mutable_configs['configurations']['webhcat-site'],
+            configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
+            owner=params.webhcat_user,
+            group=params.user_group,
+            )
+
+  File(format("{config_dir}/webhcat-env.sh"),
+       owner=params.webhcat_user,
+       group=params.user_group,
+       content=InlineTemplate(params.webhcat_env_sh_template)
+  )

ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/params.py

@@ -19,17 +19,18 @@ limitations under the License.
 """
 
 from resource_management import *
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 
 # server configurations
 config = Script.get_config()
 
-# RPM versioning support
-rpm_version = default("/configurations/cluster-env/rpm_version", None)
-
+# This is expected to be of the form #.#.#.#
 hdp_stack_version = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
 stack_is_hdp22_or_further = not (hdp_stack_version.startswith('2.0') or hdp_stack_version.startswith('2.1'))
 
-if stack_is_hdp22_or_further:  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+if compare_versions(hdp_stack_version, "2.2.0.0") >= 0:
+  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
 else:
   hadoop_bin_dir = "/usr/bin"
 hadoop_conf_dir = "/etc/hadoop/conf"

ambari-server/src/main/resources/stacks/HDP/2.2/configuration/cluster-env.xml

@@ -21,6 +21,7 @@
 -->
 
 <configuration>
+  <!-- TODO, this property needs to be removed in AMBARI-7884. -->
   <property>
     <name>rpm_version</name>
     <value>2.2.0.0</value>
@@ -28,22 +29,72 @@
   </property>
 
   <!-- The properties that end in tar_source describe the pattern of where the tar.gz files come from.
-  They will replace {{ hdp_stack_version }} with the rpm_version value followed by -* (which is the build number in HDP 2.2),
+  They will replace {{ hdp_stack_version }} with the "#.#.#.#" value followed by -* (which is the build number in HDP 2.2),
   and treat {{ component_version }} as a wildcard.
   When copying those tarballs, Ambari will look up the corresponding tar_destination_folder property to know where it
   should be copied to.
   All of the destination folders must begin with hdfs://
   Please note that the spaces inside of {{ ... }} are important.
   -->
+  <!-- Tez tarball is needed by Hive Server when using the Tez execution egine. -->
   <property>
     <name>tez_tar_source</name>
     <value>/usr/hdp/current/tez-client/lib/tez-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
-    <description></description>
+    <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description>
   </property>
   <property>
     <name>tez_tar_destination_folder</name>
     <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/</value>
-    <description></description>
+    <description>Destination HDFS folder for the file.</description>
+  </property>
+
+  <!-- Hive tarball is needed by WebHCat. -->
+  <property>
+    <name>hive_tar_source</name>
+    <value>/usr/hdp/current/hive-client/hive-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description>
+  </property>
+  <property>
+    <name>hive_tar_destination_folder</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/</value>
+    <description>Destination HDFS folder for the file.</description>
+  </property>
+
+  <!-- Pig tarball is needed by WebHCat. -->
+  <!-- TODO, pig is missing the build number. -->
+  <property>
+    <name>pig_tar_source</name>
+    <value>/usr/hdp/current/pig-client/pig-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description>
+  </property>
+  <property>
+    <name>pig_tar_destination_folder</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/</value>
+    <description>Destination HDFS folder for the file.</description>
+  </property>
+
+  <!-- Hadoop Streaming jar is needed by WebHCat. -->
+  <property>
+    <name>hadoop-streaming_tar_source</name>
+    <value>/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming-{{ component_version }}.{{ hdp_stack_version }}.jar</value>
+    <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description>
+  </property>
+  <property>
+    <name>hadoop-streaming_tar_destination_folder</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/mr/</value>
+    <description>Destination HDFS folder for the file.</description>
+  </property>
+
+  <!-- Sqoop tarball is needed by WebHCat. -->
+  <property>
+    <name>sqoop_tar_source</name>
+    <value>/usr/hdp/current/sqoop-client/sqoop-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <description>Source file path that uses dynamic variables and regex to copy the file to HDFS.</description>
+  </property>
+  <property>
+    <name>sqoop_tar_destination_folder</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/</value>
+    <description>Destination HDFS folder for the file.</description>
   </property>
 
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml

@@ -25,7 +25,7 @@ limitations under the License.
 
   <property>
     <name>templeton.jar</name>
-    <value>/usr/hdp/current/hcatalog/share/webhcat/svr/webhcat.jar</value>
+    <value>/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar</value>
     <description>The path to the Templeton jar file.</description>
   </property>
 
@@ -35,13 +35,17 @@ limitations under the License.
     <description>Jars to add the the classpath.</description>
   </property>
 
-
   <property>
     <name>templeton.hadoop</name>
     <value>/usr/hdp/current/hadoop-client/bin/hadoop</value>
     <description>The path to the Hadoop executable.</description>
   </property>
 
+  <property>
+    <name>templeton.pig.archive</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/pig-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <description>The path to the Pig archive in HDFS.</description>
+  </property>
 
   <property>
     <name>templeton.hcat</name>
@@ -49,9 +53,15 @@ limitations under the License.
     <description>The path to the hcatalog executable.</description>
   </property>
 
+  <property>
+    <name>templeton.hive.archive</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/hive-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <description>The path to the Hive archive.</description>
+  </property>
+
   <property>
     <name>templeton.sqoop.archive</name>
-    <value>hdfs:///apps/webhcat/sqoop.tar.gz</value>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/sqoop-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
     <description>The path to the Sqoop archive in HDFS.</description>
   </property>
 
@@ -61,4 +71,10 @@ limitations under the License.
     <description>The path to the Sqoop executable.</description>
   </property>
 
+  <property>
+    <name>templeton.streaming.jar</name>
+    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/mr/hadoop-streaming-{{ component_version }}.{{ hdp_stack_version }}.jar</value>
+    <description>The hdfs path to the Hadoop streaming jar file.</description>
+  </property>
+
 </configuration>

ambari-server/src/test/python/TestVersion.py

@@ -0,0 +1,61 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+import os
+
+
+class TestVersion(TestCase):
+  """
+  Class that tests the method of the version.py file used to format and compare version numbers
+  of both Ambari (which use 3 digits separated by dots) and stacks (which use 4 digits separated by dots).
+  """
+  def setUp(self):
+    import imp
+
+    self.test_directory = os.path.dirname(os.path.abspath(__file__))
+    test_file_path = os.path.join(self.test_directory, '../../../../ambari-common/src/main/python/resource_management/libraries/functions/version.py')
+    with open(test_file_path, 'rb') as fp:
+        self.version_module = imp.load_module('version', fp, test_file_path, ('.py', 'rb', imp.PY_SOURCE))
+
+  def test_format(self):
+    l = [("2.2",   "2.2.0.0"),
+         ("2.2.1", "2.2.1.0"),
+         ("2.2.1.3", "2.2.1.3")]
+
+    for input, expected in l:
+      actual = self.version_module.format_hdp_stack_version(input)
+      self.assertEqual(expected, actual)
+
+  def test_comparison(self):
+    # All versions to compare, from 1.0.0.0 to 3.0.0.0, and only include elements that are a multiple of 7.
+    versions = range(1000, 3000, 7)
+    versions = [".".join(list(str(elem))) for elem in versions]
+
+    for idx, x in enumerate(versions):
+      for idy, y in enumerate(versions):
+        # Expected value will either be -1, 0, 1, and it relies on the fact
+        # that an increasing index implies a greater version number.
+        expected_value = cmp(idx, idy)
+        actual_value = self.version_module.compare_versions(x, y)
+        self.assertEqual(expected_value, actual_value)
+
+    # Try something fancier
+    self.assertEqual(0, self.version_module.compare_versions("2.10", "2.10.0"))
+    self.assertEqual(0, self.version_module.compare_versions("2.10", "2.10.0.0"))
+    self.assertEqual(0, self.version_module.compare_versions("2.10.0", "2.10.0.0"))

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py

@@ -146,18 +146,6 @@ class TestWebHCatServer(RMFTestCase):
                               group = 'hadoop',
                               recursive = True,
                               )
-    self.assertResourceCalled('XmlConfig', 'webhcat-site.xml',
-                              owner = 'hcat',
-                              group = 'hadoop',
-                              conf_dir = '/etc/hcatalog/conf',
-                              configurations = self.getConfig()['configurations']['webhcat-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['webhcat-site']
-    )
-    self.assertResourceCalled('File', '/etc/hcatalog/conf/webhcat-env.sh',
-                              content = InlineTemplate(self.getConfig()['configurations']['webhcat-env']['content']),
-                              owner = 'hcat',
-                              group = 'hadoop',
-                              )
     self.assertResourceCalled('CopyFromLocal', '/usr/lib/hadoop-mapreduce/hadoop-streaming-*.jar',
                               owner='hcat',
                               mode=0755,
@@ -194,6 +182,18 @@ class TestWebHCatServer(RMFTestCase):
                               hadoop_conf_dir='/etc/hadoop/conf',
                               hdfs_user='hdfs'
     )
+    self.assertResourceCalled('XmlConfig', 'webhcat-site.xml',
+                              owner = 'hcat',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hcatalog/conf',
+                              configurations = self.getConfig()['configurations']['webhcat-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['webhcat-site']
+    )
+    self.assertResourceCalled('File', '/etc/hcatalog/conf/webhcat-env.sh',
+                              content = InlineTemplate(self.getConfig()['configurations']['webhcat-env']['content']),
+                              owner = 'hcat',
+                              group = 'hadoop',
+                              )
 
   def assert_configure_secured(self):
     self.assertResourceCalled('HdfsDirectory', '/apps/webhcat',
@@ -244,18 +244,6 @@ class TestWebHCatServer(RMFTestCase):
                               group = 'hadoop',
                               recursive = True,
                               )
-    self.assertResourceCalled('XmlConfig', 'webhcat-site.xml',
-                              owner = 'hcat',
-                              group = 'hadoop',
-                              conf_dir = '/etc/hcatalog/conf',
-                              configurations = self.getConfig()['configurations']['webhcat-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['webhcat-site']
-    )
-    self.assertResourceCalled('File', '/etc/hcatalog/conf/webhcat-env.sh',
-                              content = InlineTemplate(self.getConfig()['configurations']['webhcat-env']['content']),
-                              owner = 'hcat',
-                              group = 'hadoop',
-                              )
     self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs;',
                               path = ['/bin'],
                               user = 'hcat',
@@ -295,4 +283,16 @@ class TestWebHCatServer(RMFTestCase):
                               hadoop_conf_dir='/etc/hadoop/conf',
                               hadoop_bin_dir='/usr/bin',
                               hdfs_user='hdfs'
-    )
+    )
+    self.assertResourceCalled('XmlConfig', 'webhcat-site.xml',
+                              owner = 'hcat',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hcatalog/conf',
+                              configurations = self.getConfig()['configurations']['webhcat-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['webhcat-site']
+    )
+    self.assertResourceCalled('File', '/etc/hcatalog/conf/webhcat-env.sh',
+                              content = InlineTemplate(self.getConfig()['configurations']['webhcat-env']['content']),
+                              owner = 'hcat',
+                              group = 'hadoop',
+                              )

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -217,7 +217,8 @@
             "templeton.hive.path": "hive.tar.gz/hive/bin/hive", 
             "templeton.hadoop.conf.dir": "/etc/hadoop/conf", 
             "templeton.hcat": "/usr/bin/hcat", 
-            "templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz"
+            "templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz",
+            "templeton.sqoop.archive": "hdfs:///apps/webhcat/sqoop.tar.gz"
         }, 
         "capacity-scheduler": {
             "yarn.scheduler.capacity.node-locality-delay": "40", 

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -229,7 +229,8 @@
             "templeton.hive.path": "hive.tar.gz/hive/bin/hive", 
             "templeton.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab", 
             "templeton.hcat": "/usr/bin/hcat", 
-            "templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz"
+            "templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz",
+            "templeton.sqoop.archive": "hdfs:///apps/webhcat/sqoop.tar.gz"
         },
         "capacity-scheduler": {
             "yarn.scheduler.capacity.node-locality-delay": "40", 

ambari-server/src/test/python/stacks/2.2/configs/default.json

@@ -51,6 +51,13 @@
             "kerberos_domain": "EXAMPLE.COM",
             "user_group": "hadoop"
         },
+        "webhcat-site": {
+            "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+            "templeton.pig.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/pig-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.hive.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/hive-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.sqoop.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/sqoop-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.streaming.jar": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mr/hadoop-streaming-{{ component_version }}.{{ hdp_stack_version }}.jar"
+        },
         "slider-log4j": {
             "content": "log4jproperties\nline2"
         },

ambari-server/src/test/python/stacks/2.2/configs/secured.json

@@ -46,6 +46,13 @@
             "smokeuser_keytab": "/etc/security/keytabs/smokeuser.headless.keytab",
             "kinit_path_local": "/usr/bin"
         },
+        "webhcat-site": {
+            "templeton.jar": "/usr/hdp/current/hive-webhcat/share/webhcat/svr/lib/hive-webhcat-*.jar",
+            "templeton.pig.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/pig-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.hive.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/hive-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.sqoop.archive": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/sqoop-{{ component_version }}.{{ hdp_stack_version }}.tar.gz",
+            "templeton.streaming.jar": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mr/hadoop-streaming-{{ component_version }}.{{ hdp_stack_version }}.jar"
+        },
         "slider-log4j": {
             "content": "log4jproperties\nline2"
         },