AMBARI-7960. Revert injection of versioned RPMs in Tez & WebHCat, and use static *-site.xml file with ${hdp.version} variable (alejandro)

ambari-common/src/main/python/resource_management/libraries/functions/dynamic_variable_interpretation.py

@@ -18,7 +18,7 @@ limitations under the License.
 
 """
 
-__all__ = ["copy_tarballs_to_hdfs", "interpret_dynamic_version_property"]
+__all__ = ["copy_tarballs_to_hdfs", ]
 import os
 import glob
 import re
@@ -47,15 +47,7 @@ TAR_SOURCE_SUFFIX = "_tar_source"
 TAR_DESTINATION_FOLDER_SUFFIX = "_tar_destination_folder"
 
 
-def __contains_dynamic_variable(string):
-  """
-  :param string: Input string to check
-  :return: Returns True if the string contains any dynamic variables to be interpreted, otherwise False.
-  """
-  return "{{ component_version }}" in string or "{{ hdp_stack_version }}" in string
-
-
-def __get_tar_source_and_dest_folder(tarball_prefix):
+def _get_tar_source_and_dest_folder(tarball_prefix):
   """
   :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
   :return: Returns a tuple of (x, y) after verifying the properties
@@ -84,7 +76,7 @@ def __get_tar_source_and_dest_folder(tarball_prefix):
   return component_tar_source_file, component_tar_destination_folder
 
 
-def __create_regex_pattern(file_path, hdp_stack_version):
+def _create_regex_pattern(file_path, hdp_stack_version):
   """
   :param file_path: Input file path
   :param hdp_stack_version: Stack version, such as 2.2.0.0
@@ -101,7 +93,7 @@ def __create_regex_pattern(file_path, hdp_stack_version):
   return file_path_pattern
 
 
-def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, hdp_stack_version):
+def _populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, hdp_stack_version):
   """
   :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
   :param source_file_pattern: Regex pattern of the source file from the local file system
@@ -150,7 +142,7 @@ def __populate_source_and_dests(tarball_prefix, source_file_pattern, component_t
   return source_and_dest_pairs
 
 
-def __copy_files(source_and_dest_pairs, file_owner, kinit_if_needed):
+def _copy_files(source_and_dest_pairs, file_owner, kinit_if_needed):
   """
   :param source_and_dest_pairs: List of tuples (x, y), where x is the source file in the local file system,
   and y is the destination file path in HDFS
@@ -206,14 +198,14 @@ def copy_tarballs_to_hdfs(tarball_prefix, component_user, file_owner):
     Logger.warning("Could not find hdp_stack_version")
     return 1
 
-  component_tar_source_file, component_tar_destination_folder = __get_tar_source_and_dest_folder(tarball_prefix)
+  component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder(tarball_prefix)
   if not component_tar_source_file or not component_tar_destination_folder:
     return 1
 
-  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.hdp_stack_version)
+  source_file_pattern = _create_regex_pattern(component_tar_source_file, params.hdp_stack_version)
   # This is just the last segment
   file_name_pattern = source_file_pattern.split('/')[-1:][0]
-  tar_destination_folder_pattern = __create_regex_pattern(component_tar_destination_folder, params.hdp_stack_version)
+  tar_destination_folder_pattern = _create_regex_pattern(component_tar_destination_folder, params.hdp_stack_version)
 
   # Pattern for searching the file in HDFS. E.g. value, hdfs:///hdp/apps/2.2.0.0*/tez/tez-*.2.2.0.0*.tar.gz
   hdfs_file_pattern = os.path.join(tar_destination_folder_pattern, file_name_pattern)
@@ -242,73 +234,7 @@ def copy_tarballs_to_hdfs(tarball_prefix, component_user, file_owner):
     pass
 
   if not does_hdfs_file_exist:
-    source_and_dest_pairs = __populate_source_and_dests(tarball_prefix, source_file_pattern,
+    source_and_dest_pairs = _populate_source_and_dests(tarball_prefix, source_file_pattern,
                                                         component_tar_destination_folder, params.hdp_stack_version)
-    return __copy_files(source_and_dest_pairs, file_owner, kinit_if_needed)
+    return _copy_files(source_and_dest_pairs, file_owner, kinit_if_needed)
   return 1
-
-
-def __map_local_file_to_hdfs_file(tarball_prefix):
-  """
-  :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
-  :return: Using the source tarball file pattern, it finds the corresponding file in the local filesystem, and
-  maps it to its corresponding location in HDFS, while substituting the dynamic variables like {{ hdp_stack_version }}
-  and {{ component_version }}.
-  On success, returns a string with the path of where the file should be on HDFS, otherwise, returns an empty string.
-  """
-  import params
-
-  if not hasattr(params, "hdp_stack_version") or params.hdp_stack_version is None:
-    Logger.warning("Could not find hdp_stack_version")
-    return ""
-
-  component_tar_source_file, component_tar_destination_folder = __get_tar_source_and_dest_folder(tarball_prefix)
-  if not component_tar_source_file or not component_tar_destination_folder:
-    return ""
-
-  source_file_pattern = __create_regex_pattern(component_tar_source_file, params.hdp_stack_version)
-  source_and_dest_pairs = __populate_source_and_dests(tarball_prefix, source_file_pattern, component_tar_destination_folder, params.hdp_stack_version)
-  if source_and_dest_pairs and len(source_and_dest_pairs) == 1:
-    return source_and_dest_pairs[0][1]
-
-  return ""
-
-
-def interpret_dynamic_version_property(property_value, tarball_prefix, delimiter=","):
-  """
-  :param property_value: Value to scan for dynamic variables
-  :param tarball_prefix:  Prefix of the tarball must be one of tez, hive, mr, pig
-  :param delimiter: Delimiter character used in the property value, typically a comma or colon
-  :return: Returns a tuple of (x, y), where x is a bool indicating if at least one variable was substituted, and y
-  is the interpretation of the property value if an interpretation could be done, otherwise it remains unchanged.
-
-  Notice that params must have the hdp_stack_version attribute.
-  """
-  import params
-
-  found_at_least_one_replacement = False
-  versioned_tarball = __map_local_file_to_hdfs_file(tarball_prefix)
-  if versioned_tarball and versioned_tarball != "":
-    # We expect to find a file in HDFS, and must substitute it for its regex equivalent in the property inside *-site.xml
-    property_value = "" if property_value is None or property_value.strip() == "" else property_value
-
-    if property_value:
-      elements = []
-
-      for elem in property_value.split(delimiter):
-        elem = elem.strip()
-        if __contains_dynamic_variable(elem):
-          # Need to do dynamic interpretation, and slight regex escaping. Must not escape " " since it is used in
-          # the dynamic variable string.
-          elem_pattern = __create_regex_pattern(elem, params.hdp_stack_version).replace(".", "\\.").replace("*", ".*")
-          p = re.compile(elem_pattern)
-          m = p.match(versioned_tarball)
-          if m:
-            elements.append(versioned_tarball)
-            found_at_least_one_replacement = True
-        else:
-          elements.append(elem)
-
-      if found_at_least_one_replacement:
-        property_value = ",".join(elements)
-  return found_at_least_one_replacement, property_value

ambari-common/src/main/python/resource_management/libraries/script/config_dictionary.py

@@ -66,21 +66,6 @@ class ConfigDictionary(dict):
           pass
     
     return value
-  
-
-class MutableConfigDictionary(ConfigDictionary):
-  """
-  Mutable Configuration Dictionary
-  """
-  def __init__(self, dictionary):
-    d = dict()
-    for k, v in dictionary.iteritems():
-      if isinstance(v, dict):
-        d[k] = MutableConfigDictionary(v)
-      else:
-        d[k] = v
-
-    super(MutableConfigDictionary, self).__init__(d, allow_overwrite=True)
 
 
 class UnknownConfiguration():

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py

@@ -26,29 +26,6 @@ from resource_management import *
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.version import compare_versions
 from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
-from resource_management.libraries.script.config_dictionary import MutableConfigDictionary
-from resource_management.libraries.functions.dynamic_variable_interpretation import interpret_dynamic_version_property
-
-
-def __inject_config_variables(mutable_configs):
-  """
-  :param mutable_configs: Mutable Configuration Dictionary
-  :return: Returns the mutable configuration dictionary where each of the dynamic properties have been injected
-  with the value of the versioned tarball or jar in HDFS.
-  """
-  if mutable_configs is not None and "configurations" in mutable_configs and \
-          mutable_configs["configurations"] is not None and "webhcat-site" in mutable_configs["configurations"]:
-    webhcat_config = mutable_configs['configurations']['webhcat-site']
-
-    properties_and_prefix_tuple_list = [('pig', 'templeton.pig.archive'), ('hive', 'templeton.hive.archive'),
-                                        ('sqoop', 'templeton.sqoop.archive'), ('hadoop-streaming', 'templeton.streaming.jar')]
-    for (prefix, prop_name) in properties_and_prefix_tuple_list:
-      prop_value = webhcat_config[prop_name]
-      if prop_value:
-        found_at_least_one_replacement, new_value = interpret_dynamic_version_property(prop_value, prefix, ",")
-        if found_at_least_one_replacement:
-          webhcat_config[prop_name] = new_value
-  return mutable_configs
 
 
 def webhcat():
@@ -151,14 +128,9 @@ def webhcat():
                     hadoop_conf_dir=params.hadoop_conf_dir
       )
 
-  mutable_configs = MutableConfigDictionary(params.config)
-  # TODO, this is specific to HDP 2.2, but it is safe to call in earlier versions.
-  # It should eventually be moved to scripts specific to the HDP 2.2 stack.
-  mutable_configs = __inject_config_variables(mutable_configs)
-
   XmlConfig("webhcat-site.xml",
             conf_dir=params.config_dir,
-            configurations=mutable_configs['configurations']['webhcat-site'],
+            configurations=params.config['configurations']['webhcat-site'],
             configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
             owner=params.webhcat_user,
             group=params.user_group,

ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py

@@ -20,9 +20,6 @@ Ambari Agent
 """
 
 from resource_management import *
-from resource_management.libraries.script.config_dictionary import MutableConfigDictionary
-from resource_management.libraries.functions.dynamic_variable_interpretation import interpret_dynamic_version_property
-
 
 def tez():
   import params
@@ -33,16 +30,9 @@ def tez():
             recursive = True
   )
 
-  mutable_configs = MutableConfigDictionary(params.config)
-  tez_lib_uris = params.config['configurations']['tez-site']['tez.lib.uris']
-  if tez_lib_uris:
-    found_at_least_one_replacement, new_tez_lib_uris = interpret_dynamic_version_property(tez_lib_uris, "tez", ",")
-    if found_at_least_one_replacement:
-      mutable_configs['configurations']['tez-site']['tez.lib.uris'] = new_tez_lib_uris
-
   XmlConfig( "tez-site.xml",
              conf_dir = params.config_dir,
-             configurations = mutable_configs['configurations']['tez-site'],
+             configurations = params.config['configurations']['tez-site'],
              configuration_attributes=params.config['configuration_attributes']['tez-site'],
              owner = params.tez_user,
              group = params.user_group,

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml

@@ -43,10 +43,16 @@ limitations under the License.
 
   <property>
     <name>templeton.pig.archive</name>
-    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/pig-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <value>hdfs:///hdp/apps/${hdp.version}/pig/pig-0.14.0.${hdp.version}.tar.gz</value>
     <description>The path to the Pig archive in HDFS.</description>
   </property>
 
+  <property>
+    <name>templeton.pig.path</name>
+    <value>pig-0.14.0.${hdp.version}.tar.gz/pig/bin/pig</value>
+    <description>The path to the Pig executable.</description>
+  </property>
+
   <property>
     <name>templeton.hcat</name>
     <value>/usr/hdp/current/hive-client/bin/hcat</value>
@@ -55,25 +61,43 @@ limitations under the License.
 
   <property>
     <name>templeton.hive.archive</name>
-    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/hive-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <value>hdfs:///hdp/apps/${hdp.version}/hive/hive-0.14.0.${hdp.version}.tar.gz</value>
     <description>The path to the Hive archive.</description>
   </property>
 
+  <property>
+    <name>templeton.hive.home</name>
+    <value>hive-0.14.0.${hdp.version}.tar.gz/hive</value>
+    <description>The path to the Hive home within the tar. Has no effect if templeton.hive.archive is not set.</description>
+  </property>
+
+  <property>
+    <name>templeton.hcat.home</name>
+    <value>hive-0.14.0.${hdp.version}.tar.gz/hive/hcatalog</value>
+    <description>The path to the HCat home within the tar. Has no effect if templeton.hive.archive is not set.</description>
+  </property>
+
+  <property>
+    <name>templeton.hive.path</name>
+    <value>hive-0.14.0.${hdp.version}.tar.gz/hive/bin/hive</value>
+    <description>The path to the Hive executable.</description>
+  </property>
+
   <property>
     <name>templeton.sqoop.archive</name>
-    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/sqoop-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <value>hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop-1.4.5.${hdp.version}.tar.gz</value>
     <description>The path to the Sqoop archive in HDFS.</description>
   </property>
 
   <property>
     <name>templeton.sqoop.path</name>
-    <value>sqoop.tar.gz/sqoop/bin/sqoop</value>
+    <value>sqoop-1.4.5.${hdp.version}.tar.gz/sqoop/bin/sqoop</value>
     <description>The path to the Sqoop executable.</description>
   </property>
 
   <property>
     <name>templeton.streaming.jar</name>
-    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/mr/hadoop-streaming-{{ component_version }}.{{ hdp_stack_version }}.jar</value>
+    <value>hdfs:///hdp/apps/${hdp.version}/mr/hadoop-streaming-2.6.0.${hdp.version}.jar</value>
     <description>The hdfs path to the Hadoop streaming jar file.</description>
   </property>
 

ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml

@@ -21,7 +21,7 @@
 
   <property>
     <name>tez.lib.uris</name>
-    <value>hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/tez-{{ component_version }}.{{ hdp_stack_version }}.tar.gz</value>
+    <value>hdfs:///hdp/apps/${hdp.version}/tez/tez-0.6.0.${hdp.version}.tar.gz</value>
     <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs.
       Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*).
       If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive).