AMBARI-14447. Fix slider install failure and review comments for AMBARI-14430 (aonishuk)

This reverts commit 901d6afca459fa2d1bf66234f07b351f78266413.

ambari-agent/src/test/python/resource_management/TestPackagesAnalyzer.py

@@ -0,0 +1,40 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from unittest import TestCase
+from mock.mock import patch, MagicMock, call
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions import packages_analyzer
+
+class TestPackagesAnalyzer(TestCase):
+  @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  def test_get_installed_package_version_ubuntu(self, is_ubuntu_family_mock, checked_call_mock):
+    is_ubuntu_family_mock.return_value = True
+    checked_call_mock.return_value = (0, '1.2.3','')
+    result = packages_analyzer.getInstalledPackageVersion("package1")
+    self.assertEqual(result, '1.2.3')
+    self.assertEqual(checked_call_mock.call_args_list, [call("dpkg -s package1 | grep Version | awk '{print $2}'", stderr=-1)])
+    
+  @patch("resource_management.libraries.functions.packages_analyzer.rmf_shell.checked_call")
+  @patch.object(OSCheck, "is_ubuntu_family")
+  def test_get_installed_package_version_centos_suse(self, is_ubuntu_family_mock, checked_call_mock):
+    is_ubuntu_family_mock.return_value = False
+    checked_call_mock.return_value = (0, '0.0.1-SNAPSHOT','')
+    result = packages_analyzer.getInstalledPackageVersion("package1")
+    self.assertEqual(result, '0.0.1-SNAPSHOT')
+    self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)])

ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py

@@ -20,6 +20,7 @@ limitations under the License.
 
 import os
 import sys
+import re
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
 from resource_management.core.resources.system import Execute
@@ -263,3 +264,21 @@ def get_hdp_versions(stack_root):
   if not versions:
     versions = get_versions_from_stack_root(stack_root)
   return versions
+
+def get_hdp_version_before_install(component_name):
+  """
+  Works in the similar way to 'hdp-select status component', 
+  but also works for not yet installed packages.
+  
+  Note: won't work if doing initial install.
+  """
+  component_dir = HADOOP_HOME_DIR_TEMPLATE.format("current", component_name)
+  if os.path.islink(component_dir):
+    hdp_version = os.path.basename(os.path.dirname(os.readlink(component_dir)))
+    match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
+    if match is None:
+      Logger.info('Failed to get extracted version with hdp-select in method get_hdp_version_before_install')
+      return None # lazy fail
+    return hdp_version
+  else:
+    return None

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -70,7 +70,7 @@ USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEV
 
 _PASSWORD_MAP = {"/configurations/cluster-env/hadoop.user.name":"/configurations/cluster-env/hadoop.user.password"}
 DISTRO_SELECT_PACKAGE_NAME = "hdp-select"
-HDP_VERSION_PLACEHOLDER = "${hdp_version}"
+STACK_VERSION_PLACEHOLDER = "${stack_version}"
 
 def get_path_from_configuration(name, configuration):
   subdicts = filter(None, name.split('/'))
@@ -97,7 +97,7 @@ class Script(object):
   3 path to service metadata dir (Directory "package" inside service directory)
   4 path to file with structured command output (file will be created)
   """
-  stack_version_from_hdp_select = None
+  stack_version_from_distro_select = None
   structuredOut = {}
   command_data_file = ""
   basedir = ""
@@ -142,17 +142,26 @@ class Script(object):
         json.dump(Script.structuredOut, fp)
     except IOError, err:
       Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
+      
+  def get_component_name(self):
+    stack_name = Script.get_stack_name()
+    stack_to_component = self.get_stack_to_component()
+    
+    if stack_to_component and stack_name:
+      component_name = stack_to_component[stack_name] if stack_name in stack_to_component else None
+      return component_name
+    
+    return None
 
   def save_component_version_to_structured_out(self):
     """
     :param stack_name: One of HDP, HDPWIN, PHD, BIGTOP.
     :return: Append the version number to the structured out.
     """
-    from resource_management.libraries.functions.default import default
-    stack_name = default("/hostLevelParams/stack_name", None)
-    stack_to_component = self.get_stack_to_component()
-    if stack_to_component and stack_name:
-      component_name = stack_to_component[stack_name] if stack_name in stack_to_component else None
+    stack_name = Script.get_stack_name()
+    component_name = self.get_component_name()
+    
+    if component_name and stack_name:
       component_version = get_component_version(stack_name, component_name)
 
       if component_version:
@@ -240,34 +249,36 @@ class Script(object):
     method = getattr(self, command_name)
     return method
   
-  @staticmethod
-  def get_stack_version_from_hdp_select():
+  def get_stack_version_before_packages_installed(self):
     """
     This works in a lazy way (calculates the version first time and stores it). 
     If you need to recalculate the version explicitly set:
     
-    Script.stack_version_from_hdp_select = None
+    Script.stack_version_from_distro_select = None
     
     before the call. However takes a bit of time, so better to avoid.
-    
-    :param install_hdp_select: whether to ensure if hdp-select is installed, before checking the version.
-    Set this to false, if you're sure hdp-select is present at the point you call this, to save some time.
-    
+
     :return: hdp version including the build number. e.g.: 2.3.4.0-1234.
     """
-    if not Script.stack_version_from_hdp_select:
-      Script.stack_version_from_hdp_select = packages_analyzer.getInstalledPackageVersion(DISTRO_SELECT_PACKAGE_NAME)
+    # preferred way is to get the actual selected version of current component
+    component_name = self.get_component_name()
+    if not Script.stack_version_from_distro_select and component_name:
+      from resource_management.libraries.functions import hdp_select
+      Script.stack_version_from_distro_select = hdp_select.get_hdp_version_before_install(component_name)
       
-    return Script.stack_version_from_hdp_select
+    # if hdp-select has not yet been done (situations like first install), we can use hdp-select version itself.
+    if not Script.stack_version_from_distro_select:
+      Script.stack_version_from_distro_select = packages_analyzer.getInstalledPackageVersion(DISTRO_SELECT_PACKAGE_NAME)
+      
+    return Script.stack_version_from_distro_select
   
-  @staticmethod
-  def format_package_name(name):
+  def format_package_name(self, name):
     """
-    This function replaces ${hdp_version} placeholder into actual version.
+    This function replaces ${stack_version} placeholder into actual version.
     """
     package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_'
-    hdp_version_package_formatted = Script.get_stack_version_from_hdp_select().replace('.', package_delimiter).replace('-', package_delimiter) if HDP_VERSION_PLACEHOLDER in name else name
-    package_name = name.replace(HDP_VERSION_PLACEHOLDER, hdp_version_package_formatted)
+    stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name
+    package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted)
     
     return package_name
 
@@ -431,9 +442,8 @@ class Script(object):
       if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
         package_list = json.loads(package_list_str)
         for package in package_list:
-          #import pydevd;pydevd.settrace(host='192.168.64.1',stdoutToServer=True, stderrToServer=True)
           if not Script.matches_any_regexp(package['name'], exclude_packages):
-            name = Script.format_package_name(package['name'])
+            name = self.format_package_name(package['name'])
             # HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
             # TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
             # <osFamily>any<osFamily> which would cause installation failure on Windows.

ambari-server/src/main/resources/custom_actions/scripts/install_packages.py

@@ -342,9 +342,9 @@ class InstallPackages(Script):
       packages_were_checked = True
       filtered_package_list = self.filter_package_list(package_list)
       for package in filtered_package_list:
-        name = Script.format_package_name(package['name'])
+        name = self.format_package_name(package['name'])
         Package(name
-        # action="upgrade" - should we user ugrade action here? to updated not versioned packages?       
+        # action="upgrade" # should we use "upgrade" action here, to upgrade not versioned packages?       
         )
     except Exception, err:
       ret_code = 1

ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/metainfo.xml

@@ -27,7 +27,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>falcon_${hdp_version}</name>
+              <name>falcon_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -35,7 +35,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>falcon-${hdp_version}</name>
+              <name>falcon-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/FLUME/metainfo.xml

@@ -29,7 +29,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>flume_${hdp_version}</name>
+              <name>flume_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -37,7 +37,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>flume-${hdp_version}</name>
+              <name>flume-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HBASE/metainfo.xml

@@ -28,10 +28,10 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>hbase_${hdp_version}</name>
+              <name>hbase_${stack_version}</name>
             </package>
             <package>
-              <name>phoenix_${hdp_version}</name>
+              <name>phoenix_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -39,10 +39,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>hbase-${hdp_version}</name>
+              <name>hbase-${stack_version}</name>
             </package>
             <package>
-              <name>phoenix-${hdp_version}</name>
+              <name>phoenix-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml

@@ -22,20 +22,13 @@
       <name>HDFS</name>
       <displayName>HDFS</displayName>
       <version>2.6.0.2.2</version>
+
       <osSpecifics>
         <osSpecific>
-          <osFamily>any</osFamily>
-          <packages>
-            <package>
-              <name>rpcbind</name>
-            </package>
-          </packages>
-        </osSpecific>
-        <osSpecific>
-          <osFamily>redhat7,redhat6,suse11</osFamily>
+          <osFamily>redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>hadoop_${hdp_version}</name>
+              <name>hadoop_${stack_version}</name>
             </package>
             <package>
               <name>snappy</name>
@@ -48,34 +41,34 @@
               <skipUpgrade>true</skipUpgrade>
             </package>
             <package>
-              <name>hadooplzo_${hdp_version}</name>
+              <name>hadooplzo_${stack_version}</name>
             </package>
             <package>
-              <name>hadoop_${hdp_version}-libhdfs</name>
+              <name>hadoop_${stack_version}-libhdfs</name>
             </package>
           </packages>
         </osSpecific>
 
         <osSpecific>
-          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>hadoop-${hdp_version}-client</name>
+              <name>hadoop-${stack_version}-client</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-hdfs-datanode</name>
+              <name>hadoop-${stack_version}-hdfs-datanode</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-hdfs-journalnode</name>
+              <name>hadoop-${stack_version}-hdfs-journalnode</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-hdfs-namenode</name>
+              <name>hadoop-${stack_version}-hdfs-namenode</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-hdfs-secondarynamenode</name>
+              <name>hadoop-${stack_version}-hdfs-secondarynamenode</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-hdfs-zkfc</name>
+              <name>hadoop-${stack_version}-hdfs-zkfc</name>
             </package>
             <package>
               <name>libsnappy1</name>
@@ -84,14 +77,19 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>hadooplzo-${hdp_version}</name>
+              <name>hadooplzo-${stack_version}</name>
+            </package>
+            <package>
+              <name>liblzo2-2</name>
+              <skipUpgrade>true</skipUpgrade>
             </package>
             <package>
-              <name>libhdfs0-${hdp_version}</name>
+              <name>libhdfs0-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
       </osSpecifics>
+      
       <themes>
           <theme>
               <fileName>theme.json</fileName>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/metainfo.xml

@@ -55,7 +55,7 @@
         </component>
 
       </components>
-
+      
       <osSpecifics>
         <osSpecific>
           <osFamily>any</osFamily>
@@ -67,19 +67,16 @@
           </packages>
         </osSpecific>
         <osSpecific>
-          <osFamily>redhat7,redhat6,suse11</osFamily>
+          <osFamily>redhat6,redhat7,suse11</osFamily>
           <packages>
             <package>
-              <name>hive_${hdp_version}</name>
-            </package>
-            <package>
-              <name>hive_${hdp_version}-hcatalog</name>
+              <name>hive_${stack_version}</name>
             </package>
             <package>
-              <name>hive_${hdp_version}-webhcat</name>
+              <name>hive_${stack_version}-hcatalog</name>
             </package>
             <package>
-              <name>atlas-metadata_${hdp_version}-hive-plugin</name>
+              <name>hive_${stack_version}-webhcat</name>
             </package>
           </packages>
         </osSpecific>
@@ -106,24 +103,21 @@
           </packages>
         </osSpecific>
         <osSpecific>
-          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>hive-${hdp_version}</name>
-            </package>
-            <package>
-              <name>hive-${hdp_version}-hcatalog</name>
+              <name>hive-${stack_version}</name>
             </package>
             <package>
-              <name>hive-${hdp_version}-webhcat</name>
+              <name>hive-${stack_version}-hcatalog</name>
             </package>
             <package>
-              <name>atlas-metadata-${hdp_version}-hive-plugin</name>
+              <name>hive-${stack_version}-webhcat</name>
             </package>
           </packages>
         </osSpecific>
         <osSpecific>
-          <osFamily>redhat6,debian7,ubuntu12,ubuntu14</osFamily>
+          <osFamily>redhat6,ubuntu12</osFamily>
           <packages>
             <package>
               <name>mysql-server</name>

ambari-server/src/main/resources/stacks/HDP/2.2/services/KAFKA/metainfo.xml

@@ -26,7 +26,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>kafka_${hdp_version}</name>
+              <name>kafka_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -34,7 +34,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>kafka-${hdp_version}</name>
+              <name>kafka-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/KNOX/metainfo.xml

@@ -26,7 +26,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>knox_${hdp_version}</name>
+              <name>knox_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -34,7 +34,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>knox-${hdp_version}</name>
+              <name>knox-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/OOZIE/metainfo.xml

@@ -77,10 +77,10 @@
           <osFamily>redhat5,redhat6,redhat7,suse11</osFamily>
           <packages>
             <package>
-              <name>oozie_${hdp_version}</name>
+              <name>oozie_${stack_version}</name>
             </package>
             <package>
-              <name>falcon_${hdp_version}</name>
+              <name>falcon_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -88,10 +88,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>oozie-${hdp_version}</name>
+              <name>oozie-${stack_version}</name>
             </package>
             <package>
-              <name>falcon-${hdp_version}</name>
+              <name>falcon-${stack_version}</name>
             </package>
             <package>
               <name>extjs</name>

ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml

@@ -27,7 +27,7 @@
           <osFamily>redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>pig_${hdp_version}</name>
+              <name>pig_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -35,7 +35,7 @@
           <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>pig-${hdp_version}</name>
+              <name>pig-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/RANGER/metainfo.xml

@@ -35,10 +35,10 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>ranger_${hdp_version}-admin</name>
+              <name>ranger_${stack_version}-admin</name>
             </package>
             <package>
-              <name>ranger_${hdp_version}-usersync</name>
+              <name>ranger_${stack_version}-usersync</name>
             </package>
           </packages>
         </osSpecific>
@@ -46,10 +46,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>ranger-${hdp_version}-admin</name>
+              <name>ranger-${stack_version}-admin</name>
             </package>
             <package>
-              <name>ranger-${hdp_version}-usersync</name>
+              <name>ranger-${stack_version}-usersync</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/SLIDER/metainfo.xml

@@ -26,10 +26,10 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>slider_${hdp_version}</name>
+              <name>slider_${stack_version}</name>
             </package>
             <package>
-              <name>storm_${hdp_version}-slider-client</name>
+              <name>storm_${stack_version}-slider-client</name>
             </package>
           </packages>
         </osSpecific>
@@ -37,10 +37,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>slider-${hdp_version}</name>
+              <name>slider-${stack_version}</name>
             </package>
             <package>
-              <name>storm-${hdp_version}-slider-client</name>
+              <name>storm-${stack_version}-slider-client</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/SPARK/metainfo.xml

@@ -30,10 +30,10 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>spark_${hdp_version}</name>
+              <name>spark_${stack_version}</name>
             </package>
             <package>
-              <name>spark_${hdp_version}-python</name>
+              <name>spark_${stack_version}-python</name>
             </package>
           </packages>
         </osSpecific>
@@ -41,10 +41,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>spark-${hdp_version}</name>
+              <name>spark-${stack_version}</name>
             </package>
             <package>
-              <name>spark-${hdp_version}-python</name>
+              <name>spark-${stack_version}-python</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/SQOOP/metainfo.xml

@@ -35,7 +35,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>sqoop_${hdp_version}</name>
+              <name>sqoop_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -43,7 +43,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>sqoop-${hdp_version}</name>
+              <name>sqoop-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/STORM/metainfo.xml

@@ -34,7 +34,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>storm_${hdp_version}</name>
+              <name>storm_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -42,7 +42,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>storm-${hdp_version}</name>
+              <name>storm-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/metainfo.xml

@@ -28,7 +28,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>tez_${hdp_version}</name>
+              <name>tez_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -36,7 +36,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>tez-${hdp_version}</name>
+              <name>tez-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/metainfo.xml

@@ -35,13 +35,13 @@
           <osFamily>redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>hadoop_${hdp_version}-yarn</name>
+              <name>hadoop_${stack_version}-yarn</name>
             </package>
             <package>
-              <name>hadoop_${hdp_version}-mapreduce</name>
+              <name>hadoop_${stack_version}-mapreduce</name>
             </package>
             <package>
-              <name>hadoop_${hdp_version}-hdfs</name>
+              <name>hadoop_${stack_version}-hdfs</name>
             </package>
           </packages>
         </osSpecific>
@@ -49,10 +49,10 @@
           <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>hadoop-${hdp_version}-yarn</name>
+              <name>hadoop-${stack_version}-yarn</name>
             </package>
             <package>
-              <name>hadoop-${hdp_version}-mapreduce</name>
+              <name>hadoop-${stack_version}-mapreduce</name>
             </package>
           </packages>
         </osSpecific>
@@ -75,7 +75,7 @@
           <osFamily>redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>hadoop_${hdp_version}-mapreduce</name>
+              <name>hadoop_${stack_version}-mapreduce</name>
             </package>
           </packages>
         </osSpecific>
@@ -83,7 +83,7 @@
           <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>hadoop-${hdp_version}-mapreduce</name>
+              <name>hadoop-${stack_version}-mapreduce</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.3/services/ACCUMULO/metainfo.xml

@@ -27,7 +27,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>accumulo_${hdp_version}</name>
+              <name>accumulo_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -35,7 +35,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>accumulo-${hdp_version}</name>
+              <name>accumulo-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.3/services/ATLAS/metainfo.xml

@@ -27,7 +27,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>atlas-metadata_${hdp_version}</name>
+              <name>atlas-metadata_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -35,7 +35,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>atlas-metadata-${hdp_version}</name>
+              <name>atlas-metadata-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml

@@ -45,6 +45,77 @@
           </dependencies>
         </component>
       </components>
+      
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>rpcbind</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat7,redhat6,suse11</osFamily>
+          <packages>
+            <package>
+              <name>hadoop_${stack_version}</name>
+            </package>
+            <package>
+              <name>snappy</name>
+            </package>
+            <package>
+              <name>snappy-devel</name>
+            </package>
+            <package>
+              <name>lzo</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+            <package>
+              <name>hadooplzo_${stack_version}</name>
+            </package>
+            <package>
+              <name>hadoop_${stack_version}-libhdfs</name>
+            </package>
+          </packages>
+        </osSpecific>
+
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <packages>
+            <package>
+              <name>hadoop-${stack_version}-client</name>
+            </package>
+            <package>
+              <name>hadoop-${stack_version}-hdfs-datanode</name>
+            </package>
+            <package>
+              <name>hadoop-${stack_version}-hdfs-journalnode</name>
+            </package>
+            <package>
+              <name>hadoop-${stack_version}-hdfs-namenode</name>
+            </package>
+            <package>
+              <name>hadoop-${stack_version}-hdfs-secondarynamenode</name>
+            </package>
+            <package>
+              <name>hadoop-${stack_version}-hdfs-zkfc</name>
+            </package>
+            <package>
+              <name>libsnappy1</name>
+            </package>
+            <package>
+              <name>libsnappy-dev</name>
+            </package>
+            <package>
+              <name>hadooplzo-${stack_version}</name>
+            </package>
+            <package>
+              <name>libhdfs0-${stack_version}</name>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
 
       <quickLinksConfigurations>
         <quickLinksConfiguration>

ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/metainfo.xml

@@ -21,6 +21,92 @@
     <service>
       <name>HIVE</name>
       <version>1.2.1.2.3</version>
+      
+      <osSpecifics>
+        <osSpecific>
+          <osFamily>any</osFamily>
+          <packages>
+            <package>
+              <name>mysql-connector-java</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat7,redhat6,suse11</osFamily>
+          <packages>
+            <package>
+              <name>hive_${stack_version}</name>
+            </package>
+            <package>
+              <name>hive_${stack_version}-hcatalog</name>
+            </package>
+            <package>
+              <name>hive_${stack_version}-webhcat</name>
+            </package>
+            <package>
+              <name>atlas-metadata_${stack_version}-hive-plugin</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat6,suse11</osFamily>
+          <packages>
+            <package>
+              <name>mysql</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat7</osFamily>
+          <packages>
+            <package>
+              <name>mysql-community-release</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+            <package>
+              <name>mysql-community-server</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
+          <packages>
+            <package>
+              <name>hive-${stack_version}</name>
+            </package>
+            <package>
+              <name>hive-${stack_version}-hcatalog</name>
+            </package>
+            <package>
+              <name>hive-${stack_version}-webhcat</name>
+            </package>
+            <package>
+              <name>atlas-metadata-${stack_version}-hive-plugin</name>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>redhat6,debian7,ubuntu12,ubuntu14</osFamily>
+          <packages>
+            <package>
+              <name>mysql-server</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+          </packages>
+        </osSpecific>
+        <osSpecific>
+          <osFamily>suse11</osFamily>
+          <packages>
+            <package>
+              <name>mysql-client</name>
+              <skipUpgrade>true</skipUpgrade>
+            </package>
+          </packages>
+        </osSpecific>
+      </osSpecifics>
     </service>
   </services>
 </metainfo>

ambari-server/src/main/resources/stacks/HDP/2.3/services/OOZIE/metainfo.xml

@@ -48,10 +48,10 @@
           <osFamily>redhat5,redhat6,redhat7,suse11</osFamily>
           <packages>
             <package>
-              <name>oozie_${hdp_version}</name>
+              <name>oozie_${stack_version}</name>
             </package>
             <package>
-              <name>falcon_${hdp_version}</name>
+              <name>falcon_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -59,10 +59,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>oozie-${hdp_version}</name>
+              <name>oozie-${stack_version}</name>
             </package>
             <package>
-              <name>falcon-${hdp_version}</name>
+              <name>falcon-${stack_version}</name>
             </package>
             <package>
               <name>extjs</name>

ambari-server/src/main/resources/stacks/HDP/2.3/services/PIG/metainfo.xml

@@ -26,10 +26,10 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>pig_${hdp_version}</name>
+              <name>pig_${stack_version}</name>
             </package>
             <package>
-              <name>datafu_${hdp_version}</name>
+              <name>datafu_${stack_version}</name>
             </package>
           </packages>
         </osSpecific>
@@ -37,10 +37,10 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>pig-${hdp_version}</name>
+              <name>pig-${stack_version}</name>
             </package>
             <package>
-              <name>datafu-${hdp_version}</name>
+              <name>datafu-${stack_version}</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.3/services/RANGER_KMS/metainfo.xml

@@ -29,7 +29,7 @@
           <osFamily>redhat7,redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>ranger_${hdp_version}-kms</name>
+              <name>ranger_${stack_version}-kms</name>
             </package>
           </packages>
         </osSpecific>
@@ -37,7 +37,7 @@
           <osFamily>debian7,ubuntu12,ubuntu14</osFamily>
           <packages>
             <package>
-              <name>ranger-${hdp_version}-kms</name>
+              <name>ranger-${stack_version}-kms</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/main/resources/stacks/HDP/2.3/services/ZOOKEEPER/metainfo.xml

@@ -26,10 +26,10 @@
           <osFamily>redhat6,suse11</osFamily>
           <packages>
             <package>
-              <name>zookeeper_${hdp_version}</name>
+              <name>zookeeper_${stack_version}</name>
             </package>
             <package>
-              <name>zookeeper_${hdp_version}-server</name>
+              <name>zookeeper_${stack_version}-server</name>
             </package>
           </packages>
         </osSpecific>
@@ -37,10 +37,10 @@
           <osFamily>ubuntu12</osFamily>
           <packages>
             <package>
-              <name>zookeeper-${hdp_version}</name>
+              <name>zookeeper-${stack_version}</name>
             </package>
             <package>
-              <name>zookeeper-${hdp_version}-server</name>
+              <name>zookeeper-${stack_version}-server</name>
             </package>
           </packages>
         </osSpecific>

ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -131,7 +131,7 @@ class TestInstallPackages(RMFTestCase):
                             read_actual_version_from_history_file_mock,
                             hdp_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, is_suse_family_mock):
     is_suse_family_mock = True
-    Script.stack_version_from_hdp_select = VERSION_STUB
+    Script.stack_version_from_distro_select = VERSION_STUB
     hdp_versions_mock.side_effect = [
       [],  # before installation attempt
       [VERSION_STUB]
@@ -197,7 +197,7 @@ class TestInstallPackages(RMFTestCase):
       [],  # before installation attempt
       [VERSION_STUB]
     ]
-    Script.stack_version_from_hdp_select = VERSION_STUB
+    Script.stack_version_from_distro_select = VERSION_STUB
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"]
     is_redhat_family_mock.return_value = True
@@ -327,7 +327,7 @@ class TestInstallPackages(RMFTestCase):
                                hdp_versions_mock,
                                allInstalledPackages_mock, put_structured_out_mock,
                                package_mock, is_suse_family_mock):
-    Script.stack_version_from_hdp_select = VERSION_STUB
+    Script.stack_version_from_distro_select = VERSION_STUB
     hdp_versions_mock.side_effect = [
       [],  # before installation attempt
       [VERSION_STUB]

ambari-server/src/test/python/custom_actions/configs/install_packages_config.json

@@ -26,7 +26,7 @@
         "stack_id": "HDP-2.2",
         "repository_version": "2.2.0.1-885",
         "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
-        "package_list": "[{\"name\":\"hadoop_${hdp_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${hdp_version}\"},{\"name\":\"hadoop_${hdp_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]"
+        "package_list": "[{\"name\":\"hadoop_${stack_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${stack_version}\"},{\"name\":\"hadoop_${stack_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]"
     }, 
     "serviceName": "null", 
     "role": "install_packages", 
@@ -44,7 +44,7 @@
         "script_type": "PYTHON",
         "repository_version": "2.2.0.1-885",
         "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://repo1/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
-        "package_list": "[{\"name\":\"hadoop_${hdp_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${hdp_version}\"},{\"name\":\"hadoop_${hdp_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]",
+        "package_list": "[{\"name\":\"hadoop_${stack_version}\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_${stack_version}\"},{\"name\":\"hadoop_${stack_version}-libhdfs\"},{\"name\":\"ambari-log4j\"}]",
         "script": "install_packages.py"
     }, 
     "commandId": "14-1",