AMBARI-14939. [Ambari tarballs] optionally install required packages (aonishuk)

ambari-agent/pom.xml

@@ -43,10 +43,7 @@
     <jinja.install.dir>/usr/lib/ambari-agent/lib/ambari_jinja2</jinja.install.dir>
     <simplejson.install.dir>/usr/lib/ambari-agent/lib/ambari_simplejson</simplejson.install.dir>
     <lib.dir>/usr/lib/ambari-agent/lib</lib.dir>
-    <python.ver>python &gt;= 2.6</python.ver>
-    <deb.python.ver>python (&gt;= 2.6)</deb.python.ver>
     <deb.architecture>amd64</deb.architecture>
-    <deb.dependency.list>openssl, zlibc, ${deb.python.ver}</deb.dependency.list>
     <ambari.server.module>../ambari-server</ambari.server.module>
     <target.cache.dir>${project.build.directory}/cache/</target.cache.dir>
     <resource.keeper.script>${ambari.server.module}/src/main/python/ambari_server/resourceFilesKeeper.py</resource.keeper.script>
@@ -138,6 +135,23 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>properties-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>read-project-properties</goal>
+            </goals>
+            <configuration>
+              <files>
+                <file>${basedir}/src/main/package/dependencies.properties</file>
+              </files>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>rpm-maven-plugin</artifactId>
@@ -156,10 +170,7 @@
           <group>Development</group>
           <description>Maven Recipe: RPM Package.</description>
           <requires>
-            <require>openssl</require>
-            <require>rpm-python</require>
-            <require>zlib</require>
-            <require>${python.ver}</require>
+            <require>${rpm.dependency.list}</require>
           </requires>
           <postinstallScriptlet>
             <scriptFile>src/main/package/rpm/postinstall.sh</scriptFile>
@@ -355,6 +366,19 @@
                 <resource>
                   <directory>${basedir}/src/main/package/deb/control</directory>
                 </resource>
+                <resource>
+                  <directory>${basedir}/src/main/package</directory>
+                  <includes>
+                    <include>dependencies.properties</include>
+                  </includes>
+                </resource>
+                <resource>
+                  <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory>
+                  <includes>
+                    <include>os_check.py</include>
+                    <include>resources/os_family.json</include>
+                  </includes>
+                </resource>
               </resources>
             </configuration>
           </execution>

ambari-agent/src/main/package/dependencies.properties

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Please make sure the format of the file is one of the following
+# rpm.dependency.list=...
+# rpm.dependency.list.[os_family_name]=...
+# rpm.dependency.list.[os_family_name][starting_from_version]=...
+# deb.dependency.list=...
+# deb.dependency.list.[os_family_name]=...
+# deb.dependency.list.[os_family_name][starting_from_version]=...
+# 
+# Examples:
+# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os
+# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11.
+#
+# Such a format is respected by install_ambari_tarball.py by default, 
+# however should be encouraged manually in pom.xml.
+
+rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6
+deb.dependency.list=openssl, zlibc, python (>= 2.6)

ambari-common/src/main/repo/install_ambari_tarball.py

@@ -17,11 +17,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+import re
 import os
 import sys
 import logging
 import subprocess
 from optparse import OptionParser
+import ConfigParser
 
 USAGE = "Usage: %prog [OPTION]... URL"
 DESCRIPTION = "URL should point to full tar.gz location e.g.: https://public-repo-1.hortonworks.com/something/ambari-server.tar.gz"
@@ -32,9 +34,16 @@ PREINST_SCRIPT = "preinst"
 PRERM_SCRIPT = "prerm"
 POSTINST_SCRIPT = "postinst"
 POSTRM_SCRIPT = "postrm"
+OS_CHECK = "os_check.py"
+OS_PACKAGE_DEPENDENCIES = "dependencies.properties"
+OS_FAMILY_DESCRIPTION = "resources/os_family.json"
+RPM_DEPENDENCIES_PROPERTY =  "rpm.dependency.list"
+DEB_DEPENDENCIES_PROPERTY =  "deb.dependency.list"
 
-ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER"
+FILES_TO_DOWNLOAD = [PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT, OS_CHECK, OS_FAMILY_DESCRIPTION, OS_PACKAGE_DEPENDENCIES]
 
+ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER"
+          
 class Utils:
   verbose = False
   @staticmethod
@@ -65,26 +74,72 @@ class Utils:
       
     return out
   
+  @staticmethod
+  def install_package(name):
+    from os_check import OSCheck
+    
+    logger.info("Checking for existance of {0} dependency package".format(name))
+    is_rpm = not OSCheck.is_ubuntu_family()
+    
+    if is_rpm:
+      is_installed_cmd = ['rpm', '-q'] + [name]
+      install_cmd = ['sudo', 'yum', '-y', 'install'] + [name]
+    else:
+      is_installed_cmd = ['dpkg', '-s'] + [name]
+      install_cmd = ['sudo', 'apt-get', '-y', 'install'] + [name]
+      
+    try:
+      Utils.os_call(is_installed_cmd, logoutput=False)
+      logger.info("Package {0} is already installed. Skipping installation.".format(name))
+    except OsCallFailure:
+      logger.info("Package {0} is not installed. Installing it...".format(name))
+      Utils.os_call(install_cmd)
+    
+  
+class FakePropertiesHeader(object):
+  """
+  Hacky class to parse properties file without sections.
+  see http://stackoverflow.com/questions/2819696/module-to-use-when-parsing-properties-file-in-python/2819788#2819788
+  """
+  FAKE_SECTION_NAME = 'section'
+  def __init__(self, fp):
+    self.fp = fp
+    self.sechead = '[{0}]\n'.format(FakePropertiesHeader.FAKE_SECTION_NAME)
+  def readline(self):
+    if self.sechead:
+      try: 
+        return self.sechead
+      finally: 
+        self.sechead = None
+    else: 
+      return self.fp.readline()
+  
 class OsCallFailure(RuntimeError):
   pass
 
 class Installer:
-  def __init__(self, archive_url, root_folder, verbose):
+  def __init__(self, archive_url, root_folder, verbose, skip_dependencies):
     splited_url = archive_url.split('/')
     self.archive_name = splited_url[-1]
     self.base_url = '/'.join(splited_url[0:-1])
     self.root_folder = root_folder
     self.verbose = verbose
+    self.skip_dependencies = skip_dependencies
     
-  def download_files(self):
-    for name in [ self.archive_name, PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT]: 
+  def download_files(self, files_list):
+    for name in files_list:
+      dirname = os.path.dirname(name)
+      if dirname:
+        Utils.os_call(["mkdir", "-p", dirname])
+        
       url = "{0}/{1}".format(self.base_url, name)
       logger.info("Downloading {0}".format(url))
       Utils.os_call(["wget", "-O", name, url])
     
   def run(self):
-    self.download_files()
+    self.download_files([self.archive_name] +FILES_TO_DOWNLOAD) # [self.archive_name] +
     
+    self.check_dependencies()
     self.run_script(PRERM_SCRIPT, ["remove"]) # in case we are upgrading
     self.run_script(POSTRM_SCRIPT, ["remove"]) # in case we are upgrading
     
@@ -92,6 +147,54 @@ class Installer:
     self.extract_archive()
     self.run_script(POSTINST_SCRIPT, ["configure"])
     
+  def check_dependencies(self):
+    from os_check import OSCheck
+    
+    os_family = OSCheck.get_os_family()
+    os_version = OSCheck.get_os_major_version()
+    
+    is_rpm = not OSCheck.is_ubuntu_family()
+    property_prefix = RPM_DEPENDENCIES_PROPERTY if is_rpm else DEB_DEPENDENCIES_PROPERTY
+    
+    cp = ConfigParser.SafeConfigParser()
+    with open(OS_PACKAGE_DEPENDENCIES) as fp:
+      cp.readfp(FakePropertiesHeader(fp))
+      
+    properties = dict(cp.items(FakePropertiesHeader.FAKE_SECTION_NAME))
+    
+    packages_string = None
+    postfixes = [os_family+str(ver) for ver in range(int(os_version),0,-1)+['']]+['']
+    for postfix in postfixes:
+      property_name = property_prefix + postfix
+      if property_name in properties:
+        packages_string = properties[property_name]
+        break
+      
+    if packages_string is None:
+      err_msg = "No os dependencies found. "
+      if self.skip_dependencies:
+        logger.warn(err_msg)
+      else:
+        raise Exception(err_msg)
+    
+    packages_string = re.sub('Requires\s*:','',packages_string)
+    packages_string = re.sub('\\\\n','',packages_string)
+    packages_string = re.sub('\s','',packages_string)
+    packages_string = re.sub('[()]','',packages_string)
+    
+    if self.skip_dependencies:
+      var = raw_input("Please confirm you have the following packages installed {0} (y/n): ".format(packages_string))
+      if var.lower() != "y" and var.lower() != "yes":
+        raise Exception("User canceled the installation.")
+      return
+    
+    pacakges = packages_string.split(',')
+    
+    for package in pacakges:
+      split_parts = re.split('[><=]', package)
+      package_name = split_parts[0]
+      Utils.install_package(package_name)
+    
   def run_script(self, script_name, args):
     bash_args = []
     if self.verbose:
@@ -112,6 +215,8 @@ class Runner:
                       help="sets output level to more detailed")
     parser.add_option("-r", "--root-folder", dest="root_folder", default="/",
                       help="root folder to install Ambari to. E.g.: /opt")
+    parser.add_option("-d", "--dependencies-skip", dest="skip_dependencies", action="store_true",
+                  help="the script won't install the package dependencies. Please make sure to install them manually.")
     
     (self.options, args) = parser.parse_args()
     
@@ -138,7 +243,7 @@ class Runner:
     Utils.verbose = self.options.verbose
     
     # TODO: check if ends with tar.gz?
-    targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose)
+    targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose, self.options.skip_dependencies)
     targz_installer.run()
       
 if __name__ == '__main__':

ambari-server/pom.xml

@@ -26,10 +26,7 @@
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <python.ver>python &gt;= 2.6</python.ver>
     <!-- On centos the python xml's are inside python package -->
-    <python.xml.package>${python.ver}</python.xml.package>
-    <deb.python.ver>python (&gt;= 2.6)</deb.python.ver>
     <deb.architecture>amd64</deb.architecture>
-    <deb.dependency.list>openssl, postgresql (&gt;= 8.1), ${deb.python.ver}, curl</deb.dependency.list>
     <custom.tests>false</custom.tests>
     <hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6>
     <hdpLatestUrl>http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json</hdpLatestUrl>
@@ -195,6 +192,19 @@
                 <resource>
                   <directory>${basedir}/src/main/package/deb/control</directory>
                 </resource>
+                <resource>
+                  <directory>${basedir}/src/main/package</directory>
+                  <includes>
+                    <include>dependencies.properties</include>
+                  </includes>
+                </resource>
+                <resource>
+                  <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory>
+                  <includes>
+                    <include>os_check.py</include>
+                    <include>resources/os_family.json</include>
+                  </includes>
+                </resource>
               </resources>
             </configuration>
           </execution>
@@ -307,6 +317,23 @@
           </dependency>
         </dependencies>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>properties-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>read-project-properties</goal>
+            </goals>
+            <configuration>
+              <files>
+                <file>${basedir}/src/main/package/dependencies.properties</file>
+              </files>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>rpm-maven-plugin</artifactId>
@@ -326,10 +353,7 @@
           <description>Maven Recipe: RPM Package.</description>
           <autoRequires>no</autoRequires>
           <requires>
-            <require>postgresql-server &gt;= 8.1</require>
-            <require>openssl</require>
-            <require>${python.ver}</require>
-            <require>${python.xml.package}</require>
+            <require>${rpm.dependency.list}</require>
           </requires>
           <postinstallScriptlet>
             <scriptFile>src/main/package/rpm/postinstall.sh</scriptFile>
@@ -738,7 +762,7 @@
     <profile>
       <id>suse11</id>
       <properties>
-        <python.xml.package>python-xml</python.xml.package>
+        <rpm.dependency.list>${rpm.dependency.list.suse}</rpm.dependency.list>
       </properties>
     </profile>
     <profile>

ambari-server/src/main/package/dependencies.properties

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Please make sure the format of the file is one of the following
+# rpm.dependency.list=...
+# rpm.dependency.list.[os_family_name]=...
+# rpm.dependency.list.[os_family_name][starting_from_version]=...
+# deb.dependency.list=...
+# deb.dependency.list.[os_family_name]=...
+# deb.dependency.list.[os_family_name][starting_from_version]=...
+# 
+# Examples:
+# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os
+# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11.
+#
+# Such a format is respected by install_ambari_tarball.py by default, 
+# however should be encouraged manually in pom.xml.
+
+rpm.dependency.list=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python >= 2.6
+rpm.dependency.list.suse=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python-xml,\nRequires: python >= 2.6
+deb.dependency.list=openssl, postgresql (>= 8.1), python (>= 2.6), curl