AMBARI-4523. Host registering failure from primary/agent os checking on centos6 (aonishuk)

ambari-agent/pom.xml

@@ -41,6 +41,7 @@
     <puppet.tar>http://downloads.puppetlabs.com/puppet/puppet-2.7.9.tar.gz</puppet.tar>
     <agent.install.dir>/usr/lib/python2.6/site-packages/ambari_agent</agent.install.dir>
     <resmgmt.install.dir>/usr/lib/python2.6/site-packages/resource_management</resmgmt.install.dir>
+    <common_functions.install.dir>/usr/lib/python2.6/site-packages/common_functions</common_functions.install.dir>
     <jinja.install.dir>/usr/lib/python2.6/site-packages/jinja2</jinja.install.dir>
     <ruby.tar>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.15/repos/centos6/ruby-1.8.7-p370.tar.gz</ruby.tar>
     <lib.dir>/usr/lib/ambari-agent/lib</lib.dir>
@@ -131,7 +132,7 @@
                 <argument>unitTests.py</argument>
               </arguments>
               <environmentVariables>
-                <PYTHONPATH>${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/src/main/python/ambari_agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/ambari_agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python:${project.basedir}/../ambari-agent/src/main/puppet/modules/hdp-hadoop/files:$PYTHONPATH</PYTHONPATH>
+                <PYTHONPATH>${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python/common_functions:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/src/main/python/ambari_agent:${project.basedir}/src/main/python/resource_management:${project.basedir}/src/test/python/ambari_agent:${project.basedir}/src/test/python/resource_management:${project.basedir}/src/main/python:${project.basedir}/../ambari-agent/src/main/puppet/modules/hdp-hadoop/files:$PYTHONPATH</PYTHONPATH>
               </environmentVariables>
               <skip>${skipTests}</skip>
             </configuration>
@@ -235,6 +236,14 @@
                 </source>
               </sources>
             </mapping>
+            <mapping>
+              <directory>${common_functions.install.dir}</directory>
+              <sources>
+                <source>
+                  <location>${project.basedir}/../ambari-common/src/main/python/common_functions</location>
+                </source>
+              </sources>
+            </mapping>
             <mapping>
               <directory>${lib.dir}</directory>
               <filemode>755</filemode>

ambari-agent/src/main/python/ambari_agent/Facter.py

@@ -30,7 +30,7 @@ import subprocess
 
 import time
 import uuid
-import hostname
+from common_functions import OSCheck
 
 log = logging.getLogger()
 
@@ -99,7 +99,7 @@ class Facter():
 
   # Returns the short hostname
   def getHostname(self):
-    return hostname.hostname()
+    return self.getFqdn().split('.', 1)[0]
 
   # Returns the CPU hardware architecture
   def getArchitecture(self):
@@ -109,29 +109,13 @@ class Facter():
     else:
       return result
 
-  # Returns the name of the OS
+  # Returns the full name of the OS
   def getOperatingSystem(self):
-    dist = platform.linux_distribution()
-    operatingSystem = dist[0].lower()
-
-    if os.path.exists('/etc/oracle-release'):
-      return 'OracleLinux'
-    elif operatingSystem.startswith('suse linux enterprise server'):
-      return 'SLES'
-    elif operatingSystem.startswith('red hat enterprise linux server'):
-      return 'RedHat'
-    elif operatingSystem != '':
-      return operatingSystem
-    else:
-      return 'OS NOT SUPPORTED'
+    return OSCheck().get_os_type()
 
   # Returns the OS version
   def getOperatingSystemRelease(self):
-    dist = platform.linux_distribution()
-    if dist[1] != '':
-      return dist[1]
-    else:
-      return 'OS NOT SUPPORTED'
+    return OSCheck().get_os_version()
 
   # Returns the OS TimeZone
   def getTimeZone(self):
@@ -167,20 +151,7 @@ class Facter():
   # Returns the operating system family
 
   def getOsFamily(self):
-    os_family = self.getOperatingSystem().lower()
-    if os_family in ['redhat', 'fedora', 'centos', 'oraclelinux', 'ascendos',
-                     'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux',
-                     'slc', 'scientific', 'psbm']:
-      os_family = 'RedHat'
-    elif os_family in ['ubuntu', 'debian']:
-      os_family = 'Debian'
-    elif os_family in ['sles', 'sled', 'opensuse', 'suse']:
-      os_family = 'Suse'
-    elif os_family == '':
-      os_family = 'OS NOT SUPPORTED'
-    else:
-      os_family = self.getOperatingSystem()
-    return os_family
+    return OSCheck().get_os_family()
 
   def isSeLinux(self):
 

ambari-agent/src/main/python/ambari_agent/HostCleanup.py

@@ -31,6 +31,7 @@ import sys
 import datetime
 import AmbariConfig
 from pwd import getpwnam
+from common_functions import OSCheck
 
 logger = logging.getLogger()
 configFile = "/etc/ambari-agent/conf/ambari-agent.ini"
@@ -420,10 +421,7 @@ class HostCleanup:
     return os.getuid() == 0
 
   def get_os_type(self):
-    os_info = platform.linux_distribution(
-      None, None, None, ['SuSE', 'redhat'], 0
-    )
-    return os_info[0].lower()
+    return OSCheck().get_os_family()
 
 
   # Run command as sudoer by default, if root no issues

ambari-agent/src/main/python/ambari_agent/HostInfo.py

@@ -31,6 +31,7 @@ import platform
 from PackagesAnalyzer import PackagesAnalyzer
 from HostCheckReportFileHandler import HostCheckReportFileHandler
 from Hardware import Hardware
+from common_functions import OSCheck
 
 logger = logging.getLogger()
 
@@ -102,7 +103,7 @@ class HostInfo:
   TIMEOUT_SECONDS = 60
   RESULT_UNAVAILABLE = "unable_to_determine"
 
-  OS_NAME = platform.dist()[0].lower()
+  OS_NAME = OSCheck().get_os_family()
   OS_UBUNTU = 'ubuntu'
   # service cmd
   SERVICE_CMD = "/sbin/service"
@@ -163,9 +164,7 @@ class HostInfo:
         etcResults.append(result)
 
   def get_os_type(self):
-    os_info = platform.linux_distribution(None, None, None, ['SuSE',
-                                          'redhat' ], 0)
-    return os_info[0].lower()
+    return OSCheck().get_os_family()
 
   def checkLiveServices(self, services, result):
     osType = self.get_os_type()
@@ -283,12 +282,6 @@ class HostInfo:
      return self.current_umask
 
 
-  def get_os_type(self):
-    os_info = platform.linux_distribution(
-      None, None, None, ['SuSE', 'redhat' ], 0
-    )
-    return os_info[0].lower()
-
   def checkIptables(self):
     iptablesIsRunning = False
     try:

ambari-agent/src/main/python/resource_management/core/system.py

@@ -28,6 +28,7 @@ import platform
 from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 from functools import wraps
+from common_functions import OSCheck
 
 def lazy_property(undecorated):
   name = '_' + undecorated.__name__
@@ -65,13 +66,9 @@ class System(object):
     Example return value:
     "6.3" for "Centos 6.3"
     
-    In case cannot detect raises 'unknown'
+    In case cannot detect --> Fail
     """
-    dist = platform.linux_distribution()
-    if dist[1] != '':
-      return dist[1]
-    else:
-      return 'unknown'
+    return OSCheck().get_os_version()
     
   @lazy_property
   def os_type(self):
@@ -83,22 +80,7 @@ class System(object):
     
     In case cannot detect raises exception.
     """
-    dist = platform.linux_distribution()
-    operatingSystem = dist[0].lower()
-
-    # special cases
-    if os.path.exists('/etc/oracle-release'):
-      return 'oraclelinux'
-    elif operatingSystem.startswith('suse linux enterprise server'):
-      return 'sles'
-    elif operatingSystem.startswith('red hat enterprise linux server'):
-      return 'redhat'
-    
-    # in general
-    if operatingSystem:
-      return operatingSystem
-    else:
-      raise Fail("Cannot detect os type")
+    return OSCheck().get_os_type()
     
   @lazy_property
   def os_family(self):
@@ -108,19 +90,7 @@ class System(object):
     
     In case cannot detect raises exception
     """
-    os_type = self.os_type
-    if os_type in ['redhat', 'centos', 'fedora', 'oraclelinux', 'ascendos',
-                     'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux',
-                     'slc', 'scientific', 'psbm']:
-      os_family = 'redhat'
-    elif os_type in ['ubuntu', 'debian']:
-      os_family = 'debian'
-    elif os_type in ['sles', 'sled', 'opensuse', 'suse']:
-      os_family = 'suse'
-    else:
-      raise Fail("Cannot detect os family for os: {0}".format(os_type))
-      
-    return os_family
+    return OSCheck().get_os_family()
 
   @lazy_property
   def ec2(self):

ambari-agent/src/test/python/ambari_agent/TestHostInfo.py

@@ -30,7 +30,9 @@ from ambari_agent.PackagesAnalyzer import PackagesAnalyzer
 from ambari_agent.HostInfo import HostInfo
 from ambari_agent.Hardware import Hardware
 from ambari_agent.AmbariConfig import AmbariConfig
+from resource_management.core.system import System
 
+@patch.object(System, "os_family", new = 'redhat')
 class TestHostInfo(TestCase):
 
   logger = logging.getLogger()

ambari-common/src/main/python/common_functions/__init__.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from common_functions.os_check import OSCheck
+
+__all__ = [
+  'OSCheck',
+]

ambari-common/src/main/python/common_functions/os_check.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import sys
+import platform
+
+
+class OSCheck(object):
+  def __init__(self):
+    pass
+
+  def get_os_type(self):
+    """
+    Return values:
+    redhat, fedora, centos, oraclelinux, ascendos,
+    amazon, xenserver, oel, ovs, cloudlinux, slc, scientific, psbm,
+    ubuntu, debian, sles, sled, opensuse, suse ... and others
+
+    In case cannot detect - exit.
+    """
+    # Read content from /etc/*-release file
+    # Full release name
+    dist = platform.linux_distribution()
+    operatingSystem = dist[0].lower()
+
+    # special cases
+    if os.path.exists('/etc/oracle-release'):
+      return 'oraclelinux'
+    elif operatingSystem.startswith('suse linux enterprise server'):
+      return 'sles'
+    elif operatingSystem.startswith('red hat enterprise linux server'):
+      return 'redhat'
+
+    if operatingSystem != '':
+      return operatingSystem
+    else:
+      print "Cannot detect os type. Exiting..."
+      sys.exit(1)
+
+
+  def get_os_family(self):
+    """
+    Return values:
+    redhat, debian, suse ... and others
+
+    In case cannot detect raises exception( from self.get_operating_system_type() ).
+    """
+    os_family = self.get_os_type()
+    if os_family in ['redhat', 'fedora', 'centos', 'oraclelinux', 'ascendos',
+                     'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux',
+                     'slc', 'scientific', 'psbm', 'centos linux']:
+      os_family = 'RedHat'
+    elif os_family in ['ubuntu', 'debian']:
+      os_family = 'Debian'
+    elif os_family in ['sles', 'sled', 'opensuse', 'suse']:
+      os_family = 'Suse'
+    #else:  os_family = self.get_os_type()
+    return os_family.lower()
+
+
+  def get_os_version(self):
+    """
+    Returns the OS version
+
+    In case cannot detect raises exception.
+    """
+    # Read content from /etc/*-release file
+    # Full release name
+    dist = platform.linux_distribution()
+    dist = dist[1]
+
+    if dist:
+      return dist
+    else:
+      print "Cannot detect os version. Exiting..."
+      sys.exit(1)
+
+  def get_os_major_version(self):
+    """
+    Returns the main OS version like
+    Centos 6.5 --> 6
+    RedHat 1.2.3 --> 1
+    """
+    return self.get_os_version().split('.')[0]
+
+  def get_os_release_name(self):
+    """
+    Returns the OS release name
+
+    In case cannot detect raises exception.
+    """
+    dist = platform.linux_distribution()
+    dist = dist[2].lower()
+
+    if dist:
+      return dist
+    else:
+      print "Cannot detect os release name. Exiting..."
+      sys.exit(1)
+
+
+def main(argv=None):
+  # Same logic that was in "os_type_check.sh"
+  if len(sys.argv) != 2:
+    print "Usage: <cluster_os>"
+    sys.exit(2)
+    pass
+
+  cluster_os = sys.argv[1]
+  current_os = OSCheck().get_os_family() + OSCheck().get_os_major_version()
+
+  # If agent/server have the same {"family","main_version"} - then ok.
+  print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % (
+    cluster_os, current_os)
+  if current_os == cluster_os:
+    sys.exit(0)
+  else:
+    print "Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host."
+    sys.exit(1)
+
+
+if __name__ == "__main__":
+  main()

ambari-server/pom.xml

@@ -31,6 +31,7 @@
 
     <custom.tests>false</custom.tests>
     <hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6>
+    <common_functions.install.dir>/usr/lib/python2.6/site-packages/common_functions</common_functions.install.dir>
     <ambari-web-dir>${basedir}/../ambari-web/public</ambari-web-dir>
   </properties>
   <build>
@@ -227,7 +228,17 @@
                 </source>
               </sources>
             </mapping>
-            <mapping>
+              <mapping>
+                  <directory>${common_functions.install.dir}</directory>
+                  <sources>
+                      <source>
+                          <location>
+                              ${project.basedir}/../ambari-common/src/main/python/common_functions
+                          </location>
+                      </source>
+                  </sources>
+              </mapping>
+              <mapping>
               <directory>/usr/sbin</directory>
               <filemode>755</filemode>
               <username>root</username>
@@ -390,7 +401,7 @@
                   <location>src/main/python/setupAgent.py</location>
                 </source>
                 <source>
-                  <location>src/main/python/os_type_check.sh</location>
+                  <location>${project.basedir}/../ambari-common/src/main/python/common_functions/os_check.py</location>
                 </source>
               </sources>
             </mapping>
@@ -731,11 +742,11 @@
             </mapper>
           </data>
           <data>
-            <src>src/main/python/os_type_check.sh</src>
+            <src>${project.basedir}/../ambari-common/src/main/python/common_functions/os_check.py</src>
             <type>file</type>
             <mapper>
               <type>perm</type>
-              <prefix>/usr/lib/python2.6/site-packages/ambari_server</prefix>
+              <prefix>/usr/lib/python2.6/site-packages/common_functions</prefix>
               <user>root</user>
               <group>root</group>
               <filemode>755</filemode>
@@ -801,7 +812,7 @@
                 <argument>${custom.tests}</argument>
               </arguments>
               <environmentVariables>
-                <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH>
+                  <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-common/src/main/python/common_functions:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH>
               </environmentVariables>
               <skip>${skipTests}</skip>
             </configuration>

ambari-server/src/main/python/ambari-server.py

@@ -41,6 +41,7 @@ import random
 import pwd
 from ambari_server.resourceFilesKeeper import ResourceFilesKeeper, KeeperException
 import json
+from common_functions import OSCheck
 
 # debug settings
 VERBOSE = False
@@ -48,8 +49,8 @@ SILENT = False
 SERVER_START_DEBUG = False
 
 # OS info
-OS, OS_VERSION, _ = platform.linux_distribution() 
-OS = OS.lower().strip()
+OS_VERSION = OSCheck().get_os_major_version()
+OS = OSCheck().get_os_family()
 OS_UBUNTU = 'ubuntu'
 OS_FEDORA = 'fedora'
 OS_OPENSUSE = 'opensuse'
@@ -1962,13 +1963,8 @@ def configure_os_settings():
     print_error_msg ("Non-Linux systems are not supported")
     return -1
 
-  os_info = platform.linux_distribution(
-    None, None, None, ['SuSE', 'redhat' ], 0
-  )
-  os_name = os_info[0].lower()
-  if os_name == 'suse':
-    os_name = 'sles'
-  os_version = os_info[1].split('.', 1)[0]
+  os_name = OSCheck().get_os_family()
+  os_version = OS_VERSION
   master_os_type = os_name + os_version    
   write_property(OS_TYPE_PROPERTY, master_os_type)
   return 0

ambari-server/src/main/python/bootstrap.py

@@ -135,7 +135,7 @@ class SSH:
 class Bootstrap(threading.Thread):
   """ Bootstrap the agent on a separate host"""
   TEMP_FOLDER = "/tmp"
-  OS_CHECK_SCRIPT_FILENAME = "os_type_check.sh"
+  OS_CHECK_SCRIPT_FILENAME = "os_check.py"
   AMBARI_REPO_FILENAME = "ambari.repo"
   SETUP_SCRIPT_FILENAME = "setupAgent.py"
   PASSWORD_FILENAME = "host_pass"

ambari-server/src/main/python/os_type_check.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-C5="centos5"
-C6="centos6"
-RH5="redhat5"
-RH6="redhat6"
-SLES11="sles11"
-SUSE11="suse11"
-OL5="oraclelinux5"
-OL6="oraclelinux6"
-
-cluster_os=$1
-current_os="N/A"
-
-pattern='[^[:digit:]]*'
-
-# OS type detection
-if [ -f  "/etc/centos-release" ] 
-then
-  grep -sqiE "^centos ${pattern}6" /etc/centos-release && current_os=$C6
-  grep -sqiE "^centos ${pattern}5" /etc/centos-release && current_os=$C5
-elif [ -f  "/etc/oracle-release" ] || [ -f  "/etc/ovs-release" ] || [ -f  "/etc/enterprise-release" ] 
-then
-  grep -sqE "${pattern}6" /etc/oracle-release || grep -sqE "${pattern}6" /etc/ovs-release || \
-    grep -sqE "${pattern}6" /etc/enterprise-release && current_os=$OL6
-  grep -sqE "${pattern}5" /etc/oracle-release || grep -sqE "${pattern}5" /etc/ovs-release || \
-      grep -sqE "${pattern}5" /etc/enterprise-release && current_os=$OL5
-elif [ -f  "/etc/redhat-release" ]
-then
-  grep -sqiE "^centos ${pattern}5" /etc/redhat-release && current_os=$C5
-  grep -sqiE "^centos ${pattern}6" /etc/redhat-release && current_os=$C6
-  grep -sqE "^Red Hat Enterprise Linux ${pattern}6" /etc/redhat-release && current_os=$RH6
-  grep -sqE "^Red Hat Enterprise Linux ${pattern}5" /etc/redhat-release && current_os=$RH5
-elif [ -f  "/etc/SuSE-release" ]
-then
-  grep -sqE "${pattern}11" /etc/SuSE-release && current_os=$SUSE11
-  grep -sqi "SUSE LINUX Enterprise Server" /etc/SuSE-release && [ "$current_os" = "$SUSE11" ] && current_os=$SLES11
-fi
-
-
-echo "Cluster primary OS type is $cluster_os and local OS type is $current_os"
-
-# Compatibility check
-res=1
-case "$cluster_os" in
-
-  $C5|$RH5|$OL5)
-    [ "$current_os" = "$C5" ] || [ "$current_os" = "$RH5" ] || [ "$current_os" = "$OL5" ] && res=0 
-  ;;
-
-  $C6|$RH6|$OL6)
-    [ "$current_os" = "$C6" ] || [ "$current_os" = "$RH6" ] || [ "$current_os" = "$OL6" ] && res=0 
-  ;;
-
-  $SUSE11|$SLES11)
-    [ "$current_os" = "$SUSE11" ] || [ "$current_os" = "$SLES11" ] && res=0 
-  ;;
-
-  *)
-    res=1
-  ;;
-esac
-
-[[ $res -ne 0 ]] && echo "Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host."
-
-exit $res

ambari-server/src/test/python/TestBootstrap.py

@@ -317,7 +317,7 @@ class TestBootstrap(TestCase):
                                None, "8440")
     bootstrap_obj = Bootstrap("hostname", shared_state)
     ocs = bootstrap_obj.getOsCheckScript()
-    self.assertEquals(ocs, "scriptDir/os_type_check.sh")
+    self.assertEquals(ocs, "scriptDir/os_check.py")
 
 
   @patch.object(Bootstrap, "getRemoteName")
@@ -326,7 +326,7 @@ class TestBootstrap(TestCase):
                                "setupAgentFile", "ambariServer", "centos6",
                                None, "8440")
     bootstrap_obj = Bootstrap("hostname", shared_state)
-    v = "/tmp/os_type_check1374259902.sh"
+    v = "/tmp/os_check1374259902.py"
     getRemoteName_mock.return_value = v
     ocs = bootstrap_obj.getOsCheckScriptRemoteLocation()
     self.assertEquals(ocs, v)