AMBARI-2552. Ambari-agent test coverage. (Vitaly Brodetskyi via smohanty)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1499488 13f79535-47bb-0310-9956-ffa450edef68

ambari-agent/src/main/python/ambari_agent/Hardware.py

@@ -30,7 +30,8 @@ import AmbariConfig
 logger = logging.getLogger()
 
 class Hardware:
-  def __init__(self):
+  def __init__(self, config):
+    self.config = config
     self.hardware = {}
     osdisks = self.osdisks()
     self.hardware['mounts'] = osdisks
@@ -88,9 +89,9 @@ class Hardware:
     pass
   
   def configureEnviron(self, environ):
-    if not AmbariConfig.config.has_option("puppet", "ruby_home"):
+    if not self.config.has_option("puppet", "ruby_home"):
       return environ
-    ruby_home = AmbariConfig.config.get("puppet", "ruby_home")
+    ruby_home = self.config.get("puppet", "ruby_home")
     if os.path.exists(ruby_home):
       """Only update ruby home if the config is configured"""
       path = os.environ["PATH"]
@@ -143,8 +144,8 @@ class Hardware:
     logger.info("Facter info : \n" + pprint.pformat(retDict))
     return retDict  
   
-  def facterInfo(self):   
-    facterHome = AmbariConfig.config.get("puppet", "facter_home")
+  def facterInfo(self):
+    facterHome = self.config.get("puppet", "facter_home")
     facterEnv = os.environ
     logger.info("Using facter home as: " + facterHome)
     facterInfo = {}
@@ -183,7 +184,7 @@ class Hardware:
     return self.hardware
 
 def main(argv=None):
-  hardware = Hardware()
+  hardware = Hardware(AmbariConfig.config)
   print hardware.get()
 
 if __name__ == '__main__':

ambari-agent/src/main/python/ambari_agent/Register.py

@@ -31,7 +31,7 @@ class Register:
   """ Registering with the server. Get the hardware profile and 
   declare success for now """
   def __init__(self, config):
-    self.hardware = Hardware()
+    self.hardware = Hardware(config)
     self.config = config
 
   def build(self, id='-1'):
@@ -61,46 +61,4 @@ class Register:
     version = f.read().strip()
     f.close()
     return version
-
-
-def doExec(vals, key, command, preLF=False):
-  template = "{0}: {1} {2}"
-  try:
-    osStat = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = osStat.communicate()
-    if 0 != osStat.returncode or 0 == len(out.strip()):
-      print template.format(key, "UNAVAILABLE", "")
-    else:
-      if (preLF):
-        print template.format(key, "ok,\n", out.strip())
-      else:
-        print template.format(key, "ok,", out.strip())
-  except:
-    print template.format(key, "UNAVAILABLE", "")
   
-
-# Linux only
-def machineInfo():
-  vals = { }
-  doExec(vals, 'hostname', ["hostname", "-f"])
-  doExec(vals, 'ip', ["hostname", "-i"])
-  doExec(vals, 'cpu', ["sh", "-c", "cat /proc/cpuinfo | grep 'model name' | awk -F': ' '{ print $2; }'"])
-  doExec(vals, 'memory', ["sh", "-c", "cat /proc/meminfo | grep MemTotal | awk -F': ' '{ print $2/1024/1024 \" GB\"; }'"])
-  doExec(vals, 'disks', ["df", "-h"], True)
-  doExec(vals, 'os', ["sh", "-c", "cat /etc/issue.net | head -1"])
-  doExec(vals, 'iptables', ["iptables", "-vnL"], True)
-  doExec(vals, 'selinux', ["sh", "-c", "cat /etc/selinux/config | grep ^SELINUX"])
-
-  rpm_req = { }
-  for REQ in (["yum", "rpm", "openssl", "curl", "wget", "net-snmp", "net-snmp-utils", "ntpd"]):
-   doExec(rpm_req, REQ, ["rpm", "-qa", REQ])
-  vals["required_packages"] = rpm_req
-
-  rpm_opt = { }
-  for OPT in (["ruby", "puppet", "nagios", "ganglia", "passenger", "hadoop"]):
-   doExec(rpm_opt, OPT, ["rpm", "-qa", OPT])
-  vals["optional_packages"] = rpm_opt
-
-  doExec(vals, "yum_repos", ["sh", "-c", "yum -C repolist enabled | egrep \"(AMBARI|HDP)\""], True)
-  # for SUSE-based agents
-  doExec(vals, "zypper_repos", ["sh", "-c", "zypper repos | egrep \"(AMBARI|HDP)\""], True)

ambari-agent/src/test/python/TestHardware.py

@@ -18,12 +18,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+import subprocess, os
 from unittest import TestCase
 from ambari_agent.Hardware import Hardware
+from mock.mock import MagicMock, patch, ANY
+import mock.mock
+from AmbariConfig import AmbariConfig
 
 class TestHardware(TestCase):
   def test_build(self):
-    hardware = Hardware()
+    hardware = Hardware(AmbariConfig().getConfig())
     result = hardware.get()
     osdisks = hardware.osdisks()
     for dev_item in result['mounts']:
@@ -45,4 +49,116 @@ class TestHardware(TestCase):
       self.assertTrue(os_disk_item['size'] > 0)
 
     self.assertTrue(len(result['mounts']) == len(osdisks))
-    
+
+
+  @patch.object(subprocess, "Popen")
+  @patch.object(Hardware, "facterLib")
+  @patch("os.path.exists")
+  def test_facterInfo(self, os_path_exists_mock, hardware_facterLib_mock, subprocess_popen_mock):
+    config = AmbariConfig().getConfig()
+    config.set("puppet", "facter_home", AmbariConfig().getConfig().get("stack", "installprefix"))
+    hardware = Hardware(config)
+    facter = MagicMock()
+    facter.communicate.return_value = ["memoryfree => 1 GB\n memorysize => 25 MB\n memorytotal => 300 KB\n "
+                                        + "physicalprocessorcount => 25\n is_virtual => true\n", "no errors"]
+    facter.returncode = 0
+    os.environ['RUBYLIB'] = AmbariConfig().getConfig().get("stack", "installprefix");
+    subprocess_popen_mock.return_value = facter
+    os_path_exists_mock.return_value = True
+    hardware_facterLib_mock.return_value = "bla bla bla"
+    facterInfo = hardware.facterInfo()
+
+    self.assertEquals(facterInfo['memoryfree'], 1048576L)
+    self.assertEquals(facterInfo['memorysize'], 25600L)
+    self.assertEquals(facterInfo['memorytotal'], 300L)
+    self.assertEquals(facterInfo['physicalprocessorcount'], 25)
+    self.assertTrue(facterInfo['is_virtual'])
+    self.assertEquals(subprocess_popen_mock.call_args[1]['env']['RUBYLIB'],
+                                      AmbariConfig().getConfig().get("stack", "installprefix") + ":" + "bla bla bla")
+
+    facter.communicate.return_value = ["memoryfree => 1 G\n memorysize => 25 M\n memorytotal => 300 K\n "
+                                         +                                         "someinfo => 12 Byte" , "no errors"]
+    facterInfo = hardware.facterInfo()
+    facter.returncode = 1
+    self.assertEquals(facterInfo['memoryfree'], 1048576L)
+    self.assertEquals(facterInfo['memorysize'], 25600L)
+    self.assertEquals(facterInfo['memorytotal'], 300L)
+    self.assertEquals(facterInfo['someinfo'], '12 Byte')
+
+    facter.communicate.return_value = ["memoryfree => 1024 M B\n memorytotal => 1024 Byte" , "no errors"]
+
+    facterInfo = hardware.facterInfo()
+
+    self.assertEquals(facterInfo['memoryfree'], 1L)
+    self.assertEquals(facterInfo['memorytotal'], 1L)
+
+    os_path_exists_mock.return_value = False
+    facterInfo = hardware.facterInfo()
+
+    self.assertEquals(facterInfo, {})
+
+
+  @patch("os.path.exists")
+  def test_facterBin(self, ps_path_exists_mock):
+    hardware = Hardware(AmbariConfig().getConfig())
+    ps_path_exists_mock.return_value = False
+    result = hardware.facterBin("bla bla bla")
+    self.assertEquals(result, "facter")
+
+    ps_path_exists_mock.return_value = True
+    result = hardware.facterBin("bla bla bla")
+    self.assertEquals(result, "bla bla bla/bin/facter")
+
+
+  @patch("os.path.exists")
+  @patch.dict('os.environ', {"PATH": ""})
+  @patch.object(subprocess, "Popen")
+  @patch.object(Hardware, "facterInfo")
+  def test_configureEnviron(self, hrdware_facterinfo_mock, subproc_popen, os_path_exists_mock):
+    config = AmbariConfig().getConfig()
+    config.set("puppet", "ruby_home", AmbariConfig().getConfig().get("stack", "installprefix"))
+    hardware = Hardware(config)
+    os_path_exists_mock.return_value = True
+    result = hardware.configureEnviron({'PATH': ""})
+
+    self.assertEquals(result['PATH'], AmbariConfig().getConfig().get("stack", "installprefix") + "/bin:")
+    self.assertEquals(result['MY_RUBY_HOME'], AmbariConfig().getConfig().get("stack", "installprefix"))
+    config.remove_option("puppet", "ruby_home")
+
+
+  def test_facterLib(self):
+    hardware = Hardware(AmbariConfig().getConfig())
+    facterLib = hardware.facterLib("/home")
+    self.assertEquals(facterLib, "/home/lib/")
+
+
+  def test_extractMountInfo(self):
+    outputLine = "device type size used available percent mountpoint"
+    result = Hardware.extractMountInfo(outputLine)
+
+    self.assertEquals(result['device'], 'device')
+    self.assertEquals(result['type'], 'type')
+    self.assertEquals(result['size'], 'size')
+    self.assertEquals(result['used'], 'used')
+    self.assertEquals(result['available'], 'available')
+    self.assertEquals(result['percent'], 'percent')
+    self.assertEquals(result['mountpoint'], 'mountpoint')
+
+    outputLine = ""
+    result = Hardware.extractMountInfo(outputLine)
+
+    self.assertEquals(result, None)
+
+    outputLine = "device type size used available percent"
+    result = Hardware.extractMountInfo(outputLine)
+
+    self.assertEquals(result, None)
+
+    outputLine = "device type size used available percent mountpoint info"
+    result = Hardware.extractMountInfo(outputLine)
+
+    self.assertEquals(result, None)
+
+
+
+

ambari-agent/src/test/python/TestHostInfo.py

@@ -21,12 +21,15 @@ limitations under the License.
 from unittest import TestCase
 import logging
 import unittest
+import subprocess
 from mock.mock import patch
 from mock.mock import MagicMock
 from mock.mock import create_autospec
 from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
 from ambari_agent.PackagesAnalyzer import PackagesAnalyzer
 from ambari_agent.HostInfo import HostInfo
+from ambari_agent.Hardware import Hardware
+from ambari_agent.AmbariConfig import AmbariConfig
 
 class TestHostInfo(TestCase):
 
@@ -209,7 +212,7 @@ class TestHostInfo(TestCase):
     builtins_open_mock.return_value = [
       "hdfs:x:493:502:Hadoop HDFS:/usr/lib/hadoop:/bin/bash",
       "zookeeper:x:492:502:ZooKeeper:/var/run/zookeeper:/bin/bash"]
-    path_mock.return_value = True
+    path_mock.side_effect = [True, False]
 
     hostInfo = HostInfo()
     results = []
@@ -220,6 +223,9 @@ class TestHostInfo(TestCase):
     self.assertTrue(newlist[1]['name'], "zookeeper")
     self.assertTrue(newlist[0]['homeDir'], "/usr/lib/hadoop")
     self.assertTrue(newlist[1]['homeDir'], "/var/run/zookeeper")
+    self.assertTrue(newlist[0]['status'], "Available")
+    self.assertTrue(newlist[1]['status'], "Invalid home directory")
+
 
   @patch.object(HostInfo, 'osdiskAvailableSpace')
   @patch.object(HostCheckReportFileHandler, 'writeHostCheckFile')
@@ -280,5 +286,194 @@ class TestHostInfo(TestCase):
     self.assertEqual(dict['installedPackages'], [])
     self.assertEqual(1, len(dict['hostHealth']['diskStatus']))
 
+  @patch("os.path.exists")
+  @patch("os.path.islink")
+  @patch("os.path.isdir")
+  @patch("os.path.isfile")
+  def test_dirType(self, os_path_isfile_mock, os_path_isdir_mock, os_path_islink_mock, os_path_exists_mock):
+    host = HostInfo()
+
+    os_path_exists_mock.return_value = False
+    result = host.dirType("/home")
+    self.assertEquals(result, 'not_exist')
+
+    os_path_exists_mock.return_value = True
+    os_path_islink_mock.return_value = True
+    result = host.dirType("/home")
+    self.assertEquals(result, 'sym_link')
+
+    os_path_exists_mock.return_value = True
+    os_path_islink_mock.return_value = False
+    os_path_isdir_mock.return_value = True
+    result = host.dirType("/home")
+    self.assertEquals(result, 'directory')
+
+    os_path_exists_mock.return_value = True
+    os_path_islink_mock.return_value = False
+    os_path_isdir_mock.return_value = False
+    os_path_isfile_mock.return_value = True
+    result = host.dirType("/home")
+    self.assertEquals(result, 'file')
+
+    os_path_exists_mock.return_value = True
+    os_path_islink_mock.return_value = False
+    os_path_isdir_mock.return_value = False
+    os_path_isfile_mock.return_value = False
+    result = host.dirType("/home")
+    self.assertEquals(result, 'unknown')
+
+
+  @patch("os.path.exists")
+  @patch("glob.glob")
+  def test_hadoopVarRunCount(self, glob_glob_mock, os_path_exists_mock):
+    hostInfo = HostInfo()
+
+    os_path_exists_mock.return_value = True
+    glob_glob_mock.return_value = ['pid1','pid2','pid3']
+    result = hostInfo.hadoopVarRunCount()
+    self.assertEquals(result, 3)
+
+    os_path_exists_mock.return_value = False
+    result = hostInfo.hadoopVarRunCount()
+    self.assertEquals(result, 0)
+
+
+  @patch("os.path.exists")
+  @patch("glob.glob")
+  def test_hadoopVarLogCount(self, glob_glob_mock, os_path_exists_mock):
+    hostInfo = HostInfo()
+
+    os_path_exists_mock.return_value = True
+    glob_glob_mock.return_value = ['log1','log2']
+    result = hostInfo.hadoopVarLogCount()
+    self.assertEquals(result, 2)
+
+    os_path_exists_mock.return_value = False
+    result = hostInfo.hadoopVarLogCount()
+    self.assertEquals(result, 0)
+
+
+  @patch("os.listdir", create=True, autospec=True)
+  @patch("__builtin__.open", create=True, autospec=True)
+  @patch("pwd.getpwuid", create=True, autospec=True)
+  def test_javaProcs(self, pwd_getpwuid_mock, buitin_open_mock, os_listdir_mock):
+    hostInfo = HostInfo()
+    openRead = MagicMock()
+    openRead.read.return_value = '/java/;/hadoop/'
+    buitin_open_mock.side_effect = [openRead, ['Uid: 22']]
+    pwuid = MagicMock()
+    pwd_getpwuid_mock.return_value = pwuid
+    pwuid.pw_name = 'user'
+    os_listdir_mock.return_value = ['1']
+    list = []
+    hostInfo.javaProcs(list)
+
+    self.assertEquals(list[0]['command'], '/java/;/hadoop/')
+    self.assertEquals(list[0]['pid'], 1)
+    self.assertTrue(list[0]['hadoop'])
+    self.assertEquals(list[0]['user'], 'user')
+
+
+  @patch("subprocess.Popen")
+  @patch.object(Hardware, 'extractMountInfo')
+  def test_osdiskAvailableSpace(self, extract_mount_info_mock, subproc_popen_mock):
+    hostInfo = HostInfo()
+    p = MagicMock()
+    p.communicate.return_value = ['some']
+    subproc_popen_mock.return_value = p
+    extract_mount_info_mock.return_value = {'info' : 'info'}
+    result = hostInfo.osdiskAvailableSpace('')
+
+    self.assertTrue(result['info'], 'info')
+
+    p.communicate.return_value = ''
+    result = hostInfo.osdiskAvailableSpace('')
+
+    self.assertEquals(result, {})
+
+
+  @patch("subprocess.Popen")
+  def test_checkLiveServices(self, subproc_popen):
+    hostInfo = HostInfo()
+    p = MagicMock()
+    p.returncode = 0
+    p.communicate.return_value = ('', 'err')
+    subproc_popen.return_value = p
+    result = []
+    hostInfo.checkLiveServices(['service1'], result)
+
+    self.assertEquals(result[0]['status'], 'Healthy')
+    self.assertEquals(result[0]['name'], 'service1')
+    self.assertEquals(result[0]['desc'], '')
+
+    p.returncode = 1
+    p.communicate.return_value = ('out', 'err')
+    result = []
+    hostInfo.checkLiveServices(['service1'], result)
+
+    self.assertEquals(result[0]['status'], 'Unhealthy')
+    self.assertEquals(result[0]['name'], 'service1')
+    self.assertEquals(result[0]['desc'], 'out')
+
+    p.communicate.return_value = ('', 'err')
+    result = []
+    hostInfo.checkLiveServices(['service1'], result)
+
+    self.assertEquals(result[0]['status'], 'Unhealthy')
+    self.assertEquals(result[0]['name'], 'service1')
+    self.assertEquals(result[0]['desc'], 'err')
+
+    p.communicate.return_value = ('', 'err', '')
+    result = []
+    hostInfo.checkLiveServices(['service1'], result)
+
+    self.assertEquals(result[0]['status'], 'Unhealthy')
+    self.assertEquals(result[0]['name'], 'service1')
+    self.assertTrue(len(result[0]['desc']) > 0)
+
+
+  @patch("os.path.exists")
+  @patch("os.listdir", create=True, autospec=True)
+  @patch("os.path.islink")
+  @patch("os.path.realpath")
+  def test_etcAlternativesConf(self, os_path_realpath_mock, os_path_islink_mock, os_listdir_mock, os_path_exists_mock):
+    hostInfo = HostInfo()
+    os_path_exists_mock.return_value = False
+    result = hostInfo.etcAlternativesConf('',[])
+
+    self.assertEquals(result, [])
+
+    os_path_exists_mock.return_value = True
+    os_listdir_mock.return_value = ['config1']
+    os_path_islink_mock.return_value = True
+    os_path_realpath_mock.return_value = 'real_path_to_conf'
+    result = []
+    hostInfo.etcAlternativesConf('project',result)
+
+    self.assertEquals(result[0]['name'], 'config1')
+    self.assertEquals(result[0]['target'], 'real_path_to_conf')
+
+
+  @patch("subprocess.Popen")
+  def test_rpmInfo(self, subproc_popen):
+    hostInfo = HostInfo()
+    p = MagicMock()
+    p.returncode = 0
+    p.communicate.side_effect = [('out', 'err'), ('', 'err') , ('out', 'err', 'fail')]
+    subproc_popen.return_value = p
+    rpmList = []
+    hostInfo.rpmInfo(rpmList)
+
+    self.assertEquals(rpmList[0]['version'], 'out')
+    self.assertEquals(rpmList[0]['name'], 'glusterfs')
+    self.assertTrue(rpmList[0]['installed'])
+
+    self.assertEquals(rpmList[1]['name'], 'openssl')
+    self.assertFalse(rpmList[1]['installed'])
+
+    self.assertFalse(rpmList[2]['available'])
+    self.assertEquals(rpmList[2]['name'], 'wget')
+
+
 if __name__ == "__main__":
   unittest.main(verbosity=2)

ambari-agent/src/test/python/TestManifestGenerator.py

@@ -68,6 +68,7 @@ class TestManifestGenerator(TestCase):
   @patch.object(manifestGenerator, 'writeTasks')
   def testGenerateManifest(self, writeTasksMock, writeParamsMock, writeNodesMock, writeImportsMock):
     tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
+    self.parsedJson['roleParams'] = 'role param'
     manifestGenerator.generateManifest(self.parsedJson, tmpFileName, '../../main/puppet/modules', self.config.getConfig())
 
     self.assertTrue(writeParamsMock.called)
@@ -84,3 +85,42 @@ class TestManifestGenerator(TestCase):
     result = manifestGenerator.escape('\'\\')
     self.assertEqual(result, shouldBe)
 
+
+  def test_writeNodes(self):
+    tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
+    tmpFile = file(tmpFileName, 'r+')
+
+    clusterHostInfo = self.parsedJson['clusterHostInfo']
+    clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
+    manifestGenerator.writeNodes(tmpFile, clusterHostInfo)
+    tmpFile.seek(0)
+    print tmpFile.read()
+    tmpFile.close()
+    os.remove(tmpFileName)
+
+
+  def test_writeHostAttributes(self):
+    tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
+    tmpFile = file(tmpFileName, 'r+')
+
+    hostAttributes = {'HostAttr1' : '1', 'HostAttr2' : '2'}
+    manifestGenerator.writeHostAttributes(tmpFile, hostAttributes)
+    tmpFile.seek(0)
+    print tmpFile.read()
+    tmpFile.close()
+    os.remove(tmpFileName)
+
+
+  def test_writeTasks(self):
+    tmpFileName = tempfile.mkstemp(dir=self.dir, text=True)[1]
+    tmpFile = file(tmpFileName, 'r+')
+    roles = [{'role' : 'ZOOKEEPER_SERVER',
+              'cmd' : 'NONE',
+              'roleParams' : {'someRoleParams': '-x'}}]
+    clusterHostInfo = self.parsedJson['clusterHostInfo']
+    clusterHostInfo['zookeeper_hosts'] = ["h1.hortonworks.com", "h2.hortonworks.com"]
+    manifestGenerator.writeTasks(tmpFile, roles, self.config, clusterHostInfo, "h1.hortonworks.com")
+    tmpFile.seek(0)
+    print tmpFile.read()
+    tmpFile.close()
+    os.remove(tmpFileName)