AMBARI-14690. Configurable system resource values for ambari-agent. (Oliver Szabo via stoader)

ambari-agent/conf/unix/ambari-agent.ini

@@ -30,6 +30,7 @@ tolerate_download_failures=true
 run_as_user=root
 parallel_execution=0
 alert_grace_period=5
+system_resource_overrides=/etc/resource_overrides
 
 [security]
 keysdir=/var/lib/ambari-agent/keys

ambari-agent/conf/windows/ambari-agent.ini

@@ -30,6 +30,7 @@ cache_dir=cache
 tolerate_download_failures=true
 parallel_execution=0
 alert_grace_period=5
+system_resource_overrides=\\etc\\resource_overrides
 
 [security]
 keysdir=keys

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -46,6 +46,7 @@ data_cleanup_max_size_MB = 100
 ping_port=8670
 cache_dir={ps}var{ps}lib{ps}ambari-agent{ps}cache
 parallel_execution=0
+system_resource_overrides={ps}etc{ps}resource_overrides
 
 [services]
 

ambari-agent/src/main/python/ambari_agent/Facter.py

@@ -31,6 +31,8 @@ import subprocess
 from ambari_commons.shell import shellRunner
 import time
 import uuid
+import json
+import glob
 from AmbariConfig import AmbariConfig
 from ambari_commons import OSCheck, OSConst
 from ambari_commons.os_family_impl import OsFamilyImpl
@@ -150,6 +152,47 @@ class Facter(object):
   def getUptimeDays(self):
     return self.getUptimeSeconds() / (60 * 60 * 24)
 
+  def getSystemResourceIfExists(self, systemResources, key, default):
+    if key in systemResources:
+      return systemResources[key]
+    else:
+      return default
+
+  def replaceFacterInfoWithSystemResources(self, systemResources, facterInfo):
+    """
+    Replace facter info with fake system resource data (if there are any).
+    """
+    for key in facterInfo:
+      facterInfo[key] = self.getSystemResourceIfExists(systemResources, key, facterInfo[key])
+    return facterInfo
+
+  def getSystemResourceOverrides(self):
+    """
+    Read all json files from 'system_resource_overrides' directory, and later these values are used as
+    fake system data for hosts. In case of the key-value pairs cannot be loaded use default behaviour.
+    """
+    systemResources = {}
+    if self.config.has_option('agent', 'system_resource_overrides'):
+      systemResourceDir = self.config.get('agent', 'system_resource_overrides', '').strip()
+      if systemResourceDir:
+        if os.path.isdir(systemResourceDir) and os.path.exists(systemResourceDir):
+          try:
+            for filename in glob.glob('%s/*.json' % systemResourceDir):
+              with open(filename) as fp:
+                data = json.loads(fp.read())
+                for (key, value) in data.items():
+                  systemResources[key] = data[key]
+          except:
+            log.warn(
+              "Cannot read values from json files in %s. it won't be used for gathering system resources." % systemResourceDir)
+        else:
+          log.info(
+            "Directory: '%s' does not exist - it won't be used for gathering system resources." % systemResourceDir)
+      else:
+        log.info("'system_resource_dir' is not set - it won't be used for gathering system resources.")
+    return systemResources
+
+
   def facterInfo(self):
     facterInfo = {}
     facterInfo['id'] = self.getId()
@@ -290,8 +333,12 @@ class FacterWindows(Facter):
 
   def facterInfo(self):
     facterInfo = super(FacterWindows, self).facterInfo()
-    facterInfo['swapsize'] = Facter.convertSizeMbToGb(self.getSwapSize())
-    facterInfo['swapfree'] = Facter.convertSizeMbToGb(self.getSwapFree())
+    systemResourceOverrides = self.getSystemResourceOverrides()
+    facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+    facterInfo['swapsize'] = Facter.convertSizeMbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+    facterInfo['swapfree'] = Facter.convertSizeMbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
     return facterInfo
 
 
@@ -468,9 +515,14 @@ class FacterLinux(Facter):
 
   def facterInfo(self):
     facterInfo = super(FacterLinux, self).facterInfo()
+    systemResourceOverrides = self.getSystemResourceOverrides()
+    facterInfo = self.replaceFacterInfoWithSystemResources(systemResourceOverrides, facterInfo)
+
     facterInfo['selinux'] = self.isSeLinux()
-    facterInfo['swapsize'] = Facter.convertSizeKbToGb(self.getSwapSize())
-    facterInfo['swapfree'] = Facter.convertSizeKbToGb(self.getSwapFree())
+    facterInfo['swapsize'] = Facter.convertSizeKbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapsize', self.getSwapSize()))
+    facterInfo['swapfree'] = Facter.convertSizeKbToGb(
+      self.getSystemResourceIfExists(systemResourceOverrides, 'swapfree', self.getSwapFree()))
     return facterInfo
 
 

ambari-agent/src/test/python/ambari_agent/TestHardware.py

@@ -29,6 +29,7 @@ from ambari_agent.Hardware import Hardware
 from ambari_agent.AmbariConfig import AmbariConfig
 from ambari_agent.Facter import Facter, FacterLinux
 from ambari_commons import OSCheck
+from glob import glob
 
 @not_for_platform(PLATFORM_WINDOWS)
 @patch.object(platform,"linux_distribution", new = ('Suse','11','Final'))
@@ -265,6 +266,56 @@ SwapFree:        1598676 kB
     self.assertEquals(result['operatingsystem'], 'some_type_of_os')
     self.assertEquals(result['osfamily'], 'My_new_family')
 
+
+  @patch("os.path.exists")
+  @patch("os.path.isdir")
+  @patch("json.loads")
+  @patch("glob.glob")
+  @patch("__builtin__.open")
+  @patch.object(OSCheck, "get_os_type")
+  @patch.object(OSCheck, "get_os_version")
+  @patch.object(FacterLinux, "resolve_ambari_config")
+  def test_system_resource_overrides(self, resolve_ambari_config, get_os_version_mock, get_os_type_mock,
+                                     open_mock, glob_mock, json_mock, isdir, exists):
+    get_os_type_mock.return_value = "suse"
+    get_os_version_mock.return_value = "11"
+    config = MagicMock()
+    config.get.return_value = '/etc/custom_resource_overrides'
+    config.has_option.return_value = True
+    resolve_ambari_config.return_value = config
+    isdir.return_value = True
+    exists.return_value = True
+    open_mock.return_value.read = "1"
+    file_handle = open_mock.return_value.__enter__.return_value
+    file_handle.read.return_value = '1'
+    glob_mock.side_effect = \
+      [
+        [
+          "/etc/custom_resource_overrides/1.json",
+          "/etc/custom_resource_overrides/2.json"
+          ]
+      ]
+    json_data = json_mock.return_value
+    json_data.items.return_value = [('key', 'value')]
+    json_data.__getitem__.return_value = 'value'
+
+    facter = Facter()
+    facter.config = config
+    result = facter.getSystemResourceOverrides()
+
+    isdir.assert_called_with('/etc/custom_resource_overrides')
+    exists.assert_called_with('/etc/custom_resource_overrides')
+    glob_mock.assert_called_with('/etc/custom_resource_overrides/*.json')
+    self.assertTrue(config.has_option.called)
+    self.assertTrue(config.get.called)
+    self.assertTrue(glob_mock.called)
+    self.assertEquals(2, file_handle.read.call_count)
+    self.assertEquals(2, open_mock.call_count)
+    self.assertEquals(2, json_mock.call_count)
+    self.assertEquals('value', result['key'])
+
+
+
 if __name__ == "__main__":
   unittest.main()