AMBARI-12502 ambari-metrics-collector filling up root-fs causes daemon failures (dsen)

ambari-common/src/main/python/resource_management/libraries/functions/file_system.py

@@ -66,7 +66,7 @@ def get_mount_point_for_dir(dir):
       if dir.startswith(m['mount_point']):
         if best_mount_found is None:
           best_mount_found = m["mount_point"]
-        elif best_mount_found.count(os.path.sep) < m["mount_point"].count(os.path.sep):
+        elif best_mount_found.count(os.path.sep) < os.path.join(m["mount_point"]).count(os.path.sep):
           best_mount_found = m["mount_point"]
 
   Logger.info("Mount point for directory %s is %s" % (str(dir), str(best_mount_found)))

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -155,10 +155,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
 
   def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
-    putAmsEnvProperty = self.putProperty(configurations, "ams-env")
-    putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site")
-    putTimelineServiceProperty = self.putProperty(configurations, "ams-site")
-    putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env")
+    putAmsEnvProperty = self.putProperty(configurations, "ams-env", services)
+    putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site", services)
+    putTimelineServiceProperty = self.putProperty(configurations, "ams-site", services)
+    putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env", services)
 
     amsCollectorHosts = self.getComponentHostNames(services, "AMBARI_METRICS", "METRICS_COLLECTOR")
     putHbaseEnvProperty("hbase_regionserver_heapsize", "1024m")
@@ -207,6 +207,29 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         putAmsEnvProperty("metrics_collector_heapsize", "512m")
         putAmsHbaseSiteProperty("hbase_master_xmn_size", "128m")
       pass
+
+    rootDir = "file:///var/lib/ambari-metrics-collector/hbase"
+    tmpDir = "/var/lib/ambari-metrics-collector/hbase-tmp"
+    mountpoint = "/"
+    if "ams-hbase-site" in services["configurations"]:
+      if "hbase.rootdir" in services["configurations"]["ams-hbase-site"]["properties"]:
+        rootDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.rootdir"]
+      if "hbase.tmp.dir" in services["configurations"]["ams-hbase-site"]["properties"]:
+        tmpDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.tmp.dir"]
+    for collectorHostName in amsCollectorHosts:
+      for host in hosts["items"]:
+        if host["Hosts"]["host_name"] == collectorHostName:
+          mountpoint = self.getProperMountPoint(host["Hosts"])
+          break
+    if not rootDir.startswith("hdfs://") :
+      rootDir = re.sub("^file:///|/", "", rootDir, count=1)
+      rootDir = "file://" + os.path.join(mountpoint, rootDir)
+    tmpDir = re.sub("^file:///|/", "", tmpDir, count=1)
+    tmpDir = os.path.join(mountpoint, tmpDir)
+
+    putAmsHbaseSiteProperty("hbase.rootdir", rootDir)
+    putAmsHbaseSiteProperty("hbase.tmp.dir", tmpDir)
+
     pass
 
 
@@ -433,6 +456,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       for host in hosts["items"]:
         if host["Hosts"]["host_name"] == collectorHostName:
           validationItems.extend([ {"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
+          validationItems.extend([ {"config-name": 'hbase.rootdir', "item": self.validatorNotRootFs(properties, 'hbase.rootdir', host["Hosts"])}])
+          validationItems.extend([ {"config-name": 'hbase.tmp.dir', "item": self.validatorNotRootFs(properties, 'hbase.tmp.dir', host["Hosts"])}])
           break
 
     rootdir_item = None
@@ -524,12 +549,48 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
   def getErrorItem(self, message):
     return {"level": "ERROR", "message": message}
 
+  def getProperMountPoint(self, hostInfo):
+
+    # '/etc/resolv.conf', '/etc/hostname', '/etc/hosts' are docker specific mount points
+    undesirableMountPoints = ["/", "/home", "/etc/resolv.conf", "/etc/hosts",
+                              "/etc/hostname"]
+    undesirableFsTypes = ["devtmpfs", "tmpfs", "vboxsf", "CDFS"]
+    if hostInfo and "disk_info" in hostInfo:
+      mountPoints = {}
+      for mountpoint in hostInfo["disk_info"]:
+        if not (mountpoint["mountpoint"] in undesirableMountPoints or
+                mountpoint["mountpoint"].startswith(("/boot", "/mnt")) or
+                mountpoint["type"] in undesirableFsTypes or
+                mountpoint["available"] == str(0)):
+          mountPoints[mountpoint["mountpoint"]] = to_number(mountpoint["available"])
+      if mountPoints:
+        return max(mountPoints, key=mountPoints.get)
+    return "/"
+
+  def validatorNotRootFs(self, properties, propertyName, hostInfo):
+    if not propertyName in properties:
+      return self.getErrorItem("Value should be set")
+    dir = properties[propertyName]
+    if dir.startswith("hdfs://"):
+      return None
+
+    dir = re.sub("^file://", "", dir, count=1)
+    mountPoints = []
+    for mountPoint in hostInfo["disk_info"]:
+      mountPoints.append(mountPoint["mountpoint"])
+    mountPoint = getMountPointForDir(dir, mountPoints)
+
+    if "/" == mountPoint and self.getProperMountPoint(hostInfo) != mountPoint:
+      return self.getWarnItem("The root device should not be used for {0}".format(propertyName))
+
+    return None
+
   def validatorEnoughDiskSpace(self, properties, propertyName, hostInfo, reqiuredDiskSpace):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")
     dir = properties[propertyName]
     if dir.startswith("hdfs://"):
-      return None #TODO following code fails for hdfs://, is this valid check for hdfs?
+      return None
 
     dir = re.sub("^file://", "", dir, count=1)
     mountPoints = {}
@@ -806,7 +867,7 @@ def getMountPointForDir(dir, mountPoints):
       if dir.startswith(mountPoint):
         if bestMountFound is None:
           bestMountFound = mountPoint
-        elif bestMountFound.count(os.path.sep) < mountPoint.count(os.path.sep):
+        elif bestMountFound.count(os.path.sep) < os.path.join(mountPoint, "").count(os.path.sep):
           bestMountFound = mountPoint
 
   return bestMountFound

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -888,3 +888,107 @@ class TestHDP206StackAdvisor(TestCase):
 
     self.stackAdvisor.mergeValidators(parentValidators, childValidators)
     self.assertEquals(expected, parentValidators)
+
+  def test_getProperMountPoint(self):
+    hostInfo = None
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    hostInfo = {"some_key": []}
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    hostInfo["disk_info"] = []
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # root mountpoint with low space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "1",
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    )
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # tmpfs with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "2",
+        "type" : "tmpfs",
+        "mountpoint" : "/dev/shm"
+      }
+    )
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # /boot with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "3",
+        "type" : "tmpfs",
+        "mountpoint" : "/boot/grub"
+      }
+    )
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # /boot with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "4",
+        "type" : "tmpfs",
+        "mountpoint" : "/mnt/external_hdd"
+      }
+    )
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # virtualbox fs with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "5",
+        "type" : "vboxsf",
+        "mountpoint" : "/vagrant"
+      }
+    )
+    self.assertEquals("/", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # proper mountpoint with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "6",
+        "type" : "ext4",
+        "mountpoint" : "/grid/0"
+      }
+    )
+    self.assertEquals("/grid/0", self.stackAdvisor.getProperMountPoint(hostInfo))
+    # proper mountpoint with more space available
+    hostInfo["disk_info"].append(
+      {
+        "available" : "7",
+        "type" : "ext4",
+        "mountpoint" : "/grid/1"
+      }
+    )
+    self.assertEquals("/grid/1", self.stackAdvisor.getProperMountPoint(hostInfo))
+
+  def test_validateNonRootFs(self):
+    hostInfo = {"disk_info": [
+      {
+        "available" : "2",
+        "type" : "ext4",
+        "mountpoint" : "/"
+      }
+    ]}
+    properties = {"property1": "/var/dir"}
+    # only / mountpoint - no warning
+    self.assertIsNone(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo))
+    # More preferable /grid/0 mountpoint - warning
+    hostInfo["disk_info"].append(
+      {
+        "available" : "3",
+        "type" : "ext4",
+        "mountpoint" : "/grid/0"
+      }
+    )
+    warn = self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo)
+    self.assertIsNotNone(warn)
+    self.assertEquals({'message': 'The root device should not be used for property1', 'level': 'WARN'}, warn)
+
+    # Set by user /var mountpoint, which is non-root , but not preferable - no warning
+    hostInfo["disk_info"].append(
+      {
+        "available" : "1",
+        "type" : "ext4",
+        "mountpoint" : "/var"
+      }
+    )
+    self.assertIsNone(self.stackAdvisor.validatorNotRootFs(properties, 'property1', hostInfo))

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1842,7 +1842,8 @@ class TestHDP22StackAdvisor(TestCase):
           }
 
         }]
-      }]
+      }],
+      "configurations": []
     }
     hosts = {
       "items": [{
@@ -1870,6 +1871,8 @@ class TestHDP22StackAdvisor(TestCase):
           "hbase.regionserver.global.memstore.lowerLimit": "0.3",
           "hbase.regionserver.global.memstore.upperLimit": "0.35",
           "hfile.block.cache.size": "0.3",
+          "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase",
+          "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp",
           "hbase_master_xmn_size" : "128m"
         }
       },