AMBARI-8545 Create stack advisor script to recommend configurations (dsen)

ambari-server/src/main/resources/scripts/stack_advisor.py

@@ -17,7 +17,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-import StringIO
 
 import json
 import os

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/stack_advisor.py

@@ -18,7 +18,6 @@ limitations under the License.
 """
 
 import re
-import sys
 from math import ceil
 
 from stack_advisor import DefaultStackAdvisor
@@ -259,7 +258,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
       return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx)
     return None
 
-  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations):
+  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
                         {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
                         {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
@@ -269,7 +268,7 @@ class BaseBIGTOP08StackAdvisor(DefaultStackAdvisor):
                         {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
     return self.toConfigurationValidationProblems(validationItems, "mapred-site")
 
-  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
                         {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
                         {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
@@ -318,7 +317,7 @@ class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor):
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
 
-  def recommendOozieConfigurations(self, configurations, clusterData):
+  def recommendOozieConfigurations(self, configurations, clusterData, services, hosts):
     if "FALCON_SERVER" in clusterData["components"]:
       putMapredProperty = self.putProperty(configurations, "oozie-site")
       putMapredProperty("oozie.services.ext",
@@ -326,7 +325,7 @@ class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor):
                         "org.apache.oozie.service.PartitionDependencyManagerService," +
                         "org.apache.oozie.service.HCatAccessorService")
 
-  def recommendHiveConfigurations(self, configurations, clusterData):
+  def recommendHiveConfigurations(self, configurations, clusterData, services, hosts):
     containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
     containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
     putHiveProperty = self.putProperty(configurations, "hive-site")
@@ -335,7 +334,7 @@ class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor):
                     + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC")
     putHiveProperty('hive.tez.container.size', containerSize)
 
-  def recommendTezConfigurations(self, configurations, clusterData):
+  def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']))
     putTezProperty("tez.am.java.opts",
@@ -366,13 +365,13 @@ class BIGTOP08StackAdvisor(BaseBIGTOP08StackAdvisor):
     parentValidators.update(childValidators)
     return parentValidators
 
-  def validateHiveConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
                         {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
                         {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
     return self.toConfigurationValidationProblems(validationItems, "hive-site")
 
-  def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
                         {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
     return self.toConfigurationValidationProblems(validationItems, "tez-site")

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -18,7 +18,7 @@ limitations under the License.
 """
 
 import re
-import sys
+import os
 from math import ceil
 
 from stack_advisor import DefaultStackAdvisor
@@ -91,13 +91,13 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       config[configType]["properties"][key] = str(value)
     return appendProperty
 
-  def recommendYARNConfigurations(self, configurations, clusterData):
+  def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
     putYarnProperty = self.putProperty(configurations, "yarn-site")
     putYarnProperty('yarn.nodemanager.resource.memory-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
     putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
     putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(round(clusterData['containers'] * clusterData['ramPerContainer'])))
 
-  def recommendMapReduce2Configurations(self, configurations, clusterData):
+  def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
     putMapredProperty = self.putProperty(configurations, "mapred-site")
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory']))
     putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m")
@@ -196,20 +196,19 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     for service in services["services"]:
       serviceName = service["StackServices"]["service_name"]
       validator = self.validateServiceConfigurations(serviceName)
-      if validator is not None:
-        siteName = validator[0]
-        method = validator[1]
-        if siteName in recommendedDefaults:
-          siteProperties = getSiteProperties(configurations, siteName)
-          if siteProperties is not None:
-            resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"], configurations)
-            items.extend(resultItems)
+      if validator is not  None:
+        for siteName, method in validator.items():
+          if siteName in recommendedDefaults:
+            siteProperties = getSiteProperties(configurations, siteName)
+            if siteProperties is not None:
+              resultItems = method(siteProperties, recommendedDefaults[siteName]["properties"], configurations, services, hosts)
+              items.extend(resultItems)
     return items
 
   def getServiceConfigurationValidators(self):
     return {
-      "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations],
-      "YARN": ["yarn-site", self.validateYARNConfigurations]
+      "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
+      "YARN": {"yarn-site": self.validateYARNConfigurations}
     }
 
   def validateServiceConfigurations(self, serviceName):
@@ -231,6 +230,21 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
   def getErrorItem(self, message):
     return {"level": "ERROR", "message": message}
 
+  def validatorEnoughDiskSpace(self, properties, propertyName, hostInfo, reqiuredDiskSpace):
+    if not propertyName in properties:
+      return self.getErrorItem("Value should be set")
+    dir = properties[propertyName]
+    if not dir.startswith("hdfs://"):
+      dir = re.sub("^file://", "", dir, count=1)
+    mountPoints = {}
+    for mountPoint in hostInfo["disk_info"]:
+      mountPoints[mountPoint["mountpoint"]] = to_number(mountPoint["available"])
+    mountPoint = getMountPointForDir(dir, mountPoints.keys())
+
+    if mountPoints[mountPoint] < reqiuredDiskSpace:
+      return self.getWarnItem("Recommended disk space for partition {0} is {1}k".format(mountPoint, reqiuredDiskSpace))
+    return None
+
   def validatorLessThenDefaultValue(self, properties, recommendedDefaults, propertyName):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")
@@ -260,7 +274,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       return self.getWarnItem("Value is less than the recommended default of -Xmx" + defaultValueXmx)
     return None
 
-  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations):
+  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
                         {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
                         {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
@@ -270,7 +284,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                         {"config-name": 'yarn.app.mapreduce.am.command-opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
     return self.toConfigurationValidationProblems(validationItems, "mapred-site")
 
-  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
                         {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
                         {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
@@ -370,4 +384,27 @@ def isSecurePort(port):
   if port is not None:
     return port < 1024
   else:
-    return False
+    return False
+
+def getMountPointForDir(dir, mountPoints):
+  """
+  :param dir: Directory to check, even if it doesn't exist.
+  :return: Returns the closest mount point as a string for the directory.
+  if the "dir" variable is None, will return None.
+  If the directory does not exist, will return "/".
+  """
+  bestMountFound = None
+  if dir:
+    dir = dir.strip().lower()
+
+    # If the path is "/hadoop/hdfs/data", then possible matches for mounts could be
+    # "/", "/hadoop/hdfs", and "/hadoop/hdfs/data".
+    # So take the one with the greatest number of segments.
+    for mountPoint in mountPoints:
+      if dir.startswith(mountPoint):
+        if bestMountFound is None:
+          bestMountFound = mountPoint
+        elif bestMountFound.count(os.path.sep) < mountPoint.count(os.path.sep):
+          bestMountFound = mountPoint
+
+  return bestMountFound

ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py

@@ -29,7 +29,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
 
-  def recommendOozieConfigurations(self, configurations, clusterData):
+  def recommendOozieConfigurations(self, configurations, clusterData, services, hosts):
     if "FALCON_SERVER" in clusterData["components"]:
       putMapredProperty = self.putProperty(configurations, "oozie-site")
       putMapredProperty("oozie.services.ext",
@@ -37,7 +37,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
                         "org.apache.oozie.service.PartitionDependencyManagerService," +
                         "org.apache.oozie.service.HCatAccessorService")
 
-  def recommendHiveConfigurations(self, configurations, clusterData):
+  def recommendHiveConfigurations(self, configurations, clusterData, services, hosts):
     containerSize = clusterData['mapMemory'] if clusterData['mapMemory'] > 2048 else int(clusterData['reduceMemory'])
     containerSize = min(clusterData['containers'] * clusterData['ramPerContainer'], containerSize)
     putHiveProperty = self.putProperty(configurations, "hive-site")
@@ -46,7 +46,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
                     + "m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps")
     putHiveProperty('hive.tez.container.size', containerSize)
 
-  def recommendTezConfigurations(self, configurations, clusterData):
+  def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']))
     putTezProperty("tez.am.java.opts",
@@ -71,19 +71,19 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP21StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
-      "HIVE": ["hive-site", self.validateHiveConfigurations],
-      "TEZ": ["tez-site", self.validateTezConfigurations]
+      "HIVE": {"hive-site": self.validateHiveConfigurations},
+      "TEZ": {"tez-site": self.validateTezConfigurations}
     }
     parentValidators.update(childValidators)
     return parentValidators
 
-  def validateHiveConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateHiveConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'hive.tez.container.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.tez.container.size')},
                         {"config-name": 'hive.tez.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'hive.tez.java.opts')},
                         {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')} ]
     return self.toConfigurationValidationProblems(validationItems, "hive-site")
 
-  def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
                         {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')} ]
     return self.toConfigurationValidationProblems(validationItems, "tez-site")

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -25,21 +25,22 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
       "HDFS": self.recommendHDFSConfigurations,
       "MAPREDUCE2": self.recommendMapReduce2Configurations,
       "TEZ": self.recommendTezConfigurations,
+      "AMS": self.recommendAmsConfigurations,
       "YARN": self.recommendYARNConfigurations
     }
     parentRecommendConfDict.update(childRecommendConfDict)
     return parentRecommendConfDict
 
-  def recommendYARNConfigurations(self, configurations, clusterData):
-    super(HDP22StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData)
+  def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
+    super(HDP22StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
     putYarnProperty = self.putProperty(configurations, "yarn-site")
     putYarnProperty('yarn.nodemanager.resource.cpu-vcores', clusterData['cpu'])
 
-  def recommendHDFSConfigurations(self, configurations, clusterData):
+  def recommendHDFSConfigurations(self, configurations, clusterData, services, hosts):
     putHdfsPropery = self.putProperty(configurations, "hdfs-site")
     putHdfsPropery("dfs.datanode.max.transfer.threads", 16384 if clusterData["hBaseInstalled"] else 4096)
 
-  def recommendTezConfigurations(self, configurations, clusterData):
+  def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     putTezProperty = self.putProperty(configurations, "tez-site")
     putTezProperty("tez.am.resource.memory.mb", int(clusterData['amMemory']) * 2 if int(clusterData['amMemory']) < 3072 else int(clusterData['amMemory']))
     putTezProperty("tez.am.java.opts",
@@ -52,17 +53,57 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putTezProperty("tez.runtime.io.sort.mb", int(taskResourceMemory * 0.4) if int(taskResourceMemory * 0.4) <= 2147483644 else 2147483644)
     putTezProperty("tez.runtime.unordered.output.buffer.size-mb", int(taskResourceMemory * 0.075))
 
+  def recommendAmsConfigurations(self, configurations, clusterData, services, hosts):
+    putAmsHbaseSiteProperty = self.putProperty(configurations, "ams-hbase-site")
+    putTimelineServiceProperty = self.putProperty(configurations, "ams-site")
+    putHbaseEnvProperty = self.putProperty(configurations, "ams-hbase-env")
+
+    amsCollectorHosts = self.getComponentHostNames(services, "AMS", "METRIC_COLLECTOR")
+
+    # TODO recommend configuration for multiple AMS collectors
+    if len(amsCollectorHosts) > 1:
+      pass
+    else:
+      totalHostsCount = len(hosts["items"])
+      # TODO Tune values according to performance testing results
+      if totalHostsCount > 400:
+        putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.5)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.4)
+        putTimelineServiceProperty("timeline.metrics.host.aggregator.ttl", 86400)
+
+        putHbaseEnvProperty("hbase_master_heapsize", "8096m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "8096m")
+      elif totalHostsCount > 100:
+        putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.5)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.4)
+        putTimelineServiceProperty("timeline.metrics.host.aggregator.ttl", 86400)
+
+        putHbaseEnvProperty("hbase_master_heapsize", "2048m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "2048m")
+      else:
+        putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.upperLimit", 0.5)
+        putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.4)
+        putTimelineServiceProperty("timeline.metrics.host.aggregator.ttl", 86400)
+
+        putHbaseEnvProperty("hbase_master_heapsize", "1024m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "1024m")
+
   def getServiceConfigurationValidators(self):
     parentValidators = super(HDP22StackAdvisor, self).getServiceConfigurationValidators()
     childValidators = {
-      "HDFS": ["hdfs-site", self.validateHDFSConfigurations],
-      "MAPREDUCE2": ["mapred-site", self.validateMapReduce2Configurations],
-      "TEZ": ["tez-site", self.validateTezConfigurations]
+      "HDFS": {"hdfs-site": self.validateHDFSConfigurations},
+      "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
+      "AMS": {"ams-hbase-site": self.validateAmsHbaseSiteConfigurations,
+              "ams-hbase-env": self.validateAmsHbaseEnvConfigurations},
+      "TEZ": {"tez-site": self.validateTezConfigurations}
     }
     parentValidators.update(childValidators)
     return parentValidators
 
-  def validateTezConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateTezConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.am.resource.memory.mb')},
                         {"config-name": 'tez.am.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'tez.am.java.opts')},
                         {"config-name": 'tez.task.resource.memory.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.task.resource.memory.mb')},
@@ -70,7 +111,61 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                         {"config-name": 'tez.runtime.unordered.output.buffer.size-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'tez.runtime.unordered.output.buffer.size-mb')},]
     return self.toConfigurationValidationProblems(validationItems, "tez-site")
 
-  def recommendMapReduce2Configurations(self, configurations, clusterData):
+  def validateAmsHbaseSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+
+    amsCollectorHosts = self.getComponentHostNames(services, "AMS", "METRIC_COLLECTOR")
+    recommendedDiskSpace = 10485760
+    # TODO validate configuration for multiple AMS collectors
+    if len(amsCollectorHosts) > 1:
+      pass
+    else:
+      totalHostsCount = len(hosts["items"])
+      if totalHostsCount > 400:
+        recommendedDiskSpace  = 104857600  # * 1k == 100 Gb
+      elif totalHostsCount > 100:
+        recommendedDiskSpace  = 52428800  # * 1k == 50 Gb
+      elif totalHostsCount > 10:
+        recommendedDiskSpace  = 20971520  # * 1k == 20 Gb
+
+
+    validationItems = []
+    for collectorHostName in amsCollectorHosts:
+      for host in hosts["items"]:
+        if host["Hosts"]["host_name"] == collectorHostName:
+          validationItems.extend([ {"config-name": 'hbase.rootdir', "item": self.validatorEnoughDiskSpace(properties, 'hbase.rootdir', host["Hosts"], recommendedDiskSpace)}])
+          break
+
+    return self.toConfigurationValidationProblems(validationItems, "ams-hbase-site")
+
+  def validateAmsHbaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+    regionServerItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_regionserver_heapsize")
+    masterItem = self.validatorLessThenDefaultValue(properties, recommendedDefaults, "hbase_master_heapsize")
+
+    if regionServerItem is None and masterItem is None:
+      hbase_regionserver_heapsize = formatXmxSizeToBytes(properties["hbase_regionserver_heapsize"])
+      hbase_master_heapsize = formatXmxSizeToBytes(properties["hbase_master_heapsize"])
+
+      # TODO Add AMS Collector Xmx property to ams-env
+      # Collector + HBASE Master + HBASE RegionServer HeapSize
+      requiredMemory = 1073741824 + hbase_regionserver_heapsize + hbase_master_heapsize
+
+      amsCollectorHosts = self.getComponentHostNames(services, "AMS", "METRIC_COLLECTOR")
+      for collectorHostName in amsCollectorHosts:
+        for host in hosts["items"]:
+          if host["Hosts"]["host_name"] == collectorHostName:
+            if host["Hosts"]["total_mem"] * 1024 < requiredMemory:  # in bytes
+              message = "Not enough total RAM on the host {0}, " \
+                        "at least {1} MB required" \
+                        .format(collectorHostName, requiredMemory/1048576)  # MB
+              regionServerItem = self.getWarnItem(message)
+              masterItem = self.getWarnItem(message)
+              break
+
+    validationItems = [{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem},
+                       {"config-name": "hbase_master_heapsize", "item": masterItem}]
+    return self.toConfigurationValidationProblems(validationItems, "ams-hbase-env")
+
+  def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
     putMapredProperty = self.putProperty(configurations, "mapred-site")
     putMapredProperty('yarn.app.mapreduce.am.resource.mb', int(clusterData['amMemory']))
     putMapredProperty('yarn.app.mapreduce.am.command-opts', "-Xmx" + str(int(round(0.8 * clusterData['amMemory']))) + "m" + " -Dhdp.version=${hdp.version}")
@@ -80,7 +175,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putMapredProperty('mapreduce.reduce.java.opts', "-Xmx" + str(int(round(0.8 * clusterData['reduceMemory']))) + "m")
     putMapredProperty('mapreduce.task.io.sort.mb', min(int(round(0.4 * clusterData['mapMemory'])), 1024))
 
-  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations):
+  def validateMapReduce2Configurations(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'mapreduce.map.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.map.java.opts')},
                         {"config-name": 'mapreduce.reduce.java.opts', "item": self.validateXmxValue(properties, recommendedDefaults, 'mapreduce.reduce.java.opts')},
                         {"config-name": 'mapreduce.task.io.sort.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'mapreduce.task.io.sort.mb')},
@@ -89,7 +184,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                         {"config-name": 'yarn.app.mapreduce.am.resource.mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')}]
     return self.toConfigurationValidationProblems(validationItems, "mapred-site")
 
-  def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations):
+  def validateHDFSConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
     # We can not access property hadoop.security.authentication from the
     # other config (core-site). That's why we are using another heuristics here
     hdfs_site = properties
@@ -195,5 +290,27 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
                                       data_transfer_protection_value, VALID_TRANSFER_PROTECTION_VALUES))})
     return self.toConfigurationValidationProblems(validationItems, "hdfs-site")
 
+  def getMastersWithMultipleInstances(self):
+    result = super(HDP22StackAdvisor, self).getMastersWithMultipleInstances()
+    result.extend(['METRIC_COLLECTOR'])
+    return result
+
+  def getNotValuableComponents(self):
+    result = super(HDP22StackAdvisor, self).getNotValuableComponents()
+    result.extend(['METRIC_MONITOR'])
+    return result
+
+  def getNotPreferableOnServerComponents(self):
+    result = super(HDP22StackAdvisor, self).getNotPreferableOnServerComponents()
+    result.extend(['METRIC_COLLECTOR'])
+    return result
 
+  def getCardinalitiesDict(self):
+    result = super(HDP22StackAdvisor, self).getCardinalitiesDict()
+    result['METRIC_COLLECTOR'] = {"min": 1}
+    return result
 
+  def getComponentLayoutSchemes(self):
+    result = super(HDP22StackAdvisor, self).getComponentLayoutSchemes()
+    result['METRIC_COLLECTOR'] = {"else": 2}
+    return result

ambari-server/src/main/resources/stacks/stack_advisor.py

@@ -17,7 +17,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import time
 import socket
 
 class StackAdvisor(object):
@@ -480,7 +479,7 @@ class DefaultStackAdvisor(StackAdvisor):
     for service in servicesList:
       calculation = self.getServiceConfigurationRecommender(service)
       if calculation is not None:
-        calculation(configurations, clusterSummary)
+        calculation(configurations, clusterSummary, services, hosts)
 
     return recommendations
 
@@ -584,3 +583,11 @@ class DefaultStackAdvisor(StackAdvisor):
     host index where component should exist.
     """
     return {}
+
+  def getComponentHostNames(self, servicesDict, serviceName, componentName):
+    for service in servicesDict["services"]:
+      if service["StackServices"]["service_name"] == serviceName:
+        for component in service['components']:
+          if component["StackServiceComponents"]["component_name"] == componentName:
+            return component["StackServiceComponents"]["hostnames"]
+  pass

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -337,7 +337,7 @@ class TestHDP206StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendMapReduce2Configurations_mapMemoryLessThan2560(self):
@@ -361,7 +361,7 @@ class TestHDP206StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData)
+    self.stackAdvisor.recommendMapReduce2Configurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_getConfigurationClusterSummary_noHostsWithoutHBase(self):

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -47,7 +47,7 @@ class TestHDP21StackAdvisor(TestCase):
     expected = {
     }
 
-    self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendOozieConfigurations_withFalconServer(self):
@@ -65,7 +65,7 @@ class TestHDP21StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendOozieConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendHiveConfigurations_mapMemoryLessThan2048(self):
@@ -86,7 +86,7 @@ class TestHDP21StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendHiveConfigurations_mapMemoryMoreThan2048(self):
@@ -107,7 +107,7 @@ class TestHDP21StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_createComponentLayoutRecommendations_mastersIn10nodes(self):
@@ -157,5 +157,5 @@ class TestHDP21StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -16,7 +16,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
-import json
 import os
 from unittest import TestCase
 
@@ -62,7 +61,7 @@ class TestHDP22StackAdvisor(TestCase):
         }
       }
     }
-    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendTezConfigurations_amMemoryMoreThan3072(self):
@@ -85,7 +84,7 @@ class TestHDP22StackAdvisor(TestCase):
         }
       }
     }
-    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendTezConfigurations_mapMemoryLessThan768(self):
@@ -108,7 +107,7 @@ class TestHDP22StackAdvisor(TestCase):
         }
       }
     }
-    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
 
@@ -140,7 +139,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []  # No warnings
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Unsecured cluster, unsecure ports
@@ -157,7 +156,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []  # No warnings
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, invalid dfs.http.policy value
@@ -179,7 +178,7 @@ class TestHDP22StackAdvisor(TestCase):
                  'level': 'WARN',
                  'message': "Invalid property value: WRONG_VALUE. Valid values are ['HTTP_ONLY', 'HTTPS_ONLY', 'HTTP_AND_HTTPS']",
                  'type': 'configuration'}]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address not defined
@@ -196,7 +195,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = [ ]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and secure
@@ -214,7 +213,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, https address defined and non secure
@@ -232,7 +231,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, non secure dfs port, https property not defined
@@ -276,7 +275,7 @@ class TestHDP22StackAdvisor(TestCase):
                             "order to be able to use HTTPS.",
                  'type': 'configuration'}
     ]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
 
@@ -321,7 +320,7 @@ class TestHDP22StackAdvisor(TestCase):
                             "able to use HTTPS.",
                  'type': 'configuration'}
     ]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, valid non-root configuration
@@ -340,7 +339,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, insecure port
@@ -376,7 +375,7 @@ class TestHDP22StackAdvisor(TestCase):
                             "['dfs.datanode.address', 'dfs.datanode.http.address'] use secure ports.",
                  'type': 'configuration'}
                 ]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, valid configuration
@@ -394,7 +393,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, absent dfs.http.policy (typical situation)
@@ -411,7 +410,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTP_ONLY, misusage of dfs.data.transfer.protection warning
@@ -435,7 +434,7 @@ class TestHDP22StackAdvisor(TestCase):
                  'message': "dfs.data.transfer.protection property can not be used when dfs.http.policy is "
                             "set to any value other then HTTPS_ONLY. Tip: When dfs.http.policy property is not defined, it defaults to HTTP_ONLY",
                  'type': 'configuration'}]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Secure cluster, dfs.http.policy=HTTPS_ONLY, wrong dfs.data.transfer.protection value
@@ -458,7 +457,7 @@ class TestHDP22StackAdvisor(TestCase):
                  'level': 'WARN',
                  'message': "Invalid property value: WRONG_VALUE. Valid values are ['authentication', 'integrity', 'privacy'].",
                  'type': 'configuration'}]
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
     # TEST CASE: Hadoop wire encryption enabled
@@ -477,7 +476,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
     expected = []  # No warnings
-    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations)
+    validation_problems = self.stackAdvisor.validateHDFSConfigurations(properties, recommendedDefaults, configurations, None, None)
     self.assertEquals(validation_problems, expected)
 
   def test_recommendYARNConfigurations(self):
@@ -498,5 +497,54 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData)
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
+    self.assertEquals(configurations, expected)
+
+  def test_recommendAmsConfigurations(self):
+    configurations = {}
+    clusterData = {}
+
+    services = {
+      "services":  [ {
+        "StackServices": {
+          "service_name": "AMS"
+        },"components": [{
+          "StackServiceComponents": {
+            "component_name": "METRIC_COLLECTOR",
+            "hostnames": ["host1"]
+          }
+
+        }]
+      }]
+    }
+    hosts = {
+      "items": [{
+        "Hosts": {
+          "host_name": "host1",
+
+        }
+      }]
+    }
+
+    expected = {
+      "ams-hbase-env": {
+        "properties": {
+          "hbase_master_heapsize": "1024m",
+          "hbase_regionserver_heapsize": "1024m",
+        }
+      },
+      "ams-hbase-site": {
+        "properties": {
+          "hbase.regionserver.global.memstore.lowerLimit": "0.4",
+          "hbase.regionserver.global.memstore.upperLimit": "0.5",
+          "hfile.block.cache.size": "0.3"
+        }
+      },
+      "ams-site": {
+        "properties": {
+          "timeline.metrics.host.aggregator.ttl": "86400"
+        }
+      }
+    }
+    self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)