"""
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import os, imp, json
from unittest import TestCase
from mock.mock import patch, MagicMock


class TestHAWQ200ServiceAdvisor(TestCase):

  testDirectory = os.path.dirname(os.path.abspath(__file__))
  serviceAdvisorPath = '../../../../main/resources/common-services/HAWQ/2.0.0/service_advisor.py'
  hawq200ServiceAdvisorPath = os.path.join(testDirectory, serviceAdvisorPath)
  with open(hawq200ServiceAdvisorPath, 'rb') as fp:
    service_advisor_impl = imp.load_module('service_advisor_impl', fp, hawq200ServiceAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))

  def setUp(self):
    serviceAdvisorClass = getattr(self.service_advisor_impl, 'HAWQ200ServiceAdvisor')
    self.serviceAdvisor = serviceAdvisorClass()

  def fqdn_mock_result(value=None):
      return 'c6401.ambari.apache.org' if value is None else value

  def load_json(self, filename):
    file = os.path.join(self.testDirectory, "../configs", filename)
    with open(file, 'rb') as f:
      data = json.load(f)
    return data

  def prepareHosts(self, hostsNames):
    hosts = { "items": [] }
    for hostName in hostsNames:
      nextHost = {"Hosts":{"host_name" : hostName}}
      hosts["items"].append(nextHost)
    return hosts

  def getHosts(self, componentsList, componentName):
    hosts = [component["StackServiceComponents"] for component in componentsList
             if component["StackServiceComponents"]["component_name"] == componentName]
    return hosts[0] if len(hosts) > 0 else []

  def getHostsFromRecommendations(self, recommendations, componentName):
    hostGroups = [hostgroup["name"] for hostgroup in recommendations["blueprint"]["host_groups"] if
                  {"name": componentName} in hostgroup["components"]]
    return set([host["fqdn"] for hostgroup in recommendations["blueprint_cluster_binding"]["host_groups"] if
                hostgroup["name"] in hostGroups for host in hostgroup["hosts"]])

  def getComponentsListFromServices(self, services):
    componentsListList = [service["components"] for service in services["services"]]
    return [item for sublist in componentsListList for item in sublist]

  def getComponentsFromServices(self, services):
    componentsList = self.getComponentsListFromServices(services)
    return [component["StackServiceComponents"]["component_name"] for component in componentsList]

  def getComponentsFromRecommendations(self, recommendations):
    componentsListList = [hostgroup["components"] for hostgroup in
                          recommendations["recommendations"]["blueprint"]["host_groups"]]
    return [item["name"] for sublist in componentsListList for item in sublist]

  def insertHAWQServiceAdvisorInfo(self, services):
    for service in services["services"]:
      if service["StackServices"]["service_name"] == 'HAWQ':
        service["StackServices"]["advisor_name"] = "HAWQ200ServiceAdvisor"
        service["StackServices"]["advisor_path"] = self.hawq200ServiceAdvisorPath

  def getDesiredHDFSSiteValues(self, is_secure):
    hdfs_site_desired_values = {
      "dfs.allow.truncate" : "true",
      "dfs.block.access.token.enable" : str(is_secure).lower(),
      "dfs.block.local-path-access.user" : "gpadmin",
      "dfs.client.read.shortcircuit" : "true",
      "dfs.client.use.legacy.blockreader.local" : "false",
      "dfs.datanode.data.dir.perm" : "750",
      "dfs.datanode.handler.count" : "60",
      "dfs.datanode.max.transfer.threads" : "40960",
      "dfs.namenode.accesstime.precision" : "0",
      "dfs.support.append" : "true"
    }
    return hdfs_site_desired_values

  @patch("socket.getfqdn")
  def test_getHostsForMasterComponent(self, getfqdn_mock):
    getfqdn_mock.return_value = "c6401.ambari.apache.org"

    services = {
      "services": [
        {
          "StackServices": {
            "service_name": "HAWQ"
          },
          "components": [
            {
              "StackServiceComponents": {
                "component_name": "HAWQMASTER",
                "hostnames": [
                  "c6403.ambari.apache.org"
                ]
              }
            },
            {
              "StackServiceComponents": {
                "component_name": "HAWQSTANDBY",
                "hostnames": [
                ]
              }
            }
          ]
        }
      ]
    }

    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org", "c6404.ambari.apache.org"]

    component = {
      "StackServiceComponents": {
        "component_name": "HAWQSTANDBY"
      }
    }

    # Case 1:
    # Ambari Server is placed on c6401.ambari.apache.org
    # HAWQMASTER is placed on c6403.ambari.apache.org
    # There are 4 available hosts in the cluster
    # Recommend HAWQSTANDBY on next available host, c6402.ambari.apache.org
    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
    self.assertEquals(standbyHosts, ["c6402.ambari.apache.org"])

    # Case 2:
    # Ambari Server is placed on c6401.ambari.apache.org
    # HAWQMASTER is placed on c6402.ambari.apache.org
    # There are 4 available hosts in the cluster
    # Recommend HAWQSTANDBY on next available host, c6403.ambari.apache.org
    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
    self.assertEquals(standbyHosts, ["c6403.ambari.apache.org"])

    # Case 3:
    # Ambari Server is placed on c6401.ambari.apache.org
    # HAWQMASTER is placed on c6402.ambari.apache.org
    # There are 2 available hosts in the cluster
    # Recommend HAWQSTANDBY on a host which does not have HAWQMASTER, c6401.ambari.apache.org
    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org"]
    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
    self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])

    # Case 4:
    # Ambari Server is placed on c6401.ambari.apache.org
    # HAWQMASTER is placed on c6401.ambari.apache.org
    # There is 1 available host in the cluster
    # Do not recommend HAWQSTANDBY on a single node cluster
    hostsList = ["c6401.ambari.apache.org"]
    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
    self.assertEquals(standbyHosts, [])

    # Case 5:
    # Ambari Server is placed on c6401.ambari.apache.org
    # HAWQMASTER is placed on c6402.ambari.apache.org
    # HAWQSTANDBY is placed on c6401.ambari.apache.org
    # There are 3 available host in the cluster
    # Do not change HAWQSTANDBY host according to recommendation since HAWQSTANDBY has already been assigned a host
    hostsList = ["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"]
    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["c6402.ambari.apache.org"]
    services["services"][0]["components"][1]["StackServiceComponents"]["hostnames"] = ["c6401.ambari.apache.org"]
    standbyHosts = self.serviceAdvisor.getHostsForMasterComponent(services, None, component, hostsList)
    self.assertEquals(standbyHosts, ["c6401.ambari.apache.org"])


  def test_getServiceConfigurationRecommendations(self):

    configurations = {
      "hawq-sysctl-env": {
        "properties": {
          "vm.overcommit_memory": 1,
          "vm.overcommit_ratio": 50
        }
      },
      "hawq-site": {
        "properties": {
          "hawq_rm_memory_limit_perseg": "65535MB",
          "hawq_rm_nvcore_limit_perseg": "16",
          "hawq_global_rm_type": "yarn",
          "default_hash_table_bucket_number": 18
        }
      },
      "hdfs-site": {
        "properties": {
        }
      },
      "core-site": {
        "properties": {
        }
      },
      "cluster-env": {
        "properties": {
        }
      }
    }

    services = {
      "services": [
        {
          "StackServices": {
            "service_name": "HAWQ",
            "service_version": "2.0",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "component_name": "HAWQMASTER",
                "hostnames": [
                  "c6401.ambari.apache.org"
                ]
              }
            },
            {
              "StackServiceComponents": {
                "component_name": "HAWQSEGMENT",
                "hostnames": [
                  "c6402.ambari.apache.org",
                  "c6404.ambari.apache.org",
                ]
              }
            }
          ]
        }
      ],
      "configurations": configurations
    }

    hosts = {
      "items": [
        {
          "Hosts": {
            "host_name": "c6401.ambari.apache.org",
            "cpu_count" : 2,
            "total_mem": 33554432
          }
        },
        {
          "Hosts": {
            "host_name": "c6402.ambari.apache.org",
            "cpu_count" : 4,
            "total_mem": 33554433
          }
        },
        {
          "Hosts": {
            "host_name": "c6403.ambari.apache.org",
            "cpu_count" : 1,
            "total_mem": 33554434
          }
        },
        {
          "Hosts": {
            "host_name": "c6404.ambari.apache.org",
            "cpu_count" : 2,
            "total_mem": 33554435
          }
        }
      ]
    }

    ## Test that HDFS parameters required by HAWQ are recommended
    configurations["cluster-env"]["properties"]["security_enabled"]="false"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    hdfs_site_desired_values = self.getDesiredHDFSSiteValues(False)
    for property, value in hdfs_site_desired_values.iteritems():
      self.assertEquals(configurations["hdfs-site"]["properties"][property], value)
    self.assertEquals(configurations["core-site"]["properties"]["ipc.server.listen.queue.size"], "3300")

    # Kerberos causes 1 property to be recommended differently
    configurations["cluster-env"]["properties"]["security_enabled"]="true"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEquals(configurations["hdfs-site"]["properties"]["dfs.block.access.token.enable"], "true")


    ## Test if hawq_rm_nvcore_limit_perseg is set correctly

    # Case 1:
    # HAWQ Hosts Core Count: c6401.ambari.apache.org - 2, c6402.ambari.apache.org - 4, c6404.ambari.apache.org - 2
    # hawq_global_rm_type: yarn
    # Non HAWQ Hosts Core Count: c6401.ambari.apache.org - 1
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_nvcore_limit_perseg"], "2")

    ## Test if vm.overcommit_memory is set correctly

    # Case 1: All machines have total_mem above 32GB (total_mem >= 33554432)
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "2")

    # Case 2: One machine has total_mem below 32GB
    hosts["items"][0]["Hosts"]["total_mem"] = 33554431
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEquals(configurations["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"], "1")

    ## Test if hawq_rm_memory_limit_perseg is set correctly

    # Case 1: Minimum host memory is ~ 2 GB (2048MB), recommended val must be .75% of 2GB as vm.overcommit_memory = 1 and in MB
    hosts["items"][0]["Hosts"]["total_mem"] = 2097152
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1536MB")

    # Case 2: Minimum host memory is ~ 16 GB, recommended val must be .75% of 16GB as vm.overcommit_memory = 1 and in GB
    hosts["items"][0]["Hosts"]["total_mem"] = 16777216
    hosts["items"][1]["Hosts"]["total_mem"] = 26777216
    hosts["items"][3]["Hosts"]["total_mem"] = 36777216
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "12GB")

    # Case 2: Minimum host memory is ~ 64 GB, recommended val must be .75% of 32GB as vm.overcommit_memory = 2 and in GB
    hosts["items"][0]["Hosts"]["total_mem"] = 67108864
    hosts["items"][1]["Hosts"]["total_mem"] = 77108864
    hosts["items"][3]["Hosts"]["total_mem"] = 87108864
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "65535MB"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "24GB")

    # Case 4: Minimum host memory is ~ 512 GB, recommended val must be .85% of 256GB as vm.overcommit_memory = 2 and in GB
    hosts["items"][0]["Hosts"]["total_mem"] = 536870912
    hosts["items"][1]["Hosts"]["total_mem"] = 636870912
    hosts["items"][3]["Hosts"]["total_mem"] = 736870912
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "218GB")

    # Case 5: Minimum host memory is ~ 1024 GB, recommended val must be .95% of 512GB as vm.overcommit_memory = 2 and in GB
    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "436GB")

    # Case 6: Minimum host memory is ~ 1024 GB, vm.overcommit_ratio = 75, vm.overcommit_memory = 2
    # recommended val must be .95% of (1024*75)/100 and in GB
    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_ratio"] = 75
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")

    ## Test if default_hash_table_bucket_number and hawq_rm_nvseg_perquery_perseg_limit are set correctly based on low hawq_rm_memory_limit_perseg

    # Case 1: When hawq_rm_memory_limit_perseg is between 1GB and 2GB
    # Set hawq_rm_nvseg_perquery_perseg_limit to 4 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments
    hosts["items"][0]["Hosts"]["total_mem"] = 2097152
    hosts["items"][1]["Hosts"]["total_mem"] = 2097152
    hosts["items"][3]["Hosts"]["total_mem"] = 2097152
    services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none"
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "1152MB")
    self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "8")
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "4")

    # Case 2: When hawq_rm_memory_limit_perseg > 2GB
    # Set hawq_rm_nvseg_perquery_perseg_limit to 6 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments
    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512"
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"] = "4"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
    self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12")
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "6")

    # Case 3: When hawq_rm_memory_limit_perseg > 2GB
    # Set hawq_rm_nvseg_perquery_perseg_limit to 8 and default_hash_table_bucket_number as hawq_rm_nvseg_perquery_perseg_limit * numSegments
    hosts["items"][0]["Hosts"]["total_mem"] = 1073741824
    hosts["items"][1]["Hosts"]["total_mem"] = 2073741824
    hosts["items"][3]["Hosts"]["total_mem"] = 3073741824
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_limit"] = "512"
    services["configurations"]["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"] = "8"
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"], "730GB")
    self.assertEqual(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "12")
    self.assertEqual(configurations["hawq-site"]["properties"]["hawq_rm_nvseg_perquery_perseg_limit"], "8")

    ## Test if the properties are set to visible / invisible based on the value of hawq_global_rm_type

    # Case 1: When hawq_global_rm_type is yarn
    services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "yarn"
    properties_visible_status = {"hawq_rm_memory_limit_perseg": "false",
                  "hawq_rm_nvcore_limit_perseg": "false",
                  "hawq_rm_yarn_app_name":"true",
                  "hawq_rm_yarn_queue_name": "true",
                  "hawq_rm_yarn_scheduler_address": "true",
                  "hawq_rm_yarn_address": "true"}
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    for property, status in properties_visible_status.iteritems():
      self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)

    # Case 2: When hawq_global_rm_type is none
    services["configurations"]["hawq-site"]["properties"]["hawq_global_rm_type"] = "none"
    properties_visible_status = {"hawq_rm_memory_limit_perseg": "true",
                                 "hawq_rm_nvcore_limit_perseg": "true",
                                 "hawq_rm_yarn_app_name": "false",
                                 "hawq_rm_yarn_queue_name": "false",
                                 "hawq_rm_yarn_scheduler_address": "false",
                                 "hawq_rm_yarn_address": "false"}
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    for property, status in properties_visible_status.iteritems():
      self.assertEqual(configurations["hawq-site"]["property_attributes"][property]["visible"], status)

    ## Test if vm.overcommit_ratio is set to visible / invisible based on the value of vm.overcommit_memory

    # Case 1: vm.overcommit_ratio should be invisible when overcommit_memory is set as 0
    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 0
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false")

    # Case 2: vm.overcommit_ratio should be invisible when overcommit_memory is set as 1
    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 1
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "false")

    # Case 3: vm.overcommit_ratio should be visible when overcommit_memory is set as 2
    services["configurations"]["hawq-sysctl-env"]["properties"]["vm.overcommit_memory"] = 2
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, None, services, hosts)
    self.assertEqual(configurations["hawq-sysctl-env"]["property_attributes"]["vm.overcommit_ratio"]["visible"], "true")


  def test_createComponentLayoutRecommendations_hawq_3_Hosts(self):
    """ Test that HAWQSTANDBY is recommended on a 3-node cluster """

    services = self.load_json("services-hawq-3-hosts.json")
    componentNames = self.getComponentsFromServices(services)
    self.assertTrue('HAWQSTANDBY' in componentNames)

    hosts = self.load_json("hosts-3-hosts.json")
    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
    self.assertEquals(len(hostsList), 3)

    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
    self.assertTrue('HAWQMASTER' in recommendedComponents)
    self.assertTrue('HAWQSTANDBY' in recommendedComponents)
    self.assertTrue('HAWQSEGMENT' in recommendedComponents)

    # make sure master components are not collocated
    componentsListList = [hostgroup["components"] for hostgroup in
                          recommendations["recommendations"]["blueprint"]["host_groups"]]
    for sublist in componentsListList:
      hostComponents = [item["name"] for item in sublist]
      self.assertFalse({'HAWQMASTER', 'HAWQSTANDBY'}.issubset(hostComponents))

  def test_createComponentLayoutRecommendations_hawq_1_Host(self):

    services = self.load_json("services-hawq-3-hosts.json")
    componentNames = self.getComponentsFromServices(services)
    self.assertTrue('HAWQSTANDBY' in componentNames)

    hosts = self.load_json("hosts-1-host.json")
    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
    self.assertEquals(len(hostsList), 1)

    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
    self.assertTrue('HAWQMASTER' in recommendedComponents)
    self.assertFalse('HAWQSTANDBY' in recommendedComponents)
    self.assertTrue('HAWQSEGMENT' in recommendedComponents)

  def test_createComponentLayoutRecommendations_no_hawq_3_Hosts(self):
    """ Test no failures when there are no HAWQ components """

    services = self.load_json("services-nohawq-3-hosts.json")
    componentNames = self.getComponentsFromServices(services)
    self.assertFalse('HAWQMASTER' in componentNames)
    self.assertFalse('HAWQSTANDBY' in componentNames)
    self.assertFalse('HAWQSEGMENT' in componentNames)

    hosts = self.load_json("hosts-3-hosts.json")
    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
    self.assertEquals(len(hostsList), 3)

    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.recommendComponentLayout(services, hosts)
    recommendedComponents = self.getComponentsFromRecommendations(recommendations)
    self.assertFalse('HAWQMASTER' in recommendedComponents)
    self.assertFalse('HAWQSTANDBY' in recommendedComponents)
    self.assertFalse('HAWQSEGMENT' in recommendedComponents)

  def test_createComponentLayoutRecommendations_hawqsegment_cluster_install(self):
    """ Test that HAWQSEGMENT gets recommended correctly during Cluster Install Wizard, when HAWQ is selected for installation """

    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
    services =  {
                  "services" : [
                    {
                      "StackServices" : {
                        "service_name" : "HDFS"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "DATANODE",
                            "hostnames" : []
                          }
                        }
                      ]
                    },
                    {
                      "StackServices" : {
                        "service_name" : "HAWQ"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "HAWQSEGMENT",
                            "hostnames" : []
                          }
                        }
                      ]
                    }
                  ]
                }

    hawqSegmentHosts = {"c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"}
    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
    self.assertEquals(set(hostNames), hawqSegmentHosts)

  def test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_to_be_installed(self):
    """ Test that HAWQSEGMENT gets recommended correctly during Add Service Wizard, when HAWQ is selected for installation """

    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
    services =  {
                  "services" : [
                    {
                      "StackServices" : {
                        "service_name" : "HDFS"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "DATANODE",
                            "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
                          }
                        }
                      ]
                    },
                    {
                      "StackServices" : {
                        "service_name" : "HAWQ"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "HAWQSEGMENT",
                            "hostnames" : []
                          }
                        }
                      ]
                    }
                  ]
                }

    hawqSegmentHosts = {"c6401.ambari.apache.org", "c6403.ambari.apache.org"}
    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
    self.assertEquals(set(hostNames), hawqSegmentHosts)

  def test_createComponentLayoutRecommendations_hawqsegment_add_service_wizard_already_installed(self):
    """ Test that HAWQSEGMENT does not get recommended during Add Service Wizard, when HAWQ has already been installed """

    hosts = self.prepareHosts(["c6401.ambari.apache.org", "c6402.ambari.apache.org", "c6403.ambari.apache.org"])
    services =  {
                  "services" : [
                    {
                      "StackServices" : {
                        "service_name" : "HDFS"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "DATANODE",
                            "hostnames" : ["c6401.ambari.apache.org", "c6403.ambari.apache.org"]
                          }
                        }
                      ]
                    },
                    {
                      "StackServices" : {
                        "service_name" : "HAWQ"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "HAWQSEGMENT",
                            "hostnames" : ["c6402.ambari.apache.org"]
                          }
                        }
                      ]
                    },
                    {
                      "StackServices" : {
                        "service_name" : "PXF"
                      },
                      "components" : [
                        {
                          "StackServiceComponents" : {
                            "cardinality" : "1+",
                            "component_category" : "SLAVE",
                            "component_name" : "PXF",
                            "hostnames" : []
                          }
                        }
                      ]
                    }
                  ]
                }

    hawqSegmentHosts = {"c6402.ambari.apache.org"}
    self.insertHAWQServiceAdvisorInfo(services)
    recommendations = self.serviceAdvisor.createComponentLayoutRecommendations(services, hosts)
    hostNames = self.getHostsFromRecommendations(recommendations, "HAWQSEGMENT")
    self.assertEquals(set(hostNames), hawqSegmentHosts)

  def test_getComponentLayoutValidations_hawqsegment_not_co_located_with_datanode(self):
    """ Test validation warning for HAWQ segment not colocated with DATANODE """

    services = self.load_json("services-normal-hawq-3-hosts.json")
    hosts = self.load_json("hosts-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)

    hawqsegmentComponent = self.getHosts(componentsList, "HAWQSEGMENT")
    hawqsegmentComponent["hostnames"] = ['c6401.ambari.apache.org']
    datanodeComponent = self.getHosts(componentsList, "DATANODE")
    datanodeComponent["hostnames"] = ['c6402.ambari.apache.org']

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    expected = {
      'type': 'host-component',
      'level': 'WARN',
      'component-name': 'HAWQSEGMENT',
      'message': 'HAWQ Segment must be installed on all DataNodes. The following 2 host(s) do not satisfy the colocation recommendation: c6401.ambari.apache.org, c6402.ambari.apache.org',
    }
    self.assertEquals(validations[0], expected)

    datanodeComponent["hostnames"] = ['c6401.ambari.apache.org']
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 0)


  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
  def test_getComponentLayoutValidations_hawq_3_Hosts(self, socket_mock):
    """ Test layout validations for HAWQ components on a 3-node cluster """

    # case-1: normal placement, no warnings
    services = self.load_json("services-normal-hawq-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
    self.assertEquals(len(hawqMasterHosts), 1)
    self.assertEquals(len(hawqStandbyHosts), 1)
    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])

    hosts = self.load_json("hosts-3-hosts.json")
    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
    self.assertEquals(len(hostsList), 3)

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 0)

    # case-2: HAWQ masters are collocated
    services = self.load_json("services-master_standby_colo-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
    self.assertEquals(len(hawqMasterHosts), 1)
    self.assertEquals(len(hawqStandbyHosts), 1)
    self.assertEquals(hawqMasterHosts[0], hawqStandbyHosts[0])

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 1)
    expected = {
      'component-name': 'HAWQSTANDBY',
      'message': 'HAWQ Master and HAWQ Standby Master cannot be deployed on the same host.',
      'type': 'host-component',
      'host': 'c6403.ambari.apache.org',
      'level': 'ERROR'
    }
    self.assertEquals(validations[0], expected)

    # case-3: HAWQ Master and Ambari Server are collocated
    services = self.load_json("services-master_ambari_colo-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
    self.assertEquals(len(hawqMasterHosts), 1)
    self.assertEquals(len(hawqStandbyHosts), 1)
    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
    self.assertEquals(hawqMasterHosts[0], "c6401.ambari.apache.org")

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 1)
    expected = {
      'component-name': 'HAWQMASTER',
      'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. '  +
                 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Master on a different host ' +
                 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.',
      'type': 'host-component',
      'host': 'c6401.ambari.apache.org',
      'level': 'WARN'
    }
    self.assertEquals(validations[0], expected)

    # case-4: HAWQ Standby and Ambari Server are collocated
    services = self.load_json("services-standby_ambari_colo-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")["hostnames"]
    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")["hostnames"]
    self.assertEquals(len(hawqMasterHosts), 1)
    self.assertEquals(len(hawqStandbyHosts), 1)
    self.assertNotEquals(hawqMasterHosts[0], hawqStandbyHosts[0])
    self.assertEquals(hawqStandbyHosts[0], "c6401.ambari.apache.org")

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 1)
    expected = {
      'component-name': 'HAWQSTANDBY',
      'message': 'The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. '  +
                 'If you are using port 5432 for Postgres, you must either deploy the HAWQ Standby Master on a different host ' +
                 'or configure a different port for the HAWQ Masters in the HAWQ Configuration page.',
      'type': 'host-component',
      'host': 'c6401.ambari.apache.org',
      'level': 'WARN'
    }
    self.assertEquals(validations[0], expected)


  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
  def test_getComponentLayoutValidations_nohawq_3_Hosts(self, socket_mock):
    """ Test no failures when there are no HAWQ components on a 3-node cluster """

    # normal placement, no warnings
    services = self.load_json("services-normal-nohawq-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqMasterHosts = self.getHosts(componentsList, "HAWQMASTER")
    hawqStandbyHosts = self.getHosts(componentsList, "HAWQSTANDBY")
    self.assertEquals(len(hawqMasterHosts), 0)
    self.assertEquals(len(hawqStandbyHosts), 0)

    hosts = self.load_json("hosts-3-hosts.json")
    hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
    self.assertEquals(len(hostsList), 3)

    self.insertHAWQServiceAdvisorInfo(services)
    validations = self.serviceAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 0)

  def test_recommendHAWQConfigurations(self):

    hosts = {
      "items": [
        {
          "Hosts": {
            "host_name": "c6401.ambari.apache.org",
            "cpu_count" : 2,
            "total_mem": 33554432
          }
        },
        {
          "Hosts": {
            "host_name": "c6402.ambari.apache.org",
            "cpu_count" : 4,
            "total_mem": 33554433
          }
        },
        {
          "Hosts": {
            "host_name": "c6403.ambari.apache.org",
            "cpu_count" : 1,
            "total_mem": 33554434
          }
        },
        {
          "Hosts": {
            "host_name": "c6404.ambari.apache.org",
            "cpu_count" : 2,
            "total_mem": 33554435
          }
        }
      ]
    }

    # original cluster data with 3 segments
    services = self.load_json("services-normal-hawq-3-hosts.json")
    componentsList = self.getComponentsListFromServices(services)
    hawqSegmentComponent = self.getHosts(componentsList, "HAWQSEGMENT")

    # setup default configuration values
    services["configurations"]["hawq-site"] = {
      "properties": {
        "default_hash_table_bucket_number": "24",
        "hawq_rm_nvseg_perquery_limit": "512",
        "hawq_rm_yarn_address": "localhost:8032",
        "hawq_rm_yarn_scheduler_address": "localhost:8030",
        "hawq_global_rm_type":  "none",
        "hawq_rm_nvseg_perquery_perseg_limit": "6"
      }
    }

    services["configurations"]["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}}
    services["configurations"]["hawq-sysctl-env"] = {"properties": {}}

    services["configurations"]["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
                                                              "yarn.resourcemanager.scheduler.address": "host1:8030"}}
    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
    configurations = {}
    clusterData = {}
    self.insertHAWQServiceAdvisorInfo(services)

    # Test 1 - with 3 segments
    self.assertEquals(len(hawqSegmentComponent["hostnames"]), 3)
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(3 * 6))
    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")

    # check derived properties
    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_address"], "host1:8050")
    self.assertEquals(configurations["hawq-site"]["properties"]["hawq_rm_yarn_scheduler_address"], "host1:8030")

    # Test 2 - with 100 segments
    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(100)]
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], str(100 * 5))
    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")

    # Test 3 - with 512 segments
    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(512)]
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")

    # Test 4 - with 513 segments
    hawqSegmentComponent["hostnames"] = ["host" + str(i) for i in range(513)]
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations["hawq-site"]["properties"]["default_hash_table_bucket_number"], "512")
    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "true")

    # Test 5 - with no segments
    configurations = {}
    services["configurations"]["hawq-site"] = {"properties":{"hawq_global_rm_type": "none"}}
    hawqSegmentComponent["hostnames"] = []
    self.serviceAdvisor.getServiceConfigurationRecommendations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations["hdfs-client"]["properties"]["output.replace-datanode-on-failure"], "false")
    self.assertTrue("default_hash_table_bucket_number" not in configurations["hawq-site"])


  def test_validateHAWQSiteConfigurations(self):
    services = self.load_json("services-hawq-3-hosts.json")
    # setup default configuration values
    # Test hawq_rm_yarn_address and hawq_rm_scheduler_address are set correctly
    configurations = services["configurations"]
    configurations["hawq-site"] = {"properties": {"hawq_rm_yarn_address": "localhost:8032",
                                                  "hawq_rm_yarn_scheduler_address": "localhost:8030"}}
    configurations["yarn-site"] = {"properties": {"yarn.resourcemanager.address": "host1:8050",
                                                  "yarn.resourcemanager.scheduler.address": "host1:8030"}}
    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})
    properties = configurations["hawq-site"]["properties"]
    defaults = {}
    hosts = {}

    expected_warnings = {
      'hawq_rm_yarn_address': {
        'config-type': 'hawq-site',
        'message': 'Expected value: host1:8050 (this property should have the same value as the property yarn.resourcemanager.address in yarn-site)',
        'type': 'configuration',
        'config-name': 'hawq_rm_yarn_address',
        'level': 'WARN'
      },
      'hawq_rm_yarn_scheduler_address': {
        'config-type': 'hawq-site',
        'message': 'Expected value: host1:8030 (this property should have the same value as the property yarn.resourcemanager.scheduler.address in yarn-site)',
        'type': 'configuration',
        'config-name': 'hawq_rm_yarn_scheduler_address',
        'level': 'WARN'
      }
    }

    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    problems_dict = {}
    for problem in problems:
      problems_dict[problem['config-name']] = problem
    self.assertEqual(len(problems), 2)
    self.assertEqual(problems_dict, expected_warnings)

    # Test hawq_master_directory multiple directories validation
    configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master",
                                                  "hawq_segment_directory": "/data/hawq/segment"}}
    properties = configurations["hawq-site"]["properties"]
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    problems_dict = {}
    self.assertEqual(len(problems), 0)
    expected_warnings = {}
    self.assertEqual(problems_dict, expected_warnings)

    configurations["hawq-site"] = {"properties": {"hawq_master_directory": "/data/hawq/master1,/data/hawq/master2",
                                                  "hawq_segment_directory": "/data/hawq/segment1 /data/hawq/segment2"}}
    properties = configurations["hawq-site"]["properties"]
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    problems_dict = {}
    for problem in problems:
      problems_dict[problem['config-name']] = problem
    self.assertEqual(len(problems), 2)
    expected_warnings = {
      'hawq_master_directory': {
        'config-type': 'hawq-site',
        'message': 'Multiple directories for HAWQ Master directory are not allowed.',
        'type': 'configuration',
        'config-name': 'hawq_master_directory',
        'level': 'ERROR'
      },
      'hawq_segment_directory': {
        'config-type': 'hawq-site',
        'message': 'Multiple directories for HAWQ Segment directory are not allowed.',
        'type': 'configuration',
        'config-name': 'hawq_segment_directory',
        'level': 'ERROR'
      }
    }
    self.assertEqual(problems_dict, expected_warnings)

    # Test hawq_global_rm_type validation
    services = {
                 "services" : [
                   {
                     "StackServices" : {
                     "service_name" : "HAWQ"
                     },
                     "components": []
                   } ],
                 "configurations":
                   {
                     "hawq-site": {
                       "properties": {
                         "hawq_global_rm_type": "yarn"
                       }
                     }
                   }
                }
    properties = services["configurations"]["hawq-site"]["properties"]

    # case 1: hawq_global_rm_type is set as yarn, but YARN service is not installed. Validation error expected.
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
    self.assertEqual(len(problems), 1)
    expected = {
      "config-type": "hawq-site",
      "message": "hawq_global_rm_type must be set to none if YARN service is not installed",
      "type": "configuration",
      "config-name": "hawq_global_rm_type",
      "level": "ERROR"
    }
    self.assertEqual(problems[0], expected)

    # case 2: hawq_global_rm_type is set as yarn, and YARN service is installed. No validation errors expected.
    services["services"].append({"StackServices" : {"service_name" : "YARN"}, "components":[]})

    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, services["configurations"], services, hosts)
    self.assertEqual(len(problems), 0)

    # Test HAWQ Master port conflict with Ambari Server Postgres port

    # case 1: HAWQ Master is placed on Ambari Server and HAWQ Master port is same as Ambari Server Postgres Port
    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True)
    configurations = {
      "hawq-site": {
        "properties":
          {"hawq_master_address_port": "5432"}
      }
    }
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    expected = {
      "config-name": "hawq_master_address_port",
      "config-type": "hawq-site",
      "level": "WARN",
      "message": "The default Postgres port (5432) on the Ambari Server conflicts with the default HAWQ Masters port. "
                 "If you are using port 5432 for Postgres, you must either deploy the HAWQ Masters on a different host "
                 "or configure a different port for the HAWQ Masters in the HAWQ Configuration page.",
      "type": "configuration"}
    self.assertEqual(problems[0], expected)

    # case 2: HAWQ Master is placed on Ambari Server and HAWQ Master port is different from  Ambari Server Postgres Port
    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=True)
    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # case 3: HAWQ Master is not placed on Ambari Server and HAWQ Master port is same as  Ambari Server Postgres Port
    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "5432"
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # case 4: HAWQ Master is not placed on Ambari Server and HAWQ Master port is different from  Ambari Server Postgres Port
    self.serviceAdvisor.isHawqMasterComponentOnAmbariServer = MagicMock(return_value=False)
    configurations["hawq-site"]["properties"]["hawq_master_address_port"] = "10432"
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # -------- test query limits warning ----------
    services = {
      "services":  [
        { "StackServices": {"service_name": "HAWQ"},
          "components": [{
            "StackServiceComponents": {
              "component_name": "HAWQSEGMENT",
              "hostnames": []
            }}]
          }],
      "configurations": {}
    }
    # setup default configuration values
    configurations = services["configurations"]
    configurations["hawq-site"] = {
      "properties": {
        "default_hash_table_bucket_number": "600",
        "hawq_rm_nvseg_perquery_limit": "500",
        "hawq_rm_nvseg_perquery_perseg_limit": "6"
      }
    }
    properties = configurations["hawq-site"]["properties"]
    defaults = {}
    hosts = {}

    expected = {
      'config-type': 'hawq-site',
      'message': 'Default buckets for Hash Distributed tables parameter value should not be greater than the value of Virtual Segments Limit per Query (Total) parameter, currently set to 500.',
      'type': 'configuration',
      'config-name': 'default_hash_table_bucket_number',
      'level': 'ERROR'
    }
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

    configurations["hawq-site"] = {
      "properties": {
        "default_hash_table_bucket_number": "500",
        "hawq_rm_nvseg_perquery_limit": "500"
      }
    }
    properties = configurations["hawq-site"]["properties"]
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    configurations["hawq-site"] = {
      "properties":
        {
          "hawq_global_rm_type": "none",
          "hawq_rm_memory_limit_perseg": "1023MB"
        }
    }
    expected = {
      'config-type': 'hawq-site',
      'message': 'HAWQ Segment Memory less than 1GB is not sufficient',
      'type': 'configuration',
      'config-name': 'hawq_global_rm_type',
      'level': 'ERROR'
    }
    properties = configurations["hawq-site"]["properties"]
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

    configurations["hawq-site"]["properties"]["hawq_rm_memory_limit_perseg"] = "1GB"
    properties = configurations["hawq-site"]["properties"]
    problems = self.serviceAdvisor.validateHAWQSiteConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)


  def test_validateHAWQHdfsClientConfigurations(self):
    services = {
      "services":  [
        { "StackServices": {"service_name": "HAWQ"},
          "components": [{
            "StackServiceComponents": {
              "component_name": "HAWQSEGMENT",
              "hostnames": []
            }}]
          }],
      "configurations": {}
    }
    # setup default configuration values
    configurations = services["configurations"]
    configurations["hdfs-client"] = {"properties": {"output.replace-datanode-on-failure": "true"}}
    properties = configurations["hdfs-client"]["properties"]
    defaults = {}
    hosts = {}

    # 1. Try with no hosts
    expected = {
        'config-type': 'hdfs-client',
        'message': 'output.replace-datanode-on-failure should be set to false (unchecked) for clusters with 3 or less HAWQ Segments',
        'type': 'configuration',
        'config-name': 'output.replace-datanode-on-failure',
        'level': 'WARN'
    }

    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

    # 2. Try with 3 hosts
    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3"]
    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

    # 3. Try with 4 hosts - default value
    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = ["host1", "host2", "host3", "host4"]
    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # 4. Try with 4 hosts
    properties = {"output.replace-datanode-on-failure": "false"}
    expected = {
      'config-type': 'hdfs-client',
      'message': 'output.replace-datanode-on-failure should be set to true (checked) for clusters with more than 3 HAWQ Segments',
      'type': 'configuration',
      'config-name': 'output.replace-datanode-on-failure',
      'level': 'WARN'
    }
    problems = self.serviceAdvisor.validateHAWQHdfsClientConfigurations(properties, defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

  def test_validateHDFSSiteConfigurations(self):
    services = {
      "services":  [
        { "StackServices": {"service_name": "HAWQ"},
          "components": [{
            "StackServiceComponents": {
              "component_name": "HAWQSEGMENT",
              "hostnames": []
            }}]
          }],
      "configurations": {"hdfs-site": {}, "core-site": {}}
    }

    # setup default configuration values for non-kerberos case
    configurations = services["configurations"]
    configurations["cluster-env"] = {"properties": {"security_enabled": "false"}}
    defaults = {}
    hosts = {}
    desired_values = self.getDesiredHDFSSiteValues(False)

    # check all properties setup correctly in hdfs-site
    configurations["hdfs-site"]["properties"] = desired_values.copy()
    problems = self.serviceAdvisor.validateHDFSSiteConfigurations(configurations["hdfs-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # check overall number of validations for hdfs-site
    configurations["hdfs-site"]["properties"] = {}
    problems = self.serviceAdvisor.validateHDFSSiteConfigurations(configurations["hdfs-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 10)

    # check individual properties
    for property in desired_values.keys():
      # populate all properties as to desired configuration
      configurations["hdfs-site"]["properties"] = desired_values.copy()
      # test when the given property is missing
      configurations["hdfs-site"]["properties"].pop(property)
      expected = {
        'config-type': 'hdfs-site',
        'message': 'HAWQ requires this property to be set to the recommended value of ' + desired_values[property],
        'type': 'configuration',
        'config-name': property,
        'level': 'ERROR' if property == 'dfs.allow.truncate' else 'WARN'
      }
      problems = self.serviceAdvisor.validateHDFSSiteConfigurations(configurations["hdfs-site"]["properties"], defaults, configurations, services, hosts)
      self.assertEqual(len(problems), 1)
      self.assertEqual(problems[0], expected)

      # test when the given property has a non-desired value
      configurations["hdfs-site"]["properties"][property] = "foo"
      problems = self.serviceAdvisor.validateHDFSSiteConfigurations(configurations["hdfs-site"]["properties"], defaults, configurations, services, hosts)
      self.assertEqual(len(problems), 1)
      self.assertEqual(problems[0], expected)

    # check all properties setup correctly in core-site
    configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size" : "3300"}
    problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 0)

    # check overall number of validations for core-site
    configurations["core-site"]["properties"] = {}
    problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)

    # check incorrect core-site property
    expected = {
      'config-type': 'core-site',
      'message': 'HAWQ requires this property to be set to the recommended value of 3300',
      'type': 'configuration',
      'config-name': 'ipc.server.listen.queue.size',
      'level': 'WARN'
    }
    configurations["core-site"]["properties"] = {"ipc.server.listen.queue.size" : "0"}
    problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)

    # check missing core-site property
    configurations["core-site"]["properties"].pop("ipc.server.listen.queue.size")
    problems = self.serviceAdvisor.validateCORESiteConfigurations(configurations["core-site"]["properties"], defaults, configurations, services, hosts)
    self.assertEqual(len(problems), 1)
    self.assertEqual(problems[0], expected)