'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''

import json
import os
import socket
from unittest import TestCase
from mock.mock import patch


class TestHDP23StackAdvisor(TestCase):

  def setUp(self):
    import imp
    self.maxDiff = None
    self.testDirectory = os.path.dirname(os.path.abspath(__file__))
    stackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
    hdp206StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
    hdp21StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.1/services/stack_advisor.py')
    hdp22StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.2/services/stack_advisor.py')
    hdp23StackAdvisorPath = os.path.join(self.testDirectory, '../../../../../main/resources/stacks/HDP/2.3/services/stack_advisor.py')
    hdp23StackAdvisorClassName = 'HDP23StackAdvisor'
    with open(stackAdvisorPath, 'rb') as fp:
      imp.load_module('stack_advisor', fp, stackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
    with open(hdp206StackAdvisorPath, 'rb') as fp:
      imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
    with open(hdp21StackAdvisorPath, 'rb') as fp:
      imp.load_module('stack_advisor_impl', fp, hdp21StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
    with open(hdp22StackAdvisorPath, 'rb') as fp:
      imp.load_module('stack_advisor_impl', fp, hdp22StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
    with open(hdp23StackAdvisorPath, 'rb') as fp:
      stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp23StackAdvisorPath, ('.py', 'rb', imp.PY_SOURCE))
    clazz = getattr(stack_advisor_impl, hdp23StackAdvisorClassName)
    self.stackAdvisor = clazz()

    # substitute method in the instance
    self.get_system_min_uid_real = self.stackAdvisor.get_system_min_uid
    self.stackAdvisor.get_system_min_uid = self.get_system_min_uid_magic

  def load_json(self, filename):
    file = os.path.join(self.testDirectory, filename)
    with open(file, 'rb') as f:
      data = json.load(f)
    return data

  def prepareHosts(self, hostsNames):
    hosts = { "items": [] }
    for hostName in hostsNames:
      nextHost = {"Hosts":{"host_name" : hostName}}
      hosts["items"].append(nextHost)
    return hosts

  @patch('__builtin__.open')
  @patch('os.path.exists')
  def get_system_min_uid_magic(self, exists_mock, open_mock):
    class MagicFile(object):
      def read(self):
        return """
        #test line UID_MIN 200
        UID_MIN 500
        """

      def __exit__(self, exc_type, exc_val, exc_tb):
        pass

      def __enter__(self):
        return self

    exists_mock.return_value = True
    open_mock.return_value = MagicFile()
    return self.get_system_min_uid_real()


  def fqdn_mock_result(value=None):
      return 'c6401.ambari.apache.org' if value is None else value


  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
  def test_getComponentLayoutValidations_sparkts_no_hive(self, socket_mock):
    """ Test SparkTS is picked when Hive is not installed """

    hosts = self.load_json("sparkts-host.json")
    services = self.load_json("services-sparkts.json")
    componentsListList = [service["components"] for service in services["services"]]
    componentsList = [item for sublist in componentsListList for item in sublist]

    sparkTS = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SPARK_THRIFTSERVER"]
    hiveMetaStore = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HIVE_METASTORE"]
    self.assertEquals(len(sparkTS), 1)
    self.assertEquals(len(hiveMetaStore), 0)

    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
    expected = {'component-name': 'SPARK_THRIFTSERVER', 'message': 'SPARK_THRIFTSERVER requires HIVE_METASTORE to be selected/deployed.', 'type': 'host-component', 'level': 'ERROR'}
    self.assertEquals(validations[0], expected)


  @patch('socket.getfqdn', side_effect=fqdn_mock_result)
  def test_getComponentLayoutValidations_sparkts_with_hive(self, socket_mock):
    """ Test SparkTS is picked when Hive is installed """

    hosts = self.load_json("sparkts-host.json")
    services = self.load_json("services-sparkts-hive.json")
    componentsListList = [service["components"] for service in services["services"]]
    componentsList = [item for sublist in componentsListList for item in sublist]

    sparkTS = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "SPARK_THRIFTSERVER"]
    hiveMetaStore = [component["StackServiceComponents"]["hostnames"] for component in componentsList if component["StackServiceComponents"]["component_name"] == "HIVE_METASTORE"]
    self.assertEquals(len(sparkTS), 1)
    self.assertEquals(len(hiveMetaStore), 1)

    validations = self.stackAdvisor.getComponentLayoutValidations(services, hosts)
    self.assertEquals(len(validations), 0)


  def test_recommendHDFSConfigurations(self):
    configurations = {
      "hdfs-site": {
        "properties": {
          "dfs.namenode.inode.attributes.provider.class": "org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer",
        }
      },
      "ranger-hdfs-plugin-properties": {
        "properties": {
          "ranger-hdfs-plugin-enabled": "No"
        }
      }
    }
    clusterData = {
      "totalAvailableRam": 2048,
      "hBaseInstalled": True,
      "hbaseRam": 112,
      "reservedRam": 128
    }
    hosts = {
      "items": [
        {
          "Hosts": {
            "disk_info": [{
              "size": '8',
              "mountpoint": "/"
            }]
          }
        }]}
    services = {
      "services":
        [
          {
            "StackServices": {
              "service_name" : "HDFS",
              "service_version" : "2.6.0.2.2"
            },
            "components": [
            ]
          }
        ],
      "Versions": {
        "stack_version": "2.3"
      },
      "configurations": configurations,
      "ambari-server-properties": {"ambari-server.user":"ambari_user"}
    }

    # Test with Ranger HDFS plugin disabled
    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations['hdfs-site']['property_attributes']['dfs.namenode.inode.attributes.provider.class'], {'delete': 'true'}, "Test with Ranger HDFS plugin is disabled")

    # Test with Ranger HDFS plugin is enabled
    configurations['hdfs-site']['properties'] = {}
    configurations['hdfs-site']['property_attributes'] = {}
    services['configurations']['ranger-hdfs-plugin-properties']['properties']['ranger-hdfs-plugin-enabled'] = 'Yes'
    self.stackAdvisor.recommendHDFSConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations['hdfs-site']['properties']['dfs.namenode.inode.attributes.provider.class'], 'org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer', "Test with Ranger HDFS plugin is enabled")

  def test_recommendYARNConfigurations(self):
    configurations = {}
    servicesList = ["YARN"]
    components = []
    hosts = {
      "items" : [
        {
          "Hosts" : {
            "cpu_count" : 6,
            "total_mem" : 50331648,
            "disk_info" : [
              {"mountpoint" : "/"},
              {"mountpoint" : "/dev/shm"},
              {"mountpoint" : "/vagrant"},
              {"mountpoint" : "/"},
              {"mountpoint" : "/dev/shm"},
              {"mountpoint" : "/vagrant"}
            ],
            "public_host_name" : "c6401.ambari.apache.org",
            "host_name" : "c6401.ambari.apache.org"
          }
        }
      ]
    }
    services = {
      "services" : [ {
        "StackServices":{
          "service_name": "YARN",
        },
        "Versions": {
          "stack_version": "2.3"
        },
        "components": [
          {
            "StackServiceComponents": {
              "component_name": "NODEMANAGER",
              "hostnames": ["c6401.ambari.apache.org"]
            }
          }
        ]
      }
      ],
      "configurations": {
        "yarn-site": {
          "properties": {
            "yarn.authorization-provider": "org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer"
          }
        },
        "ranger-yarn-plugin-properties": {
          "properties": {
            "ranger-yarn-plugin-enabled": "No"
          }
        }
      }
    }

    clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None)
    # Test with Ranger YARN plugin disabled
    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['yarn-site']['property_attributes']['yarn.authorization-provider'], {'delete': 'true'}, "Test with Ranger HDFS plugin is disabled")

    # Test with Ranger YARN plugin is enabled
    configurations['yarn-site']['properties'] = {}
    configurations['yarn-site']['property_attributes'] = {}
    services['configurations']['ranger-yarn-plugin-properties']['properties']['ranger-yarn-plugin-enabled'] = 'Yes'
    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['yarn-site']['properties']['yarn.authorization-provider'], 'org.apache.ranger.authorization.yarn.authorizer.RangerYarnAuthorizer', "Test with Ranger YARN plugin enabled")


  def test_recommendKAFKAConfigurations(self):
    configurations = {}
    clusterData = {
      "totalAvailableRam": 2048,
      "hBaseInstalled": True,
      "hbaseRam": 112,
      "reservedRam": 128
    }
    services = {
      "services":
        [
          {
            "StackServices": {
              "service_name" : "KAFKA",
              "service_version" : "2.6.0.2.2"
            }
          },
          {
            "StackServices": {
              "service_name": "RANGER",
              "service_version": "0.5.0.2.3"
            }
          },
          {
            "StackServices": {
              "service_name": "AMBARI_METRICS"
            },
            "components": [{
              "StackServiceComponents": {
                "component_name": "METRICS_COLLECTOR",
                "hostnames": ["host1"]
              }

            }, {
              "StackServiceComponents": {
                "component_name": "METRICS_MONITOR",
                "hostnames": ["host1"]
              }

            }]
          }
        ],
      "Versions": {
        "stack_version": "2.3"
      },
      "configurations": {
        "core-site": {
          "properties": { },
        },
        "cluster-env": {
          "properties": {
            "security_enabled" : "true"
          },
          "property_attributes": {}
        },
        "kafka-broker": {
          "properties": {
            "authorizer.class.name" : "kafka.security.auth.SimpleAclAuthorizer"
          },
          "property_attributes": {}
        },
        "ranger-kafka-plugin-properties": {
          "properties": {
            "ranger-kafka-plugin-enabled": "No"
          }
        },
        "kafka-log4j": {
          "properties": {
            "content": "kafka.logs.dir=logs"
          }
        }
      }
    }

    # Test authorizer.class.name with Ranger Kafka plugin disabled in non-kerberos environment
    services['configurations']['cluster-env']['properties']['security_enabled'] = "false"
    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['kafka-broker']['property_attributes']['authorizer.class.name'], {'delete': 'true'}, "Test authorizer.class.name with Ranger Kafka plugin is disabled in non-kerberos environment")

    # Test authorizer.class.name with Ranger Kafka plugin disabled in kerberos environment
    services['configurations']['cluster-env']['properties']['security_enabled'] = "true"
    configurations['kafka-broker']['properties'] = {}
    configurations['kafka-broker']['property_attributes'] = {}
    services['configurations']['kafka-broker']['properties']['security.inter.broker.protocol'] = 'PLAINTEXTSASL'
    services['configurations']['kafka-broker']['properties']['authorizer.class.name'] = 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer'
    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['kafka-broker']['properties']['authorizer.class.name'], 'kafka.security.auth.SimpleAclAuthorizer' , "Test authorizer.class.name with Ranger Kafka plugin disabled in kerberos environment")

    # Test authorizer.class.name with Ranger Kafka plugin enabled in non-kerberos environment
    services['configurations']['cluster-env']['properties']['security_enabled'] = "false"
    configurations['kafka-broker']['properties'] = {}
    configurations['kafka-broker']['property_attributes'] = {}
    del services['configurations']['kafka-broker']['properties']['security.inter.broker.protocol']
    services['configurations']['kafka-broker']['properties']['authorizer.class.name'] = 'kafka.security.auth.SimpleAclAuthorizer'
    services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'] = 'Yes'
    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['kafka-broker']['properties']['authorizer.class.name'], 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer', "Test authorizer.class.name with Ranger Kafka plugin enabled in kerberos environment")

    services['configurations']['cluster-env']['properties']['security_enabled'] = "false"
    configurations['kafka-broker']['properties'] = {}
    configurations['kafka-broker']['property_attributes'] = {}
    services['configurations']['kafka-broker']['properties']['security.inter.broker.protocol'] = 'PLAINTEXTSASL'
    services['configurations']['kafka-broker']['properties']['authorizer.class.name'] = 'kafka.security.auth.SimpleAclAuthorizer'
    services['configurations']['ranger-kafka-plugin-properties']['properties']['ranger-kafka-plugin-enabled'] = 'Yes'
    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['kafka-broker']['properties']['authorizer.class.name'], 'org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer', "Test authorizer.class.name with Ranger Kafka plugin enabled in kerberos environment")

    # Test kafka-log4j content when Ranger plugin for Kafka is enabled

    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    log4jContent = services['configurations']['kafka-log4j']['properties']['content']
    newRangerLog4content = "\nlog4j.appender.rangerAppender=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.rangerAppender.DatePattern='.'yyyy-MM-dd-HH\n" \
                     "log4j.appender.rangerAppender.File=${kafka.logs.dir}/ranger_kafka.log\nlog4j.appender.rangerAppender.layout" \
                     "=org.apache.log4j.PatternLayout\nlog4j.appender.rangerAppender.layout.ConversionPattern=%d{ISO8601} %p [%t] %C{6} (%F:%L) - %m%n\n" \
                     "log4j.logger.org.apache.ranger=INFO, rangerAppender"
    expectedLog4jContent = log4jContent + newRangerLog4content
    self.assertEquals(configurations['kafka-log4j']['properties']['content'], expectedLog4jContent, "Test kafka-log4j content when Ranger plugin for Kafka is enabled")

    # Test kafka.metrics.reporters when AMBARI_METRICS is present in services
    self.stackAdvisor.recommendKAFKAConfigurations(configurations, clusterData, services, None)
    self.assertEqual(configurations['kafka-broker']['properties']['kafka.metrics.reporters'],
                                              'org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter')

  def test_recommendHBASEConfigurations(self):
    configurations = {}
    clusterData = {
      "totalAvailableRam": 2048,
      "hBaseInstalled": True,
      "hbaseRam": 112,
      "reservedRam": 128
    }
    expected = {
      "hbase-site": {
        "properties": {
          "hbase.bucketcache.size": "92160",
          "hbase.bucketcache.percentage.in.combinedcache": "1.0000",
          "hbase.regionserver.global.memstore.size": "0.4",
          "hfile.block.cache.size": "0.4",
          "hbase.coprocessor.region.classes": "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint",
          "hbase.coprocessor.master.classes": "",
          "hbase.coprocessor.regionserver.classes": "",
          "hbase.rpc.controllerfactory.class": "org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory",
          "hbase.region.server.rpc.scheduler.factory.class": "org.apache.hadoop.hbase.ipc.PhoenixRpcSchedulerFactory",
          'hbase.regionserver.wal.codec': 'org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec',
          "hbase.bucketcache.ioengine": "offheap",
          "phoenix.functions.allowUserDefinedFunctions": "true"
        },
        "property_attributes": {
          "hbase.coprocessor.regionserver.classes": {
            "delete": "true"
          },
          "hbase.bucketcache.percentage.in.combinedcache": {
            "delete": "true"
          }
        }
      },
      "hbase-env": {
        "properties": {
          "hbase_master_heapsize": "1024",
          "hbase_max_direct_memory_size": "94208",
          "hbase_regionserver_heapsize": "20480"
        }
      }
    }
    services = {
      "services":
        [{"StackServices":
            {"service_name" : "HDFS",
             "service_version" : "2.6.0.2.2"
             },
          "components":[
            {
              "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/DATANODE",
              "StackServiceComponents":{
                "advertise_version":"true",
                "cardinality":"1+",
                "component_category":"SLAVE",
                "component_name":"DATANODE",
                "custom_commands":[

                ],
                "display_name":"DataNode",
                "is_client":"false",
                "is_master":"false",
                "service_name":"HDFS",
                "stack_name":"HDP",
                "stack_version":"2.2",
                "hostnames":[
                  "host1"
                ]
              },
              "dependencies":[

              ]
            },
            {
              "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/JOURNALNODE",
              "StackServiceComponents":{
                "advertise_version":"true",
                "cardinality":"0+",
                "component_category":"SLAVE",
                "component_name":"JOURNALNODE",
                "custom_commands":[

                ],
                "display_name":"JournalNode",
                "is_client":"false",
                "is_master":"false",
                "service_name":"HDFS",
                "stack_name":"HDP",
                "stack_version":"2.2",
                "hostnames":[
                  "host1"
                ]
              },
              "dependencies":[
                {
                  "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/JOURNALNODE/dependencies/HDFS_CLIENT",
                  "Dependencies":{
                    "component_name":"HDFS_CLIENT",
                    "dependent_component_name":"JOURNALNODE",
                    "dependent_service_name":"HDFS",
                    "stack_name":"HDP",
                    "stack_version":"2.2"
                  }
                }
              ]
            },
            {
              "href":"/api/v1/stacks/HDP/versions/2.2/services/HDFS/components/NAMENODE",
              "StackServiceComponents":{
                "advertise_version":"true",
                "cardinality":"1-2",
                "component_category":"MASTER",
                "component_name":"NAMENODE",
                "custom_commands":[
                  "DECOMMISSION",
                  "REBALANCEHDFS"
                ],
                "display_name":"NameNode",
                "is_client":"false",
                "is_master":"true",
                "service_name":"HDFS",
                "stack_name":"HDP",
                "stack_version":"2.2",
                "hostnames":[
                  "host2"
                ]
              },
              "dependencies":[

              ]
            },
            ],
          }],
      "Versions": {
        "stack_version": "2.3"
      },
      "configurations": {
        "yarn-site": {
          "properties": {
            "yarn.scheduler.minimum-allocation-mb": "256",
            "yarn.scheduler.maximum-allocation-mb": "2048"
            }
          },
        "hbase-env": {
          "properties": {
            "phoenix_sql_enabled": "true"
          }
        },
        "hbase-site": {
          "properties": {
            "hbase.coprocessor.regionserver.classes": ""
          }
        }
      }
    }

    # Test
    self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations, expected)

    # Test
    clusterData['hbaseRam'] = '4'
    expected["hbase-site"]["property_attributes"]["hbase.bucketcache.size"] = {"delete": "true"}
    expected["hbase-site"]["property_attributes"]["hbase.bucketcache.ioengine"] = {"delete": "true"}
    expected["hbase-site"]["property_attributes"]["hbase.bucketcache.percentage.in.combinedcache"] = {"delete": "true"}
    expected["hbase-env"]["property_attributes"] = {"hbase_max_direct_memory_size" : {"delete": "true"}}
    expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "1024"
    expected["hbase-env"]["properties"]["hbase_regionserver_heapsize"] = "4096"
    self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations, expected)

    # Test - default recommendations should have certain configs deleted. HAS TO BE LAST TEST.
    services["configurations"] = {"hbase-site": {"properties": {"phoenix.functions.allowUserDefinedFunctions": '', "hbase.rpc.controllerfactory.class": '', "hbase.region.server.rpc.scheduler.factory.class": ''}}}
    configurations = {}
    self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
    self.assertEquals(configurations['hbase-site']['property_attributes']['phoenix.functions.allowUserDefinedFunctions'], {'delete': 'true'})
    self.assertEquals(configurations['hbase-site']['property_attributes']['hbase.rpc.controllerfactory.class'], {'delete': 'true'})
    self.assertEquals(configurations['hbase-site']['property_attributes']['hbase.region.server.rpc.scheduler.factory.class'], {'delete': 'true'})
    self.assertEquals(configurations['hbase-site']['properties']['hbase.regionserver.wal.codec'], "org.apache.hadoop.hbase.regionserver.wal.WALCellCodec")


  def test_recommendHiveConfigurations(self):
    self.maxDiff = None
    configurations = {
      "yarn-site": {
        "properties": {
          "yarn.scheduler.minimum-allocation-mb": "256",
          "yarn.scheduler.maximum-allocation-mb": "8192",
        },
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'yarn-site': {
        'properties': {
          'yarn.scheduler.minimum-allocation-mb': '256',
          'yarn.scheduler.maximum-allocation-mb': '8192'
        }
      },
      'hive-env': {
        'properties': {
          'hive_exec_orc_storage_strategy': 'SPEED',
          'hive_security_authorization': 'None',
          'hive_timeline_logging_enabled': 'true',
          'hive_txn_acid': 'off'
        }
      },
      'hive-site': {
        'properties': {
          'hive.server2.enable.doAs': 'true',
          'hive.server2.tez.default.queues': "queue1,queue2",
          'hive.server2.tez.initialize.default.sessions': 'false',
          'hive.server2.tez.sessions.per.default.queue': '1',
          'hive.auto.convert.join.noconditionaltask.size': '214748364',
          'hive.compactor.initiator.on': 'false',
          'hive.compactor.worker.threads': '0',
          'hive.compute.query.using.stats': 'true',
          'hive.enforce.bucketing': 'false',
          'hive.exec.dynamic.partition.mode': 'strict',
          'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
          'hive.exec.orc.compression.strategy': 'SPEED',
          'hive.exec.orc.default.compress': 'ZLIB',
          'hive.exec.orc.default.stripe.size': '67108864',
          'hive.exec.orc.encoding.strategy': 'SPEED',
          'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
          'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
          'hive.exec.reducers.bytes.per.reducer': '67108864',
          'hive.execution.engine': 'mr',
          'hive.optimize.index.filter': 'true',
          'hive.optimize.sort.dynamic.partition': 'false',
          'hive.prewarm.enabled': 'false',
          'hive.prewarm.numcontainers': '3',
          'hive.security.authorization.enabled': 'false',
          'hive.server2.use.SSL': 'false',
          'hive.stats.fetch.column.stats': 'true',
          'hive.stats.fetch.partition.stats': 'true',
          'hive.support.concurrency': 'false',
          'hive.tez.auto.reducer.parallelism': 'true',
          'hive.tez.container.size': '768',
          'hive.tez.dynamic.partition.pruning': 'true',
          'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
          'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager',
          'hive.vectorized.execution.enabled': 'true',
          'hive.vectorized.execution.reduce.enabled': 'false',
          'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
          'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
        },
       'property_attributes': {
         'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
         'hive.server2.authentication.pam.services': {'delete': 'true'},
         'hive.server2.custom.authentication.class': {'delete': 'true'},
         'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
         'hive.server2.authentication.kerberos.keytab': {'delete': 'true'},
         'hive.server2.authentication.ldap.url': {'delete': 'true'},
         'hive.server2.tez.default.queues': {
           'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}]
          },
         'atlas.cluster.name': {'delete': 'true'},
         'atlas.rest.address': {'delete': 'true'},
         'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'},
         'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'}
        }
      },
      'hiveserver2-site': {
        'properties': {
        },
        'property_attributes': {
         'hive.security.authorization.manager': {'delete': 'true'},
         'hive.security.authenticator.manager': {'delete': 'true'}
        }
      },
      'webhcat-site': {
        'properties': {
          'templeton.hadoop.queue.name': 'queue1'
        }
      }
    }
    services = {
      "Versions": {
        "parent_stack_version": "2.2",
        "stack_name": "HDP",
        "stack_version": "2.3",
        "stack_hierarchy": {
          "stack_name": "HDP",
          "stack_versions": ["2.2", "2.1", "2.0.6"]
        }
      },
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
          "StackServices": {
            "service_name": "YARN",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.2"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "APP_TIMELINE_SERVER",
                "display_name": "App Timeline Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "SLAVE",
                "component_name": "NODEMANAGER",
                "display_name": "NodeManager",
                "is_client": "false",
                "is_master": "false",
                "hostnames": [
                  "c6403.ambari.apache.org"
                ]
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1-2",
                "component_category": "MASTER",
                "component_name": "RESOURCEMANAGER",
                "display_name": "ResourceManager",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "CLIENT",
                "component_name": "YARN_CLIENT",
                "display_name": "YARN Client",
                "is_client": "true",
                "is_master": "false",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": {
        "capacity-scheduler": {
          "properties": {
            "capacity-scheduler" :"yarn.scheduler.capacity.root.queues=queue1,queue2"
          }
        },
        "hive-env": {
          "properties": {
          }
        },
        "hive-site": {
          "properties": {
            "hive.server2.authentication": "none",
            "hive.server2.authentication.ldap.url": "",
            "hive.server2.authentication.ldap.baseDN": "",
            "hive.server2.authentication.kerberos.keytab": "",
            "hive.server2.authentication.kerberos.principal": "",
            "hive.server2.authentication.pam.services": "",
            "hive.server2.custom.authentication.class": "",
            "hive.cbo.enable": "true"
          }
        },
        "hiveserver2-site": {
          "properties": {
            "hive.security.authorization.manager": "",
            "hive.security.authenticator.manager": ""
          }
        }
      },
      "changed-configurations": [ ]

    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6402.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6402.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6402.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6403.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6403.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6403.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    # Test JDK1.7
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    # Test JDK1.8
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.8_44'}
    expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    # Test JDK1.9
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'}
    expected['hive-site']['properties']['hive.tez.java.opts'] = "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps"
    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_recommendHiveConfigurations_with_atlas(self):
    self.maxDiff = None
    configurations = {
      "yarn-site": {
        "properties": {
          "yarn.scheduler.minimum-allocation-mb": "256",
          "yarn.scheduler.maximum-allocation-mb": "8192",
        },
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'yarn-site': {
        'properties': {
          'yarn.scheduler.minimum-allocation-mb': '256',
          'yarn.scheduler.maximum-allocation-mb': '8192'
        }
      },
      'hive-env': {
        'properties': {
          'hive_exec_orc_storage_strategy': 'SPEED',
          'hive_security_authorization': 'None',
          'hive_timeline_logging_enabled': 'true',
          'hive_txn_acid': 'off'
        }
      },
      'hive-site': {
        'properties': {
          'hive.server2.enable.doAs': 'true',
          'hive.server2.tez.default.queues': "queue1,queue2",
          'hive.server2.tez.initialize.default.sessions': 'false',
          'hive.server2.tez.sessions.per.default.queue': '1',
          'hive.auto.convert.join.noconditionaltask.size': '214748364',
          'hive.compactor.initiator.on': 'false',
          'hive.compactor.worker.threads': '0',
          'hive.compute.query.using.stats': 'true',
          'hive.enforce.bucketing': 'false',
          'hive.exec.dynamic.partition.mode': 'strict',
          'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
          'hive.exec.orc.compression.strategy': 'SPEED',
          'hive.exec.orc.default.compress': 'ZLIB',
          'hive.exec.orc.default.stripe.size': '67108864',
          'hive.exec.orc.encoding.strategy': 'SPEED',
          'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.atlas.hive.hook.HiveHook',
          'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
          'hive.exec.reducers.bytes.per.reducer': '67108864',
          'hive.execution.engine': 'mr',
          'hive.optimize.index.filter': 'true',
          'hive.optimize.sort.dynamic.partition': 'false',
          'hive.prewarm.enabled': 'false',
          'hive.prewarm.numcontainers': '3',
          'hive.security.authorization.enabled': 'false',
          'hive.server2.use.SSL': 'false',
          'hive.stats.fetch.column.stats': 'true',
          'hive.stats.fetch.partition.stats': 'true',
          'hive.support.concurrency': 'false',
          'hive.tez.auto.reducer.parallelism': 'true',
          'hive.tez.container.size': '768',
          'hive.tez.dynamic.partition.pruning': 'true',
          'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
          'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager',
          'hive.vectorized.execution.enabled': 'true',
          'hive.vectorized.execution.reduce.enabled': 'false',
          'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
          'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
        },
        'property_attributes': {
          'hive.auto.convert.join.noconditionaltask.size': {'maximum': '644245094'},
          'hive.tez.container.size': {'maximum': '8192', 'minimum': '256'},
          'hive.server2.authentication.pam.services': {'delete': 'true'},
          'hive.server2.custom.authentication.class': {'delete': 'true'},
          'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
          'hive.server2.authentication.kerberos.keytab': {'delete': 'true'},
          'hive.server2.authentication.ldap.url': {'delete': 'true'},
          'hive.server2.tez.default.queues': {
            'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}]
          },
          'atlas.cluster.name': {'delete': 'true'},
          'atlas.rest.address': {'delete': 'true'},
          'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'}
        }
      },
      'hiveserver2-site': {
        'properties': {
        },
        'property_attributes': {
          'hive.security.authorization.manager': {'delete': 'true'},
          'hive.security.authenticator.manager': {'delete': 'true'}
        }
      },
      'webhcat-site': {
        'properties': {
          'templeton.hadoop.queue.name': 'queue1'
        }
      }
    }
    services = {
      "Versions": {
        "parent_stack_version": "2.2",
        "stack_name": "HDP",
        "stack_version": "2.3",
        "stack_hierarchy": {
          "stack_name": "HDP",
          "stack_versions": ["2.2", "2.1", "2.0.6"]
        }
      },
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
          "StackServices": {
            "service_name": "YARN",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.2"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "APP_TIMELINE_SERVER",
                "display_name": "App Timeline Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "SLAVE",
                "component_name": "NODEMANAGER",
                "display_name": "NodeManager",
                "is_client": "false",
                "is_master": "false",
                "hostnames": [
                  "c6403.ambari.apache.org"
                ]
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1-2",
                "component_category": "MASTER",
                "component_name": "RESOURCEMANAGER",
                "display_name": "ResourceManager",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "CLIENT",
                "component_name": "YARN_CLIENT",
                "display_name": "YARN Client",
                "is_client": "true",
                "is_master": "false",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
          "StackServices": {
            "service_name": "ATLAS",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "ATLAS_SERVER",
                "display_name": "Atlas Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        }
      ],
      "configurations": {
        "capacity-scheduler": {
          "properties": {
            "capacity-scheduler" :"yarn.scheduler.capacity.root.queues=queue1,queue2"
          }
        },
        "hive-env": {
          "properties": {
          }
        },
        "hive-site": {
          "properties": {
            "hive.server2.authentication": "none",
            "hive.server2.authentication.ldap.url": "",
            "hive.server2.authentication.ldap.baseDN": "",
            "hive.server2.authentication.kerberos.keytab": "",
            "hive.server2.authentication.kerberos.principal": "",
            "hive.server2.authentication.pam.services": "",
            "hive.server2.custom.authentication.class": "",
            "hive.cbo.enable": "true"
          }
        },
        "hiveserver2-site": {
          "properties": {
            "hive.security.authorization.manager": "",
            "hive.security.authenticator.manager": ""
          }
        }
      },
      "changed-configurations": [ ]

    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6402.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6402.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6402.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6403.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6403.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6403.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  @patch('os.path.exists')
  @patch('os.path.isdir')
  @patch('os.listdir')
  def test_recommendTezConfigurations(self, os_listdir_mock, os_isdir_mock, os_exists_mock):

    os_exists_mock.return_value = True
    os_isdir_mock.return_value = True
    os_listdir_mock.return_value = ['TEZ{0.7.0.2.3.0.0-2155}']

    self.maxDiff = None
    configurations = {
      "yarn-site": {
        "properties": {
          "yarn.scheduler.minimum-allocation-mb": "256",
          "yarn.scheduler.maximum-allocation-mb": "8192",
        },
      },
      "capacity-scheduler": {
        "properties": {
          "yarn.scheduler.capacity.root.queues": "queue1,queue2"
        }
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      "capacity-scheduler": {
        "properties": {
          "yarn.scheduler.capacity.root.queues": "queue1,queue2"
        }
      },
      "tez-site": {
        "properties": {
          "tez.task.resource.memory.mb": "768",
          "tez.am.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
          "tez.task.launch.cmd-opts": "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC",
          "tez.runtime.io.sort.mb": "202",
          "tez.session.am.dag.submit.timeout.secs": "600",
          "tez.runtime.unordered.output.buffer.size-mb": "57",
          "tez.am.resource.memory.mb": "4000",
          "tez.queue.name": "queue1",
        }
      },
      "yarn-site": {
        "properties": {
          "yarn.scheduler.minimum-allocation-mb": "256",
          "yarn.scheduler.maximum-allocation-mb": "8192"
        }
      }
    }
    services = {
      "Versions": {
        "parent_stack_version": "2.2",
        "stack_name": "HDP",
        "stack_version": "2.3",
        "stack_hierarchy": {
          "stack_name": "HDP",
          "stack_versions": ["2.2", "2.1", "2.0.6"]
        }
      },
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
          "StackServices": {
            "service_name": "YARN",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.2"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "APP_TIMELINE_SERVER",
                "display_name": "App Timeline Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "SLAVE",
                "component_name": "NODEMANAGER",
                "display_name": "NodeManager",
                "is_client": "false",
                "is_master": "false",
                "hostnames": [
                  "c6403.ambari.apache.org"
                ]
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1-2",
                "component_category": "MASTER",
                "component_name": "RESOURCEMANAGER",
                "display_name": "ResourceManager",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            },
            {
              "StackServiceComponents": {
                "advertise_version": "true",
                "cardinality": "1+",
                "component_category": "CLIENT",
                "component_name": "YARN_CLIENT",
                "display_name": "YARN Client",
                "is_client": "true",
                "is_master": "false",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": configurations,
      "changed-configurations": [ ],
      "ambari-server-properties": {}
    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6402.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6402.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6402.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        },
        {
          "href" : "/api/v1/hosts/c6403.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6403.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6403.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    server_host = socket.getfqdn()
    for host in hosts["items"]:
      if server_host == host["Hosts"]["host_name"]:
        server_host = host["Hosts"]["public_host_name"]
    tez_ui_url =  "http://" + server_host + ":8080/#/main/views/TEZ/0.7.0.2.3.0.0-2155/TEZ_CLUSTER_INSTANCE"

    # Test JDK1.7
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
    expected['tez-site']['properties']['tez.tez-ui.history-url.base'] = tez_ui_url
    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    # Test JDK1.8
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.8_44'}
    expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
    expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
    expected['tez-site']['properties']['tez.tez-ui.history-url.base'] = tez_ui_url
    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    # Test JDK1.9
    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.9.2_44'}
    expected['tez-site']['properties']['tez.am.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
    expected['tez-site']['properties']['tez.task.launch.cmd-opts'] = "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseG1GC -XX:+ResizeTLAB"
    expected['tez-site']['properties']['tez.tez-ui.history-url.base'] = tez_ui_url
    self.stackAdvisor.recommendTezConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_validateHiveConfigurations(self):
    properties = {"hive_security_authorization": "None",
                  "hive.exec.orc.default.stripe.size": "8388608",
                  'hive.tez.container.size': '2048',
                  'hive.tez.java.opts': '-server -Xmx546m -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
                  'hive.auto.convert.join.noconditionaltask.size': '1100000000'}
    recommendedDefaults = {'hive.tez.container.size': '1024',
                           'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
                           'hive.auto.convert.join.noconditionaltask.size': '1000000000'}
    configurations = {
      "hive-site": {
        "properties": {"hive.security.authorization.enabled": "true", 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true'}
      },
      "hive-env": {
        "properties": {"hive_security_authorization": "None"}
      }
    }
    services = {
      "services": []
    }

    # Test for 'ranger-hive-plugin-properties' not being in configs
    res_expected = []
    res = self.stackAdvisor.validateHiveConfigurations(properties, recommendedDefaults, configurations, services, {})
    self.assertEquals(res, res_expected)

  # This test intentionally calls all validate methods with
  # incorrect parameters (empty configs)
  def test_noRiskyDictLookups(self):
    properties = {}
    recommendedDefaults = {}
    configurations = {"core-site": {"properties": {}}}
    services = {
      "services": [],
      "Versions": {
        "stack_name": "HDP",
        "stack_version": "2.3"
      },
      "configurations": configurations
    }

    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "disk_info" : [
              {
                "available" : "4564632",
                "used" : "5230344",
                "percent" : "54%",
                "size" : "10319160",
                "type" : "ext4",
                "mountpoint" : "/"
              },
              {
                "available" : "1832436",
                "used" : "0",
                "percent" : "0%",
                "size" : "1832436",
                "type" : "tmpfs",
                "mountpoint" : "/dev/shm"
              }
            ],
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    def return_c6401_hostname(services, service_name, component_name):
      return ["c6401.ambari.apache.org"]
    self.stackAdvisor.getComponentHostNames = return_c6401_hostname

    validators = self.stackAdvisor.getServiceConfigurationValidators()

    # Setting up empty configs and services info
    for serviceName, validator in validators.items():
      services["services"].extend([{"StackServices": {"service_name": serviceName},
                                    "components": []}])
      for siteName in validator.keys():
        configurations[siteName] = {"properties": {}}

    # Emulate enabled RANGER
    services["services"].extend([{"StackServices": {"service_name": "RANGER"},
                                "components": []}])
    configurations["ranger-hbase-plugin-properties"] = {
      "ranger-hbase-plugin-enabled": "Yes"
    }

    exceptionThrown = False
    try:
      recommendations = self.stackAdvisor.recommendConfigurations(services, hosts)
    except Exception as e:
      exceptionThrown = True
    self.assertTrue(exceptionThrown)

    pass

  def test_recommendRangerConfigurations(self):
    clusterData = {}
    # Recommend for not existing DB_FLAVOR and http enabled, HDP-2.3
    services = {
      "Versions": {
        "parent_stack_version": "2.2",
        "stack_name": "HDP",
        "stack_version": "2.3",
        "stack_hierarchy": {
          "stack_name": "HDP",
          "stack_versions": ["2.2", "2.1", "2.0.6"]
        }
      },
      "services":  [
        {
          "StackServices": {
            "service_name": "RANGER",
            "service_version": "0.5.0.2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "component_name": "RANGER_ADMIN",
                "hostnames": ["host1"]
              }
            }
          ]
        },
        ],
      "configurations": {
        "admin-properties": {
          "properties": {
            "DB_FLAVOR": "NOT_EXISTING",
            }
        },
        "ranger-admin-site": {
          "properties": {
            "ranger.service.http.port": "7777",
            "ranger.service.http.enabled": "true",
            }
        }
      },
      "ambari-server-properties": {
        "ambari.ldap.isConfigured" : "true",
        "authentication.ldap.bindAnonymously" : "false",
        "authentication.ldap.baseDn" : "dc=apache,dc=org",
        "authentication.ldap.groupNamingAttr" : "cn",
        "authentication.ldap.primaryUrl" : "c6403.ambari.apache.org:389",
        "authentication.ldap.userObjectClass" : "posixAccount",
        "authentication.ldap.secondaryUrl" : "c6403.ambari.apache.org:389",
        "authentication.ldap.usernameAttribute" : "uid",
        "authentication.ldap.dnAttribute" : "dn",
        "authentication.ldap.useSSL" : "false",
        "authentication.ldap.managerPassword" : "/etc/ambari-server/conf/ldap-password.dat",
        "authentication.ldap.groupMembershipAttr" : "memberUid",
        "authentication.ldap.groupObjectClass" : "posixGroup",
        "authentication.ldap.managerDn" : "uid=hdfs,ou=people,ou=dev,dc=apache,dc=org"
      }
    }

    expected = {
      'admin-properties': {
        'properties': {
          'policymgr_external_url': 'http://host1:7777'
        }
      },
      'ranger-ugsync-site': {
        'properties': {
          'ranger.usersync.group.objectclass': 'posixGroup',
          'ranger.usersync.group.nameattribute': 'cn',
          'ranger.usersync.group.memberattributename': 'memberUid',
          'ranger.usersync.ldap.binddn': 'uid=hdfs,ou=people,ou=dev,dc=apache,dc=org',
          'ranger.usersync.ldap.user.nameattribute': 'uid',
          'ranger.usersync.ldap.user.objectclass': 'posixAccount',
          'ranger.usersync.ldap.url': 'ldap://c6403.ambari.apache.org:389',
          'ranger.usersync.ldap.searchBase': 'dc=apache,dc=org'
        }
      },
      'ranger-admin-site': {
        'properties': {
          "ranger.audit.solr.zookeepers": "NONE",
          "ranger.audit.source.type": "solr"
        }
      },
      'ranger-env': {
        'properties': {
          'ranger-storm-plugin-enabled': 'No',
        }
      }
    }

    recommendedConfigurations = {}
    self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None)
    self.assertEquals(recommendedConfigurations, expected)

    # Recommend ranger.audit.solr.zookeepers when solrCloud is disabled
    services['configurations']['ranger-env'] = {
      "properties": {
        "is_solrCloud_enabled": "false"
      }
    }

    recommendedConfigurations = {}
    self.stackAdvisor.recommendRangerConfigurations(recommendedConfigurations, clusterData, services, None)
    self.assertEquals(recommendedConfigurations['ranger-admin-site']['properties']['ranger.audit.solr.zookeepers'], 'NONE')

  def test_recommendRangerKMSConfigurations(self):
    clusterData = {}
    services = {
      "Versions": {
        "stack_version" : "2.3",
        },
      "services": [
        {
          "StackServices": {
            "service_name": "RANGER_KMS",
            "service_version": "0.5.0.2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "component_name": "RANGER_KMS_SERVER",
                "hostnames": ["host1"]
              }
            }
          ]
        }
        ],
      "configurations": {
        "kms-env": {
          "properties": {
            "kms_user": "kmsname"
            }
        },
        "core-site": {
          "properties": {
            "fs.defaultFS": "hdfs://host1:8020"
          }
        },
        'ranger-kms-audit': {
          'properties': {
          }
        },
        'kms-properties': {
          'properties': {
            'DB_FLAVOR': 'ORACLE',
            'db_host' : 'c6401.ambari.apache.org:1521:XE',
            'db_name' : "XE"
          }
        }
      },
      "forced-configurations": []
    }
    expected = {
      'kms-properties': {
        'properties': {}
      },
      'dbks-site': {
        'properties': {
          "ranger.ks.jpa.jdbc.driver" : "oracle.jdbc.driver.OracleDriver",
          "ranger.ks.jpa.jdbc.url" : "jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE"
        }
      },
      'core-site': {
        'properties': {
        }
      },
      'ranger-kms-audit': {
          'properties': {
          }
      }
    }

    # non kerberized cluster. There should be no proxyuser configs
    recommendedConfigurations = {}
    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
    self.assertEquals(recommendedConfigurations, expected)

    # kerberized cluster
    services['services'].append({
      "StackServices": {
        "service_name": "KERBEROS"
      }
    })

    expected = {
      'kms-properties': {
        'properties': {}
      },
      'dbks-site': {
        'properties': {
          "ranger.ks.jpa.jdbc.driver" : "oracle.jdbc.driver.OracleDriver",
          "ranger.ks.jpa.jdbc.url" : "jdbc:oracle:thin:@c6401.ambari.apache.org:1521:XE"
        }
      },
      'core-site': {
        'properties': {
          'hadoop.proxyuser.kmsname.groups': '*'
        }
      },
      'ranger-kms-audit': {
          'properties': {
          }
      }
    }

    # on kerberized cluster property should be recommended
    recommendedConfigurations = {}
    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
    self.assertEquals(recommendedConfigurations, expected)

    recommendedConfigurations = {}
    services['changed-configurations'] = [
      {
        'type': 'kms-env',
        'name': 'kms_user',
        'old_value': 'kmsname'
      }
    ]
    services['configurations']['kms-env']['properties']['kms_user'] = 'kmsnew'

    expected['core-site'] = {
      'properties': {
        'hadoop.proxyuser.kmsnew.groups': '*'
      },
      'property_attributes': {
        'hadoop.proxyuser.kmsname.groups': {
          'delete': 'true'
        }
      }
    }

    # kms_user was changed, old property should be removed
    self.stackAdvisor.recommendRangerKMSConfigurations(recommendedConfigurations, clusterData, services, None)
    self.assertEquals(recommendedConfigurations, expected)

  def test_recommendStormConfigurations(self):
    self.maxDiff = None
    configurations = {
      "storm-site": {
        "properties": {
          "storm.topology.submission.notifier.plugin.class": "foo"
        }
      },
      "ranger-storm-plugin-properties": {
        "properties": {
          "ranger-storm-plugin-enabled": "No"
        }
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'storm-site': {
        'properties': {
          'storm.topology.submission.notifier.plugin.class': 'foo,org.apache.atlas.storm.hook.StormAtlasHook',
        },
        "property_attributes":{
          'nimbus.authorizer': {'delete':'true'}
        }
      },
      "ranger-storm-plugin-properties": {
        "properties": {
          "ranger-storm-plugin-enabled": "No"
        }
      }
    }
    services = {
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
          "StackServices": {
            "service_name": "ATLAS",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "ATLAS_SERVER",
                "display_name": "Atlas Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": {
        "storm-site": {
          "properties": {
            "storm.topology.submission.notifier.plugin.class": "foo"
          },
          "property-attributes":{}
        },
        "ranger-storm-plugin-properties": {
          "properties": {
            "ranger-storm-plugin-enabled": "No"
          }
        }
      },
      "changed-configurations": [ ]

    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
    self.stackAdvisor.recommendStormConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_recommendFalconConfigurations(self):
    self.maxDiff = None
    configurations = {
      "falcon-startup.properties": {
        "properties": {
          "*.application.services": "foo"
        }
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'falcon-startup.properties': {
        'properties': {
          '*.application.services': 'foo,org.apache.falcon.atlas.service.AtlasService',
        }
      }
    }
    services = {
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
          "StackServices": {
            "service_name": "ATLAS",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "ATLAS_SERVER",
                "display_name": "Atlas Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": {
        "falcon-startup.properties": {
          "properties": {
            "*.application.services": "foo"
          }
        }
      },
      "changed-configurations": [ ]

    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
    self.stackAdvisor.recommendFalconConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_recommendSqoopConfigurations(self):
    self.maxDiff = None
    configurations = {
      "sqoop-site": {
        "properties": {
          "sqoop.job.data.publish.class": "foo"
        }
      }
    }
    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'sqoop-site': {
        'properties': {
          'sqoop.job.data.publish.class': 'org.apache.atlas.sqoop.hook.SqoopHook',
        }
      }
    }
    services = {
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
          "StackServices": {
            "service_name": "ATLAS",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "ATLAS_SERVER",
                "display_name": "Atlas Server",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": {
        "sqoop-site": {
          "properties": {
            "sqoop.job.data.publish.class": "foo"
          }
        }
      },
      "changed-configurations": [ ]

    }
    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }

    self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

    services['ambari-server-properties'] = {'java.home': '/usr/jdk64/jdk1.7.3_23'}
    self.stackAdvisor.recommendSqoopConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_recommendLogsearchConfiguration(self):
    configurations = {
      "logsearch-properties": {
        "properties": {
          "logsearch.collection.service.logs.numshards" : "5",
          "logsearch.collection.service.logs.replication.factor": "0",
          "logsearch.collection.audit.logs.numshards" : "5",
          "logsearch.collection.audit.logs.replication.factor": "0"
        }
      }
    }

    clusterData = {
      "cpu": 4,
      "mapMemory": 3000,
      "amMemory": 2000,
      "reduceMemory": 2056,
      "containers": 3,
      "ramPerContainer": 256
    }
    expected = {
      'logsearch-properties': {
        'properties': {
          "logsearch.collection.service.logs.numshards" : "2",
          "logsearch.collection.service.logs.replication.factor": "1",
          "logsearch.collection.audit.logs.numshards" : "2",
          "logsearch.collection.audit.logs.replication.factor": "1"
        },
        "property_attributes": {
          "logsearch.collection.service.logs.numshards": {
            "minimum": "1",
            "maximum": "3"
          },
          "logsearch.collection.audit.logs.numshards": {
            "minimum": "1",
            "maximum": "3"
          }
        }
      }
    }
    services = {
      "services": [
        {
          "href": "/api/v1/stacks/HDP/versions/2.3/services/AMBARI_INFRA",
          "StackServices": {
            "service_name": "AMBARI_INFRA",
            "service_version": "2.6.0.2.2",
            "stack_name": "HDP",
            "stack_version": "2.3"
          },
          "components": [
            {
              "StackServiceComponents": {
                "advertise_version": "false",
                "cardinality": "1",
                "component_category": "MASTER",
                "component_name": "INFRA_SOLR",
                "display_name": "Infra Solr Instance",
                "is_client": "false",
                "is_master": "true",
                "hostnames": []
              },
              "dependencies": []
            }
          ]
        },
      ],
      "configurations": {
        "logsearch-properties": {
          "properties": {
            "logsearch.collection.numshards" : "5",
            "logsearch.collection.replication.factor": "0"
          }
        }
      },
      "changed-configurations": [ ]

    }

    hosts = {
      "items" : [
        {
          "href" : "/api/v1/hosts/c6401.ambari.apache.org",
          "Hosts" : {
            "cpu_count" : 1,
            "host_name" : "c6401.ambari.apache.org",
            "os_arch" : "x86_64",
            "os_type" : "centos6",
            "ph_cpu_count" : 1,
            "public_host_name" : "c6401.ambari.apache.org",
            "rack_info" : "/default-rack",
            "total_mem" : 1922680
          }
        }
      ]
    }
    def return_c6401_hostname(services, service_name, component_name):
      return ["c6401.ambari.apache.org"]
    self.stackAdvisor.getComponentHostNames = return_c6401_hostname
    self.stackAdvisor.recommendLogsearchConfigurations(configurations, clusterData, services, hosts)
    self.assertEquals(configurations, expected)

  def test_validateRangerConfigurationsEnv(self):
    properties = {
      "ranger-kafka-plugin-enabled": "Yes",
      }
    recommendedDefaults = {
      "ranger-kafka-plugin-enabled": "No",
      }

    configurations = {
      "cluster-env": {
        "properties": {
          "security_enabled": "false",
          }
      }
    }
    services = {
      "services":
        [
          {
            "StackServices": {
              "service_name" : "RANGER"
            }
          }
        ],
      "configurations": {
        "cluster-env": {
          "properties": {
            "security_enabled" : "false"
          },
          "property_attributes": {}
        }
      }
    }

    # Test with ranger plugin enabled, validation fails
    res_expected = [{'config-type': 'ranger-env', 'message': 'Ranger Kafka plugin should not be enabled in non-kerberos environment.', 'type': 'configuration', 'config-name': 'ranger-kafka-plugin-enabled', 'level': 'WARN'}]
    res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {})
    self.assertEquals(res, res_expected)

    # Test for security_enabled is true
    services['configurations']['cluster-env']['properties']['security_enabled'] = "true"
    configurations['cluster-env']['properties']['security_enabled'] = "true"
    res_expected = []
    res = self.stackAdvisor.validateRangerConfigurationsEnv(properties, recommendedDefaults, configurations, services, {})
    self.assertEquals(res, res_expected)
