|
|
@@ -1472,6 +1472,274 @@ class TestHDP23StackAdvisor(TestCase):
|
|
|
self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
|
|
|
self.assertEquals(configurations, expected)
|
|
|
|
|
|
+ def test_recommendHiveConfigurations_with_atlas(self):
|
|
|
+ self.maxDiff = None
|
|
|
+ configurations = {
|
|
|
+ "yarn-site": {
|
|
|
+ "properties": {
|
|
|
+ "yarn.scheduler.minimum-allocation-mb": "256",
|
|
|
+ "yarn.scheduler.maximum-allocation-mb": "8192",
|
|
|
+ },
|
|
|
+ }
|
|
|
+ }
|
|
|
+ clusterData = {
|
|
|
+ "cpu": 4,
|
|
|
+ "mapMemory": 3000,
|
|
|
+ "amMemory": 2000,
|
|
|
+ "reduceMemory": 2056,
|
|
|
+ "containers": 3,
|
|
|
+ "ramPerContainer": 256
|
|
|
+ }
|
|
|
+ expected = {
|
|
|
+ 'yarn-site': {
|
|
|
+ 'properties': {
|
|
|
+ 'yarn.scheduler.minimum-allocation-mb': '256',
|
|
|
+ 'yarn.scheduler.maximum-allocation-mb': '8192'
|
|
|
+ }
|
|
|
+ },
|
|
|
+ 'hive-env': {
|
|
|
+ 'properties': {
|
|
|
+ 'hive_exec_orc_storage_strategy': 'SPEED',
|
|
|
+ 'hive_security_authorization': 'None',
|
|
|
+ 'hive_timeline_logging_enabled': 'true',
|
|
|
+ 'hive_txn_acid': 'off'
|
|
|
+ }
|
|
|
+ },
|
|
|
+ 'hive-site': {
|
|
|
+ 'properties': {
|
|
|
+ 'hive.server2.enable.doAs': 'true',
|
|
|
+ 'hive.server2.tez.default.queues': "queue1,queue2",
|
|
|
+ 'hive.server2.tez.initialize.default.sessions': 'false',
|
|
|
+ 'hive.server2.tez.sessions.per.default.queue': '1',
|
|
|
+ 'hive.auto.convert.join.noconditionaltask.size': '268435456',
|
|
|
+ 'hive.compactor.initiator.on': 'false',
|
|
|
+ 'hive.compactor.worker.threads': '0',
|
|
|
+ 'hive.compute.query.using.stats': 'true',
|
|
|
+ 'hive.enforce.bucketing': 'false',
|
|
|
+ 'hive.exec.dynamic.partition.mode': 'strict',
|
|
|
+ 'hive.exec.failure.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
|
|
|
+ 'hive.exec.orc.compression.strategy': 'SPEED',
|
|
|
+ 'hive.exec.orc.default.compress': 'ZLIB',
|
|
|
+ 'hive.exec.orc.default.stripe.size': '67108864',
|
|
|
+ 'hive.exec.orc.encoding.strategy': 'SPEED',
|
|
|
+ 'hive.exec.post.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook,org.apache.atlas.hive.hook.HiveHook',
|
|
|
+ 'hive.exec.pre.hooks': 'org.apache.hadoop.hive.ql.hooks.ATSHook',
|
|
|
+ 'hive.exec.reducers.bytes.per.reducer': '67108864',
|
|
|
+ 'hive.execution.engine': 'mr',
|
|
|
+ 'hive.optimize.index.filter': 'true',
|
|
|
+ 'hive.optimize.sort.dynamic.partition': 'false',
|
|
|
+ 'hive.prewarm.enabled': 'false',
|
|
|
+ 'hive.prewarm.numcontainers': '3',
|
|
|
+ 'hive.security.authorization.enabled': 'false',
|
|
|
+ 'hive.server2.use.SSL': 'false',
|
|
|
+ 'hive.stats.fetch.column.stats': 'true',
|
|
|
+ 'hive.stats.fetch.partition.stats': 'true',
|
|
|
+ 'hive.support.concurrency': 'false',
|
|
|
+ 'hive.tez.auto.reducer.parallelism': 'true',
|
|
|
+ 'hive.tez.container.size': '768',
|
|
|
+ 'hive.tez.dynamic.partition.pruning': 'true',
|
|
|
+ 'hive.tez.java.opts': '-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps',
|
|
|
+ 'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager',
|
|
|
+ 'hive.vectorized.execution.enabled': 'true',
|
|
|
+ 'hive.vectorized.execution.reduce.enabled': 'false',
|
|
|
+ 'hive.security.metastore.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider',
|
|
|
+ 'hive.security.authorization.manager': 'org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory'
|
|
|
+ },
|
|
|
+ 'property_attributes': {
|
|
|
+ 'hive.auto.convert.join.noconditionaltask.size': {'maximum': '805306368'},
|
|
|
+ 'hive.server2.authentication.pam.services': {'delete': 'true'},
|
|
|
+ 'hive.server2.custom.authentication.class': {'delete': 'true'},
|
|
|
+ 'hive.server2.authentication.kerberos.principal': {'delete': 'true'},
|
|
|
+ 'hive.server2.authentication.kerberos.keytab': {'delete': 'true'},
|
|
|
+ 'hive.server2.authentication.ldap.url': {'delete': 'true'},
|
|
|
+ 'hive.server2.tez.default.queues': {
|
|
|
+ 'entries': [{'value': 'queue1', 'label': 'queue1 queue'}, {'value': 'queue2', 'label': 'queue2 queue'}]
|
|
|
+ },
|
|
|
+ 'atlas.cluster.name': {'delete': 'true'},
|
|
|
+ 'atlas.rest.address': {'delete': 'true'},
|
|
|
+ 'datanucleus.rdbms.datastoreAdapterClassName': {'delete': 'true'}
|
|
|
+ }
|
|
|
+ },
|
|
|
+ 'hiveserver2-site': {
|
|
|
+ 'properties': {
|
|
|
+ },
|
|
|
+ 'property_attributes': {
|
|
|
+ 'hive.security.authorization.manager': {'delete': 'true'},
|
|
|
+ 'hive.security.authenticator.manager': {'delete': 'true'}
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ services = {
|
|
|
+ "services": [
|
|
|
+ {
|
|
|
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/YARN",
|
|
|
+ "StackServices": {
|
|
|
+ "service_name": "YARN",
|
|
|
+ "service_version": "2.6.0.2.2",
|
|
|
+ "stack_name": "HDP",
|
|
|
+ "stack_version": "2.2"
|
|
|
+ },
|
|
|
+ "components": [
|
|
|
+ {
|
|
|
+ "StackServiceComponents": {
|
|
|
+ "advertise_version": "false",
|
|
|
+ "cardinality": "1",
|
|
|
+ "component_category": "MASTER",
|
|
|
+ "component_name": "APP_TIMELINE_SERVER",
|
|
|
+ "display_name": "App Timeline Server",
|
|
|
+ "is_client": "false",
|
|
|
+ "is_master": "true",
|
|
|
+ "hostnames": []
|
|
|
+ },
|
|
|
+ "dependencies": []
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "StackServiceComponents": {
|
|
|
+ "advertise_version": "true",
|
|
|
+ "cardinality": "1+",
|
|
|
+ "component_category": "SLAVE",
|
|
|
+ "component_name": "NODEMANAGER",
|
|
|
+ "display_name": "NodeManager",
|
|
|
+ "is_client": "false",
|
|
|
+ "is_master": "false",
|
|
|
+ "hostnames": [
|
|
|
+ "c6403.ambari.apache.org"
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ "dependencies": []
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "StackServiceComponents": {
|
|
|
+ "advertise_version": "true",
|
|
|
+ "cardinality": "1-2",
|
|
|
+ "component_category": "MASTER",
|
|
|
+ "component_name": "RESOURCEMANAGER",
|
|
|
+ "display_name": "ResourceManager",
|
|
|
+ "is_client": "false",
|
|
|
+ "is_master": "true",
|
|
|
+ "hostnames": []
|
|
|
+ },
|
|
|
+ "dependencies": []
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "StackServiceComponents": {
|
|
|
+ "advertise_version": "true",
|
|
|
+ "cardinality": "1+",
|
|
|
+ "component_category": "CLIENT",
|
|
|
+ "component_name": "YARN_CLIENT",
|
|
|
+ "display_name": "YARN Client",
|
|
|
+ "is_client": "true",
|
|
|
+ "is_master": "false",
|
|
|
+ "hostnames": []
|
|
|
+ },
|
|
|
+ "dependencies": []
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "href": "/api/v1/stacks/HDP/versions/2.2/services/ATLAS",
|
|
|
+ "StackServices": {
|
|
|
+ "service_name": "ATLAS",
|
|
|
+ "service_version": "2.6.0.2.2",
|
|
|
+ "stack_name": "HDP",
|
|
|
+ "stack_version": "2.3"
|
|
|
+ },
|
|
|
+ "components": [
|
|
|
+ {
|
|
|
+ "StackServiceComponents": {
|
|
|
+ "advertise_version": "false",
|
|
|
+ "cardinality": "1",
|
|
|
+ "component_category": "MASTER",
|
|
|
+ "component_name": "ATLAS_SERVER",
|
|
|
+ "display_name": "Atlas Server",
|
|
|
+ "is_client": "false",
|
|
|
+ "is_master": "true",
|
|
|
+ "hostnames": []
|
|
|
+ },
|
|
|
+ "dependencies": []
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ }
|
|
|
+ ],
|
|
|
+ "configurations": {
|
|
|
+ "capacity-scheduler": {
|
|
|
+ "properties": {
|
|
|
+ "capacity-scheduler" :"yarn.scheduler.capacity.root.queues=queue1,queue2"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "hive-env": {
|
|
|
+ "properties": {
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "hive-site": {
|
|
|
+ "properties": {
|
|
|
+ "hive.server2.authentication": "none",
|
|
|
+ "hive.server2.authentication.ldap.url": "",
|
|
|
+ "hive.server2.authentication.ldap.baseDN": "",
|
|
|
+ "hive.server2.authentication.kerberos.keytab": "",
|
|
|
+ "hive.server2.authentication.kerberos.principal": "",
|
|
|
+ "hive.server2.authentication.pam.services": "",
|
|
|
+ "hive.server2.custom.authentication.class": "",
|
|
|
+ "hive.cbo.enable": "true"
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "hiveserver2-site": {
|
|
|
+ "properties": {
|
|
|
+ "hive.security.authorization.manager": "",
|
|
|
+ "hive.security.authenticator.manager": ""
|
|
|
+ }
|
|
|
+ }
|
|
|
+ },
|
|
|
+ "changed-configurations": [ ]
|
|
|
+
|
|
|
+ }
|
|
|
+ hosts = {
|
|
|
+ "items" : [
|
|
|
+ {
|
|
|
+ "href" : "/api/v1/hosts/c6401.ambari.apache.org",
|
|
|
+ "Hosts" : {
|
|
|
+ "cpu_count" : 1,
|
|
|
+ "host_name" : "c6401.ambari.apache.org",
|
|
|
+ "os_arch" : "x86_64",
|
|
|
+ "os_type" : "centos6",
|
|
|
+ "ph_cpu_count" : 1,
|
|
|
+ "public_host_name" : "c6401.ambari.apache.org",
|
|
|
+ "rack_info" : "/default-rack",
|
|
|
+ "total_mem" : 1922680
|
|
|
+ }
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "href" : "/api/v1/hosts/c6402.ambari.apache.org",
|
|
|
+ "Hosts" : {
|
|
|
+ "cpu_count" : 1,
|
|
|
+ "host_name" : "c6402.ambari.apache.org",
|
|
|
+ "os_arch" : "x86_64",
|
|
|
+ "os_type" : "centos6",
|
|
|
+ "ph_cpu_count" : 1,
|
|
|
+ "public_host_name" : "c6402.ambari.apache.org",
|
|
|
+ "rack_info" : "/default-rack",
|
|
|
+ "total_mem" : 1922680
|
|
|
+ }
|
|
|
+ },
|
|
|
+ {
|
|
|
+ "href" : "/api/v1/hosts/c6403.ambari.apache.org",
|
|
|
+ "Hosts" : {
|
|
|
+ "cpu_count" : 1,
|
|
|
+ "host_name" : "c6403.ambari.apache.org",
|
|
|
+ "os_arch" : "x86_64",
|
|
|
+ "os_type" : "centos6",
|
|
|
+ "ph_cpu_count" : 1,
|
|
|
+ "public_host_name" : "c6403.ambari.apache.org",
|
|
|
+ "rack_info" : "/default-rack",
|
|
|
+ "total_mem" : 1922680
|
|
|
+ }
|
|
|
+ }
|
|
|
+ ]
|
|
|
+ }
|
|
|
+
|
|
|
+ self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, services, hosts)
|
|
|
+ self.assertEquals(configurations, expected)
|
|
|
+
|
|
|
@patch('os.path.exists')
|
|
|
@patch('os.path.isdir')
|
|
|
@patch('os.listdir')
|
tbeerbower