AMBARI-4964. hive.tez.container.size should be in MB and dynamically calculated. (srimanth)

ambari-web/app/assets/test/tests.js

@@ -64,6 +64,8 @@ require('test/mappers/service_mapper_test');
 require('test/mappers/status_mapper_test');
 require('test/mappers/users_mapper_test');
 require('test/utils/configs/defaults_providers/yarn_defaults_provider_test');
+require('test/utils/configs/defaults_providers/tez_defaults_provider_test');
+require('test/utils/configs/defaults_providers/hive_defaults_provider_test');
 require('test/utils/configs/validators/service_configs_validator_test');
 require('test/utils/ajax_test');
 require('test/utils/batch_scheduled_requests_test');

ambari-web/app/data/service_configs.js

@@ -20,8 +20,10 @@ var App = require('app');
 require('models/service_config');
 require('utils/configs/defaults_providers/yarn_defaults_provider');
 require('utils/configs/defaults_providers/tez_defaults_provider');
+require('utils/configs/defaults_providers/hive_defaults_provider');
 require('utils/configs/defaults_providers/storm_defaults_provider');
 require('utils/configs/validators/yarn_configs_validator');
+require('utils/configs/validators/hive_configs_validator');
 require('utils/configs/validators/tez_configs_validator');
 require('utils/configs/validators/mapreduce2_configs_validator');
 require('utils/configs/validators/storm_configs_validator');
@@ -113,6 +115,8 @@ module.exports = [
     serviceName: 'HIVE',
     displayName: 'Hive',
     filename: 'hive-site',
+    configsValidator: App.HiveConfigsValidator,
+    defaultsProviders: [App.HiveDefaultsProvider.create()],
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'Hive Metastore', displayName : 'Hive Metastore'}),
       App.ServiceConfigCategory.create({ name: 'Advanced', displayName : 'Advanced'}),

ambari-web/app/utils/configs/defaults_providers/hive_defaults_provider.js

@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+
+App.HiveDefaultsProvider = App.YARNDefaultsProvider.extend({
+
+  configsTemplate: {
+    'hive.tez.container.size': null
+  },
+
+  getDefaults : function(localDB) {
+    var configs = this._super(localDB);
+    if (configs['yarn.scheduler.maximum-allocation-mb'] != null && configs['mapreduce.map.memory.mb'] != null
+        && configs['mapreduce.reduce.memory.mb'] != null) {
+      var containerSize = configs['mapreduce.map.memory.mb'] >= 2048 ? configs['mapreduce.map.memory.mb'] : configs['mapreduce.reduce.memory.mb'];
+      configs['hive.tez.container.size'] = Math.min(configs['yarn.scheduler.maximum-allocation-mb'], containerSize);
+    } else {
+      jQuery.extend(configs, this.get('configsTemplate'));
+    }
+    return configs;
+  }
+
+});

ambari-web/app/utils/configs/validators/hive_configs_validator.js

@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/validators/service_configs_validator');
+
+App.HiveConfigsValidator = App.ServiceConfigsValidator.create({
+  /**
+   * List of the configs that should be validated
+   */
+  configValidators: {
+    'hive.tez.container.size': 'hiveTezContainerMb',
+  },
+
+  hiveTezContainerMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  }
+});

ambari-web/test/utils/configs/defaults_providers/hive_defaults_provider_test.js

@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/defaults_providers/defaultsProvider');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+
+describe('HiveDefaultsProvider', function() {
+
+  describe('#getDefaults', function() {
+    var tests = [
+      {
+        localDB: {},
+        m: 'Empty localDB',
+        e: null
+      },
+      {
+        localDB: {
+          "masterComponentHosts": []
+        },
+        m: 'localDB without hosts',
+        e: null
+      },
+      {
+        localDB: {
+          "hosts": {}
+        },
+        m: 'localDB without masterComponentHosts amd slaveComponentHosts',
+        e: null
+      },
+      {
+        localDB: {
+          "hosts": {
+            "host1": {"name": "host1","cpu": 8,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]},
+            "host2": {"name": "host2","cpu": 4,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]}
+          },
+          "masterComponentHosts": [],
+          "slaveComponentHosts": [
+            {
+              "componentName": "NODEMANAGER",
+              "hosts": [{"hostName": "host2"}]
+            }
+          ]
+        },
+        m: 'Without HBase',
+        e: {
+          'mapreduce.map.java.opts': '-Xmx1024m',
+          'mapreduce.map.memory.mb': 1280,
+          'mapreduce.reduce.java.opts': '-Xmx2048m',
+          'mapreduce.reduce.memory.mb': 2560,
+          'yarn.app.mapreduce.am.command-opts': '-Xmx2048m',
+          'yarn.app.mapreduce.am.resource.mb': 2560,
+          'yarn.nodemanager.resource.memory-mb': 20480,
+          'yarn.scheduler.maximum-allocation-mb': 20480,
+          'yarn.scheduler.minimum-allocation-mb': 2560,
+          'mapreduce.task.io.sort.mb': 512,
+          'hive.tez.container.size': 2560
+        }
+      },
+      {
+        localDB: {
+          "hosts": {
+            "host1": {"name": "host1","cpu": 8,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]},
+            "host2": {"name": "host2","cpu": 4,"memory": "12582912.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]}
+          },
+          "masterComponentHosts": [
+            {"component": "HBASE_MASTER","hostName": "host1","serviceId": "HDFS"}
+          ],
+          "slaveComponentHosts": [
+            {
+              "componentName": "NODEMANAGER",
+              "hosts": [{"hostName": "host2"}]
+            }
+          ]
+        },
+        m: 'With HBase (low memory - pick mapreduce.reduce.memory.mb)',
+        e: {
+          'mapreduce.map.java.opts': '-Xmx410m',
+          'mapreduce.map.memory.mb': 512,
+          'mapreduce.reduce.java.opts': '-Xmx819m',
+          'mapreduce.reduce.memory.mb': 1024,
+          'yarn.app.mapreduce.am.command-opts': '-Xmx819m',
+          'yarn.app.mapreduce.am.resource.mb': 1024,
+          'yarn.nodemanager.resource.memory-mb': 8192,
+          'yarn.scheduler.maximum-allocation-mb': 8192,
+          'yarn.scheduler.minimum-allocation-mb': 1024,
+          'mapreduce.task.io.sort.mb': 205,
+          'hive.tez.container.size': 1024
+        }
+      },
+      {
+        localDB: {
+          "hosts": {
+            "host1": {"name": "host1","cpu": 8,"memory": "100165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]},
+            "host2": {"name": "host2","cpu": 4,"memory": "100165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]}
+          },
+          "masterComponentHosts": [
+            {"component": "HBASE_MASTER","hostName": "host1","serviceId": "HDFS"}
+          ],
+          "slaveComponentHosts": [
+            {
+              "componentName": "NODEMANAGER",
+              "hosts": [{"hostName": "host2"}]
+            }
+          ]
+        },
+        m: 'With HBase (high memory - pick mapreduce.map.memory.mb)',
+        e: {
+          'mapreduce.map.java.opts': '-Xmx3482m',
+          'mapreduce.map.memory.mb': 4352,
+          'mapreduce.reduce.java.opts': '-Xmx6963m',
+          'mapreduce.reduce.memory.mb': 8704,
+          'yarn.app.mapreduce.am.command-opts': '-Xmx6963m',
+          'yarn.app.mapreduce.am.resource.mb': 8704,
+          'yarn.nodemanager.resource.memory-mb': 69632,
+          'yarn.scheduler.maximum-allocation-mb': 69632,
+          'yarn.scheduler.minimum-allocation-mb': 8704,
+          'mapreduce.task.io.sort.mb': 1024,
+          'hive.tez.container.size': 4352
+        }
+      }
+    ];
+    var defaultsProvider = App.HiveDefaultsProvider.create();
+    tests.forEach(function(test) {
+      it(test.m, function() {
+        defaultsProvider.set('clusterData', null);
+        var configs = defaultsProvider.getDefaults(test.localDB);
+        for(var config in configs) {
+          if (test.e) {
+            expect(configs[config]).to.equal(test.e[config]);
+          }
+          else {
+            expect(configs[config] == 0 || configs[config] == null).to.equal(true);
+          }
+        }
+      });
+    });
+  });
+
+});

ambari-web/test/utils/configs/defaults_providers/tez_defaults_provider_test.js

@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/defaults_providers/defaultsProvider');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+
+describe('TezDefaultsProvider', function() {
+
+  describe('#getDefaults', function() {
+    var tests = [
+      {
+        localDB: {},
+        m: 'Empty localDB',
+        e: null
+      },
+      {
+        localDB: {
+          "masterComponentHosts": []
+        },
+        m: 'localDB without hosts',
+        e: null
+      },
+      {
+        localDB: {
+          "hosts": {}
+        },
+        m: 'localDB without masterComponentHosts amd slaveComponentHosts',
+        e: null
+      },
+      {
+        localDB: {
+          "hosts": {
+            "host1": {"name": "host1","cpu": 8,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]},
+            "host2": {"name": "host2","cpu": 4,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]}
+          },
+          "masterComponentHosts": [],
+          "slaveComponentHosts": [
+            {
+              "componentName": "NODEMANAGER",
+              "hosts": [{"hostName": "host2"}]
+            }
+          ]
+        },
+        m: 'Without HBase',
+        e: {
+          'mapreduce.map.java.opts': '-Xmx1024m',
+          'mapreduce.map.memory.mb': 1280,
+          'mapreduce.reduce.java.opts': '-Xmx2048m',
+          'mapreduce.reduce.memory.mb': 2560,
+          'yarn.app.mapreduce.am.command-opts': '-Xmx2048m',
+          'yarn.app.mapreduce.am.resource.mb': 2560,
+          'yarn.nodemanager.resource.memory-mb': 20480,
+          'yarn.scheduler.maximum-allocation-mb': 20480,
+          'yarn.scheduler.minimum-allocation-mb': 2560,
+          'mapreduce.task.io.sort.mb': 512,
+          'tez.am.resource.memory.mb': 2560,
+          'tez.am.java.opts': '-server -Xmx2048m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC'
+        }
+      },
+      {
+        localDB: {
+          "hosts": {
+            "host1": {"name": "host1","cpu": 8,"memory": "25165824.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]},
+            "host2": {"name": "host2","cpu": 4,"memory": "12582912.00","disk_info": [{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'},{mountpoint:'/'}]}
+          },
+          "masterComponentHosts": [
+            {"component": "HBASE_MASTER","hostName": "host1","serviceId": "HDFS"}
+          ],
+          "slaveComponentHosts": [
+            {
+              "componentName": "NODEMANAGER",
+              "hosts": [{"hostName": "host2"}]
+            }
+          ]
+        },
+        m: 'With HBase',
+        e: {
+          'mapreduce.map.java.opts': '-Xmx410m',
+          'mapreduce.map.memory.mb': 512,
+          'mapreduce.reduce.java.opts': '-Xmx819m',
+          'mapreduce.reduce.memory.mb': 1024,
+          'yarn.app.mapreduce.am.command-opts': '-Xmx819m',
+          'yarn.app.mapreduce.am.resource.mb': 1024,
+          'yarn.nodemanager.resource.memory-mb': 8192,
+          'yarn.scheduler.maximum-allocation-mb': 8192,
+          'yarn.scheduler.minimum-allocation-mb': 1024,
+          'mapreduce.task.io.sort.mb': 205,
+          'tez.am.resource.memory.mb': 1024,
+          'tez.am.java.opts': '-server -Xmx819m -Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC'
+        }
+      }
+    ];
+    var defaultsProvider = App.TezDefaultsProvider.create();
+    tests.forEach(function(test) {
+      it(test.m, function() {
+        defaultsProvider.set('clusterData', null);
+        var configs = defaultsProvider.getDefaults(test.localDB);
+        for ( var config in configs) {
+          if (test.e) {
+            expect(configs[config]).to.equal(test.e[config]);
+          } else {
+            expect(configs[config] == 0 || configs[config] == null).to.equal(true);
+          }
+        }
+      });
+    });
+  });
+
+});

ambari-web/test/utils/configs/validators/service_configs_validator_test.js

@@ -138,7 +138,17 @@ describe('App.ServiceConfigsValidator', function() {
       {value: '-server-Xmx1024m-Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC', e: false},
       {value: '-server-Xmx1024-Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC', e: false},
       {value: '-Xmx1024-Djava.net.preferIPv4Stack=true -XX:+UseNUMA -XX:+UseParallelGC', e: false},
-      {value: '-server-Xmx1024', e: false}
+      {value: '-server-Xmx1024', e: false},
+      {value: '-server    -Xmx1024m   -Da=b',e: true},
+      {value: '-server -Xmx1024m -Da=b',e: true},
+      {value: '-server -XMx1024m -Da=b',e: false},
+      {value: '-server -Xmx1024M -Da=b',e: true},
+      {value: '-server -Xmx1 -Da=b',e: true},
+      {value: '-server -Xmx1100MBPS -Da=b',e: false},
+      {value: '-server -Xmx1100M -Xmx200 -Da=b',e: false},
+      {value: '-server --Xmx1100M -Da=b',e: false},
+      {value: '-Xmx1024m -server -Da=b',e: true},
+      {value: ' -server -Da=b -Xmx1024m',e: true}
     ]);
     tests.forEach(function(test) {
       it(test.value, function() {