AMBARI-3699. App.ServiceConfig needs 'configsValidator' validating across service values. (onechiporenko, srimanth via srimanth)

ambari-web/app/assets/test/tests.js

@@ -61,7 +61,7 @@ require('test/mappers/runs_mapper_test');
 require('test/mappers/service_mapper_test');
 require('test/mappers/status_mapper_test');
 require('test/mappers/users_mapper_test');
-require('test/utils/defaults_providers/yarn_defaults_provider_test');
+require('test/utils/configs/defaults_providers/yarn_defaults_provider_test');
 require('test/utils/config_test');
 require('test/utils/date_test');
 require('test/utils/config_test');

ambari-web/app/controllers/global/cluster_controller.js

@@ -320,7 +320,7 @@ App.ClusterController = Em.Controller.extend({
     }
     var clusterUrl = this.getUrl('/data/clusters/cluster.json', '?fields=Clusters');
     var hostsRealUrl = '/hosts?fields=Hosts/host_name,Hosts/public_host_name,Hosts/cpu_count,Hosts/total_mem,' +
-      'Hosts/host_status,Hosts/last_heartbeat_time,Hosts/os_arch,Hosts/os_type,Hosts/ip,host_components,' +
+      'Hosts/host_status,Hosts/last_heartbeat_time,Hosts/os_arch,Hosts/os_type,Hosts/ip,host_components,Hosts/disk_info,' +
       'metrics/disk,metrics/load/load_one,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free';
     var usersUrl = App.testMode ? '/data/users/users.json' : App.apiPrefix + '/users/?fields=*';
     var racksUrl = "/data/racks/racks.json";

ambari-web/app/controllers/global/update_controller.js

@@ -56,7 +56,7 @@ App.UpdateController = Em.Controller.extend({
 
   updateHost:function(callback) {
     var testUrl = App.get('isHadoop2Stack') ? '/data/hosts/HDP2/hosts.json' : '/data/hosts/hosts.json';
-    var hostsUrl = this.getUrl(testUrl, '/hosts?fields=Hosts/host_name,Hosts/last_heartbeat_time,' +
+    var hostsUrl = this.getUrl(testUrl, '/hosts?fields=Hosts/host_name,Hosts/last_heartbeat_time,Hosts/disk_info,' +
       'metrics/disk,metrics/load/load_one,metrics/cpu/cpu_system,metrics/cpu/cpu_user,metrics/memory/mem_total,metrics/memory/mem_free');
     App.HttpClient.get(hostsUrl, App.hostsMapper, {
       complete: callback

ambari-web/app/controllers/main/service/info/configs.js

@@ -30,6 +30,7 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({
   globalConfigs: [],
   uiConfigs: [],
   customConfig: [],
+  serviceConfigsData: require('data/service_configs'),
   isApplyingChanges: false,
   serviceConfigs: function () {
     return App.config.get('preDefinedServiceConfigs');
@@ -429,6 +430,56 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({
     }, this)
   },
 
+  /**
+   * Get info about hosts and host components to configDefaultsProviders
+   * @returns {{masterComponentHosts: Array, slaveComponentHosts: Array, hosts: {}}}
+   */
+  getInfoForDefaults: function() {
+
+    var slaveComponentHosts = [];
+    var slaves = App.HostComponent.find().filterProperty('isSlave', true).map(function(item) {
+      return Em.Object.create({
+        host: item.get('host.hostName'),
+        componentName: item.get('componentName')
+      });
+    });
+    slaves.forEach(function(slave) {
+      var s = slaveComponentHosts.findProperty('componentName', slave.componentName);
+      if (s) {
+        s.hosts.push({hostName: slave.host});
+      }
+      else {
+        slaveComponentHosts.push({
+          componentName: slave.get('componentName'),
+          hosts: [{hostName: slave.host}]
+        });
+      }
+    });
+
+    var masterComponentHosts = App.HostComponent.find().filterProperty('isMaster', true).map(function(item) {
+      return {
+        component: item.get('componentName'),
+        serviceId: item.get('service.serviceName'),
+        host: item.get('host.hostName')
+      }
+    });
+    var hosts = {};
+    App.Host.find().map(function(host) {
+      hosts[host.get('hostName')] = {
+        name: host.get('hostName'),
+        cpu: host.get('cpu'),
+        memory: host.get('memory'),
+        disk_info: host.get('diskInfo')
+      };
+    });
+
+    return {
+      masterComponentHosts: masterComponentHosts,
+      slaveComponentHosts: slaveComponentHosts,
+      hosts: hosts
+    };
+  },
+
   /**
    * Load child components to service config object
    * @param configs
@@ -436,6 +487,26 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({
    * @param restartData
    */
   loadComponentConfigs: function (configs, componentConfig, restartData) {
+
+    var localDB = this.getInfoForDefaults();
+    var recommendedDefaults = {};
+    var s = this.get('serviceConfigsData').findProperty('serviceName', this.get('content.serviceName'));
+
+    var defaults = [];
+    if (s.defaultsProviders) {
+      s.defaultsProviders.forEach(function(defaultsProvider) {
+        var d = defaultsProvider.getDefaults(localDB);
+        defaults.push(d);
+        for (var name in d) {
+          recommendedDefaults[name] = d[name];
+        }
+      });
+    }
+    if (s.configsValidator) {
+      s.configsValidator.set('recommendedDefaults', recommendedDefaults);
+    }
+
+
     configs.forEach(function (_serviceConfigProperty) {
       console.log("config", _serviceConfigProperty);
       if (!_serviceConfigProperty) return;
@@ -464,6 +535,25 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({
         serviceConfigProperty.set('restartRequiredMessage', message);
       }
       if (serviceConfigProperty.get('serviceName') === this.get('content.serviceName')) {
+
+        defaults.forEach(function(defaults) {
+          for(var name in defaults) {
+            if (serviceConfigProperty.name == name) {
+              serviceConfigProperty.set('value', defaults[name]);
+              serviceConfigProperty.set('defaultValue', defaults[name]);
+            }
+          }
+        });
+
+        if (s.configsValidator) {
+          var validators = s.configsValidator.get('configValidators');
+          for (var validatorName in validators) {
+            if (serviceConfigProperty.name == validatorName) {
+              serviceConfigProperty.set('serviceValidator', s.configsValidator);
+            }
+          }
+        }
+
         // serviceConfigProperty.serviceConfig = componentConfig;
         if (App.get('isAdmin')) {
           serviceConfigProperty.set('isEditable', serviceConfigProperty.get('isReconfigurable'));

ambari-web/app/data/HDP2/site_properties.js

@@ -201,6 +201,7 @@ module.exports =
       "id": "site property",
       "name": "yarn.scheduler.minimum-allocation-mb",
       "displayName": "yarn.scheduler.minimum-allocation-mb",
+      "displayType": "int",
       "value": "",
       "defaultValue": "",
       "serviceName": "YARN",
@@ -210,6 +211,7 @@ module.exports =
       "id": "site property",
       "name": "yarn.scheduler.maximum-allocation-mb",
       "displayName": "yarn.scheduler.maximum-allocation-mb",
+      "displayType": "int",
       "value": "",
       "defaultValue": "",
       "serviceName": "YARN",
@@ -219,6 +221,7 @@ module.exports =
       "id": "site property",
       "name": "yarn.nodemanager.resource.memory-mb",
       "displayName": "yarn.nodemanager.resource.memory-mb",
+      "displayType": "int",
       "value": "",
       "defaultValue": "",
       "serviceName": "YARN",
@@ -356,6 +359,16 @@ module.exports =
       "value": "",
       "defaultValue": ""
     },
+    {
+      "id": "site property",
+      "name": "yarn.app.mapreduce.am.resource.mb",
+      "displayName": "yarn.app.mapreduce.am.resource.mb",
+      "value": "",
+      "defaultValue": "",
+      "displayType": "int",
+      "category": "Advanced",
+      "serviceName": "MAPREDUCE2"
+    },
 
   /**********************************************oozie-site***************************************/
     {

ambari-web/app/data/service_configs.js

@@ -18,7 +18,9 @@
 
 var App = require('app');
 require('models/service_config');
-require('utils/defaults_providers/yarn_defaults_provider');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
+require('utils/configs/validators/yarn_configs_validator');
+require('utils/configs/validators/mapreduce2_configs_validator');
 
 module.exports = [
   {
@@ -66,6 +68,7 @@ module.exports = [
   {
     serviceName: 'YARN',
     displayName: 'YARN',
+    configsValidator: App.YARNConfigsValidator,
     defaultsProviders: [App.YARNDefaultsProvider],
     filename: 'yarn-site',
     configCategories: [
@@ -84,6 +87,7 @@ module.exports = [
     serviceName: 'MAPREDUCE2',
     displayName: 'MapReduce 2',
     filename: 'mapred-site',
+    configsValidator: App.MapReduce2ConfigsValidator,
     defaultsProviders: [App.YARNDefaultsProvider],
     configCategories: [
       App.ServiceConfigCategory.create({ name: 'HistoryServer', displayName : 'History Server', hostComponentNames : ['HISTORYSERVER']}),

ambari-web/app/models/service_config.js

@@ -147,6 +147,7 @@ App.ServiceConfigProperty = Ember.Object.extend({
   isVisible: true,
   isSecureConfig: false,
   errorMessage: '',
+  warnMessage: '',
   serviceConfig: null, // points to the parent App.ServiceConfig object
   filename: '',
   isOriginalSCP : true, // if true, then this is original SCP instance and its value is not overridden value.
@@ -156,11 +157,13 @@ App.ServiceConfigProperty = Ember.Object.extend({
   isUserProperty: null, // This property was added by user. Hence they get removal actions etc.
   isOverridable: true,
   error: false,
+  warn: false,
   overrideErrorTrigger: 0, //Trigger for overrridable property error
   isRestartRequired: false,
   restartRequiredMessage: 'Restart required',
   index: null, //sequence number in category
   editDone: false, //Text field: on focusOut: true, on focusIn: false
+  serviceValidator: null,
 
   /**
    * On Overridable property error message, change overrideErrorTrigger value to recount number of errors service have
@@ -486,6 +489,7 @@ App.ServiceConfigProperty = Ember.Object.extend({
     var values = [];//value split by "," to check UNIX users, groups list
 
     var isError = false;
+    var isWarn = false;
 
     if (typeof value === 'string' && value.length === 0) {
       if (this.get('isRequired')) {
@@ -619,13 +623,29 @@ App.ServiceConfigProperty = Ember.Object.extend({
         }
       }
     }
+    
+    var serviceValidator = this.get('serviceValidator');
+    if (serviceValidator!=null) {
+      var validationIssue = serviceValidator.validateConfig(this);
+      if (validationIssue) {
+    	this.set('warnMessage', validationIssue);
+    	isWarn = true;
+      }
+    }
 
-    if (!isError) {
-      this.set('errorMessage', '');
-      this.set('error', false);
+    if (!isWarn || isError) { // Errors get priority
+        this.set('warnMessage', '');
+        this.set('warn', false);
     } else {
-      this.set('error', true);
+        this.set('warn', true);
     }
+    
+    if (!isError) {
+        this.set('errorMessage', '');
+        this.set('error', false);
+      } else {
+        this.set('error', true);
+      }
   }.observes('value', 'retypedPassword')
 
 });

ambari-web/app/templates/common/configs/service_config.hbs

@@ -106,7 +106,7 @@
                           </span>
                           <div class="controls">
                             {{! Here serviceConfigBinding should ideally be serviceConfigPropertyBinding }}
-                              <div {{bindAttr class="errorMessage:error: :control-group"}}>
+                              <div {{bindAttr class="errorMessage:error: warnMessage:warning: :control-group"}}>
                                 {{view viewClass serviceConfigBinding="this" categoryConfigsAllBinding="view.categoryConfigsAll" }}
                                 {{#if view.canEdit}}
 	                                {{#if isPropertyOverridable}}
@@ -124,6 +124,7 @@
 	                                {{/if}}
                                 {{/if}}
                                   <span class="help-inline">{{errorMessage}}</span>
+                                  <span class="help-inline">{{warnMessage}}</span>
                               </div>
                             {{#if this.isOverridden}}
                               {{view App.ServiceConfigView.SCPOverriddenRowsView serviceConfigPropertyBinding="this"}}

ambari-web/app/utils/config.js

@@ -501,11 +501,12 @@ App.config = Em.Object.create({
       serviceConfig.set('showConfig', service.showConfig);
 
       // Use calculated default values for some configs
-
-      if (!storedConfigs && service.defaultsProviders)  {
+      var recommendedDefaults = {};
+      if (!storedConfigs && service.defaultsProviders) {
         service.defaultsProviders.forEach(function(defaultsProvider) {
           var defaults = defaultsProvider.getDefaults(localDB);
           for(var name in defaults) {
+        	recommendedDefaults[name] = defaults[name];
             var config = configsByService.findProperty('name', name);
             if (config) {
               config.set('value', defaults[name]);
@@ -514,6 +515,16 @@ App.config = Em.Object.create({
           }
         });
       }
+      if (service.configsValidator) {
+    	service.configsValidator.set('recommendedDefaults', recommendedDefaults);
+    	var validators = service.configsValidator.get('configValidators');
+    	for (var validatorName in validators) {
+        var c = configsByService.findProperty('name', validatorName);
+          if (c) {
+            c.set('serviceValidator', service.configsValidator);
+          }
+        }
+      }
 
       serviceConfig.set('configs', configsByService);
       renderedServiceConfigs.push(serviceConfig);

ambari-web/app/utils/defaults_providers/yarn_defaults_provider.js → ambari-web/app/utils/configs/defaults_providers/yarn_defaults_provider.js

@@ -16,7 +16,7 @@
  */
 
 var App = require('app');
-require('utils/defaults_providers/defaultsProvider');
+require('utils/configs/defaults_providers/defaultsProvider');
 
 App.YARNDefaultsProvider = App.DefaultsProvider.create({
 

ambari-web/app/utils/configs/validators/mapreduce2_configs_validator.js

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/validators/service_configs_validator');
+
+App.MapReduce2ConfigsValidator = App.ServiceConfigsValidator.create({
+
+  /**
+   * List of the configs that should be validated
+   */
+  configValidators: {
+    'mapreduce.map.java.opts': 'mapreduceMapJavaOpts',
+    'mapreduce.reduce.java.opts': 'mapreduceReduceJavaOpts',
+    'mapreduce.task.io.sort.mb': 'mapreduceTaskIoSortMb',
+    'mapreduce.map.memory.mb': 'mapreduceMapMemoryMb',
+    'mapreduce.reduce.memory.mb': 'mapreduceReduceMemoryMb',
+    'yarn.app.mapreduce.am.resource.mb': 'yarnAppMapreduceAmResourceMb',
+    'yarn.app.mapreduce.am.command-opts': 'yarnAppMapreduceAmCommandOpts'
+  },
+
+  /**
+   * List of the real configProperty objects
+   */
+  configProperties: [],
+
+  mapreduceMapJavaOpts: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  mapreduceReduceJavaOpts: function(config) {
+	return this.validatorLessThenDefaultValue(config);
+  },
+
+  mapreduceTaskIoSortMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  mapreduceMapMemoryMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  mapreduceReduceMemoryMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  yarnAppMapreduceAmResourceMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  yarnAppMapreduceAmCommandOpts: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  }
+
+});

ambari-web/app/utils/configs/validators/service_configs_validator.js

@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+
+App.ServiceConfigsValidator = Em.Object.extend({
+
+  /**
+   * Defaults recommended for various properties. This is to be used 
+   * by this validator to validate a property. Generally this value should be 
+   * set to values given by the defaults provider of the service.
+   * 
+   * The key is the property name, and the value is the recommended default.
+   */
+  recommendedDefaults: {},
+  
+  /**
+   * Per property validators where key is config name and value 
+   * is the validation function. This field is expected to be
+   * overridden by extending classes.
+   */
+  configValidators: {},
+  
+  /**
+   * Validate the given config property with the available  
+   * {@param recommendedDefaults}. This can do cross-property
+   * validations also. 
+   * 
+   * @param config  {App.ServiceConfigProperty}
+   * @return {string}  No validation issues when <code>null</code> returned.
+   */
+  validateConfig: function(config) {
+    var validatorFunction = this.get('configValidators')[config.get('name')];
+    if (validatorFunction) {
+      return this[validatorFunction](config);
+    }
+    return null;
+  },
+  
+  /**
+   * Check if provided <code>config.value</code> is less than <code>recommendedDefaults</code>
+   * @param {object} config - configProperty name
+   */
+  validatorLessThenDefaultValue: function(config) {
+    var defaultValue = this.recommendedDefaults[config.get('name')];
+    var currentValue = parseInt(config.get('value').toString().replace( /\D+/g, ''));
+    defaultValue = parseInt(defaultValue.toString().replace( /\D+/g, ''));
+    if (defaultValue && currentValue &&  currentValue < defaultValue) {
+      return "Value is less than the recommended default of "+defaultValue;
+    }
+    return null;
+  }
+
+});

ambari-web/app/utils/configs/validators/yarn_configs_validator.js

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+var App = require('app');
+require('utils/configs/validators/service_configs_validator');
+
+App.YARNConfigsValidator = App.ServiceConfigsValidator.create({
+  /**
+   * List of the configs that should be validated
+   */
+  configValidators: {
+    'yarn.nodemanager.resource.memory-mb': 'yarnNodemanagerResourceMemoryMb',
+    'yarn.scheduler.minimum-allocation-mb': 'yarnSchedulerMinimumAllocationMb',
+    'yarn.scheduler.maximum-allocation-mb': 'yarnSchedulerMaximumAllocationMb'
+  },
+
+  /**
+   * List of the real configProperty objects
+   */
+  configProperties: [],
+
+  yarnNodemanagerResourceMemoryMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  yarnSchedulerMinimumAllocationMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  },
+
+  yarnSchedulerMaximumAllocationMb: function(config) {
+    return this.validatorLessThenDefaultValue(config);
+  }
+
+});

ambari-web/test/utils/defaults_providers/yarn_defaults_provider_test.js → ambari-web/test/utils/configs/defaults_providers/yarn_defaults_provider_test.js

@@ -16,8 +16,8 @@
  */
 
 var App = require('app');
-require('utils/defaults_providers/defaultsProvider');
-require('utils/defaults_providers/yarn_defaults_provider');
+require('utils/configs/defaults_providers/defaultsProvider');
+require('utils/configs/defaults_providers/yarn_defaults_provider');
 
 describe('YARNDefaultsProvider', function() {