AMBARI-12817 YARN ownership is not set correctly on container-executor (dsen)

ambari-server/src/main/java/org/apache/ambari/server/api/resources/StackLevelConfigurationResourceDefinition.java

@@ -27,7 +27,7 @@ import org.apache.ambari.server.controller.spi.Resource.Type;
 public class StackLevelConfigurationResourceDefinition extends BaseResourceDefinition {
 
   public StackLevelConfigurationResourceDefinition(Type resourceType) {
-    super(Resource.Type.StackLevelConfiguration);
+    super(resourceType);
   }
 
   public StackLevelConfigurationResourceDefinition() {
@@ -43,11 +43,12 @@ public class StackLevelConfigurationResourceDefinition extends BaseResourceDefin
   public String getSingularName() {
     return "configuration";
   }
-  /*
+
   @Override
   public Set<SubResourceDefinition> getSubResourceDefinitions() {
-    Set<SubResourceDefinition> setChildren = new HashSet<SubResourceDefinition>();
-    return setChildren;
+    Set<SubResourceDefinition> subs = new HashSet<SubResourceDefinition>();
+    subs.add(new SubResourceDefinition(Resource.Type.StackConfigurationDependency));
+
+    return subs;
   }
-  */
 }

ambari-server/src/main/java/org/apache/ambari/server/api/services/AmbariMetaInfo.java

@@ -631,7 +631,10 @@ public class AmbariMetaInfo {
 
   public Set<PropertyInfo> getPropertiesByName(String stackName, String version, String serviceName, String propertyName)
       throws AmbariException {
-    Set<PropertyInfo> properties = getServiceProperties(stackName, version, serviceName);
+
+    Set<PropertyInfo> properties = serviceName == null ?
+      getStackProperties(stackName, version)
+      : getServiceProperties(stackName, version, serviceName);
 
     if (properties.size() == 0) {
       throw new StackAccessException("stackName=" + stackName

ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java

@@ -590,7 +590,8 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
 
     // Stack-definition has 'depends-on' relationship specified.
     // We have a map to construct the 'depended-by' relationship.
-    Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap = new HashMap<PropertyDependencyInfo, Set<PropertyDependencyInfo>>();
+    Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap =
+      new HashMap<PropertyDependencyInfo, Set<PropertyDependencyInfo>>();
 
     // Go through all service-configs and gather the reversed 'depended-by'
     // relationship into map. Since we do not have the reverse {@link PropertyInfo},
@@ -616,14 +617,26 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
 
     // Go through all service-configs again and set their 'depended-by' if necessary.
     for (ServiceModule serviceModule : serviceModules.values()) {
-      for (PropertyInfo pi : serviceModule.getModuleInfo().getProperties()) {
-        String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
-        String name = pi.getName();
-        Set<PropertyDependencyInfo> set =
-          dependedByMap.remove(new PropertyDependencyInfo(type, name));
-        if (set != null) {
-          pi.getDependedByProperties().addAll(set);
-        }
+      addDependedByProperties(dependedByMap, serviceModule.getModuleInfo().getProperties());
+    }
+    // Go through all stack-configs again and set their 'depended-by' if necessary.
+    addDependedByProperties(dependedByMap, stackInfo.getProperties());
+  }
+
+  /**
+   * Add dependendByProperties to property info's
+   * @param dependedByMap Map containing the 'depended-by' relationships
+   * @param properties properties to check against dependedByMap
+   */
+  private void addDependedByProperties(Map<PropertyDependencyInfo, Set<PropertyDependencyInfo>> dependedByMap,
+                                  Collection<PropertyInfo> properties) {
+    for (PropertyInfo pi : properties) {
+      String type = ConfigHelper.fileNameToConfigType(pi.getFilename());
+      String name = pi.getName();
+      Set<PropertyDependencyInfo> set =
+        dependedByMap.remove(new PropertyDependencyInfo(type, name));
+      if (set != null) {
+        pi.getDependedByProperties().addAll(set);
       }
     }
   }

ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-site.xml

@@ -178,6 +178,12 @@
     <name>yarn.nodemanager.linux-container-executor.group</name>
     <value>hadoop</value>
     <description>Unix group of the NodeManager</description>
+    <depends-on>
+      <property>
+        <type>cluster-env</type>
+        <name>user_group</name>
+      </property>
+    </depends-on>
   </property>
 
   <property>

ambari-server/src/main/resources/key_properties.json

@@ -70,6 +70,7 @@
     "StackVersion": "StackConfigurationDependency/stack_version",
     "StackService": "StackConfigurationDependency/service_name",
     "StackConfiguration": "StackConfigurationDependency/property_name",
+    "StackLevelConfiguration": "StackConfigurationDependency/property_name",
     "StackConfigurationDependency": "StackConfigurationDependency/dependency_name"
   },
   "StackServiceComponent": {

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -130,6 +130,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putYarnProperty('yarn.scheduler.minimum-allocation-mb', int(clusterData['ramPerContainer']))
     putYarnProperty('yarn.scheduler.maximum-allocation-mb', int(configurations["yarn-site"]["properties"]["yarn.nodemanager.resource.memory-mb"]))
     putYarnEnvProperty('min_user_id', self.get_system_min_uid())
+    containerExecutorGroup = 'hadoop'
+    if 'cluster-env' in services['configurations'] and 'user_group' in services['configurations']['cluster-env']['properties']:
+      containerExecutorGroup = services['configurations']['cluster-env']['properties']['user_group']
+    putYarnProperty("yarn.nodemanager.linux-container-executor.group", containerExecutorGroup)
 
   def recommendMapReduce2Configurations(self, configurations, clusterData, services, hosts):
     putMapredProperty = self.putProperty(configurations, "mapred-site", services)
@@ -722,8 +726,10 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     return self.toConfigurationValidationProblems(validationItems, "mapred-site")
 
   def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+    clusterEnv = getSiteProperties(configurations, "cluster-env")
     validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
                         {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+                        {"config-name": 'yarn.nodemanager.linux-container-executor.group', "item": self.validatorEqualsPropertyItem(properties, "yarn.nodemanager.linux-container-executor.group", clusterEnv, "user_group")},
                         {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
     return self.toConfigurationValidationProblems(validationItems, "yarn-site")
 

ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml

@@ -452,6 +452,10 @@
         <type>yarn-env</type>
         <name>yarn_cgroups_enabled</name>
       </property>
+        <property>
+          <type>cluster-env</type>
+          <name>user_group</name>
+        </property>
     </depends-on>
   </property>
   <property>

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -202,6 +202,7 @@ class TestHDP206StackAdvisor(TestCase):
 
     expectedItems = [
       {"message": "Value is less than the recommended default of 512", "level": "WARN"},
+      {'message': 'Value should be set for yarn.nodemanager.linux-container-executor.group', 'level': 'ERROR'},
       {"message": "Value should be integer", "level": "ERROR"},
       {"message": "Value should be set", "level": "ERROR"}
     ]
@@ -466,6 +467,7 @@ class TestHDP206StackAdvisor(TestCase):
 
   def test_recommendYARNConfigurations(self):
     configurations = {}
+    services = {"configurations": configurations}
     clusterData = {
       "containers" : 5,
       "ramPerContainer": 256
@@ -478,6 +480,7 @@ class TestHDP206StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.resource.memory-mb": "1280",
           "yarn.scheduler.minimum-allocation-mb": "256",
           "yarn.scheduler.maximum-allocation-mb": "1280"
@@ -485,7 +488,7 @@ class TestHDP206StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendMapReduce2Configurations_mapMemoryLessThan2560(self):

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -690,6 +690,7 @@ class TestHDP22StackAdvisor(TestCase):
 
   def test_recommendYARNConfigurations(self):
     configurations = {}
+    services = {"configurations": configurations}
     clusterData = {
       "cpu": 4,
       "containers" : 5,
@@ -703,6 +704,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.resource.memory-mb": "1280",
           "yarn.scheduler.minimum-allocation-mb": "256",
           "yarn.scheduler.maximum-allocation-mb": "1280",
@@ -713,7 +715,7 @@ class TestHDP22StackAdvisor(TestCase):
       }
     }
 
-    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, None, None)
+    self.stackAdvisor.recommendYARNConfigurations(configurations, clusterData, services, None)
     self.assertEquals(configurations, expected)
 
   def test_recommendYARNConfigurationAttributes(self):
@@ -745,6 +747,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.resource.memory-mb": "1280",
           "yarn.scheduler.minimum-allocation-mb": "256",
           "yarn.scheduler.maximum-allocation-vcores": "2",
@@ -1303,7 +1306,12 @@ class TestHDP22StackAdvisor(TestCase):
           "yarn.scheduler.maximum-allocation-mb": "2048",
           "yarn.nodemanager.resource.cpu-vcores": "2"
         },
-        }
+      },
+      "cluster-env": {
+        "properties": {
+          "user_group": "hadoopcustom",
+          }
+      }
     }
     clusterData = {
       "cpu": 4,
@@ -1312,6 +1320,11 @@ class TestHDP22StackAdvisor(TestCase):
       "totalAvailableRam": 4096,
     }
     expected = {
+      "cluster-env": {
+        "properties": {
+          "user_group": "hadoopcustom"
+        }
+      },
       "yarn-env": {
         "properties": {
           "min_user_id": "500"
@@ -1339,6 +1352,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoopcustom",
           "yarn.nodemanager.resource.memory-mb": "1792",
           "yarn.scheduler.minimum-allocation-mb": "100",
           "yarn.scheduler.maximum-allocation-vcores": "1",
@@ -1597,6 +1611,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.resource.memory-mb": "1280",
           "yarn.scheduler.minimum-allocation-mb": "100",
           "yarn.scheduler.maximum-allocation-vcores": "1",
@@ -1807,6 +1822,7 @@ class TestHDP22StackAdvisor(TestCase):
         },
         "yarn-site": {
             "properties": {
+                "yarn.nodemanager.linux-container-executor.group": "hadoop",
                 "yarn.nodemanager.resource.memory-mb": "1280",
                 "yarn.scheduler.minimum-allocation-mb": "700",
                 "yarn.scheduler.maximum-allocation-vcores": "1",
@@ -2687,6 +2703,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "yarn-site": {
         "properties": {
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.container-executor.group": "hadoop",
           "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor",
           "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup",
@@ -2743,6 +2760,7 @@ class TestHDP22StackAdvisor(TestCase):
           "yarn.nodemanager.container-executor.group": "hadoop",
           "yarn.nodemanager.container-executor.class": "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
           "yarn.nodemanager.linux-container-executor.cgroups.mount-path": "/cgroup",
+          "yarn.nodemanager.linux-container-executor.group": "hadoop",
           "yarn.nodemanager.container-executor.cgroups.mount": "true",
           "yarn.nodemanager.resource.memory-mb": "39424",
           "yarn.scheduler.minimum-allocation-mb": "3584",

ambari-web/app/controllers/wizard/step7_controller.js

@@ -883,7 +883,7 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E
     var serviceConfigTags = [];
     for (var site in data.Clusters.desired_configs) {
       if (data.Clusters.desired_configs.hasOwnProperty(site)) {
-        if (installedServiceSites.contains(site)) {
+        if (installedServiceSites.contains(site) || site == 'cluster-env') {
           serviceConfigTags.push({
             siteName: site,
             tagName: data.Clusters.desired_configs[site].tag,
@@ -1183,8 +1183,14 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, App.E
       var serviceToShow = this.get('selectedServiceNames').concat('MISC');
       var miscConfigs = this.get('stepConfigs').findProperty('serviceName', 'MISC').configs;
       if (this.get('wizardController.name') == "addServiceController") {
-        miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser')).set('isEditable', false);
-        miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group')).set('isEditable', false);
+        miscConfigs.findProperty('name', 'smokeuser').set('isEditable', false);
+        miscConfigs.findProperty('name', 'user_group').set('isEditable', false);
+        if (this.get('content.smokeuser')) {
+          miscConfigs.findProperty('name', 'smokeuser').set('value', this.get('content.smokeuser'));
+        }
+        if (this.get('content.group')) {
+          miscConfigs.findProperty('name', 'user_group').set('value', this.get('content.group'));
+        }
       }
       App.config.miscConfigVisibleProperty(miscConfigs, serviceToShow);
     }

ambari-web/app/mixins/common/serverValidator.js

@@ -129,7 +129,17 @@ App.ServerValidatorMixin = Em.Mixin.create({
       recommendations.blueprint.configurations = blueprintUtils.buildConfigsJSON(this.get('services'), this.get('stepConfigs').filter(function(serviceConfigs) {
         return self.get('installedServiceNames').contains(serviceConfigs.get('serviceName'));
       }));
+      // include cluster-env site to recommendations call
+      var miscService = this.get('services').findProperty('serviceName', 'MISC');
+      if (miscService) {
+        var miscConfigs = blueprintUtils.buildConfigsJSON([miscService], [this.get('stepConfigs').findProperty('serviceName', 'MISC')]);
+        var clusterEnv = App.permit(miscConfigs, 'cluster-env');
+        if (!App.isEmptyObject(clusterEnv)) {
+          $.extend(recommendations.blueprint.configurations, clusterEnv);
+        }
+      }
     }
+
     return App.ajax.send({
       'name': 'config.recommendations',
       'sender': this,