AMBARI-7756 Oozie server start fails (dsen)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py

@@ -58,6 +58,15 @@ def hdfs(name=None):
             group=params.user_group
   )
 
+  XmlConfig("core-site.xml",
+            conf_dir=params.hadoop_conf_dir,
+            configurations=params.config['configurations']['core-site'],
+            configuration_attributes=params.config['configuration_attributes']['core-site'],
+            owner=params.hdfs_user,
+            group=params.user_group,
+            mode=0644
+  )
+
   File(os.path.join(params.hadoop_conf_dir, 'slaves'),
        owner=tc_owner,
        content=Template("slaves.j2")

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -291,6 +291,14 @@ class TestDatanode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'hdfs',
@@ -323,6 +331,14 @@ class TestDatanode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+    )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'root',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -160,6 +160,14 @@ class TestJournalnode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'hdfs',
@@ -184,6 +192,14 @@ class TestJournalnode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+    )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'root',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -449,6 +449,14 @@ class TestNamenode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'hdfs',
@@ -474,6 +482,14 @@ class TestNamenode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'root',
@@ -522,4 +538,4 @@ class Popen_Mock:
     popen = MagicMock()
     popen.returncode = Popen_Mock.return_value
     popen.stdout.readline = MagicMock(side_effect = Popen_Mock.lines)
-    return popen
+    return popen

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_snamenode.py

@@ -175,6 +175,14 @@ class TestSNamenode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'hdfs',
@@ -200,6 +208,14 @@ class TestSNamenode(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'root',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_zkfc.py

@@ -42,6 +42,14 @@ class TestZkfc(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'hdfs',
@@ -109,6 +117,14 @@ class TestZkfc(RMFTestCase):
                               configurations = self.getConfig()['configurations']['hdfs-site'],
                               configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
                               )
+    self.assertResourceCalled('XmlConfig', 'core-site.xml',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              conf_dir = '/etc/hadoop/conf',
+                              configurations = self.getConfig()['configurations']['core-site'],
+                              configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+                              mode = 0644
+    )
     self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
                               content = Template('slaves.j2'),
                               owner = 'root',

ambari-web/app/controllers/main/service/add_controller.js

@@ -377,16 +377,28 @@ App.AddServiceController = App.WizardController.extend({
     App.router.get('updateController').updateAll();
   },
 
-  installServices: function (callback) {
-    this.set('content.cluster.oldRequestsId', []);
-    this.installAdditionalClients();
-    var selectedServices = this.get('content.services').filterProperty('isInstalled', false).filterProperty('isSelected', true).mapProperty('serviceName');
-    var name = 'common.services.update';
-    var data = {
+  /**
+   * genarates data for ajax request to launch install services
+   * @method generateDataForInstallServices
+   * @param {Array} selectedServices
+   * @returns {{context: *, ServiceInfo: {state: string}, urlParams: string}}
+   */
+  generateDataForInstallServices: function(selectedServices) {
+    if (selectedServices.contains('OOZIE')) {
+      selectedServices = selectedServices.concat(['HDFS', 'YARN', 'MAPREDUCE', 'MAPREDUCE2']);
+    }
+    return {
       "context": Em.I18n.t('requestInfo.installServices'),
       "ServiceInfo": {"state": "INSTALLED"},
       "urlParams": "ServiceInfo/service_name.in(" + selectedServices.join(',')  + ")"
     };
+  },
+  installServices: function (callback) {
+    this.set('content.cluster.oldRequestsId', []);
+    this.installAdditionalClients();
+    var name = 'common.services.update';
+    var selectedServices = this.get('content.services').filterProperty('isInstalled', false).filterProperty('isSelected', true).mapProperty('serviceName');
+    var data = this.generateDataForInstallServices(selectedServices);
     this.installServicesRequest(name, data, callback);
   },
 

ambari-web/app/controllers/wizard/step8_controller.js

@@ -1439,7 +1439,19 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, {
         }
       }));
     }, this);
-
+    /**
+     * if some services change core-site we should put updates to cluster
+     */
+    if (this.get('content.controllerName') == 'addServiceController' && !selectedServices.someProperty('serviceName', 'HDFS')) {
+      var coreConfig = serviceConfigTags.findProperty('type', 'core-site');
+      if (coreConfig) {
+        allConfigData.pushObject(JSON.stringify({
+          Clusters: {
+            desired_config: [coreConfig]
+          }
+        }));
+      }
+    }
     var clusterConfig = serviceConfigTags.findProperty('type', 'cluster-env');
     if (clusterConfig) {
       allConfigData.pushObject(JSON.stringify({

ambari-web/app/controllers/wizard/step9_controller.js

@@ -479,12 +479,15 @@ App.WizardStep9Controller = Em.Controller.extend({
         };
         break;
       case 'addServiceController':
-        var servicesList = this.get('content.services').filterProperty('isSelected').filterProperty('isInstalled', false).mapProperty('serviceName').join(",");
+        var servicesList = this.get('content.services').filterProperty('isSelected').filterProperty('isInstalled', false).mapProperty('serviceName');
+        if (servicesList.contains('OOZIE')) {
+          servicesList = servicesList.concat(['HDFS', 'YARN', 'MAPREDUCE', 'MAPREDUCE2']);
+        }
         name = 'common.services.update';
         data = {
           "context": Em.I18n.t("requestInfo.startAddedServices"),
           "ServiceInfo": { "state": "STARTED" },
-          "urlParams": "ServiceInfo/state=INSTALLED&ServiceInfo/service_name.in(" + servicesList + ")&params/reconfigure_client=false"
+          "urlParams": "ServiceInfo/state=INSTALLED&ServiceInfo/service_name.in(" + servicesList.join(",") + ")&params/reconfigure_client=false"
         };
         break;
       default:
@@ -1180,4 +1183,4 @@ App.WizardStep9Controller = Em.Controller.extend({
     }
   }
 
-});
+});

ambari-web/test/controllers/main/service/add_controller_test.js

@@ -78,4 +78,30 @@ describe('App.AddServiceController', function() {
 
   });
 
+  describe('#generateDataForInstallServices', function() {
+    var tests = [{
+      selected: ["YARN","HBASE"],
+      res: {
+        "context": Em.I18n.t('requestInfo.installServices'),
+        "ServiceInfo": {"state": "INSTALLED"},
+        "urlParams": "ServiceInfo/service_name.in(YARN,HBASE)"
+      }
+    },
+    {
+      selected: ['OOZIE'],
+      res: {
+        "context": Em.I18n.t('requestInfo.installServices'),
+        "ServiceInfo": {"state": "INSTALLED"},
+        "urlParams": "ServiceInfo/service_name.in(OOZIE,HDFS,YARN,MAPREDUCE,MAPREDUCE2)"
+      }
+    }
+    ]
+    tests.forEach(function(t){
+      it('should generate data with ' + t.selected.join(","), function () {
+        expect(addServiceController.generateDataForInstallServices(t.selected)).to.be.eql(t.res);
+      });
+    });
+
+
+  });
 });

ambari-web/test/controllers/wizard/step8_test.js

@@ -1291,6 +1291,61 @@ describe('App.WizardStep8Controller', function () {
         installerStep8Controller.applyConfigurationsToCluster(serviceConfigTags);
         expect(installerStep8Controller.addRequestToAjaxQueue.args[0][0].data).to.eql({data: data});
       });
+
+      it('should call addRequestToAjaxQueue with core-site if addServiceController', function() {
+        var serviceConfigTags = [
+            {
+              type: 'oozie',
+              tag: 'tag1',
+              properties: [
+                {},
+                {}
+              ]
+            },
+            {
+              type: 'core-site',
+              tag: 'tag1',
+              properties: [
+                {}
+              ]
+            },
+            {
+              type: 'cluster-env',
+              tag: 'tag1',
+              properties: [
+                {}
+              ]
+            }
+          ];
+
+        data = '['+ JSON.stringify({Clusters: {
+              desired_config: [serviceConfigTags[0]]
+            }
+          }) + ',' +
+          JSON.stringify({Clusters: {
+              desired_config: [serviceConfigTags[1]]
+            }
+          }) + ',' +
+          JSON.stringify({Clusters: {
+              desired_config: [serviceConfigTags[2]]
+            }
+          }) + ']';
+        installerStep8Controller.reopen({
+          selectedServices: [
+            Em.Object.create({
+              isSelected: true,
+              isInstalled: false,
+              configTypesRendered: {'oozie': 'tag1' }
+            })
+          ],
+          content: {
+            controllerName: 'addServiceController'
+          }
+        });
+        installerStep8Controller.applyConfigurationsToCluster(serviceConfigTags);
+        var sendData = installerStep8Controller.addRequestToAjaxQueue.args[0][0].data;
+        expect(sendData).to.eql({data: data});
+      });
     });
 
     describe('#applyConfigurationGroups', function() {