AMBARI-7887 Add support for YARN registry - required for Slider. (ababiichuk)

ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration/yarn-site.xml

@@ -26,5 +26,18 @@
     <value>/etc/hadoop/conf,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*,/usr/hdp/current/hadoop-mapreduce-client/*,/usr/hdp/current/hadoop-mapreduce-client/lib/*</value>
     <description>Classpath for typical applications.</description>
   </property>
-
+  <property>
+    <name>hadoop.registry.rm.enabled</name>
+    <value>false</value>
+    <description>
+      Is the registry enabled: does the RM start it up, create the user and system paths, and purge service records when containers, application attempts and applications complete
+    </description>
+  </property>
+  <property>
+    <name>hadoop.registry.zk.quorum</name>
+    <value>localhost:2181</value>
+    <description>
+      List of hostname:port pairs defining the zookeeper quorum binding for the registry
+    </description>
+  </property>
 </configuration>

ambari-web/app/controllers/main/host/details.js

@@ -658,7 +658,7 @@ App.MainHostDetailsController = Em.Controller.extend({
     if (services.someProperty('serviceName', 'SLIDER')) {
       urlParams.push('(type=slider-client&tag=' + data.Clusters.desired_configs['slider-client'].tag + ')');
     }
-    if (App.get('isRMHaEnabled')) {
+    if ((services.someProperty('serviceName', 'YARN') && App.get('isHadoop22Stack')) || App.get('isRMHaEnabled')) {
       urlParams.push('(type=yarn-site&tag=' + data.Clusters.desired_configs['yarn-site'].tag + ')');
     }
     return urlParams;
@@ -712,9 +712,6 @@ App.MainHostDetailsController = Em.Controller.extend({
       configs['webhcat-site']['templeton.zookeeper.hosts'] = zksWithPort;
     }
     if (configs['hive-site']) {
-      if (App.get('isHadoop22Stack')) {
-        configs['hive-site']['hive.zookeeper.quorum'] = zksWithPort;
-      }
       configs['hive-site']['hive.cluster.delegation.token.store.zookeeper.connectString'] = zksWithPort;
     }
     if (configs['storm-site']) {
@@ -726,6 +723,14 @@ App.MainHostDetailsController = Em.Controller.extend({
     if (App.get('isRMHaEnabled')) {
       configs['yarn-site']['yarn.resourcemanager.zk-address'] = zks.join(',');
     }
+    if (App.get('isHadoop22Stack')) {
+      if (configs['hive-site']) {
+        configs['hive-site']['hive.zookeeper.quorum'] = zksWithPort;
+      }
+      if (configs['yarn-site']) {
+        configs['yarn-site']['hadoop.registry.zk.quorum'] = zksWithPort;
+      }
+    }
     return true;
   },
   /**

ambari-web/app/controllers/wizard.js

@@ -870,7 +870,7 @@ App.WizardController = Em.Controller.extend(App.LocalStorage, {
         // watch for properties that are not modified but have to be updated
         if (_content.get('configs').someProperty('forceUpdate')) {
           // check for already added modified properties
-          var forceUpdatedFileNames = configs.filterProperty('forceUpdate', true).mapProperty('filename').uniq();
+          var forceUpdatedFileNames = _content.get('configs').filterProperty('forceUpdate', true).mapProperty('filename').uniq();
           fileNamesToUpdate = fileNamesToUpdate.concat(forceUpdatedFileNames).uniq();
         }
       }

ambari-web/app/controllers/wizard/step7_controller.js

@@ -668,6 +668,8 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, {
       case 'STORM':
         this.resolveStormConfigs(configs);
         break;
+      case 'YARN':
+        this.resolveYarnConfigs(configs);
       default:
         break;
     }
@@ -706,6 +708,24 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, {
     }
   },
 
+  /**
+   * Update some Storm configs
+   * If SLIDER is selected to install or already installed,
+   * some Yarn properties must be changed
+   * @param {Ember.Enumerable} configs
+   * @method resolveYarnConfigs
+   */
+  resolveYarnConfigs: function(configs) {
+    var cfgToChange = configs.findProperty('name', 'hadoop.registry.rm.enabled');
+    if (cfgToChange) {
+      var res = this.get('allSelectedServiceNames').contains('SLIDER');
+      if (Em.get(cfgToChange, 'value') !== res) {
+        Em.set(cfgToChange, 'defaultValue', res);
+        Em.set(cfgToChange, 'value', res);
+        Em.set(cfgToChange, 'forceUpdate', true);
+      }
+    }
+  },
   /**
    * On load function
    * @method loadStep
@@ -750,9 +770,12 @@ App.WizardStep7Controller = Em.Controller.extend(App.ServerValidatorMixin, {
     if (this.get('allSelectedServiceNames').contains('YARN') && !App.get('supports.capacitySchedulerUi')) {
       configs = App.config.fileConfigsIntoTextarea(configs, 'capacity-scheduler.xml');
     }
-    if (this.get('allSelectedServiceNames').contains('STORM')) {
-      this.resolveServiceDependencyConfigs('STORM', configs);
-    }
+    var dependendServices = ["STORM", "YARN"];
+    dependendServices.forEach(function(serviceName){
+      if (this.get('allSelectedServiceNames').contains(serviceName)) {
+        this.resolveServiceDependencyConfigs(serviceName, configs);
+      }
+    }, this);
     //STEP 6: Distribute configs by service and wrap each one in App.ServiceConfigProperty (configs -> serviceConfigs)
     var self = this;
     this.loadServerSideConfigsRecommendations().always(function() {

ambari-web/app/controllers/wizard/step8_controller.js

@@ -146,11 +146,11 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, {
   }.property('content.services.@each.isSelected','content.services.@each.isInstalled').cacheable(),
 
   /**
-   * List of installed and selected services
+   * List of installed services
    * @type {Object[]}
    */
   installedServices: function () {
-    return this.get('content.services').filterProperty('isSelected').filterProperty('isInstalled');
+    return this.get('content.services').filterProperty('isInstalled');
   }.property('content.services').cacheable(),
 
   /**
@@ -1436,9 +1436,9 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, {
    * @method applyConfigurationsToCluster
    */
   applyConfigurationsToCluster: function (serviceConfigTags) {
-    var selectedServices = this.get('selectedServices');
+    var allServices = this.get('installedServices').concat(this.get('selectedServices'));
     var allConfigData = [];
-    selectedServices.forEach(function (service) {
+    allServices.forEach(function (service) {
       var serviceConfigData = [];
       Object.keys(service.get('configTypesRendered')).forEach(function (type) {
         var serviceConfigTag = serviceConfigTags.findProperty('type', type);
@@ -1446,25 +1446,14 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, {
           serviceConfigData.pushObject(serviceConfigTag);
         }
       }, this);
-      allConfigData.pushObject(JSON.stringify({
-        Clusters: {
-          desired_config: serviceConfigData
-        }
-      }));
-    }, this);
-    /**
-     * if some services change core-site we should put updates to cluster
-     */
-    if (this.get('content.controllerName') == 'addServiceController' && !selectedServices.someProperty('serviceName', 'HDFS')) {
-      var coreConfig = serviceConfigTags.findProperty('type', 'core-site');
-      if (coreConfig) {
+      if (serviceConfigData.length) {
         allConfigData.pushObject(JSON.stringify({
           Clusters: {
-            desired_config: [coreConfig]
+            desired_config: serviceConfigData
           }
         }));
       }
-    }
+    }, this);
     var clusterConfig = serviceConfigTags.findProperty('type', 'cluster-env');
     if (clusterConfig) {
       allConfigData.pushObject(JSON.stringify({

ambari-web/app/data/HDP2.2/site_properties.js

@@ -18,21 +18,40 @@
 
 var App = require('app');
 var hdp2properties = require('data/HDP2/site_properties').configProperties;
-var hdp22properties = hdp2properties.filter(function (item){
+var hdp22properties = hdp2properties.filter(function (item) {
   //In HDP2.2 storm.thrift.transport property is computed on server
   return item.name !== 'storm.thrift.transport' && item.name !== 'storm_rest_api_host';
 });
 
 hdp22properties.push(
-{
-  "id": "site property",
-  "name": "hive.zookeeper.quorum",
-  "displayName": "hive.zookeeper.quorum",
-  "defaultValue": "localhost:2181",
-  "displayType": "multiLine",
-  "isVisible": true,
-  "serviceName": "HIVE",
-  "category": "Advanced hive-site"
+  {
+    "id": "site property",
+    "name": "hive.zookeeper.quorum",
+    "displayName": "hive.zookeeper.quorum",
+    "defaultValue": "localhost:2181",
+    "displayType": "multiLine",
+    "isVisible": true,
+    "serviceName": "HIVE",
+    "category": "Advanced hive-site"
+  },
+  {
+    "id": "site property",
+    "name": "hadoop.registry.rm.enabled",
+    "displayName": "hadoop.registry.rm.enabled",
+    "defaultValue": "false",
+    "displayType": "checkbox",
+    "isVisible": true,
+    "serviceName": "YARN",
+    "category": "Advanced yarn-site"
+  },
+  {
+    "id": "site property",
+    "name": "hadoop.registry.zk.quorum",
+    "displayName": "hadoop.registry.zk.quorum",
+    "defaultValue": "localhost:2181",
+    "isVisible": true,
+    "serviceName": "YARN",
+    "category": "Advanced yarn-site"
 });
 
 module.exports =

ambari-web/app/models/service_config.js

@@ -498,6 +498,7 @@ App.ServiceConfigProperty = Ember.Object.extend({
       case 'hive.zookeeper.quorum':
       case 'templeton.zookeeper.hosts':
       case 'slider.zookeeper.quorum':
+      case 'hadoop.registry.zk.quorum':
       case 'hive.cluster.delegation.token.store.zookeeper.connectString':
         var zkHosts = masterComponentHostsInDB.filterProperty('component', 'ZOOKEEPER_SERVER').mapProperty('hostName');
         var zkHostPort = zkHosts;

ambari-web/test/controllers/main/host/details_test.js

@@ -686,6 +686,17 @@ describe('App.MainHostDetailsController', function () {
       expect(controller.constructConfigUrlParams(data)).to.eql(['(type=slider-client&tag=1)']);
       App.Service.find().clear();
     });
+    it('YARN for 2.2 stack is installed', function () {
+      App.set('currentStackVersion', 'HDP-2.2.0');
+      App.store.load(App.Service, {
+        id: 'YARN',
+        service_name: 'YARN'
+      });
+      var data = {Clusters: {desired_configs: {'yarn-site': {tag: 1}}}};
+      expect(controller.constructConfigUrlParams(data)).to.eql(['(type=yarn-site&tag=1)']);
+      App.set('currentStackVersion', 'HDP-2.0.1');
+      App.Service.find().clear();
+    });
     it('isRMHaEnabled true', function () {
       sinon.stub(App, 'get').withArgs('isRMHaEnabled').returns(true);
       var data = {Clusters: {desired_configs: {'yarn-site': {tag: 1}}}};
@@ -821,6 +832,16 @@ describe('App.MainHostDetailsController', function () {
       }});
       App.set('currentStackVersion', version);
     });
+    it('yarn-site is present and stack > 2.2', function () {
+      var version = App.get('currentStackVersion');
+      var configs = {'yarn-site': {}};
+      App.set('currentStackVersion', 'HDP-2.2.0');
+      expect(controller.setZKConfigs(configs, 'host1:2181', [])).to.be.true;
+      expect(configs).to.eql({"yarn-site": {
+        'hadoop.registry.zk.quorum': "host1:2181"
+      }});
+      App.set('currentStackVersion', version);
+    });
     it('storm-site is present', function () {
       var configs = {'storm-site': {}};
       expect(controller.setZKConfigs(configs, '', ["host1", 'host2'])).to.be.true;

ambari-web/test/controllers/wizard/step7_test.js

@@ -316,6 +316,47 @@ describe('App.InstallerStep7Controller', function () {
     });
   });
 
+  describe('#resolveYarnConfigs', function () {
+    it('should set property to true', function () {
+      var allSelectedServiceNames = ['SLIDER', 'YARN'],
+        configs = [
+          {name: 'hadoop.registry.rm.enabled', value: false, defaultValue: false}
+        ],
+        expected = [
+          {name: 'hadoop.registry.rm.enabled', value: true, defaultValue: true, forceUpdate: true}
+        ];
+      installerStep7Controller.reopen({allSelectedServiceNames: allSelectedServiceNames});
+      installerStep7Controller.resolveYarnConfigs(configs);
+      expect(configs[0]).to.eql(expected[0]);
+    });
+
+    it('should set property to false', function () {
+      var allSelectedServiceNames = ['YARN'],
+        configs = [
+          {name: 'hadoop.registry.rm.enabled', value: true, defaultValue: true},
+        ],
+        expected = [
+          {name: 'hadoop.registry.rm.enabled', value: false, defaultValue: false, forceUpdate: true}
+        ];
+      installerStep7Controller.reopen({allSelectedServiceNames: allSelectedServiceNames});
+      installerStep7Controller.resolveYarnConfigs(configs);
+      expect(configs[0]).to.eql(expected[0]);
+    });
+
+    it('should skip setting property', function () {
+      var allSelectedServiceNames = ['YARN', 'SLIDER'],
+        configs = [
+          {name: 'hadoop.registry.rm.enabled', value: true, defaultValue: true},
+        ],
+        expected = [
+          {name: 'hadoop.registry.rm.enabled', value: true, defaultValue: true}
+        ];
+      installerStep7Controller.reopen({allSelectedServiceNames: allSelectedServiceNames});
+      installerStep7Controller.resolveYarnConfigs(configs);
+      expect(configs[0]).to.eql(expected[0]);
+    });
+  });
+
   describe('#resolveStormConfigs', function () {
 
     beforeEach(function () {
@@ -421,18 +462,23 @@ describe('App.InstallerStep7Controller', function () {
   describe('#resolveServiceDependencyConfigs', function () {
     beforeEach(function () {
       sinon.stub(installerStep7Controller, 'resolveStormConfigs', Em.K);
+      sinon.stub(installerStep7Controller, 'resolveYarnConfigs', Em.K);
     });
     afterEach(function () {
       installerStep7Controller.resolveStormConfigs.restore();
+      installerStep7Controller.resolveYarnConfigs.restore();
     });
-    it('should call resolveStormConfigs if serviceName is STORM', function () {
-      var serviceName = 'STORM', configs = [
-        {},
-        {},
-        {}
-      ];
-      installerStep7Controller.resolveServiceDependencyConfigs(serviceName, configs);
-      expect(installerStep7Controller.resolveStormConfigs.calledWith(configs)).to.equal(true);
+    var serviceNames = [
+      {serviceName: 'STORM', method: "resolveStormConfigs"},
+      {serviceName: 'YARN', method: "resolveYarnConfigs"}].forEach(function(t) {
+      it("should call " + t.method + " if serviceName is " + t.serviceName, function () {
+        var configs = [
+          {},
+          {}
+        ];
+        installerStep7Controller.resolveServiceDependencyConfigs(t.serviceName, configs);
+        expect(installerStep7Controller[t.method].calledWith(configs)).to.equal(true);
+      });
     });
   });
 
@@ -1014,7 +1060,7 @@ describe('App.InstallerStep7Controller', function () {
   describe('#applyServicesConfigs', function() {
     beforeEach(function() {
       installerStep7Controller.reopen({
-        allSelectedServiceNames: [],
+        allSelectedServiceNames: []
       });
       sinon.stub(App.config, 'fileConfigsIntoTextarea', function(configs) {
         return configs;
@@ -1054,12 +1100,14 @@ describe('App.InstallerStep7Controller', function () {
       {
         allSelectedServiceNames: ['YARN'],
         fileConfigsIntoTextarea: true,
-        m: 'should run fileConfigsIntoTextarea',
+        m: 'should run fileConfigsIntoTextarea and resolveServiceDependencyConfigs',
+        resolveServiceDependencyConfigs: true,
         capacitySchedulerUi: false
       },
       {
         allSelectedServiceNames: ['YARN'],
-        m: 'shouldn\'t run fileConfigsIntoTextarea',
+        m: 'shouldn\'t run fileConfigsIntoTextarea but  run resolveServiceDependencyConfigs',
+        resolveServiceDependencyConfigs: true,
         capacitySchedulerUi: true
       },
       {
@@ -1083,7 +1131,7 @@ describe('App.InstallerStep7Controller', function () {
           expect(App.config.fileConfigsIntoTextarea.calledOnce).to.equal(false);
         }
         if (t.resolveServiceDependencyConfigs) {
-          expect(installerStep7Controller.resolveServiceDependencyConfigs.calledWith('STORM', {name: 'configs'})).to.equal(true);
+          expect(installerStep7Controller.resolveServiceDependencyConfigs.calledWith(t.allSelectedServiceNames[0], {name: 'configs'})).to.equal(true);
         } else {
           expect(installerStep7Controller.resolveServiceDependencyConfigs.calledOnce).to.equal(false);
         }

ambari-web/test/controllers/wizard/step8_test.js

@@ -1287,72 +1287,17 @@ describe('App.WizardStep8Controller', function () {
             }
           })+']';
         installerStep8Controller.reopen({
-          selectedServices: [
+          installedServices: [
               Em.Object.create({
                 isSelected: true,
                 isInstalled: false,
                 configTypesRendered: {hdfs:'tag1'}
               })
-            ]
+            ], selectedServices: []
         });
         installerStep8Controller.applyConfigurationsToCluster(serviceConfigTags);
         expect(installerStep8Controller.addRequestToAjaxQueue.args[0][0].data).to.eql({data: data});
       });
-
-      it('should call addRequestToAjaxQueue with core-site if addServiceController', function() {
-        var serviceConfigTags = [
-            {
-              type: 'oozie',
-              tag: 'tag1',
-              properties: [
-                {},
-                {}
-              ]
-            },
-            {
-              type: 'core-site',
-              tag: 'tag1',
-              properties: [
-                {}
-              ]
-            },
-            {
-              type: 'cluster-env',
-              tag: 'tag1',
-              properties: [
-                {}
-              ]
-            }
-          ];
-
-        data = '['+ JSON.stringify({Clusters: {
-              desired_config: [serviceConfigTags[0]]
-            }
-          }) + ',' +
-          JSON.stringify({Clusters: {
-              desired_config: [serviceConfigTags[1]]
-            }
-          }) + ',' +
-          JSON.stringify({Clusters: {
-              desired_config: [serviceConfigTags[2]]
-            }
-          }) + ']';
-        installerStep8Controller.reopen({
-          selectedServices: [
-            Em.Object.create({
-              isSelected: true,
-              isInstalled: false,
-              configTypesRendered: {'oozie': 'tag1' }
-            })
-          ],
-          content: {
-            controllerName: 'addServiceController'
-          }
-        });
-        installerStep8Controller.applyConfigurationsToCluster(serviceConfigTags);
-        var sendData = installerStep8Controller.addRequestToAjaxQueue.args[0][0].data;
-        expect(sendData).to.eql({data: data});
-      });
     });
 
     describe('#applyConfigurationGroups', function() {