AMBARI-8943. Modify configs to use correct Hive hostnames list (alexantonenko)

ambari-web/app/controllers/main/admin/security/add/step2.js

@@ -69,7 +69,7 @@ App.MainAdminSecurityAddStep2Controller = Em.Controller.extend({
     {
       serviceName: 'HIVE',
       configName: 'hive_metastore',
-      components: ['HIVE_SERVER']
+      components: ['HIVE_METASTORE', 'HIVE_SERVER']
     },
     {
       serviceName: 'HIVE',

ambari-web/app/controllers/main/admin/security/add/step3.js

@@ -38,6 +38,7 @@ App.MainAdminSecurityAddStep3Controller = Em.Controller.extend({
       'NODEMANAGER': 'yarn_user',
       'ZOOKEEPER_SERVER': 'zk_user',
       'HIVE_SERVER': 'hive_user',
+      'HIVE_METASTORE': 'hive_user',
       'OOZIE_SERVER': 'oozie_user',
       'NAGIOS_SERVER': 'nagios_user',
       'HBASE_MASTER': 'hbase_user',
@@ -346,9 +347,9 @@ App.MainAdminSecurityAddStep3Controller = Em.Controller.extend({
    * @param hadoopGroupId
    */
   setHostComponentsSecureValue: function (result, host, addedPrincipalsHost, securityUsers, hadoopGroupId) {
-    var componentsToDisplay = ['NAMENODE', 'SECONDARY_NAMENODE', 'DATANODE', 'JOBTRACKER', 'ZOOKEEPER_SERVER', 'HIVE_SERVER', 'TASKTRACKER',
-      'OOZIE_SERVER', 'NAGIOS_SERVER', 'HBASE_MASTER', 'HBASE_REGIONSERVER', 'HISTORYSERVER', 'RESOURCEMANAGER', 'NODEMANAGER', 'JOURNALNODE',
-      'SUPERVISOR', 'NIMBUS', 'STORM_UI_SERVER','FALCON_SERVER', 'KNOX_GATEWAY', 'APP_TIMELINE_SERVER'];
+    var componentsToDisplay = ['NAMENODE', 'SECONDARY_NAMENODE', 'DATANODE', 'JOBTRACKER', 'ZOOKEEPER_SERVER', 'HIVE_SERVER', 'HIVE_METASTORE',
+      'TASKTRACKER', 'OOZIE_SERVER', 'NAGIOS_SERVER', 'HBASE_MASTER', 'HBASE_REGIONSERVER', 'HISTORYSERVER', 'RESOURCEMANAGER', 'NODEMANAGER',
+      'JOURNALNODE', 'SUPERVISOR', 'NIMBUS', 'STORM_UI_SERVER','FALCON_SERVER', 'KNOX_GATEWAY', 'APP_TIMELINE_SERVER'];
     if (App.get('isHadoop22Stack')) {
       componentsToDisplay.push('DRPC_SERVER');
     }
@@ -439,7 +440,7 @@ App.MainAdminSecurityAddStep3Controller = Em.Controller.extend({
    * @return {*}
    */
   changeDisplayName: function (name) {
-    if (name === 'HiveServer2') {
+    if (name === 'HiveServer2' || name === 'Hive Metastore') {
       return 'Hive Metastore and HiveServer2';
     } else {
       return name;

ambari-web/app/controllers/main/host/details.js

@@ -538,7 +538,7 @@ App.MainHostDetailsController = Em.Controller.extend({
     }
 
     this.showBackgroundOperationsPopup(function () {
-      if (params.componentName === 'ZOOKEEPER_SERVER' || params.componentName === 'HIVE_METASTORE') {
+      if (params.componentName === 'ZOOKEEPER_SERVER' || params.componentName === 'HIVE_METASTORE' ||  params.componentName === 'HIVE_SERVER') {
         self.set(params.componentName === 'ZOOKEEPER_SERVER' ? 'zkRequestId' : 'hiveRequestId', data.Requests.id);
         self.addObserver(
           'App.router.backgroundOperationsController.serviceTimestamp',
@@ -576,21 +576,26 @@ App.MainHostDetailsController = Em.Controller.extend({
       name: 'admin.get.all_configurations',
       sender: this,
       data: {
-        urlParams: '(type=hive-site&tag='+data.Clusters.desired_configs['hive-site'].tag+')|(type=webhcat-site&tag=' + data.Clusters.desired_configs['webhcat-site'].tag + ')'
+        urlParams: '(type=hive-site&tag=' + data.Clusters.desired_configs['hive-site'].tag + ')|(type=webhcat-site&tag=' +
+          data.Clusters.desired_configs['webhcat-site'].tag + ')|(type=hive-env&tag=' + data.Clusters.desired_configs['hive-env'].tag +
+          ')|(type=core-site&tag=' + data.Clusters.desired_configs['core-site'].tag + ')'
       },
       success: 'onLoadHiveConfigs'
     });
   },
 
   /**
-   * update and save Hive hive.metastore.uris config to server
+   * update and save Hive related configs to server
    * @param {object} data
    * @method onLoadHiveConfigs
    */
   onLoadHiveConfigs: function (data) {
     var hiveMSHosts = this.getHiveHosts();
+    var hiveMasterHosts = hiveMSHosts.concat([App.HostComponent.find().findProperty('componentName', 'HIVE_SERVER').get('hostName')]).uniq().join(',');
     var configs = {};
     var port = "";
+    var hiveUser = "";
+    var webhcatUser = "";
 
     data.items.forEach(function (item) {
       configs[item.type] = item.properties;
@@ -599,11 +604,16 @@ App.MainHostDetailsController = Em.Controller.extend({
     port = configs['hive-site']['hive.metastore.uris'].match(/:[0-9]{2,4}/);
     port = port ? port[0].slice(1) : "9083";
 
+    hiveUser = configs['hive-env']['hive_user'];
+    webhcatUser = configs['hive-env']['webhcat_user'];
+
     for (var i = 0; i < hiveMSHosts.length; i++) {
       hiveMSHosts[i] = "thrift://" + hiveMSHosts[i] + ":" + port;
     }
     configs['hive-site']['hive.metastore.uris'] = hiveMSHosts.join(',');
     configs['webhcat-site']['templeton.hive.properties'] = configs['webhcat-site']['templeton.hive.properties'].replace(/thrift.+[0-9]{2,},/i, hiveMSHosts.join('\\,') + ",");
+    configs['core-site']['hadoop.proxyuser.' + hiveUser + '.hosts'] = hiveMasterHosts;
+    configs['core-site']['hadoop.proxyuser.' + webhcatUser + '.hosts'] = hiveMasterHosts;
 
     for (var site in configs) {
       if (!configs.hasOwnProperty(site)) continue;

ambari-web/app/controllers/wizard/step8_controller.js

@@ -305,6 +305,7 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, App.wiz
           break;
       }
 
+      hive_properties.push('hive_master_hosts');
       hive_properties.forEach(function (property) {
         configs = configs.without(configs.findProperty('name', property));
       });

ambari-web/app/data/BIGTOP/site_properties.js

@@ -2508,6 +2508,16 @@ module.exports =
       "category": "HIVE_METASTORE",
       "index": 0
     },
+    {
+      "name": "hive_master_hosts",
+      "value": "",
+      "defaultValue": "",
+      "isOverridable": false,
+      "isRequiredByAgent": false,
+      "isVisible": false,
+      "serviceName": "HIVE",
+      "filename": "hive-env.xml"
+    },
     // for existing MySQL
     {
       "id": "puppet var",

ambari-web/app/data/HDP2/config_mapping.js

@@ -29,7 +29,7 @@ var configs = [
   },
   {
     "name": "hadoop.proxyuser.<foreignKey[0]>.hosts",
-    "templateName": ["hivemetastore_host"],
+    "templateName": ["hive_master_hosts"],
     "foreignKey": ["hive_user"],
     "value": "<templateName[0]>",
     "filename": "core-site.xml",
@@ -61,7 +61,7 @@ var configs = [
   },
   {
     "name": "hadoop.proxyuser.<foreignKey[0]>.hosts",
-    "templateName": ["hivemetastore_host"],
+    "templateName": ["hive_master_hosts"],
     "foreignKey": ["webhcat_user"],
     "value": "<templateName[0]>",
     "filename": "core-site.xml",

ambari-web/app/data/HDP2/secure_mapping.js

@@ -543,9 +543,9 @@ var props = [
   },
   {
     "name": "templeton.hive.properties",
-    "templateName": ["hivemetastore_host","hive_metastore_principal_name","kerberos_domain"],
+    "templateName": ["hive_metastore","hive_metastore_principal_name","kerberos_domain"],
     "foreignKey": null,
-    "value": "hive.metastore.local=false,hive.metastore.uris=thrift://<templateName[0]>:9083,hive." +
+    "value": "hive.metastore.local=false,hive.metastore.uris=<templateName[0]>,hive." +
       "metastore.sasl.enabled=true,hive.metastore.execute.setugi=true,hive.metastore.warehouse.dir=/apps/hive/warehouse,hive.exec.mode.local.auto=false,hive.metastore.kerberos.principal=<templateName[1]>@<templateName[2]>",
     "filename": "webhcat-site.xml",
     "serviceName": "HIVE"
@@ -747,7 +747,7 @@ var props = [
   },
   {
     "name": "hadoop.proxyuser.<foreignKey[0]>.hosts",
-    "templateName": ["hivemetastore_host"],
+    "templateName": ["hive_metastore"],
     "foreignKey": ["hive_metastore_primary_name"],
     "value": "<templateName[0]>",
     "filename": "core-site.xml",

ambari-web/app/data/HDP2/secure_properties.js

@@ -802,11 +802,11 @@ var props = {
     {
       "id": "puppet var",
       "name": "hive_metastore",
-      "displayName": "Hive Metastore host",
+      "displayName": "Hive Metastore hosts",
       "value": "",
       "defaultValue": "localhost",
-      "description": "The host that has been assigned to run Hive Metastore and HiveServer2",
-      "displayType": "masterHost",
+      "description": "The hosts that have been assigned to run Hive Metastore and HiveServer2",
+      "displayType": "masterHosts",
       "isVisible": true,
       "isOverridable": false,
       "serviceName": "HIVE",
@@ -824,7 +824,7 @@ var props = {
       "isOverridable": false,
       "serviceName": "HIVE",
       "category": "Hive Metastore",
-      "component": "HIVE_SERVER"
+      "components": ["HIVE_SERVER", "HIVE_METASTORE"]
     },
     {
       "id": "puppet var",
@@ -838,7 +838,7 @@ var props = {
       "isOverridable": false,
       "serviceName": "HIVE",
       "category": "Hive Metastore",
-      "component": "HIVE_SERVER"
+      "components": ["HIVE_SERVER", "HIVE_METASTORE"]
     },
     {
       "id": "puppet var",

ambari-web/app/data/HDP2/site_properties.js

@@ -3232,6 +3232,16 @@ module.exports =
       "category": "HIVE_METASTORE",
       "index": 0
     },
+    {
+      "name": "hive_master_hosts",
+      "value": "",
+      "defaultValue": "",
+      "isOverridable": false,
+      "isRequiredByAgent": false,
+      "isVisible": false,
+      "serviceName": "HIVE",
+      "filename": "hive-env.xml"
+    },
     // for existing MySQL
     {
       "id": "puppet var",

ambari-web/app/data/PHD/site_properties.js

@@ -2356,6 +2356,16 @@ module.exports =
       "category": "HIVE_METASTORE",
       "index": 0
     },
+    {
+      "name": "hive_master_hosts",
+      "value": "",
+      "defaultValue": "",
+      "isOverridable": false,
+      "isRequiredByAgent": false,
+      "isVisible": false,
+      "serviceName": "HIVE",
+      "filename": "hive-env.xml"
+    },
     // for existing MySQL
     {
       "id": "puppet var",

ambari-web/app/mixins/wizard/addSecurityConfigs.js

@@ -39,11 +39,6 @@ App.AddSecurityConfigs = Em.Mixin.create({
       componentName: 'OOZIE_SERVER',
       configName: 'oozieserver_host'
     },
-    {
-      serviceName: 'HIVE',
-      componentName: 'HIVE_METASTORE',
-      configName: 'hivemetastore_host'
-    },
     {
       serviceName: 'HIVE',
       componentName: 'WEBHCAT_SERVER',
@@ -248,7 +243,7 @@ App.AddSecurityConfigs = Em.Mixin.create({
       if (_config.hasOwnProperty('dependedServiceName')) {
         value = this.checkServiceForConfigValue(value, _config.dependedServiceName);
       }
-      value = this.getConfigValue(_config.templateName, value);
+      value = this.getConfigValue(_config.templateName, value, _config.name);
       uiConfig.push({
         "id": "site property",
         "name": _config.name,
@@ -291,9 +286,10 @@ App.AddSecurityConfigs = Em.Mixin.create({
    * Set all site property that are derived from other puppet-variable
    * @param templateName
    * @param expression
+   * @param name
    * @return {String|null}
    */
-  getConfigValue: function (templateName, expression) {
+  getConfigValue: function (templateName, expression, name) {
     var express = expression.match(/<(.*?)>/g);
     var value = expression;
     if (Em.isNone(express)) return expression;
@@ -301,9 +297,13 @@ App.AddSecurityConfigs = Em.Mixin.create({
     express.forEach(function (_express) {
       var index = parseInt(_express.match(/\[([\d]*)(?=\])/)[1]);
       var configs = this.get('configs').findProperty('name', templateName[index]);
+      var configValue = templateName[index] == 'hive_metastore' ?
+        configs.value.map(function (hostName) {
+        return 'thrift://' + hostName + ':9083';
+      }).join(',') : configs.value;
 
       if (!!value) {
-        value = (configs) ? value.replace(_express, configs.value) : null;
+        value = (configs) ? App.config.replaceConfigValues(name, _express, value, configValue) : null;
       }
     }, this);
     return value;
@@ -349,9 +349,14 @@ App.AddSecurityConfigs = Em.Mixin.create({
     if (templateValue) {
       templateValue.forEach(function (_value) {
         var index = parseInt(_value.match(/\[([\d]*)(?=\])/)[1]);
-        var cfgValue = this.get('configs').findProperty('name', config.templateName[index]);
+        var cfg = this.get('configs').findProperty('name', config.templateName[index]);
 
-        config.value = (cfgValue) ? config.value.replace(_value, cfgValue.value) : null;
+        if (cfg) {
+          var cfgValue = config.templateName[index] == 'hive_metastore' ? cfg.value.join(',') : cfg.value;
+          config.value = config.value.replace(_value, cfgValue);
+        } else {
+          config.value = null;
+        }
       }, this);
       return true;
     }

ambari-web/app/models/service_config.js

@@ -439,6 +439,12 @@ App.ServiceConfigProperty = Em.Object.extend({
       case 'hive_ambari_host':
         this.set('value', masterComponentHostsInDB.findProperty('component', 'HIVE_SERVER').hostName);
         break;
+      case 'hive_master_hosts':
+        var hostNames = masterComponentHostsInDB.filter(function (masterComponent) {
+          return ['HIVE_METASTORE', 'HIVE_SERVER'].contains(masterComponent.component);
+        });
+        this.set('value', hostNames.mapProperty('hostName').uniq().join(','));
+        break;
       case 'hive_database':
         var newMySQLDBOption = this.get('options').findProperty('displayName', 'New MySQL Database');
         if (newMySQLDBOption) {

ambari-web/test/controllers/wizard/step8_test.js

@@ -293,7 +293,7 @@ describe('App.WizardStep8Controller', function () {
             {name: 'hive_ambari_host', value: 'h1'},
             {name: 'hive_hostname', value: 'h2'}
           ],
-          removed: Em.A(['hive_existing_mysql_database', 'hive_existing_oracle_database', 'hive_existing_postgresql_database']),
+          removed: Em.A(['hive_existing_mysql_database', 'hive_existing_oracle_database', 'hive_existing_postgresql_database', 'hive_master_hosts']),
           m: 'hive_database: New MySQL Database',
           host: 'h1'
         },
@@ -304,7 +304,7 @@ describe('App.WizardStep8Controller', function () {
             {name: 'hive_existing_mysql_host', value: 'h1'},
             {name: 'hive_hostname', value: 'h2'}
           ],
-          removed: Em.A(['hive_ambari_database', 'hive_existing_oracle_database', 'hive_existing_postgresql_database']),
+          removed: Em.A(['hive_ambari_database', 'hive_existing_oracle_database', 'hive_existing_postgresql_database', 'hive_master_hosts']),
           m: 'hive_database: Existing MySQL Database',
           host: 'h1'
         },
@@ -315,7 +315,7 @@ describe('App.WizardStep8Controller', function () {
             {name: 'hive_existing_postgresql_host', value: 'h1'},
             {name: 'hive_hostname', value: 'h2'}
           ],
-          removed: Em.A(['hive_ambari_database', 'hive_existing_oracle_database', 'hive_existing_mysql_database']),
+          removed: Em.A(['hive_ambari_database', 'hive_existing_oracle_database', 'hive_existing_mysql_database', 'hive_master_hosts']),
           m: 'hive_database: Existing PostgreSQL Database',
           host: 'h1'
         },
@@ -326,7 +326,7 @@ describe('App.WizardStep8Controller', function () {
             {name: 'hive_existing_oracle_host', value: 'h1'},
             {name: 'hive_hostname', value: 'h2'}
           ],
-          removed: Em.A(['hive_ambari_database', 'hive_existing_mysql_database', 'hive_existing_postgresql_database']),
+          removed: Em.A(['hive_ambari_database', 'hive_existing_mysql_database', 'hive_existing_postgresql_database', 'hive_master_hosts']),
           m: 'hive_database: Existing Oracle Database',
           host: 'h1'
         }

ambari-web/test/mixins/wizard/addSeccurityConfigs_test.js

@@ -220,6 +220,23 @@ describe('App.AddSecurityConfigs', function () {
       expect(config.value).to.be.null;
     });
 
+    it('Hive Metastore hostname array is converted to string', function () {
+      var config = {
+        value: '<templateName[0]>',
+        templateName: ['hive_metastore']
+      };
+      controller.set('globalProperties', []);
+      controller.set('configs', [
+        {
+          name: 'hive_metastore',
+          value: ['h0', 'h1', 'h2']
+        }
+      ]);
+
+      expect(controller.setConfigValue(config)).to.be.true;
+      expect(config.value).to.equal('h0,h1,h2');
+    });
+
   });
 
   describe('#addHostConfig()', function() {

ambari-web/test/models/service_config_test.js

@@ -752,6 +752,30 @@ describe('App.ServiceConfigProperty', function () {
         },
         value: ['h0', 'h1'],
         title: 'array that contains names of hosts with Hive Metastore'
+      },
+      'hive_master_hosts': {
+        localDB: {
+          masterComponentHosts: [
+            {
+              component: 'HIVE_SERVER',
+              hostName: 'h0'
+            },
+            {
+              component: 'HIVE_METASTORE',
+              hostName: 'h0'
+            },
+            {
+              component: 'HIVE_METASTORE',
+              hostName: 'h1'
+            },
+            {
+              component: 'WEBHCAT_SERVER',
+              hostName: 'h2'
+            }
+          ]
+        },
+        value: 'h0,h1',
+        title: 'comma separated list of hosts with Hive Server and Metastore'
       }
     };
 
@@ -831,5 +855,11 @@ describe('App.ServiceConfigProperty', function () {
       expect(serviceConfigProperty.get('value')).to.eql(cases['hivemetastore_host'].value);
     });
 
+    it(cases['hive_master_hosts'].title, function () {
+      serviceConfigProperty.set('name', 'hive_master_hosts');
+      serviceConfigProperty.initialValue(cases['hive_master_hosts'].localDB);
+      expect(serviceConfigProperty.get('value')).to.equal(cases['hive_master_hosts'].value);
+    });
+
   });
 });