AMBARI-11320. Issues with Hive/Oozie database properties and Blueprint cluster install (alexantonenko)

ambari-web/app/assets/test/tests.js

@@ -168,6 +168,7 @@ var files = ['test/init_model_test',
   'test/utils/object_utils_test',
   'test/utils/ui_effects_test',
   'test/utils/updater_test',
+  'test/utils/configs/database_test',
   'test/utils/configs/config_property_helper_test',
   'test/utils/configs/modification_handlers/modification_handler_test',
   'test/views/common/chart/linear_time_test',

ambari-web/app/controllers/main/service/info/configs.js

@@ -19,6 +19,7 @@
 var App = require('app');
 require('controllers/wizard/slave_component_groups_controller');
 var batchUtils = require('utils/batch_scheduled_requests');
+var databaseUtils = require('utils/configs/database');
 
 App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorMixin, App.EnhancedConfigsMixin, App.PreloadRequestsChainMixin, App.ThemesMappingMixin, App.VersionsMappingMixin, App.ConfigsSaverMixin, {
 
@@ -823,6 +824,7 @@ App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorM
       var serviceConfig = App.config.createServiceConfig(serviceName);
       //Make SecondaryNameNode invisible on enabling namenode HA
       var configsByService = this.get('allConfigs').filterProperty('serviceName', serviceName);
+      databaseUtils.bootstrapDatabaseProperties(configsByService, serviceName);
       this.loadConfigs(configsByService, serviceConfig);
       if (serviceConfig.get('serviceName') === 'HDFS') {
         if (App.get('isHaEnabled')) {
@@ -1207,19 +1209,6 @@ App.MainServiceInfoConfigsController = Em.Controller.extend(App.ServerValidatorM
     hostProperties.forEach(function (h) {
       this.setHostForService(h.serviceName, h.componentName, h.hostProperty, h.m);
     }, this);
-
-    if (serviceName === 'HIVE') {
-      var hiveDb = configs.findProperty('name', 'hive_database').value;
-      if (['Existing MySQL Database', 'Existing Oracle Database', 'Existing PostgreSQL Database', 'Existing MSSQL Server database with SQL authentication', 'Existing MSSQL Server database with integrated authentication'].contains(hiveDb)) {
-        configs.findProperty('name', 'hive_hostname').isVisible = true;
-      }
-    }
-    if (serviceName === 'OOZIE') {
-      var oozieDb = configs.findProperty('name', 'oozie_database').value;
-      if (['Existing MySQL Database', 'Existing Oracle Database', 'Existing PostgreSQL Database', 'Existing MSSQL Server database with SQL authentication', 'Existing MSSQL Server database with integrated authentication'].contains(oozieDb)) {
-        configs.findProperty('name', 'oozie_hostname').isVisible = true;
-      }
-    }
   },
 
   /**

ambari-web/app/data/HDP2/site_properties.js

@@ -3443,6 +3443,7 @@ var hdp2properties = [
     "isReconfigurable": true,
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3458,6 +3459,7 @@ var hdp2properties = [
     "recommendedValue": "",
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3473,6 +3475,7 @@ var hdp2properties = [
     "recommendedValue": "",
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3488,6 +3491,7 @@ var hdp2properties = [
     "recommendedValue": "",
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3520,6 +3524,7 @@ var hdp2properties = [
     "recommendedValue": "",
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3535,6 +3540,7 @@ var hdp2properties = [
     "recommendedValue": "",
     "displayType": "host",
     "isOverridable": false,
+    "isRequiredByAgent": false,
     "isVisible": false,
     "isObserved": true,
     "serviceName": "HIVE",
@@ -3898,6 +3904,7 @@ var hdp2properties = [
     "isOverridable": false,
     "displayType": "host",
     "isVisible": false,
+    "isRequiredByAgent": false,
     "isObserved": true,
     "serviceName": "OOZIE",
     "filename": "oozie-env.xml",
@@ -3913,6 +3920,7 @@ var hdp2properties = [
     "isOverridable": false,
     "displayType": "host",
     "isVisible": false,
+    "isRequiredByAgent": false,
     "isObserved": true,
     "serviceName": "OOZIE",
     "filename": "oozie-env.xml",
@@ -3928,6 +3936,7 @@ var hdp2properties = [
     "isOverridable": false,
     "displayType": "host",
     "isVisible": false,
+    "isRequiredByAgent": false,
     "isObserved": true,
     "serviceName": "OOZIE",
     "filename": "oozie-env.xml",

ambari-web/app/utils/configs/database.js

@@ -0,0 +1,190 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var validators = require('utils/validator');
+/**
+ * Helper methods to process database values and properties
+ * @module utils/configs/database
+ */
+module.exports = {
+  /**
+   * Database type properties with <code>options</code> attribute usually displayed as radio-button.
+   * <code>options</code> attribute contains mapping of database type to special properties name such as
+   * database host and database name.
+   * @type {object}
+   */
+  dbServicePropertyMap: {
+    HIVE: {
+      dbType: 'hive_database',
+      databaseName: 'ambari.hive.db.schema.name',
+      connectionUrl: 'javax.jdo.option.ConnectionURL',
+      fallbackHostName: 'hive_ambari_host'
+    },
+    OOZIE: {
+      dbType: 'oozie_database',
+      connectionUrl: 'oozie.service.JPAService.jdbc.url',
+      databaseName: 'oozie.db.schema.name',
+      fallbackHostName: 'oozieserver_host'
+    }
+  },
+
+  /**
+   * Map of jdbc url patterns related to supported database types.
+   *
+   * @type {object}
+   */
+  DB_JDBC_PATTERNS: {
+    mysql: 'jdbc:mysql://{0}/{1}',
+    mssql: 'jdbc:sqlserver://{0};databaseName={1}',
+    postgres: 'jdbc:postgresql://{0}:5432/{1}',
+    derby: 'jdbc:derby:{0}/{1}',
+    oracle: 'jdbc:oracle:thin:@//{0}:1521/{1}'
+  },
+
+  /**
+   * Setup database related properties.
+   *
+   * @method bootstrapDatabaseProperties
+   * @param {App.ServiceConfigProperty[]} serviceConfigs
+   * @param {string} [serviceName=false]
+   */
+  bootstrapDatabaseProperties: function(serviceConfigs, serviceName) {
+    var self = this;
+    var supportedServices = Em.keys(this.dbServicePropertyMap);
+    if (serviceName && !supportedServices.contains(serviceName)) return;
+    var serviceNames = serviceName ? [serviceName] : serviceConfigs.mapProperty('serviceName').uniq();
+    serviceNames.forEach(function(serviceName) {
+      if (!supportedServices.contains(serviceName)) return;
+      var configs = serviceConfigs.filterProperty('serviceName', serviceName) || [];
+      var connectionConfigs = self.dbServicePropertyMap[serviceName];
+      var databaseTypeProperty = configs.findProperty('name', connectionConfigs.dbType);
+      if (!databaseTypeProperty) return;
+      var databaseTypePropertyIndex = configs.indexOf(databaseTypeProperty);
+      var generatedProperties = self.getPropsByOptions(databaseTypeProperty, configs);
+      var jdbcObject = self.parseJdbcUrl(Em.get(configs.findProperty('name', connectionConfigs.connectionUrl), 'value'));
+      generatedProperties.forEach(function(property) {
+        if (Em.get(property, 'name').endsWith('_host')) {
+          // set UI host names for each database type with value parsed from jdbc connection url
+          // if value is not ip or hostname (in case of New Derby Database) for Oozie set <code>fallbackUrl</code>
+          // from <code>dbServicePropertyMap</code>
+          var dbHostName;
+          if (self.isValidHostname(jdbcObject.location)) {
+            dbHostName = jdbcObject.location;
+          } else {
+            dbHostName = Em.get(configs.findProperty('name', self.dbServicePropertyMap[serviceName].fallbackHostName), 'recommendedValue');
+            dbHostName = Em.isArray(dbHostName) ? dbHostName[0] : dbHostName;
+          }
+          Em.setProperties(property, {
+            value: dbHostName
+          });
+          self.addPropertyToServiceConfigs(property, serviceConfigs, databaseTypePropertyIndex);
+        }
+      });
+    });
+  },
+
+  isValidHostname: function(value) {
+    return validators.isHostname(value) || validators.isIpAddress(value);
+  },
+
+  /**
+   * Add UI specific property to serviceConfigObject if it does not exist, and update value for existed property.
+   * This code affects properties related to `_host` and `_database` which are hardcoded on UI.
+   *
+   * @param {object} property - property to append/update
+   * @param {App.ServiceConfigProperty[]} configs - loaded and processed service configs
+   * @param {integer} index of first occurrence of database type in config
+   */
+  addPropertyToServiceConfigs: function(property, configs, index) {
+    var configProperty = configs.findProperty('name', Em.get(property, 'name'));
+    if (configProperty) {
+      Em.set(configProperty, 'value', Em.get(property, 'value'));
+    } else {
+      if (index) {
+        configs.insertAt(index, App.ServiceConfigProperty.create(property));
+      } else {
+        configs.pushObject(App.ServiceConfigProperty.create(property));
+      }
+    }
+  },
+
+  /**
+   * Get hardcoded properties from site_properties.js which UI use to display host name and database name.
+   *
+   * @method getPropsByOptions
+   * @param {object} databaseTypeProperty - hardcoded property from site_properties.js usualy used as radiobutton
+   * @param {App.ServiceConfigProperty[]} configs - loaded and processed configs
+   * @returns {object[]} - hardcoded properties from site_properties.js related to database name and location
+   */
+  getPropsByOptions: function(databaseTypeProperty) {
+    Em.assert('Property related to database type should contains `options` attribute', databaseTypeProperty.get('options'));
+    return databaseTypeProperty.options.mapProperty('foreignKeys').reduce(function(p,c) {
+      return p.concat(c);
+    }).uniq().map(function(name) {
+      return App.config.get('preDefinedSiteProperties').findProperty('name', name) || null;
+    }).compact();
+  },
+
+  /**
+   * Get database location from jdbc url value.
+   *
+   * @method getDBLocationFromJDBC
+   * @param {string} jdbcUrl - url to parse
+   * @returns {string|null} 
+   */
+  getDBLocationFromJDBC: function(jdbcUrl) {
+    var self = this;
+    var matches = Em.keys(this.DB_JDBC_PATTERNS).map(function(key) {
+      var reg = new RegExp(self.DB_JDBC_PATTERNS[key].format('(.*)', '(.*)'));
+      return jdbcUrl.match(reg);
+    }).compact();
+    if (matches.length) {
+      var dbLocation = Em.get(matches, '0.1');
+      if (dbLocation.startsWith('${')) {
+        return Em.getWithDefault(matches, '0.0', '').match(/\${[^}]+}/)[0];
+      }
+      return dbLocation != '{0}' ? dbLocation : null;
+    } else {
+      return null;
+    }
+  },
+
+  parseJdbcUrl: function(jdbcUrl) {
+    var self = this;
+    var result = {
+      dbType: null,
+      location: null,
+      databaseName: null
+    };
+    var dbName;
+
+    result.dbType = Em.keys(this.DB_JDBC_PATTERNS).filter(function(key) {
+      var scheme = self.DB_JDBC_PATTERNS[key].match(/^jdbc:(\w+):/)[1];
+      return new RegExp('jdbc:' + scheme).test(jdbcUrl);
+    })[0];
+
+    result.location = this.getDBLocationFromJDBC(jdbcUrl);
+    if (!jdbcUrl.endsWith('{1}')) {
+      dbName = jdbcUrl.replace(this.DB_JDBC_PATTERNS[result.dbType].format(result.location,''), '');
+      if (dbName) {
+        result.databaseName = dbName.split(/[;|?]/)[0];
+      }
+    }
+    return result;
+  }
+};

ambari-web/test/utils/configs/database_test.js

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+var App = require('app');
+var dbUtils = require('utils/configs/database');
+
+describe('Database Utils', function() {
+  describe('#getDBLocationFromJDBC', function() {
+    [
+      {
+        jdbcUrl: 'jdbc:mysql://localhost/somedb',
+        e: 'localhost'
+      },
+      {
+        jdbcUrl: 'jdbc:postgresql://some.hostname.com:5432/somedb',
+        e: 'some.hostname.com'
+      },
+      {
+        jdbcUrl: 'jdbc:derby:/some/dir/another_dir/somedb',
+        e: '/some/dir/another_dir'
+      },
+      {
+        jdbcUrl: 'jdbc:derby:${oozie-env/data-dir}/${oozie-env/database_name}-db',
+        e: '${oozie-env/data-dir}'
+      },
+      {
+        jdbcUrl: 'jdbc:sqlserver://127.0.0.1;databaseName=some-db;integratedSecurity=true',
+        e: '127.0.0.1'
+      },
+      {
+        jdbcUrl: 'jdbc:oracle:thin:@//localhost.com:1521/someDb',
+        e: 'localhost.com'
+      },
+      {
+        jdbcUrl: 'jdbc:oracle:thin:@//{0}:1521/{1}',
+        e: null
+      }
+    ].forEach(function(test) {
+      it('when jdbc url is ' + test.jdbcUrl + ' host name is ' + test.e, function() {
+        expect(dbUtils.getDBLocationFromJDBC(test.jdbcUrl)).to.eql(test.e);
+      });
+    });
+  });
+
+  describe('#parseJdbcUrl', function() {
+    [
+      {
+        jdbcUrl: 'jdbc:mysql://localhost/somedb',
+        e: {
+          dbType: 'mysql',
+          location: 'localhost',
+          databaseName: 'somedb'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:postgresql://some.hostname.com:5432/somedb',
+        e: {
+          dbType: 'postgres',
+          location: 'some.hostname.com',
+          databaseName: 'somedb'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:derby:/some/dir/another_dir/somedb',
+        e: {
+          dbType: 'derby',
+          location: '/some/dir/another_dir',
+          databaseName: 'somedb'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:derby:${oozie-env/data-dir}/${oozie-env/database_name}-db',
+        e: {
+          dbType: 'derby',
+          location: '${oozie-env/data-dir}',
+          databaseName: '${oozie-env/database_name}-db'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true',
+        e: {
+          dbType: 'derby',
+          location: '${oozie.data.dir}',
+          databaseName: '${oozie.db.schema.name}-db'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:sqlserver://127.0.0.1;databaseName=some-db;integratedSecurity=true',
+        e: {
+          dbType: 'mssql',
+          location: '127.0.0.1',
+          databaseName: 'some-db'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:oracle:thin:@//localhost.com:1521/someDb',
+        e: {
+          dbType: 'oracle',
+          location: 'localhost.com',
+          databaseName: 'someDb'
+        }
+      },
+      {
+        jdbcUrl: 'jdbc:oracle:thin:@//{0}:1521/{1}',
+        e: {
+          dbType: 'oracle',
+          location: null,
+          databaseName: null
+        }
+      }
+    ].forEach(function(test) {
+      it('when jdbc url is ' + test.jdbcUrl + ' result is ' + JSON.stringify(test.e), function() {
+        expect(dbUtils.parseJdbcUrl(test.jdbcUrl)).to.be.eql(test.e);
+      });
+    });
+  });
+});