AMBARI-9173 : AMS: Cleanup Backend logic for MetricsSink based metrics reporting on Windows (Eugene Chekanskiy via jluniya)

ambari-server/conf/windows/ambari.properties

@@ -79,6 +79,3 @@ ulimit.open.files=10000
 #server.jdbc.schema=ambari
 ##server.jdbc.user.passwd=etc\\ambari-server\\conf\\password.dat
 ##server.jdbc.user.name=ambari
-#scom.sink.db.driver=com.microsoft.sqlserver.jdbc.SQLServerDriver
-##scom.sink.db.url=jdbc:sqlserver://[server]:[port];databaseName=[databaseName];user=[user];password=[password]
-#scom.sink.db.url=jdbc:sqlserver://localhost\\SQLEXPRESS;databaseName=HadoopMetrics;integratedSecurity=true

ambari-server/pom.xml

@@ -1375,11 +1375,6 @@
           <version>1.4.0-incubating</version>
           <scope>test</scope>
         </dependency>
-        <dependency>
-          <groupId>${pom.groupId}</groupId>
-          <artifactId>metrics-sink</artifactId>
-          <version>1.0.0</version>
-        </dependency>
       </dependencies>
     </profile>
     <profile>

ambari-server/src/main/assemblies/server-windows.xml

@@ -67,22 +67,6 @@
       <source>${project.build.directory}/version</source>
       <outputDirectory>ambari-server-${project.version}/</outputDirectory>
     </file>
-    <file>
-      <source>${basedir}/../contrib/ambari-scom/metrics-sink/db/Hadoop-Metrics-SQLServer-CREATE.sql</source>
-      <outputDirectory>/ambari-server-${project.version}/resources</outputDirectory>
-    </file>
-    <file>
-      <source>${basedir}/../contrib/ambari-scom/metrics-sink/db/Hadoop-Metrics-SQLServer-CREATELOCAL.sql</source>
-      <outputDirectory>/ambari-server-${project.version}/resources</outputDirectory>
-    </file>
-    <file>
-      <source>${basedir}/../contrib/ambari-scom/metrics-sink/db/Hadoop-Metrics-SQLServer-DROP.sql</source>
-      <outputDirectory>/ambari-server-${project.version}/resources</outputDirectory>
-    </file>
-    <file>
-      <source>${basedir}/../contrib/ambari-scom/metrics-sink/target/metrics-sink-1.0.0.jar</source>
-      <outputDirectory>/ambari-server-${project.version}/resources</outputDirectory>
-    </file>
     <file>
       <source>${basedir}/src/main/package/msi/ambari-server.wxs</source>
       <outputDirectory>../../</outputDirectory>

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -176,11 +176,6 @@ public class Configuration {
   public static final String SERVER_JDBC_RCA_USER_PASSWD_KEY = "server.jdbc.rca.user.passwd";
   public static final String SERVER_JDBC_RCA_DRIVER_KEY = "server.jdbc.rca.driver";
   public static final String SERVER_JDBC_RCA_URL_KEY = "server.jdbc.rca.url";
-  public static final String SCOM_JDBC_SINK_USER_NAME_KEY = "scom.sink.db.username";
-  public static final String SCOM_JDBC_SINK_USER_PASSWD_KEY = "scom.sink.db.password";
-  public static final String SCOM_JDBC_SINK_DRIVER_KEY = "scom.sink.db.driver";
-  public static final String SCOM_JDBC_SINK_URL_KEY = "scom.sink.db.url";
-  public static final String SCOM_JDBC_SINK_INT_AUTH_KEY = "scom.sink.db.use.integrated.auth";
   public static final String SERVER_JDBC_GENERATE_TABLES_KEY = "server.jdbc.generateTables";
   public static final String JDBC_UNIT_NAME = "ambari-server";
   public static final String JDBC_LOCAL_URL = "jdbc:postgresql://localhost/";
@@ -823,33 +818,6 @@ public class Configuration {
     return readPasswordFromFile(passwdProp, SERVER_JDBC_RCA_USER_PASSWD_DEFAULT);
   }
 
-  public String getSinkDatabaseDriver() {
-    return properties.getProperty(SCOM_JDBC_SINK_DRIVER_KEY);
-  }
-
-  public String getSinkDatabaseUrl() {
-    return properties.getProperty(SCOM_JDBC_SINK_URL_KEY);
-  }
-
-  public boolean getSinkUseIntegratedAuth() {
-      return "true".equalsIgnoreCase(properties.getProperty(SCOM_JDBC_SINK_INT_AUTH_KEY));
-  }
-
-  public String getSinkDatabaseUser() {
-    return properties.getProperty(SCOM_JDBC_SINK_USER_NAME_KEY, SCOM_JDBC_SINK_USER_NAME_DEFAULT);
-  }
-
-  public String getSinkDatabasePassword() {
-    String passwdProp = properties.getProperty(SCOM_JDBC_SINK_USER_PASSWD_KEY);
-    if (passwdProp != null) {
-      String dbpasswd = readPasswordFromStore(passwdProp);
-      if (dbpasswd != null) {
-        return dbpasswd;
-      }
-    }
-    return readPasswordFromFile(passwdProp, SCOM_JDBC_SINK_USER_PASSWD_DEFAULT);
-  }
-
   private String readPasswordFromFile(String filePath, String defaultPassword) {
     if (filePath == null) {
       LOG.debug("DB password file not specified - using default");

ambari-server/src/main/python/ambari-server-windows.py

@@ -291,11 +291,6 @@ def ensure_dbms_is_running(options, properties, scmStatus):
 
   dbms.ensure_dbms_is_running(options, properties, scmStatus)
 
-  dbms2 = DBMSConfig.create(options, properties, "Metrics")
-  if dbms2.database_host.lower() != dbms.database_host.lower():
-    dbms2.ensure_dbms_is_running(options, properties, scmStatus)
-  pass
-
 def ensure_resources_are_organized(properties):
   resources_location = get_resources_location(properties)
   resource_files_keeper = ResourceFilesKeeper(resources_location)
@@ -399,25 +394,6 @@ def init_options_parser():
                     help="Database user login")
   parser.add_option('-p', '--databasepassword', dest="database_password", default=None,
                     help="Database user password")
-
-  parser.add_option('-t', '--init-metrics-script-file', dest="init_metrics_db_script_file", default=None,
-                    help="File with metrics database setup script")
-  parser.add_option('-c', '--drop-metrics-script-file', dest="cleanup_metrics_db_script_file", default=None,
-                    help="File with metrics database cleanup script")
-
-  parser.add_option('-m', '--metricsdatabasehost', dest="metrics_database_host", default=None,
-                    help="Hostname of metrics database server")
-  parser.add_option('-o', '--metricsdatabaseport', dest="metrics_database_port", default=None,
-                    help="Metrics database server listening port")
-  parser.add_option('-e', '--metricsdatabasename', dest="metrics_database_name", default=None,
-                    help="Metrics database/Schema/Service name or ServiceID")
-  parser.add_option('-z', '--metricswindowsauth', action="store_true", dest="metrics_database_windows_auth", default=None,
-                    help="Integrated Windows authentication for the metrics database")
-  parser.add_option('-q', '--metricsdatabaseusername', dest="metrics_database_username", default=None,
-                    help="Metrics database user login")
-  parser.add_option('-l', '--metricsdatabasepassword', dest="metrics_database_password", default=None,
-                    help="Metrics database user password")
-
   parser.add_option('--jdbc-driver', default=None, dest="jdbc_driver",
                     help="Specifies the path to the JDBC driver JAR file for the " \
                          "database type specified with the --jdbc-db option. Used only with --jdbc-db option.")
@@ -430,12 +406,7 @@ def are_cmd_line_db_args_blank(options):
     and options.database_name is None \
     and options.database_windows_auth is None \
     and options.database_username is None \
-    and options.database_password is None \
-    and options.metrics_database_host is None \
-    and options.metrics_database_name is None \
-    and options.metrics_database_windows_auth is None \
-    and options.metrics_database_username is None \
-    and options.metrics_database_password is None):
+    and options.database_password is None):
     return True
   return False
 
@@ -453,12 +424,7 @@ def are_cmd_line_db_args_valid(options):
       #and options.database_name is not None \         # ambari by default is ok
       and are_db_auth_options_ok(options.database_windows_auth,
                                options.database_username,
-                               options.database_password) \
-    and options.metrics_database_host is not None and options.metrics_database_host is not "" \
-    #and options.metrics_database_name is not None \  # HadoopMetrics by default is ok
-    and are_db_auth_options_ok(options.metrics_database_windows_auth,
-                               options.metrics_database_username,
-                               options.metrics_database_password)):
+                               options.database_password)):
     return True
   return False
 

ambari-server/src/main/python/ambari_server/dbConfiguration.py

@@ -95,8 +95,6 @@ class DBMSConfig(object):
     #if OSCheck.is_windows_family():
     if dbId == "Ambari":
       return SQLServerAmbariDBConfig(options, properties)
-    elif dbId == "Metrics":
-      return SQLServerMetricsDBConfig(options, properties)
     else:
       raise FatalException(-1, "Invalid database requested: " + str(dbId))
     #else:
@@ -208,6 +206,6 @@ class DBMSConfig(object):
     pass
 
 if OSCheck.is_windows_family():
-  from ambari_server.dbConfiguration_windows import SQLServerAmbariDBConfig, SQLServerMetricsDBConfig
+  from ambari_server.dbConfiguration_windows import SQLServerAmbariDBConfig
 #else:
 #  from ambari_server.dbConfiguration_linux import PostgreSQLConfig #and potentially MySQLConfig, OracleConfig

ambari-server/src/main/python/ambari_server/dbConfiguration_windows.py

@@ -35,12 +35,6 @@ from ambari_server.serverConfiguration import JDBC_DRIVER_PROPERTY, JDBC_DRIVER_
   JDBC_RCA_HOSTNAME_PROPERTY, JDBC_RCA_PORT_PROPERTY, JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY, \
   JDBC_RCA_USER_NAME_PROPERTY, JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_FILENAME, JDBC_RCA_PASSWORD_ALIAS, \
   PERSISTENCE_TYPE_PROPERTY, \
-  JDBC_METRICS_DRIVER_PROPERTY, JDBC_METRICS_URL_PROPERTY, \
-  JDBC_METRICS_DATABASE_PROPERTY, METRICS_DATABASE_NAME, JDBC_METRICS_SCHEMA_PROPERTY, \
-  JDBC_METRICS_HOSTNAME_PROPERTY, JDBC_METRICS_PORT_PROPERTY, \
-  JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_USER_NAME_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, \
-  JDBC_METRICS_PASSWORD_FILENAME, JDBC_METRICS_PASSWORD_ALIAS, \
-  METRICS_PERSISTENCE_TYPE_PROPERTY, \
   PRESS_ENTER_MSG
 from ambari_server.setupSecurity import encrypt_password, store_password_file
 from dbConfiguration import DBMSConfig, DB_STATUS_RUNNING_DEFAULT
@@ -57,7 +51,6 @@ DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
 LOCAL_DATABASE_SERVER = "localhost\\SQLEXPRESS"
 AMBARI_DATABASE_NAME = "ambari"
 
-
 class DbPropKeys:
   def __init__(self, i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key):
     self.reset(i_dbms_key, i_driver_key, i_server_key, i_port_key, i_db_name_key, i_db_url_key)
@@ -419,48 +412,6 @@ class SQLServerAmbariDBConfig(SQLServerConfig):
     properties.process_pair(JDBC_RCA_URL_PROPERTY, self.database_url)
     pass
 
-
-# SQL Server Metrics database configuration and setup
-class SQLServerMetricsDBConfig(SQLServerConfig):
-  def __init__(self, options, properties):
-    self.dbPropKeys = DbPropKeys(
-      JDBC_METRICS_DATABASE_PROPERTY,
-      JDBC_METRICS_DRIVER_PROPERTY,
-      JDBC_METRICS_HOSTNAME_PROPERTY,
-      JDBC_METRICS_PORT_PROPERTY,
-      JDBC_METRICS_SCHEMA_PROPERTY,
-      JDBC_METRICS_URL_PROPERTY)
-    self.dbAuthKeys = AuthenticationKeys(
-      JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY,
-      JDBC_METRICS_USER_NAME_PROPERTY,
-      JDBC_METRICS_PASSWORD_PROPERTY,
-      JDBC_METRICS_PASSWORD_ALIAS,
-      JDBC_METRICS_PASSWORD_FILENAME
-    )
-
-    super(SQLServerMetricsDBConfig, self).__init__(options, properties)
-
-    self.database_name = METRICS_DATABASE_NAME
-
-    self.persistence_property = METRICS_PERSISTENCE_TYPE_PROPERTY
-
-    self.db_title = "metrics"
-
-    self.env_var_db_name ='METRICSDBNAME'
-    self.env_var_db_log_name = 'METRICSDBLOGNAME'
-    self.env_var_db_owner = 'METRICSDBOWNER'
-
-    if options.init_metrics_db_script_file is not None and options.init_metrics_db_script_file is not "":
-      self.init_script_file = compress_backslashes(options.init_db_script_file)
-    else:
-      self.init_script_file = "resources" + os.sep + "Hadoop-Metrics-SQLServer-CREATE.sql"
-    if options.cleanup_metrics_db_script_file is not None and options.cleanup_metrics_db_script_file is not "":
-      self.drop_tables_script_file = compress_backslashes(options.cleanup_db_script_file)
-    else:
-      self.drop_tables_script_file = "resources" + os.sep + "Hadoop-Metrics-SQLServer-DROP.sql"
-    pass
-
-
 # SQL Server database
 class SQLServerDatabase:
   def __init__(self):

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -120,28 +120,11 @@ JDBC_RCA_PASSWORD_FILE_PROPERTY = "server.jdbc.rca.user.passwd"
 JDBC_RCA_PASSWORD_ALIAS = "ambari.db.password"
 
 #Windows-specific settings
-METRICS_DATABASE_NAME = "HadoopMetrics"
 
 JDBC_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.use.integrated.auth"
 
 JDBC_RCA_USE_INTEGRATED_AUTH_PROPERTY = "server.jdbc.rca.use.integrated.auth"
 
-JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY = "scom.sink.db.use.integrated.auth"
-
-METRICS_PERSISTENCE_TYPE_PROPERTY = "metrics.persistence.type"
-
-JDBC_METRICS_DATABASE_PROPERTY = "scom.sink.db.database"
-JDBC_METRICS_HOSTNAME_PROPERTY = "scom.sink.db.hostname"
-JDBC_METRICS_PORT_PROPERTY = "scom.sink.db.port"
-JDBC_METRICS_SCHEMA_PROPERTY = "scom.sink.db.schema"
-
-JDBC_METRICS_DRIVER_PROPERTY = "scom.sink.db.driver"
-JDBC_METRICS_URL_PROPERTY = "scom.sink.db.url"
-JDBC_METRICS_USER_NAME_PROPERTY = "scom.sink.db.username"
-JDBC_METRICS_PASSWORD_PROPERTY = "scom.sink.db.password"
-JDBC_METRICS_PASSWORD_FILENAME = "scom_password.dat"
-
-JDBC_METRICS_PASSWORD_ALIAS = "scom.db.password"
 
 # resources repo configuration
 RESOURCES_DIR_PROPERTY = "resources.dir"

ambari-server/src/main/python/ambari_server/serverSetup_windows.py

@@ -32,7 +32,6 @@ from ambari_commons.exceptions import *
 from ambari_commons.logging_utils import *
 from ambari_commons.os_windows import run_powershell_script, UserHelper, CHECK_FIREWALL_SCRIPT
 from ambari_server.dbConfiguration import DBMSConfig
-from ambari_server.dbConfiguration_windows import JDBC_METRICS_URL_PROPERTY
 from ambari_server.serverConfiguration import *
 from ambari_server.userInput import get_validated_string_input
 
@@ -79,8 +78,6 @@ DATABASE_NAME = "ambari"
 DATABASE_SERVER = "localhost\\\\SQLEXPRESS"
 DATABASE_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
 
-METRICS_DATABASE_NAME = "HadoopMetrics"
-
 JDBC_PATTERNS = {"sqlserver": "sqljdbc*.jar"}
 DATABASE_FULL_NAMES = {"sqlserver": "SQL Server"}
 JDBC_DB_OPTION_VALUES = ["sqlserver"]
@@ -152,7 +149,7 @@ def os_create_custom_user():
   print_info_msg("User configuration is done.")
   print_warning_msg("When using non SYSTEM user make sure that your user have read\write access to log directories and "
                     "all server directories. In case of integrated authentication for SQL Server make sure that your "
-                    "user properly configured to use ambari and metric database.")
+                    "user properly configured to use ambari database.")
   #storing username and password in os.environ temporary to pass them to service
   os.environ[SERVICE_USERNAME_KEY] = user
   os.environ[SERVICE_PASSWORD_KEY] = password
@@ -271,15 +268,11 @@ def os_setup_database(options):
   #By default, use the same server for Metrics
   options.default_database_host = dbmsAmbari.database_host
 
-  dbmsMetrics = DBMSConfig.create(options, properties, "Metrics")
-  resultM = dbmsMetrics.configure_database(options, properties)
-
   # Now save the properties file
-  if resultA or resultM:
+  if resultA:
     update_properties(properties)
 
     dbmsAmbari.setup_database()
-    dbmsMetrics.setup_database()
 
 def os_reset_database(options):
   properties = get_ambari_properties()
@@ -287,8 +280,7 @@ def os_reset_database(options):
     raise FatalException(-1, "Error getting ambari properties")
 
   if not (properties.getPropertyDict().has_key(JDBC_URL_PROPERTY) and
-            properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY) and
-            properties.getPropertyDict().has_key(JDBC_METRICS_URL_PROPERTY)):
+            properties.getPropertyDict().has_key(JDBC_RCA_URL_PROPERTY)):
     raise FatalException(-1, "Ambari Server not set up yet. Nothing to reset.")
 
   empty_options = optparse.Values()
@@ -301,13 +293,8 @@ def os_reset_database(options):
   empty_options.database_password = ""
   empty_options.init_db_script_file = ""
   empty_options.cleanup_db_script_file = ""
-  empty_options.init_metrics_db_script_file = ""
-  empty_options.cleanup_metrics_db_script_file = ""
 
   #Only support SQL Server
   dbmsAmbari = DBMSConfig.create(empty_options, properties, "Ambari")
   dbmsAmbari.reset_database()
-
-  dbmsMetrics = DBMSConfig.create(empty_options, properties, "Metrics")
-  dbmsMetrics.reset_database()
   pass

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -39,10 +39,7 @@ from ambari_server.serverConfiguration import configDefaults, get_ambari_propert
   get_value_from_properties, find_jdk, get_ambari_classpath, get_conf_dir, is_alias_string, find_properties_file, \
   update_properties_2, \
   JDBC_USE_INTEGRATED_AUTH_PROPERTY, JDBC_PASSWORD_PROPERTY, JDBC_PASSWORD_FILENAME, \
-  JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, \
-  JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY, JDBC_METRICS_PASSWORD_PROPERTY, JDBC_METRICS_PASSWORD_FILENAME, \
-  JDBC_METRICS_PASSWORD_ALIAS, \
-  BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, BLIND_PASSWORD
+  JDBC_RCA_PASSWORD_FILE_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, BOOTSTRAP_DIR_PROPERTY, GET_FQDN_SERVICE_URL, BLIND_PASSWORD
 from setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
 from ambari_server.userInput import get_YN_input, get_validated_string_input, get_validated_filepath_input, \
   get_prompt_default
@@ -607,10 +604,6 @@ def get_original_master_key(properties):
     alias = JDBC_RCA_PASSWORD_ALIAS
 
   alias = None
-  property = properties.get_property(JDBC_METRICS_PASSWORD_PROPERTY)
-  if property and is_alias_string(property):
-    alias = JDBC_METRICS_PASSWORD_ALIAS
-
   if not alias:
     property = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY)
     if property and is_alias_string(property):
@@ -884,14 +877,6 @@ def setup_master_key():
     with open(db_password, 'r') as passwdfile:
       db_password = passwdfile.read()
 
-  # Read clear text metrics password from file
-  db_metrics_windows_auth_prop = properties.get_property(JDBC_METRICS_USE_INTEGRATED_AUTH_PROPERTY)
-  db_metrics_sql_auth = False if db_metrics_windows_auth_prop and db_metrics_windows_auth_prop.lower() == 'true' else True
-  metrics_password = properties.get_property(JDBC_METRICS_PASSWORD_PROPERTY)
-  if db_metrics_sql_auth and not is_alias_string(metrics_password) and os.path.isfile(metrics_password):
-    with open(metrics_password, 'r') as passwdfile:
-      metrics_password = passwdfile.read()
-
   ldap_password = properties.get_property(LDAP_MGR_PASSWORD_PROPERTY)
   ts_password = properties.get_property(SSL_TRUSTSTORE_PASSWORD_PROPERTY)
   resetKey = False
@@ -922,8 +907,6 @@ def setup_master_key():
               " password and call 'encrypt-passwords' again."
         if db_sql_auth and db_password and is_alias_string(db_password):
           print err.format('- Database password', "'" + SETUP_ACTION + "'")
-        if db_metrics_sql_auth and metrics_password and is_alias_string(metrics_password):
-            print err.format('- Metrics Database password', "'" + SETUP_ACTION + "'")
         if ldap_password and is_alias_string(ldap_password):
           print err.format('- LDAP manager password', "'" + LDAP_SETUP_ACTION + "'")
         if ts_password and is_alias_string(ts_password):
@@ -937,8 +920,6 @@ def setup_master_key():
   # Read back any encrypted passwords
   if db_sql_auth  and db_password and is_alias_string(db_password):
     db_password = read_passwd_for_alias(JDBC_RCA_PASSWORD_ALIAS, masterKey)
-  if db_metrics_sql_auth and metrics_password and is_alias_string(metrics_password):
-      metrics_password = read_passwd_for_alias(JDBC_METRICS_PASSWORD_ALIAS, masterKey)
   if ldap_password and is_alias_string(ldap_password):
     ldap_password = read_passwd_for_alias(LDAP_MGR_PASSWORD_ALIAS, masterKey)
   if ts_password and is_alias_string(ts_password):
@@ -986,15 +967,6 @@ def setup_master_key():
         propertyMap[JDBC_RCA_PASSWORD_FILE_PROPERTY] = get_alias_string(JDBC_RCA_PASSWORD_ALIAS)
   pass
 
-  if metrics_password and not is_alias_string(metrics_password):
-    retCode = save_passwd_for_alias(JDBC_METRICS_PASSWORD_ALIAS, metrics_password, masterKey)
-    if retCode != 0:
-      print 'Failed to save secure metrics database password.'
-    else:
-      propertyMap[JDBC_METRICS_PASSWORD_PROPERTY] = get_alias_string(JDBC_METRICS_PASSWORD_ALIAS)
-      remove_password_file(JDBC_METRICS_PASSWORD_FILENAME)
-  pass
-
   if ldap_password and not is_alias_string(ldap_password):
     retCode = save_passwd_for_alias(LDAP_MGR_PASSWORD_ALIAS, ldap_password, masterKey)
     if retCode != 0: