AMBARI-3909. Upgrade local repo definitions when upgrading Ambari. (ncole)

ambari-server/src/main/python/ambari-server.py

@@ -226,6 +226,7 @@ UPGRADE_STACK_CMD = ['su', 'postgres',
         '--command=psql -f {0} -v stack_name="\'{1}\'"  -v stack_version="\'{2}\'" -v dbname="{3}"']
 UPDATE_METAINFO_CMD = 'curl -X PUT "http://{0}:{1}/api/v1/stacks2" -u "{2}":"{3}"'
 
+
 PG_ERROR_BLOCKED = "is being accessed by other users"
 PG_STATUS_RUNNING = "running"
 PG_DEFAULT_PASSWORD = "bigdata"
@@ -377,6 +378,15 @@ JCE_MIN_FILESIZE = 5000
 
 DEFAULT_DB_NAME = "ambari"
 
+# stack repo upgrade
+STACK_LOCATION_KEY = 'metadata.path'
+STACK_LOCATION_DEFAULT = '/var/lib/ambari-server/resources/stacks'
+DATABASE_INSERT_METAINFO_SCRIPTS = ['/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql',
+                                  '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql',
+                                  '/var/lib/ambari-server/resources/upgrade/dml/Ambari-DML-MySQL-INSERT_METAINFO.sql']
+INSERT_METAINFO_CMD = ['su', 'postgres',
+        '--command=psql -f {0} -v metainfo_key="\'{1}\'" -v metainfo_value="\'{2}\'" -v dbname="{3}"']
+
 #Apache License Header
 ASF_LICENSE_HEADER = '''
 # Copyright 2011 The Apache Software Foundation
@@ -2428,6 +2438,134 @@ def upgrade_stack(args, stack_id):
       print_warning_msg(errdata)
     return retcode
 
+def load_stack_values(version, filename):
+  import xml.etree.ElementTree as ET
+  values = {}
+  root = ET.parse(filename).getroot()
+  for ostag in root:
+    ostype = ostag.attrib['type']
+    for repotag in ostag:
+      reponametag = repotag.find('reponame')
+      repoidtag = repotag.find('repoid')
+      baseurltag = repotag.find('baseurl')
+      if reponametag is not None and repoidtag is not None and baseurltag is not None:
+        key = "repo:/" + reponametag.text
+        key += "/" + version
+        key += "/" + ostype
+        key += "/" + repoidtag.text
+        key += ":baseurl"
+        values[key] = baseurltag.text
+
+  return values
+
+def upgrade_local_repo_remote_db(args, sqlfile, dbkey, dbvalue):
+  tool = get_db_cli_tool(args)
+  if not tool:
+    args.warnings.append('{0} not found. Please, run DDL script manually'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+    if VERBOSE:
+      print_warning_msg('{0} not found'.format(DATABASE_CLI_TOOLS[DATABASE_INDEX]))
+    return -1, "Client wasn't found", "Client wasn't found"
+
+  #TODO add support of other databases with scripts
+  if args.database == "oracle":
+    sid_or_sname = "sid"
+    if (hasattr(args, 'sid_or_sname') and args.sid_or_sname == "sname") or \
+        (hasattr(args, 'jdbc_url') and args.jdbc_url and re.match(ORACLE_SNAME_PATTERN, args.jdbc_url)):
+      print_info_msg("using SERVICE_NAME instead of SID for Oracle")
+      sid_or_sname = "service_name"
+
+    retcode, out, err = run_in_shell('{0} {1}'.format(tool, ORACLE_UPGRADE_STACK_ARGS.format(
+      args.database_username,
+      args.database_password,
+      args.database_host,
+      args.database_port,
+      args.database_name,
+      sqlfile,
+      sid_or_sname,
+      dbkey,
+      dbvalue
+    )))
+    return retcode, out, err
+
+  return -2, "Wrong database", "Wrong database"
+  pass
+
+def upgrade_local_repo_db(args, dbkey, dbvalue):
+  if not is_root():
+    err = 'Ambari-server upgrade_local_repo_db should be run with ' \
+          'root-level privileges'
+    raise FatalException(4, err)
+  check_database_name_property()
+
+  parse_properties_file(args)
+  if args.persistence_type == "remote":
+    client_desc = DATABASE_NAMES[DATABASE_INDEX] + ' ' + DATABASE_CLI_TOOLS_DESC[DATABASE_INDEX]
+    client_usage_cmd = DATABASE_CLI_TOOLS_USAGE[DATABASE_INDEX].format(DATABASE_INSERT_METAINFO_SCRIPTS[DATABASE_INDEX], args.database_username,
+                                                                       BLIND_PASSWORD, args.database_name)
+    #TODO temporarty code
+    if not args.database == "oracle":
+      raise FatalException(-20, "Upgrade for remote database only supports Oracle.")
+
+    if get_db_cli_tool(args):
+      retcode, out, err = upgrade_local_repo_remote_db(args, DATABASE_INSERT_METAINFO_SCRIPTS[DATABASE_INDEX],
+        dbkey, dbvalue)
+      if not retcode == 0:
+        raise NonFatalException(err)
+
+    else:
+      err = 'Cannot find ' + client_desc + ' client in the path to upgrade the local ' + \
+            'repo information.'
+      raise NonFatalException(err)
+
+    pass
+  else:
+    #password access to ambari-server and mapred
+    configure_database_username_password(args)
+    dbname = args.database_name
+    sqlfile = DATABASE_INSERT_METAINFO_SCRIPTS[0]
+    command = INSERT_METAINFO_CMD[:]
+    command[-1] = command[-1].format(sqlfile, dbkey, dbvalue, dbname)
+    retcode, outdata, errdata = run_os_command(command)
+    if not retcode == 0:
+      raise FatalException(retcode, errdata)
+    if errdata:
+      print_warning_msg(errdata)
+    return retcode
+  pass
+
+def upgrade_local_repo(args):
+  properties = get_ambari_properties()
+  if properties == -1:
+    print_error_msg ("Error getting ambari properties")
+    return -1
+
+  stack_location = properties[STACK_LOCATION_KEY]
+  if stack_location is None:
+    stack_location = STACK_LOCATION_DEFAULT
+
+  stack_root_local = os.path.join(stack_location, "HDPLocal")
+  if not os.path.exists(stack_root_local):
+    print_info_msg("HDPLocal stack directory does not exist, skipping")
+    return
+
+  stack_root = os.path.join(stack_location, "HDP")
+  if not os.path.exists(stack_root):
+    print_info_msg("HDP stack directory does not exist, skipping")
+    return
+
+  for stack_version_local in os.listdir(stack_root_local):
+    repo_file_local = os.path.join(stack_root_local, stack_version_local, "repos", "repoinfo.xml")
+    repo_file = os.path.join(stack_root, stack_version_local, "repos", "repoinfo.xml")
+
+    if os.path.exists(repo_file_local) and os.path.exists(repo_file):
+      local_values = load_stack_values(stack_version_local, repo_file_local)
+      repo_values = load_stack_values(stack_version_local, repo_file)
+      for k, v in local_values.iteritems():
+        if repo_values.has_key(k):
+          local_url = local_values[k]
+          repo_url = repo_values[k]
+          if repo_url != local_url:
+            upgrade_local_repo_db(args, k, local_url)
 
 #
 # Upgrades the Ambari Server.
@@ -2521,6 +2659,8 @@ def upgrade(args):
   else:
     adjust_directory_permissions(user)
 
+  # local repo
+  upgrade_local_repo(args)
 
 #
 # The Ambari Server status.

ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Oracle-INSERT_METAINFO.sql

@@ -0,0 +1,24 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Update stack_name and stack_version
+-- &1 - metainfo_key
+-- &2 - metainfo_value
+
+INSERT INTO ambari.metainfo (metainfo_key, metainfo_value) SELECT "&1", "&2" FROM dual WHERE
+  NOT EXISTS (SELECT metainfo_key FROM ambari.metainfo WHERE metainfo_key = "&1")

ambari-server/src/main/resources/upgrade/dml/Ambari-DML-Postgres-INSERT_METAINFO.sql

@@ -0,0 +1,22 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+\connect :dbname;
+
+INSERT INTO ambari.metainfo (metainfo_key, metainfo_value)
+  SELECT :metainfo_key, :metainfo_value
+  WHERE NOT EXISTS (SELECT metainfo_key FROM ambari.metainfo WHERE metainfo_key = :metainfo_key)

ambari-server/src/test/python/TestAmbariServer.py

@@ -2695,7 +2695,9 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
   @patch.object(ambari_server, "get_ambari_properties")
   @patch.object(ambari_server, "get_db_cli_tool")
   @patch.object(ambari_server, "execute_remote_script")
-  def test_upgrade(self, execute_remote_script_mock, get_db_cli_tool_mock, get_ambari_properties_mock, is_root_mock,
+  @patch.object(ambari_server, "upgrade_local_repo")
+  def test_upgrade(self, upgrade_local_repo_mock, execute_remote_script_mock,
+                   get_db_cli_tool_mock, get_ambari_properties_mock, is_root_mock,
                    parse_properties_file_mock,
                    update_ambari_properties_mock,
                    check_postgre_up_mock, execute_db_script_mock,
@@ -2733,6 +2735,7 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     self.assertTrue(print_warning_msg_mock.called)
     warning_args = print_warning_msg_mock.call_args[0][0]
     self.assertTrue("custom ambari user" in warning_args)
+    self.assertTrue(upgrade_local_repo_mock.called)
 
     # Testing with defined custom user
     read_ambari_user_mock.return_value = "ambari-custom-user"
@@ -4460,4 +4463,72 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
     self.assertTrue(get_validated_string_input_mock.called)
     self.assertEqual(get_validated_string_input_mock.call_count, 2)
 
+  @patch("os.listdir")
+  @patch("os.path.exists")
+  @patch.object(ambari_server, "load_stack_values")
+  @patch.object(ambari_server, "get_ambari_properties")
+  @patch.object(ambari_server, "upgrade_local_repo_db")
+  def test_upgrade_local_repo(self,
+                         upgrade_local_repo_db_mock,
+                         get_ambari_properties_mock,
+                         load_stack_values_mock,
+                         os_path_exists_mock,
+                         os_listdir_mock):
+
+    from mock.mock import call
+    args = MagicMock()
+    args.persistence_type = "local"
+
+    def load_values_side_effect(*args, **kwargs):
+      res = {}
+      res['a'] = 'http://oldurl'
+      if -1 != args[1].find("HDPLocal"):
+        res['a'] = 'http://newurl'
+      return res
+
+    load_stack_values_mock.side_effect = load_values_side_effect
+
+    properties = ambari_server.Properties()
+    get_ambari_properties_mock.return_value = properties
+    os_path_exists_mock.return_value = 1
+    os_listdir_mock.return_value = ['1.1']
+
+    ambari_server.upgrade_local_repo(args)
+
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(load_stack_values_mock.called)
+    self.assertTrue(upgrade_local_repo_db_mock.called)
 
+  @patch("os.listdir")
+  @patch("os.path.exists")
+  @patch.object(ambari_server, "load_stack_values")
+  @patch.object(ambari_server, "get_ambari_properties")
+  @patch.object(ambari_server, "upgrade_local_repo_db")
+  def test_upgrade_local_repo_nochange(self,
+                         upgrade_local_repo_db_mock,
+                         get_ambari_properties_mock,
+                         load_stack_values_mock,
+                         os_path_exists_mock,
+                         os_listdir_mock):
+
+    from mock.mock import call
+    args = MagicMock()
+    args.persistence_type = "local"
+
+    def load_values_side_effect(*args, **kwargs):
+      res = {}
+      res['a'] = 'http://oldurl'
+      return res
+
+    load_stack_values_mock.side_effect = load_values_side_effect
+
+    properties = ambari_server.Properties()
+    get_ambari_properties_mock.return_value = properties
+    os_path_exists_mock.return_value = 1
+    os_listdir_mock.return_value = ['1.1']
+
+    ambari_server.upgrade_local_repo(args)
+
+    self.assertTrue(get_ambari_properties_mock.called)
+    self.assertTrue(load_stack_values_mock.called)
+    self.assertFalse(upgrade_local_repo_db_mock.called)