AMBARI-5446. Upgrade Helper should add ATS wehn upgrading directly to 2.1. (aonishuk)

ambari-server/src/main/python/UpgradeHelper_HDP2.py

@@ -56,7 +56,9 @@ REPLACE_WITH_TAG = "REPLACE_WITH_"
 DELETE_OLD_TAG = "DELETE_OLD"
 
 AUTH_FORMAT = '{0}:{1}'
-URL_FORMAT = 'http://{0}:8080/api/v1/clusters/{1}'
+ROOT_FORMAT = 'http://{0}:8080/api/v1'
+URL_FORMAT = ROOT_FORMAT+'/clusters/{1}'
+
 
 logger = logging.getLogger()
 
@@ -736,7 +738,36 @@ def delete_mr(options):
     raise FatalException(retcode, errdata)
   pass
 
-
+def get_cluster_stackname(options):
+  VERSION_URL_FORMAT = URL_FORMAT + '?fields=Clusters/version'
+  
+  response = curl(False, '-u',
+                AUTH_FORMAT.format(options.user, options.password),
+                VERSION_URL_FORMAT.format(options.hostname, options.clustername))
+  retcode, errdata = validate_response(response, True)
+  
+  if not retcode == 0:
+    raise FatalException(retcode, errdata)
+  
+  structured_resp = json.loads(response)
+  
+  if 'Clusters' in structured_resp:
+    if 'version' in structured_resp['Clusters']:
+      return structured_resp['Clusters']['version']
+        
+  raise FatalException(-1, "Unable to get the cluster version")
+
+def has_component_in_stack_def(options, stack_name, service_name, component_name):
+  STACK_COMPONENT_URL_FORMAT = ROOT_FORMAT+'/stacks2/{1}/versions/{2}/stackServices/{3}/serviceComponents/{4}'
+  stack, stack_version = stack_name.split('-')
+  
+  response = curl(False, '-u',
+                AUTH_FORMAT.format(options.user, options.password),
+                STACK_COMPONENT_URL_FORMAT.format(options.hostname, stack,
+                                                  stack_version, service_name, component_name))
+  retcode, errdata = validate_response(response, True)
+  return not bool(retcode)
+  
 def add_services(options):
   SERVICE_URL_FORMAT = URL_FORMAT + '/services/{2}'
   COMPONENT_URL_FORMAT = SERVICE_URL_FORMAT + '/components/{3}'
@@ -750,6 +781,15 @@ def add_services(options):
     "RESOURCEMANAGER": "JOBTRACKER",
     "YARN_CLIENT": "MAPREDUCE_CLIENT",
     "MAPREDUCE2_CLIENT": "MAPREDUCE_CLIENT"}
+  
+  stack_name = get_cluster_stackname(options)
+  stack_has_ats = has_component_in_stack_def(options, stack_name, "YARN", "APP_TIMELINE_SERVER")
+  
+  # if upgrading to stack > 2.1 (which has ats)
+  if stack_has_ats:
+    service_comp["YARN"].append("APP_TIMELINE_SERVER")
+    new_old_host_map["APP_TIMELINE_SERVER"] = "JOBTRACKER"
+    
   hostmapping = read_mapping()
 
   for service in service_comp.keys():

ambari-server/src/test/python/TestUpgradeScript_HDP2.py

@@ -195,19 +195,78 @@ class TestUpgradeHDP2Script(TestCase):
     pass
 
 
+  @patch.object(UpgradeHelper_HDP2, "has_component_in_stack_def")
+  @patch.object(UpgradeHelper_HDP2, "get_cluster_stackname")
   @patch.object(UpgradeHelper_HDP2, "read_mapping")
   @patch.object(logging, 'FileHandler')
   @patch.object(UpgradeHelper_HDP2, "backup_file")
   @patch.object(UpgradeHelper_HDP2, 'curl')
   @patch('optparse.OptionParser')
-  def test_add_yarn_mr(self, option_parser_mock, curl_mock,
-                       backup_file_mock, file_handler_mock, read_mapping_mock):
+  def test_add_yarn_mr_with_ATS(self, option_parser_mock, curl_mock,
+                       backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock,  has_comp_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
     options = self.get_mock_options()
     args = ["add-yarn-mr2"]
     opm.parse_args.return_value = (options, args)
     curl_mock.return_value = ''
+    has_comp_mock.return_value = True
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"],
+      "JOBTRACKER": ["c6401"],
+      "HISTORYSERVER": ["c6401"],
+      "MAPREDUCE_CLIENT": ["c6403"]}
+    UpgradeHelper_HDP2.main()
+    expected_curl_calls = [
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
+      call(False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari', '-X', 'POST',
+           'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
+      call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER")]
+    curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    pass
+  
+  @patch.object(UpgradeHelper_HDP2, "has_component_in_stack_def")
+  @patch.object(UpgradeHelper_HDP2, "get_cluster_stackname")
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  @patch('optparse.OptionParser')
+  def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
+                       backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock,  has_comp_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    opm = option_parser_mock.return_value
+    options = self.get_mock_options()
+    args = ["add-yarn-mr2"]
+    opm.parse_args.return_value = (options, args)
+    curl_mock.return_value = ''
+    has_comp_mock.return_value = False
     read_mapping_mock.return_value = {
       "TASKTRACKER": ["c6401", "c6402"],
       "JOBTRACKER": ["c6401"],
@@ -240,8 +299,15 @@ class TestUpgradeHDP2Script(TestCase):
       call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
       call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
-           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER")]
+           "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
+      ]
     curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
+    
+    # assert no ATS was added
+    self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+        "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")     
+    self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
+        "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")  
     pass