AMBARI-17791 Spark service check fails on node with stopped Livy server (dsen)

ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py

@@ -219,7 +219,6 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stac
   livy_livyserver_hosts = default("/clusterHostInfo/livy_server_hosts", [])
 
   if len(livy_livyserver_hosts) > 0:
-    livy_livyserver_host = livy_livyserver_hosts[0]
     has_livyserver = True
 
   livy_livyserver_port = default('configurations/livy-conf/livy.server.port',8998)

ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/service_check.py

@@ -41,13 +41,22 @@ class SparkServiceCheck(Script):
       try_sleep=3,
       logoutput=True
     )
-    if params.has_livyserver and params.livy_livyserver_host != "localhost" and params.livy_livyserver_host != "0.0.0.0":
-      Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{livy_livyserver_host}:{livy_livyserver_port}/sessions | grep 200"),
-              tries=5,
-              try_sleep=3,
+    if params.has_livyserver:
+      live_livyserver_host = "";
+      for livyserver_host in params.livy_livyserver_hosts:
+        try:
+          Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{livyserver_host}:{livy_livyserver_port}/sessions | grep 200"),
+              tries=3,
+              try_sleep=1,
               logoutput=True,
               user=params.livy_user
               )
+          live_livyserver_host = livyserver_host
+          break
+        except:
+          pass
+      if len(params.livy_livyserver_hosts) > 0 and live_livyserver_host == "":
+        raise Fail(format("Connection to all Livy servers failed"))
 
 if __name__ == "__main__":
   SparkServiceCheck().execute()

ambari-server/src/main/resources/stacks/HDP/2.5/role_command_order.json

@@ -11,8 +11,8 @@
     "HIVE_SERVER_INTERACTIVE-RESTART": ["NODEMANAGER-RESTART", "MYSQL_SERVER-RESTART"],
     "HIVE_SERVICE_CHECK-SERVICE_CHECK": ["HIVE_SERVER-START", "HIVE_METASTORE-START", "WEBHCAT_SERVER-START", "HIVE_SERVER_INTERACTIVE-START"],
     "RANGER_ADMIN-START": ["ZOOKEEPER_SERVER-START", "LOGSEARCH_SOLR-START"],
-    "LIVY_SERVER-START" : ["NAMENODE-START", "DATANODE-START"],
-    "SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START","LIVY_SERVER-START"],
+    "LIVY_SERVER-START" : ["NAMENODE-START", "DATANODE-START", "APP_TIMELINE_SERVER-START"],
+    "SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "LIVY_SERVER-START"],
     "SPARK2_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK2_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"],
     "ATLAS_SERVER-START": ["KAFKA_BROKER-START", "LOGSEARCH_SOLR-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"]
   },