AMBARI-19577. Add Livy session recovery configurations in Ambari (Saisai Shao via smohanty)

ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py

@@ -223,6 +223,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stac
   user_group = status_params.user_group
   livy_hdfs_user_dir = format("/user/{livy_user}")
   livy_server_pid_file = status_params.livy_server_pid_file
+  livy_recovery_dir = default("/configurations/livy-conf/livy.server.recovery.state-store.url", "/livy-recovery")
 
   livy_server_start = format("{livy_home}/bin/livy-server start")
   livy_server_stop = format("{livy_home}/bin/livy-server stop")

ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_livy.py

@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
     )
     params.HdfsResource(None, action="execute")
 
+    params.HdfsResource(params.livy_recovery_dir,
+                       type="directory",
+                       action="create_on_execute",
+                       owner=params.livy_user,
+                       mode=0775
+    )
+    params.HdfsResource(None, action="execute")
+
   # create livy-env.sh in etc/conf dir
   File(os.path.join(params.livy_conf, 'livy-env.sh'),
        owner=params.livy_user,

ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/params.py

@@ -211,6 +211,7 @@ if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY2, sta
   user_group = status_params.user_group
   livy2_hdfs_user_dir = format("/user/{livy2_user}")
   livy2_server_pid_file = status_params.livy2_server_pid_file
+  livy2_recovery_dir = default("/configurations/livy2-conf/livy.server.recovery.state-store.url", "/livy2-recovery")
 
   livy2_server_start = format("{livy2_home}/bin/livy-server start")
   livy2_server_stop = format("{livy2_home}/bin/livy-server stop")

ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_livy2.py

@@ -40,6 +40,14 @@ def setup_livy(env, type, upgrade_type = None, action = None):
     )
     params.HdfsResource(None, action="execute")
 
+    params.HdfsResource(params.livy2_recovery_dir,
+                        type="directory",
+                        action="create_on_execute",
+                        owner=params.livy2_user,
+                        mode=0775
+       )
+    params.HdfsResource(None, action="execute")
+
   # create livy-env.sh in etc/conf dir
   File(os.path.join(params.livy2_conf, 'livy-env.sh'),
        owner=params.livy2_user,

ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/livy-conf.xml

@@ -70,4 +70,28 @@
      </description>
      <on-ambari-upgrade add="true"/>
   </property>
+  <property>
+    <name>livy.server.recovery.mode</name>
+    <value>recovery</value>
+    <description>
+          Recovery mode for livy, either be "off" or "recovery".
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>livy.server.recovery.state-store</name>
+    <value>filesystem</value>
+    <description>
+          Where Livy should store state for recovery.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>livy.server.recovery.state-store.url</name>
+    <value>/livy-recovery</value>
+    <description>
+          Where Livy should store state for recovery.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/livy2-conf.xml

@@ -78,5 +78,29 @@
         </description>
         <on-ambari-upgrade add="false"/>
     </property>
+    <property>
+        <name>livy.server.recovery.mode</name>
+        <value>recovery</value>
+        <description>
+            Recovery mode for livy, either be "off" or "recovery".
+        </description>
+        <on-ambari-upgrade add="false"/>
+    </property>
+    <property>
+        <name>livy.server.recovery.state-store</name>
+        <value>filesystem</value>
+        <description>
+            Where Livy should store state for recovery.
+        </description>
+        <on-ambari-upgrade add="false"/>
+    </property>
+    <property>
+        <name>livy.server.recovery.state-store.url</name>
+        <value>/livy2-recovery</value>
+        <description>
+            Where Livy should store state for recovery.
+        </description>
+        <on-ambari-upgrade add="false"/>
+  </property>
 </configuration>
 

ambari-server/src/test/python/stacks/2.5/SPARK/test_spark_livy.py

@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
                                   dfs_type = '',
                                   hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
                                   )
+        self.assertResourceCalled('HdfsResource', '/livy-recovery',
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  owner = 'livy',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  type = 'directory',
+                                  action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  mode = 0775,
+                                  )
+        self.assertResourceCalled('HdfsResource', None,
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  )
         self.assertResourceCalled('File', '/usr/hdp/current/livy-server/conf/livy-env.sh',
                                   content = InlineTemplate(self.getConfig()['configurations']['livy-env']['content']),
                                   owner = 'livy',

ambari-server/src/test/python/stacks/2.6/SPARK2/test_spark_livy2.py

@@ -84,6 +84,37 @@ class TestSparkClient(RMFTestCase):
                                   dfs_type = '',
                                   hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
                                   )
+        self.assertResourceCalled('HdfsResource', '/livy2-recovery',
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  owner = 'livy',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  type = 'directory',
+                                  action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  mode = 0775,
+                                  )
+        self.assertResourceCalled('HdfsResource', None,
+                                  immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
+                                  security_enabled = False,
+                                  hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+                                  keytab = UnknownConfigurationMock(),
+                                  default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                                  hdfs_site = {u'a': u'b'},
+                                  kinit_path_local = '/usr/bin/kinit',
+                                  principal_name = UnknownConfigurationMock(),
+                                  user = 'hdfs',
+                                  action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                                  dfs_type = '',
+                                  hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+                                  )
         self.assertResourceCalled('File', '/usr/hdp/current/livy2-server/conf/livy-env.sh',
                                   content = InlineTemplate(self.getConfig()['configurations']['livy2-env']['content']),
                                   owner = 'livy',