AMBARI-11298. Ambari set up for Falcon Hive DR. (mpapirkovskyy)

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/configuration/falcon-env.xml

@@ -111,5 +111,10 @@ export FALCON_DATA_DIR={{falcon_embeddedmq_data}}
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
 #export FALCON_EXPANDED_WEBAPP_DIR=
     </value>
-  </property>  
+  </property>
+  <property>
+    <name>supports_hive_dr</name>
+    <value>false</value>
+    <description>Enable support of Falcon Hive DR</description>
+  </property>
 </configuration>

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py

@@ -120,6 +120,17 @@ def falcon(type, action = None):
                   owner=params.falcon_user,
                   recursive=True
         )
+      if params.supports_hive_dr:
+        params.HdfsResource(params.dfs_data_mirroring_dir,
+                            type="directory",
+                            action="create_on_execute",
+                            owner=params.hdfs_user,
+                            group=params.user_group,
+                            recursive_chown = True,
+                            recursive_chmod = True,
+                            mode=0770,
+                            source=params.local_data_mirroring_dir
+        )
       params.HdfsResource(None, action="execute")
       Directory(params.falcon_local_dir,
                 owner=params.falcon_user,

ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py

@@ -101,6 +101,11 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
 smokeuser_principal =  config['configurations']['cluster-env']['smokeuser_principal_name']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+
+supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr']
+local_data_mirroring_dir = "/usr/hdp/current/falcon-server/data-mirroring"
+dfs_data_mirroring_dir = "/apps/data-mirroring"
+
 import functools
 #create partial functions with common arguments for every HdfsResource call
 #to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code

ambari-server/src/main/resources/stacks/HDP/2.3/services/FALCON/configuration/falcon-env.xml

@@ -0,0 +1,27 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+  <property>
+    <name>supports_hive_dr</name>
+    <value>true</value>
+    <description>Enable support of Falcon Hive DR</description>
+  </property>
+</configuration>

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -138,6 +138,23 @@ class TestFalconServer(RMFTestCase):
         owner = 'falcon',
         recursive = True,
     )
+    self.assertResourceCalled('HdfsResource', '/apps/data-mirroring',
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/bin',
+        keytab = UnknownConfigurationMock(),
+        kinit_path_local = '/usr/bin/kinit',
+        user = 'hdfs',
+        owner = 'hdfs',
+        group='hadoop',
+        hadoop_conf_dir = '/etc/hadoop/conf',
+        type = 'directory',
+        recursive_chown = True,
+        recursive_chmod = True,
+        action = ['create_on_execute'],
+        mode = 0770,
+        source='/usr/hdp/current/falcon-server/data-mirroring'
+    )
+
     self.assertResourceCalled('HdfsResource', None,
         security_enabled = False,
         hadoop_bin_dir = '/usr/bin',

ambari-server/src/test/python/stacks/2.1/configs/default.json

@@ -666,7 +666,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falcon home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", 
             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "oozie-env": {
             "oozie_derby_database": "Derby", 

ambari-server/src/test/python/stacks/2.1/configs/secured.json

@@ -658,7 +658,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falcon home dir. Default is the base locaion of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", 
             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "oozie-env": {
             "oozie_derby_database": "Derby", 

ambari-server/src/test/python/stacks/2.2/configs/falcon-upgrade.json

@@ -176,7 +176,8 @@
             "content": "\n# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java_home}}\n\n# any additional java opts you want to set. This will apply to both client and server operations\n#export FALCON_OPTS=\n\n# any additional java opts that you want to set for client only\n#export FALCON_CLIENT_OPTS=\n\n# java heap size we want to set for the client. Default is 1024MB\n#export FALCON_CLIENT_HEAP=\n\n# any additional opts you want to set for prisim service.\n#export FALCON_PRISM_OPTS=\n\n# java heap size we want to set for the prisim service. Default is 1024MB\n#export FALCON_PRISM_HEAP=\n\n# any additional opts you want to set for falcon service.\nexport FALCON_SERVER_OPTS=\"-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}\"\n\n# java heap size we want to set for the falcon server. Default is 1024MB\n#export FALCON_SERVER_HEAP=\n\n# What is is considered as falcon home dir. Default is the base location of the installed software\n#export FALCON_HOME_DIR=\n\n# Where log files are stored. Defatult is logs directory under the base install location\nexport FALCON_LOG_DIR={{falcon_log_dir}}\n\n# Where pid files are stored. Defatult is logs directory under the base install location\nexport FALCON_PID_DIR={{falcon_pid_dir}}\n\n# where the falcon active mq data is stored. Defatult is logs/data directory under the base install location\nexport FALCON_DATA_DIR={{falcon_embeddedmq_data}}\n\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\n#export FALCON_EXPANDED_WEBAPP_DIR=", 
             "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data", 
             "falcon.embeddedmq": "true", 
-            "falcon_store_uri": "file:///hadoop/falcon/store"
+            "falcon_store_uri": "file:///hadoop/falcon/store",
+            "supports_hive_dr": "true"
         }, 
         "core-site": {
             "proxyuser_group": "users",