AMBARI-5278. Review Falcon Configs in Ambari. (dlysnichenko)

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/configuration/falcon-startup.properties.xml

@@ -19,11 +19,7 @@
  */
 -->
 <configuration>
-  <property>
-    <name>*.domain</name>
-    <value>${falcon.app.type}</value>
-    <description></description>
-  </property>
+  <!--advanced properties-->
   <property>
     <name>*.workflow.engine.impl</name>
     <value>org.apache.falcon.workflow.engine.OozieWorkflowEngine</value>
@@ -70,11 +66,6 @@
     </value>
     <description></description>
   </property>
-  <property>
-    <name>prism.application.services</name>
-    <value>org.apache.falcon.entity.store.ConfigurationStore</value>
-    <description></description>
-  </property>
   <property>
     <name>*.configstore.listeners</name>
     <value>org.apache.falcon.entity.v0.EntityGraph,\
@@ -84,14 +75,6 @@
     </value>
     <description></description>
   </property>
-  <property>
-    <name>prism.configstore.listeners</name>
-    <value>org.apache.falcon.entity.v0.EntityGraph,\
-      org.apache.falcon.entity.ColoClusterRelation,\
-      org.apache.falcon.group.FeedGroupMap
-    </value>
-    <description></description>
-  </property>
   <property>
     <name>*.broker.impl.class</name>
     <value>org.apache.activemq.ActiveMQConnectionFactory</value>
@@ -102,23 +85,24 @@
     <value>activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms</value>
     <description></description>
   </property>
+  <!--common properties-->
+  <property>
+    <name>*.domain</name>
+    <value>${falcon.app.type}</value>
+    <description></description>
+  </property>
   <property>
     <name>*.config.store.uri</name>
-    <value>file://${falcon.home}/store</value>
+    <value>file:///hadoop/falcon/store</value>
     <description>Location to store user entity configurations</description>
   </property>
   <property>
   </property>
   <property>
     <name>*.system.lib.location</name>
-    <value>${falcon.home}/server/webapp/falcon/WEB-INF/lib</value>
+    <value>${falcon.home}/server/webapp/${falcon.app.type}/WEB-INF/lib</value>
     <description>Location of libraries that is shipped to Hadoop</description>
   </property>
-  <property>
-    <name>prism.system.lib.location</name>
-    <value>${falcon.home}/server/webapp/prism/WEB-INF/lib</value>
-    <description></description>
-  </property>
   <property>
     <name>*.retry.recorder.path</name>
     <value>${falcon.log.dir}/retry</value>

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/configuration/global.xml

@@ -44,4 +44,20 @@
     <value>/hadoop/falcon</value>
     <description>Directory where Falcon data, such as activemq data, is stored.</description>
   </property>
+  <!--embeddedmq properties-->
+  <property>
+    <name>falcon.embeddedmq.data</name>
+    <value>/hadoop/falcon/embeddedmq/data</value>
+    <description></description>
+  </property>
+  <property>
+    <name>falcon.embeddedmq</name>
+    <value>true</value>
+    <description></description>
+  </property>
+  <property>
+    <name>falcon.emeddedmq.port</name>
+    <value>61616</value>
+    <description></description>
+  </property>
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/falcon.py

@@ -64,20 +64,18 @@ def falcon(type, action = None):
                 owner=params.falcon_user,
                 recursive=True
       )
-      Directory(params.falcon_data_dir,
-                owner=params.falcon_user,
-                recursive=True
-      )
+      if params.falcon_embeddedmq_enabled == True:
+        Directory(params.falcon_embeddedmq_data,
+                  owner=params.falcon_user,
+                  recursive=True
+        )
+
     if action == 'start':
-      Execute(format('env JAVA_HOME={java_home} FALCON_LOG_DIR={falcon_log_dir} '
-                     'FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} '
-                     '{falcon_home}/bin/falcon-start -port {falcon_port}'),
+      Execute(format('{falcon_home}/bin/falcon-start -port {falcon_port}'),
               user=params.falcon_user
       )
     if action == 'stop':
-      Execute(format('env JAVA_HOME={java_home} FALCON_LOG_DIR={falcon_log_dir} '
-                     'FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} '
-                     '{falcon_home}/bin/falcon-stop'),
+      Execute(format('{falcon_home}/bin/falcon-stop'),
               user=params.falcon_user
       )
       File(params.server_pid_file,

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/params.py

@@ -35,14 +35,18 @@ falcon_home = '/usr/lib/falcon'
 falcon_conf_dir = '/etc/falcon/conf'
 falcon_local_dir = config['configurations']['global']['falcon_local_dir']
 falcon_log_dir = config['configurations']['global']['falcon_log_dir']
-falcon_data_dir = format('{falcon_local_dir}/activemq')
 store_uri = config['configurations']['falcon-startup.properties']['*.config.store.uri']
+
+falcon_embeddedmq_data = config['configurations']['global']['falcon.embeddedmq.data']
+falcon_embeddedmq_enabled = config['configurations']['global']['falcon.embeddedmq']
+falcon_emeddedmq_port = config['configurations']['global']['falcon.emeddedmq.port']
+
 falcon_host = config['clusterHostInfo']['falcon_server_hosts'][0]
 falcon_port = config['configurations']['global']['falcon_port']
 falcon_runtime_properties = config['configurations']['falcon-runtime.properties']
 falcon_startup_properties = config['configurations']['falcon-startup.properties']
-http_keytab = falcon_startup_properties['*.falcon.http.authentication.kerberos.keytab']
-http_principal = falcon_startup_properties['*.falcon.http.authentication.kerberos.principal']
+smokeuser_keytab = config['configurations']['global']['smokeuser_keytab']
+
 falcon_webapp_dir = '/var/lib/falcon/webapp'
 flacon_apps_dir = '/apps/falcon'
 #for create_hdfs_directory

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/service_check.py

@@ -27,12 +27,9 @@ class FalconServiceCheck(Script):
 
     env.set_params(params)
     if params.security_enabled:
-      principal_replaced = params.http_principal.replace("_HOST", params.hostname)
-      Execute(format("{kinit_path_local} -kt {http_keytab} {principal_replaced}"),
+      Execute(format("{kinit_path_local} -kt {smokeuser_keytab} {smoke_user}"),
               user=params.smoke_user)
-    Execute(format("env JAVA_HOME={java_home} FALCON_LOG_DIR=/var/log/falcon "
-                   "FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR={falcon_data_dir} "
-                   "{falcon_home}/bin/falcon admin -version"),
+    Execute(format("{falcon_home}/bin/falcon admin -version"),
             user=params.smoke_user,
             logoutput=True,
             tries = 3,

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/templates/falcon-env.sh.j2

@@ -34,7 +34,7 @@ export JAVA_HOME={{java_home}}
 #export FALCON_PRISM_HEAP=
 
 # any additional opts you want to set for falcon service.
-#export FALCON_SERVER_OPTS=
+export FALCON_SERVER_OPTS="-Dfalcon.embeddedmq={{falcon_embeddedmq_enabled}} -Dfalcon.emeddedmq.port={{falcon_emeddedmq_port}}"
 
 # java heap size we want to set for the falcon server. Default is 1024MB
 #export FALCON_SERVER_HEAP=
@@ -43,13 +43,13 @@ export JAVA_HOME={{java_home}}
 #export FALCON_HOME_DIR=
 
 # Where log files are stored. Defatult is logs directory under the base install location
-#export FALCON_LOG_DIR=
+export FALCON_LOG_DIR={{falcon_log_dir}}
 
 # Where pid files are stored. Defatult is logs directory under the base install location
-#export FALCON_PID_DIR=
+export FALCON_PID_DIR={{falcon_pid_dir}}
 
 # where the falcon active mq data is stored. Defatult is logs/data directory under the base install location
-#export FALCON_DATA_DIR=
+export FALCON_DATA_DIR={{falcon_embeddedmq_data}}
 
 # Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
 #export FALCON_EXPANDED_WEBAPP_DIR=

ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py

@@ -30,7 +30,7 @@ class TestFalconServer(RMFTestCase):
                        config_file="default.json"
     )
     self.assert_configure_default()
-    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR=/hadoop/falcon/activemq /usr/lib/falcon/bin/falcon-start -port 15000',
+    self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-start -port 15000',
                               user = 'falcon',
                               )
     self.assertNoMoreResources()
@@ -41,7 +41,7 @@ class TestFalconServer(RMFTestCase):
                        command="stop",
                        config_file="default.json"
     )
-    self.assertResourceCalled('Execute', 'env JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon FALCON_PID_DIR=/var/run/falcon FALCON_DATA_DIR=/hadoop/falcon/activemq /usr/lib/falcon/bin/falcon-stop',
+    self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon-stop',
                               user = 'falcon',
                               )
     self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
@@ -105,7 +105,7 @@ class TestFalconServer(RMFTestCase):
                               owner = 'falcon',
                               recursive = True,
                               )
-    self.assertResourceCalled('Directory', '/hadoop/falcon/activemq',
+    self.assertResourceCalled('Directory', '/hadoop/falcon/embeddedmq/data',
                               owner = 'falcon',
                               recursive = True,
                               )

ambari-server/src/test/python/stacks/2.1/FALCON/test_service_check.py

@@ -29,11 +29,7 @@ class TestFalconServer(RMFTestCase):
                        command="service_check",
                        config_file="default.json"
     )
-    self.assertResourceCalled('Execute', 'env '
-                                         'JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon '
-                                         'FALCON_PID_DIR=/var/run/falcon '
-                                         'FALCON_DATA_DIR=/hadoop/falcon/activemq '
-                                         '/usr/lib/falcon/bin/falcon admin -version',
+    self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon admin -version',
                               logoutput = True,
                               tries = 3,
                               user = 'ambari-qa',
@@ -45,14 +41,10 @@ class TestFalconServer(RMFTestCase):
                        command="service_check",
                        config_file="secured.json"
     )
-    self.assertResourceCalled('Execute','/usr/bin/kinit -kt /etc/security/keytabs/spnego.service.keytab HTTP/c6401.ambari.apache.org@EXAMPLE.COM',
+    self.assertResourceCalled('Execute','/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa',
                               user='ambari-qa'
     )
-    self.assertResourceCalled('Execute', 'env '
-                                         'JAVA_HOME=/usr/jdk64/jdk1.7.0_45 FALCON_LOG_DIR=/var/log/falcon '
-                                         'FALCON_PID_DIR=/var/run/falcon '
-                                         'FALCON_DATA_DIR=/hadoop/falcon/activemq '
-                                         '/usr/lib/falcon/bin/falcon admin -version',
+    self.assertResourceCalled('Execute', '/usr/lib/falcon/bin/falcon admin -version',
                               logoutput = True,
                               tries = 3,
                               user = 'ambari-qa',

ambari-server/src/test/python/stacks/2.1/configs/default.json

@@ -37,28 +37,25 @@
             "*.ProcessInstanceManager.impl": "org.apache.falcon.resource.InstanceManager",
             "*.falcon.http.authentication.simple.anonymous.allowed": "true",
             "*.broker.url": "tcp://localhost:61616",
-            "*.system.lib.location": "${falcon.home}/server/webapp/falcon/WEB-INF/lib",
+            "*.system.lib.location": "${falcon.home}/server/webapp/${falcon.app.type}/WEB-INF/lib",
             "*.entity.topic": "FALCON.ENTITY.TOPIC",
             "*.domain": "${falcon.app.type}",
             "*.ConfigSyncService.impl": "org.apache.falcon.resource.ConfigSyncService",
+            "*.workflow.engine.impl": "org.apache.falcon.workflow.engine.OozieWorkflowEngine",
             "*.falcon.http.authentication.signature.secret": "falcon",
             "*.retry.recorder.path": "${falcon.log.dir}/retry",
             "*.broker.ttlInMins": "4320",
             "*.falcon.authentication.type": "simple",
             "*.catalog.service.impl": "org.apache.falcon.catalog.HiveCatalogService",
-            "prism.system.lib.location": "${falcon.home}/server/webapp/prism/WEB-INF/lib",
             "*.falcon.http.authentication.kerberos.name.rules": "DEFAULT",
-            "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n      org.apache.falcon.service.ProcessSubscriberService,\\\n      org.apache.falcon.entity.store.ConfigurationStore,\\\n      org.apache.falcon.rerun.service.RetryService,\\\n      org.apache.falcon.rerun.service.LateRunService,\\\n      org.apache.falcon.service.SLAMonitoringService,\\\n      org.apache.falcon.service.LogCleanupService",
-            "*.workflow.engine.impl": "org.apache.falcon.workflow.engine.OozieWorkflowEngine",
+            "*.application.services": "org.apache.falcon.security.AuthenticationInitializationService,\\\n      org.apache.falcon.service.ProcessSubscriberService,\\\n      org.apache.falcon.entity.store.ConfigurationStore,\\\n      org.apache.falcon.rerun.service.RetryService,\\\n      org.apache.falcon.rerun.service.LateRunService,\\\n      org.apache.falcon.metadata.MetadataMappingService,\\\n      org.apache.falcon.service.LogCleanupService",
             "*.SchedulableEntityManager.impl": "org.apache.falcon.resource.SchedulableEntityManager",
             "*.max.retry.failure.count": "1",
             "*.falcon.http.authentication.type": "simple",
             "*.oozie.process.workflow.builder": "org.apache.falcon.workflow.OozieProcessWorkflowBuilder",
-            "prism.application.services": "org.apache.falcon.entity.store.ConfigurationStore",
             "*.shared.libs": "activemq-core,ant,geronimo-j2ee-management,hadoop-distcp,jms,json-simple,oozie-client,spring-jms",
             "*.falcon.cleanup.service.frequency": "days(1)",
-            "prism.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n      org.apache.falcon.entity.ColoClusterRelation,\\\n      org.apache.falcon.group.FeedGroupMap",
-            "*.config.store.uri": "file://${falcon.home}/store",
+            "*.config.store.uri": "file:///hadoop/falcon/store",
             "*.broker.impl.class": "org.apache.activemq.ActiveMQConnectionFactory",
             "*.configstore.listeners": "org.apache.falcon.entity.v0.EntityGraph,\\\n      org.apache.falcon.entity.ColoClusterRelation,\\\n      org.apache.falcon.group.FeedGroupMap,\\\n      org.apache.falcon.service.SharedLibraryHostingService"
       },
@@ -252,88 +249,91 @@
             "templeton.pig.archive": "hdfs:///apps/webhcat/pig.tar.gz"
         }, 
         "global": {
+            "falcon.embeddedmq": "true",
+            "falcon.emeddedmq.port": "61616",
+            "falcon.embeddedmq.data": "/hadoop/falcon/embeddedmq/data",
             "falcon_log_dir": "/var/log/falcon",
             "falcon_pid_dir": "/var/run/falcon",
-            "security_enabled": "false", 
-            "hbase_pid_dir": "/var/run/hbase", 
-            "proxyuser_group": "users", 
-            "zk_user": "zookeeper", 
-            "rrdcached_base_dir": "/var/lib/ganglia/rrds", 
-            "syncLimit": "5", 
-            "storm_pid_dir": "/var/run/storm", 
-            "oozie_pid_dir": "/var/run/oozie", 
-            "stormuiserver_host": "c6401.ambari.apache.org", 
-            "hbase_regionserver_heapsize": "1024m", 
-            "logviewerserver_host": "c6402.ambari.apache.org", 
-            "yarn_heapsize": "1024", 
-            "hcat_log_dir": "/var/log/webhcat", 
-            "oozie_hostname": "c6402.ambari.apache.org", 
-            "yarn_log_dir_prefix": "/var/log/hadoop-yarn", 
-            "zk_log_dir": "/var/log/zookeeper", 
-            "hive_aux_jars_path": "/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar", 
-            "nagios_web_password": "asd", 
-            "tickTime": "2000", 
-            "hive_ambari_database": "MySQL", 
-            "falcon_user": "falcon", 
-            "oozie_log_dir": "/var/log/oozie", 
-            "oozie_user": "oozie", 
-            "hcat_conf_dir": "", 
-            "oozie_data_dir": "/hadoop/oozie/data", 
-            "ganglia_runtime_dir": "/var/run/ganglia/hdp", 
-            "drpcserver_host": "c6402.ambari.apache.org", 
-            "lzo_enabled": "true", 
-            "dtnode_heapsize": "1024m", 
-            "namenode_opt_maxnewsize": "200m", 
-            "smokeuser": "ambari-qa", 
-            "namenode_heapsize": "1024m", 
-            "hive_user": "hive", 
-            "hdfs_log_dir_prefix": "/var/log/hadoop", 
-            "hive_hostname": "c6402.ambari.apache.org", 
-            "mapred_pid_dir_prefix": "/var/run/hadoop-mapreduce", 
-            "hive_metastore_port": "9083", 
-            "hbase_master_heapsize": "1024m", 
-            "zk_data_dir": "/hadoop/zookeeper", 
-            "hcat_pid_dir": "/var/run/webhcat", 
-            "oozie_jdbc_driver": "org.apache.derby.jdbc.EmbeddedDriver", 
-            "yarn_pid_dir_prefix": "/var/run/hadoop-yarn", 
-            "initLimit": "10", 
-            "hive_database_type": "mysql", 
-            "oozie_database": "New Derby Database", 
-            "zk_pid_dir": "/var/run/zookeeper", 
-            "user_group": "hadoop", 
-            "yarn_user": "yarn", 
-            "gmond_user": "nobody", 
-            "nagios_web_login": "nagiosadmin", 
-            "storm_user": "storm", 
-            "nagios_contact": "asd@asd.asd", 
-            "hive_database": "New MySQL Database", 
-            "storm_log_dir": "/var/log/storm", 
-            "clientPort": "2181", 
-            "oozie_derby_database": "Derby", 
-            "mapred_log_dir_prefix": "/var/log/hadoop-mapreduce", 
-            "nagios_group": "nagios", 
-            "hdfs_user": "hdfs", 
-            "hbase_user": "hbase", 
-            "oozie_database_type": "derby", 
-            "webhcat_user": "hcat", 
-            "nodemanager_heapsize": "1024", 
-            "gmetad_user": "nobody", 
-            "hive_log_dir": "/var/log/hive", 
-            "namenode_opt_newsize": "200m", 
-            "mapred_user": "mapred", 
-            "resourcemanager_heapsize": "1024", 
-            "hive_pid_dir": "/var/run/hive", 
-            "hcat_user": "hcat", 
-            "hadoop_heapsize": "1024", 
-            "hadoop_pid_dir_prefix": "/var/run/hadoop", 
-            "nagios_user": "nagios", 
+            "security_enabled": "false",
+            "hbase_pid_dir": "/var/run/hbase",
+            "proxyuser_group": "users",
+            "zk_user": "zookeeper",
+            "rrdcached_base_dir": "/var/lib/ganglia/rrds",
+            "syncLimit": "5",
+            "storm_pid_dir": "/var/run/storm",
+            "oozie_pid_dir": "/var/run/oozie",
+            "stormuiserver_host": "c6401.ambari.apache.org",
+            "hbase_regionserver_heapsize": "1024m",
+            "logviewerserver_host": "c6402.ambari.apache.org",
+            "yarn_heapsize": "1024",
+            "hcat_log_dir": "/var/log/webhcat",
+            "oozie_hostname": "c6402.ambari.apache.org",
+            "yarn_log_dir_prefix": "/var/log/hadoop-yarn",
+            "zk_log_dir": "/var/log/zookeeper",
+            "hive_aux_jars_path": "/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar",
+            "nagios_web_password": "asd",
+            "tickTime": "2000",
+            "hive_ambari_database": "MySQL",
+            "falcon_user": "falcon",
+            "oozie_log_dir": "/var/log/oozie",
+            "oozie_user": "oozie",
+            "hcat_conf_dir": "",
+            "oozie_data_dir": "/hadoop/oozie/data",
+            "ganglia_runtime_dir": "/var/run/ganglia/hdp",
+            "drpcserver_host": "c6402.ambari.apache.org",
+            "lzo_enabled": "true",
+            "dtnode_heapsize": "1024m",
+            "namenode_opt_maxnewsize": "200m",
+            "smokeuser": "ambari-qa",
+            "namenode_heapsize": "1024m",
+            "hive_user": "hive",
+            "hdfs_log_dir_prefix": "/var/log/hadoop",
+            "hive_hostname": "c6402.ambari.apache.org",
+            "mapred_pid_dir_prefix": "/var/run/hadoop-mapreduce",
+            "hive_metastore_port": "9083",
+            "hbase_master_heapsize": "1024m",
+            "zk_data_dir": "/hadoop/zookeeper",
+            "hcat_pid_dir": "/var/run/webhcat",
+            "oozie_jdbc_driver": "org.apache.derby.jdbc.EmbeddedDriver",
+            "yarn_pid_dir_prefix": "/var/run/hadoop-yarn",
+            "initLimit": "10",
+            "hive_database_type": "mysql",
+            "oozie_database": "New Derby Database",
+            "zk_pid_dir": "/var/run/zookeeper",
+            "user_group": "hadoop",
+            "yarn_user": "yarn",
+            "gmond_user": "nobody",
+            "nagios_web_login": "nagiosadmin",
+            "storm_user": "storm",
+            "nagios_contact": "asd@asd.asd",
+            "hive_database": "New MySQL Database",
+            "storm_log_dir": "/var/log/storm",
+            "clientPort": "2181",
+            "oozie_derby_database": "Derby",
+            "mapred_log_dir_prefix": "/var/log/hadoop-mapreduce",
+            "nagios_group": "nagios",
+            "hdfs_user": "hdfs",
+            "hbase_user": "hbase",
+            "oozie_database_type": "derby",
+            "webhcat_user": "hcat",
+            "nodemanager_heapsize": "1024",
+            "gmetad_user": "nobody",
+            "hive_log_dir": "/var/log/hive",
+            "namenode_opt_newsize": "200m",
+            "mapred_user": "mapred",
+            "resourcemanager_heapsize": "1024",
+            "hive_pid_dir": "/var/run/hive",
+            "hcat_user": "hcat",
+            "hadoop_heapsize": "1024",
+            "hadoop_pid_dir_prefix": "/var/run/hadoop",
+            "nagios_user": "nagios",
             "hbase_log_dir": "/var/log/hbase",
             "falcon_user": "falcon",
             "falcon_port": "15000",
             "falcon_local_dir": "/hadoop/falcon",
             "tez_user": "tez",
             "rca_enabled": "false"
-        }, 
+        },
         "capacity-scheduler": {
             "yarn.scheduler.capacity.node-locality-delay": "40", 
             "yarn.scheduler.capacity.root.capacity": "100",