AMBARI-8379. Falcon lineage needs changes to startup.properties (alejandro)

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/falcon.py

@@ -54,6 +54,23 @@ def falcon(type, action = None):
                    properties=params.falcon_startup_properties,
                    mode=0644
     )
+
+    if params.falcon_graph_storage_directory:
+      Directory(params.falcon_graph_storage_directory,
+                owner=params.falcon_user,
+                group=params.user_group,
+                mode=0775,
+                recursive=True
+      )
+
+    if params.falcon_graph_serialize_path:
+      Directory(params.falcon_graph_serialize_path,
+                owner=params.falcon_user,
+                group=params.user_group,
+                mode=0775,
+                recursive=True
+      )
+
   if type == 'server':
     if action == 'config':
       if params.store_uri[0:4] == "hdfs":

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/package/scripts/params.py

@@ -50,7 +50,12 @@ proxyuser_group =  config['configurations']['hadoop-env']['proxyuser_group']
 java_home = config['hostLevelParams']['java_home']
 falcon_local_dir = config['configurations']['falcon-env']['falcon_local_dir']
 falcon_log_dir = config['configurations']['falcon-env']['falcon_log_dir']
+
+# falcon-startup.properties
 store_uri = config['configurations']['falcon-startup.properties']['*.config.store.uri']
+# If these properties are present, the directories need to be created.
+falcon_graph_storage_directory = default("/configurations/falcon-startup.properties/*.falcon.graph.storage.directory", None)  # explicitly set in HDP 2.2 and higher
+falcon_graph_serialize_path = default("/configurations/falcon-startup.properties/*.falcon.graph.serialize.path", None)        # explicitly set in HDP 2.2 and higher
 
 falcon_embeddedmq_data = config['configurations']['falcon-env']['falcon.embeddedmq.data']
 falcon_embeddedmq_enabled = config['configurations']['falcon-env']['falcon.embeddedmq']

ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml

@@ -33,7 +33,8 @@
       org.apache.falcon.entity.store.ConfigurationStore,\
       org.apache.falcon.rerun.service.RetryService,\
       org.apache.falcon.rerun.service.LateRunService,\
-      org.apache.falcon.service.LogCleanupService
+      org.apache.falcon.service.LogCleanupService,\
+      org.apache.falcon.metadata.MetadataMappingService
     </value>
     <description>Falcon Services</description>
   </property>
@@ -86,4 +87,31 @@
     <description>Authorization Provider Implementation Fully Qualified Class Name</description>
   </property>
 
+  <!-- Falcon Graph and Storage -->
+  <property>
+    <name>*.falcon.graph.blueprints.graph</name>
+    <value>com.thinkaurelius.titan.core.TitanFactory</value>
+    <description></description>
+  </property>
+  <property>
+    <name>*.falcon.graph.storage.directory</name>
+    <value>/hadoop/falcon/data/lineage/graphdb</value>
+    <description></description>
+  </property>
+  <property>
+    <name>*.falcon.graph.storage.backend</name>
+    <value>berkeleyje</value>
+    <description></description>
+  </property>
+  <property>
+    <name>*.falcon.graph.serialize.path</name>
+    <value>/hadoop/falcon/data/lineage</value>
+    <description></description>
+  </property>
+  <property>
+    <name>*.falcon.graph.preserve.history</name>
+    <value>false</value>
+    <description></description>
+  </property>
+
 </configuration>

ambari-web/app/data/HDP2/site_properties.js

@@ -1903,6 +1903,54 @@ module.exports =
       "filename": "falcon-startup.properties.xml"
     },
 
+    // Falcon Graph and Storage
+    {
+      "id": "site property",
+      "name": "*.falcon.graph.blueprints.graph",
+      "displayName": "*.falcon.graph.blueprints.graph",
+      "category": "FalconStartupSite",
+      "serviceName": "FALCON",
+      "filename": "falcon-startup.properties.xml"
+    },
+    {
+      "id": "site property",
+      "name": "*.falcon.graph.storage.directory",
+      "displayName": "*.falcon.graph.storage.directory",
+      "defaultDirectory": "/hadoop/falcon/data/lineage/graphdb",
+      "displayType": "directory",
+      "category": "FalconStartupSite",
+      "serviceName": "FALCON",
+      "filename": "falcon-startup.properties.xml"
+    },
+    {
+      "id": "site property",
+      "name": "*.falcon.graph.storage.backend",
+      "displayName": "*.falcon.graph.storage.backend",
+      "category": "FalconStartupSite",
+      "serviceName": "FALCON",
+      "filename": "falcon-startup.properties.xml"
+    },
+    {
+      "id": "site property",
+      "name": "*.falcon.graph.serialize.path",
+      "displayName": "*.falcon.graph.serialize.path",
+      "defaultDirectory": "/hadoop/falcon/data/lineage",
+      "displayType": "directory",
+      "category": "FalconStartupSite",
+      "serviceName": "FALCON",
+      "filename": "falcon-startup.properties.xml"
+    },
+    {
+      "id": "site property",
+      "name": "*.falcon.graph.preserve.history",
+      "displayName": "*.falcon.graph.preserve.history",
+      "defaultValue": false,
+      "displayType": "checkbox",
+      "category": "FalconStartupSite",
+      "serviceName": "FALCON",
+      "filename": "falcon-startup.properties.xml"
+    },
+
   /**********************************************webhcat-site***************************************/
     {
       "id": "site property",

ambari-web/app/models/service_config.js

@@ -575,6 +575,8 @@ App.ServiceConfigProperty = Ember.Object.extend({
       case 'oozie_data_dir':
       case 'hbase.tmp.dir':
       case 'storm.local.dir':
+      case '*.falcon.graph.storage.directory':
+      case '*.falcon.graph.serialize.path':
         this.unionAllMountPoints(isOnlyFirstOneNeeded, localDB);
         break;
       case '*.broker.url':
@@ -690,6 +692,13 @@ App.ServiceConfigProperty = Ember.Object.extend({
           setOfHostNames.push(component.hostName);
         }, this);
         break;
+      case '*.falcon.graph.storage.directory':
+      case '*.falcon.graph.serialize.path':
+        components = masterComponentHostsInDB.findProperty('componentName', 'FALCON_SERVER');
+        components.hosts.forEach(function (host) {
+          setOfHostNames.push(host.hostName);
+        }, this);
+        break;
       case 'log.dirs':
         components = masterComponentHostsInDB.filterProperty('component', 'KAFKA_BROKER');
         components.forEach(function (component) {