AMBARI-19001. Support configurable grok filters (output + ambari) (oleewere)

Change-Id: Ibf02fc6ca6759435ac14dc59de9f71782d7ea709

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-ambari-config.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <display-name>Ambari LogFeeder config</display-name>
+    <description>Grok filters and input file locations for ambari related log files</description>
+    <value/>
+    <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+      <property-file-name>input.config-ambari.json.j2</property-file-name>
+      <property-file-type>text</property-file-type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-output-config.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>content</name>
+    <display-name>LogFeeder output config</display-name>
+    <description>Output configuration for LogFeeder (data shipping)</description>
+    <value/>
+    <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+      <property-file-name>output.config.json.j2</property-file-name>
+      <property-file-type>text</property-file-type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml

@@ -72,6 +72,7 @@
             <config-type>logsearch-admin-json</config-type>
             <config-type>logsearch-service_logs-solrconfig.xml</config-type>
             <config-type>logsearch-audit_logs-solrconfig.xml</config-type>
+            <config-type>logfeeder-custom-logsearch-config</config-type>
           </configuration-dependencies>
         </component>
 
@@ -111,6 +112,10 @@
             <config-type>logfeeder-env</config-type>
             <config-type>logfeeder-grok</config-type>
             <config-type>logfeeder-log4j</config-type>
+            <config-type>logfeeder-system_log-env</config-type>
+            <config-type>logfeeder-ambari-config</config-type>
+            <config-type>logfeeder-output-config</config-type>
+            <config-type>logfeeder-custom-logsearch-config</config-type>
           </configuration-dependencies>
         </component>
 

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py

@@ -251,16 +251,19 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe
 logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password']
 logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type']
 
-logfeeder_default_services = ['ambari', 'logsearch']
-logfeeder_default_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services]
+logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content']
+logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content']
+
+logfeeder_default_services = ['logsearch']
+logfeeder_default_config_file_names = ['global.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_default_services]
 logfeeder_custom_config_file_names = ['input.config-%s.json' % (tag.replace('-logsearch-conf', ''))
                                       for tag, content in logfeeder_metadata.iteritems() if any(logfeeder_metadata)]
 
 if logfeeder_system_log_enabled:
-  default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names
+  default_config_files = ','.join(['output.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names
                                   + ['input.config-system_messages.json', 'input.config-secure_log.json'])
 else:
-  default_config_files = ','.join(logfeeder_default_config_file_names + logfeeder_custom_config_file_names)
+  default_config_files = ','.join(['output.config.json','input.config-ambari.json'] + logfeeder_default_config_file_names + logfeeder_custom_config_file_names)
 
 
 logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns']
@@ -293,8 +296,10 @@ logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['lo
 logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
 
 if security_enabled:
-  logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
-  logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file
+  if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties:
+    logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
+  if 'logfeeder.solr.jaas.file' not in logfeeder_properties:
+    logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file
 
 logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder']
 

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py

@@ -62,6 +62,16 @@ def setup_logfeeder():
        encoding="utf-8"
        )
 
+  File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"),
+       content=InlineTemplate(params.logfeeder_ambari_config_content),
+       encoding="utf-8"
+       )
+
+  File(format("{logsearch_logfeeder_conf}/output.config.json"),
+       content=InlineTemplate(params.logfeeder_output_config_content),
+       encoding="utf-8"
+       )
+
   for file_name in params.logfeeder_default_config_file_names:
     File(format("{logsearch_logfeeder_conf}/" + file_name),
          content=Template(file_name + ".j2")

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/output.config.json.j2 → ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/output.config.json.j2

@@ -57,5 +57,5 @@
     }
 
   ]
-  
+
 }

ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py

@@ -62,7 +62,7 @@ class TestLogFeeder(RMFTestCase):
                               )
     self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
                               properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
-                                          'logfeeder.config.files': 'global.config.json,output.config.json,input.config-ambari.json,input.config-logsearch.json,input.config-zookeeper.json',
+                                          'logfeeder.config.files': 'output.config.json,input.config-ambari.json,global.config.json,input.config-logsearch.json,input.config-zookeeper.json',
                                           'logfeeder.metrics.collector.hosts': '',
                                           'logfeeder.solr.core.config.name': 'history',
                                           'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
@@ -79,10 +79,18 @@ class TestLogFeeder(RMFTestCase):
                               content=InlineTemplate('GP'),
                               encoding='utf-8'
                               )
+    self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-ambari.json',
+                              content=InlineTemplate('ambari-grok-filter'),
+                              encoding='utf-8'
+                              )
+    self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/output.config.json',
+                              content=InlineTemplate('output-grok-filter'),
+                              encoding='utf-8'
+                              )
 
-    logfeeder_supported_services = ['ambari','logsearch']
+    logfeeder_supported_services = ['logsearch']
 
-    logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \
+    logfeeder_config_file_names = ['global.config.json'] + \
                                   ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
 
     for file_name in logfeeder_config_file_names:

ambari-server/src/test/python/stacks/2.4/configs/default.json

@@ -313,6 +313,12 @@
         "logfeeder_max_mem": "512m",
         "content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements.  See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License.  You may obtain a copy of the License at\n#\n#     http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nlogsearch.solr.metrics.collector.hosts=http://{{metrics_collector_hosts}}:{{metrics_collector_port}}/ws/v1/timeline/metrics\n{% if logsearch_solr_ssl_enabled %}\nexport LOGFEEDER_SSL=\"true\"\nexport LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}}\nexport LOGFEEDER_KEYSTORE_PASSWORD={{logfeeder_keystore_password}}\nexport LOGFEEDER_KEYSTORE_TYPE={{logfeeder_keystore_type}}\nexport LOGFEEDER_TRUSTSTORE_LOCATION={{logfeeder_truststore_location}}\nexport LOGFEEDER_TRUSTSTORE_PASSWORD={{logfeeder_truststore_password}}\nexport LOGFEEDER_TRUSTSTORE_TYPE={{logfeeder_truststore_type}}\n{% endif %}"
       },
+      "logfeeder-output-config" : {
+        "content" : "output-grok-filter"
+      },
+      "logfeeder-ambari-config" : {
+        "content" : "ambari-grok-filter"
+      },
       "logfeeder-grok": {
         "default_grok_patterns": "GP",
         "custom_grok_patterns": ""