AMBARI-18298. Add ability to add custom grok filters and custom grok patterns with Ambari (Miklos Gergely via oleewere)

Change-Id: I03852666f85185be80787824422c74eb0e437757

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-grok.xml

@@ -0,0 +1,49 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<!-- This is a special config file for properties used to monitor status of the service -->
+<configuration supports_adding_forbidden="true">
+  <property>
+    <name>default_grok_patterns</name>
+    <display-name>Default Grok Patterns</display-name>
+    <description>This is the jinja template for default grok patterns</description>
+    <value/>
+    <property-type>VALUE_FROM_PROPERTY_FILE</property-type>
+    <value-attributes>
+      <read-only>true</read-only>
+      <property-file-name>logfeeder-default_grok_patterns.j2</property-file-name>
+      <property-file-type>text</property-file-type>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>custom_grok_patterns</name>
+    <display-name>Custom Grok Patterns</display-name>
+    <description>This is the jinja template for custom grok patterns</description>
+    <value/>
+    <property-type>TEXT</property-type>
+    <value-attributes>
+      <type>multiLine</type>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-log4j.xml

@@ -21,7 +21,6 @@
 -->
 <!-- This is a special config file for properties used to monitor status of the service -->
 <configuration supports_adding_forbidden="true">
-  <!-- solrconfig.xml -->
   <property>
     <name>content</name>
     <display-name>logfeeder-log4j template</display-name>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml

@@ -109,6 +109,7 @@
             <config-type>infra-solr-env</config-type>
             <config-type>logfeeder-properties</config-type>
             <config-type>logfeeder-env</config-type>
+            <config-type>logfeeder-grok</config-type>
             <config-type>logfeeder-log4j</config-type>
           </configuration-dependencies>
         </component>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py

@@ -273,6 +273,17 @@ logfeeder_config_file_names = \
 
 default_config_files = ','.join(logfeeder_config_file_names)
 
+logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns']
+if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip():
+  logfeeder_grok_patterns = \
+    logfeeder_grok_patterns + '\n' + \
+    '\n' + \
+    '########################\n' +\
+    '# Custom Grok Patterns #\n' +\
+    '########################\n' +\
+    '\n' + \
+    config['configurations']['logfeeder-grok']['custom_grok_patterns']
+
 logfeeder_properties = dict(config['configurations']['logfeeder-properties'])
 
 logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py

@@ -57,7 +57,7 @@ def setup_logfeeder():
        )
 
   File(format("{logsearch_logfeeder_conf}/grok-patterns"),
-       content=Template("grok-patterns.j2"),
+       content=InlineTemplate(params.logfeeder_grok_patterns),
        encoding="utf-8"
        )
 
@@ -65,6 +65,7 @@ def setup_logfeeder():
     File(format("{logsearch_logfeeder_conf}/" + file_name),
          content=Template(file_name + ".j2")
          )
+
   if params.security_enabled:
     File(format("{logfeeder_jaas_file}"),
          content=Template("logfeeder_jaas.conf.j2")

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 → ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-default_grok_patterns.j2

@@ -141,7 +141,7 @@ HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
 LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
 
 
-# Custom
+# Log specific patterns
 USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}
 SPARK_DATESTAMP %{YEAR}[/-]%{MONTHNUM2}[/-]%{MONTHDAY} %{TIME}
 CUSTOM_DATESTAMP %{MONTHDAY} %{MONTH} %{YEAR} %{TIME}

ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py

@@ -76,7 +76,7 @@ class TestLogFeeder(RMFTestCase):
                               content=InlineTemplate(self.getConfig()['configurations']['logfeeder-log4j']['content'])
                               )
     self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/grok-patterns',
-                              content=Template('grok-patterns.j2'),
+                              content=InlineTemplate('GP'),
                               encoding='utf-8'
                               )
 

ambari-web/app/data/HDP2/site_properties.js

@@ -1962,6 +1962,21 @@ var hdp2properties = [
     "category": "Advanced logfeeder-env",
     "index": 14
   },
+  /*logfeeder-grok*/
+  {
+    "name": "default_grok_patterns",
+    "serviceName": "LOGSEARCH",
+    "filename": "logfeeder-grok.xml",
+    "category": "Advanced logfeeder-grok",
+    "index": 1
+  },
+  {
+    "name": "custom_grok_patterns",
+    "serviceName": "LOGSEARCH",
+    "filename": "logfeeder-grok.xml",
+    "category": "Advanced logfeeder-grok",
+    "index": 2
+  },
   /*logfeeder-log4j*/
   {
     "name": "content",