AMBARI-18248. Parsing of /var/log/messages and /var/log/secure (Hayat Behlim via oleewere)

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java

@@ -20,10 +20,12 @@
 package org.apache.ambari.logfeeder.mapper;
 
 import java.text.SimpleDateFormat;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.Map;
 
 import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.lang.time.DateUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
 import org.apache.log4j.Logger;
@@ -31,9 +33,9 @@ import org.apache.log4j.Logger;
 public class MapperDate extends Mapper {
   private static final Logger logger = Logger.getLogger(MapperDate.class);
 
-  private String dateFormat = null;
-  private SimpleDateFormat dateFormatter = null;
+  private SimpleDateFormat targetDateFormatter = null;
   private boolean isEpoch = false;
+  private SimpleDateFormat srcDateFormatter=null;
 
   @Override
   public boolean init(String inputDesc, String fieldName,
@@ -48,24 +50,28 @@ public class MapperDate extends Mapper {
     }
     @SuppressWarnings("unchecked")
     Map<String, Object> mapObjects = (Map<String, Object>) mapConfigs;
-    dateFormat = (String) mapObjects.get("date_pattern");
-    if (StringUtils.isEmpty(dateFormat)) {
+    String targetDateFormat = (String) mapObjects.get("target_date_pattern");
+    String srcDateFormat = (String) mapObjects.get("src_date_pattern");
+    if (StringUtils.isEmpty(targetDateFormat)) {
       logger.fatal("Date format for map is empty. " + this.toString());
     } else {
-      logger.info("Date mapper format is " + dateFormat);
+      logger.info("Date mapper format is " + targetDateFormat);
 
-      if (dateFormat.equalsIgnoreCase("epoch")) {
+      if (targetDateFormat.equalsIgnoreCase("epoch")) {
         isEpoch = true;
         return true;
       } else {
         try {
-          dateFormatter = new SimpleDateFormat(dateFormat);
+          targetDateFormatter = new SimpleDateFormat(targetDateFormat);
+          if (!StringUtils.isEmpty(srcDateFormat)) {
+            srcDateFormatter = new SimpleDateFormat(srcDateFormat);
+          }
           return true;
         } catch (Throwable ex) {
           logger.fatal("Error creating date format. format="
-            + dateFormat + ". " + this.toString());
+            + targetDateFormat + ". " + this.toString());
         }
-      }
+      } 
     }
     return false;
   }
@@ -77,8 +83,26 @@ public class MapperDate extends Mapper {
         if (isEpoch) {
           long ms = Long.parseLong(value.toString()) * 1000;
           value = new Date(ms);
-        } else if (dateFormatter != null) {
-          value = dateFormatter.parse(value.toString());
+        } else if (targetDateFormatter != null) {
+          if(srcDateFormatter!=null){
+            Date srcDate = srcDateFormatter.parse(value.toString());
+            //set year in src_date when src_date does not have year component
+            if (!srcDateFormatter.toPattern().contains("yy")) {
+              Calendar currentCalendar = Calendar.getInstance();
+              Calendar logDateCalendar = Calendar.getInstance();
+              logDateCalendar.setTimeInMillis(srcDate.getTime());
+              if (logDateCalendar.get(Calendar.MONTH) > currentCalendar.get(Calendar.MONTH)) {
+                // set previous year as a log year  when log month is grater than current month
+                srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR) - 1);
+              } else {
+                // set current year as a log year
+                srcDate = DateUtils.setYears(srcDate, currentCalendar.get(Calendar.YEAR));
+              }
+            }
+            value = targetDateFormatter.format(srcDate);
+          } else {
+            value = targetDateFormatter.parse(value.toString());
+          }
         } else {
           return value;
         }
@@ -87,7 +111,7 @@ public class MapperDate extends Mapper {
         LogFeederUtil.logErrorMessageByInterval(this.getClass()
             .getSimpleName() + ":apply",
           "Error applying date transformation. isEpoch="
-            + isEpoch + ", dateFormat=" + dateFormat
+            + isEpoch + ", targetateFormat=" + (targetDateFormatter!=null ?targetDateFormatter.toPattern():"")
             + ", value=" + value + ". " + this.toString(),
           t, logger, Level.ERROR);
       }

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java

@@ -38,10 +38,4 @@ public class DateUtil {
     }
     return "";
   }
-
-  public static void main(String[] args) {
-    Date currentDate = new Date();
-    String fileDateFormat = "yyyy-MM-dd-HH-mm-ss";
-    System.out.println(dateToString(currentDate, fileDateFormat));
-  }
 }

ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java

@@ -41,7 +41,7 @@ public class MapperDateTest {
     LOG.info("testMapperDate_epoch()");
 
     Map<String, Object> mapConfigs = new HashMap<>();
-    mapConfigs.put("date_pattern", "epoch");
+    mapConfigs.put("target_date_pattern", "epoch");
 
     MapperDate mapperDate = new MapperDate();
     assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -61,7 +61,7 @@ public class MapperDateTest {
     LOG.info("testMapperDate_pattern()");
 
     Map<String, Object> mapConfigs = new HashMap<>();
-    mapConfigs.put("date_pattern", LogFeederUtil.DATE_FORMAT);
+    mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
 
     MapperDate mapperDate = new MapperDate();
     assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -101,7 +101,7 @@ public class MapperDateTest {
     LOG.info("testMapperDate_notParsableDatePattern()");
 
     Map<String, Object> mapConfigs = new HashMap<>();
-    mapConfigs.put("date_pattern", "not_parsable_content");
+    mapConfigs.put("target_date_pattern", "not_parsable_content");
 
     MapperDate mapperDate = new MapperDate();
     assertFalse("Was able to initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -112,7 +112,7 @@ public class MapperDateTest {
     LOG.info("testMapperDate_invalidEpochValue()");
 
     Map<String, Object> mapConfigs = new HashMap<>();
-    mapConfigs.put("date_pattern", "epoch");
+    mapConfigs.put("target_date_pattern", "epoch");
 
     MapperDate mapperDate = new MapperDate();
     assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -130,7 +130,7 @@ public class MapperDateTest {
     LOG.info("testMapperDate_invalidDateStringValue()");
 
     Map<String, Object> mapConfigs = new HashMap<>();
-    mapConfigs.put("date_pattern", LogFeederUtil.DATE_FORMAT);
+    mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
 
     MapperDate mapperDate = new MapperDate();
     assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -142,4 +142,60 @@ public class MapperDateTest {
     assertEquals("Invalid value wasn't returned as it is", invalidValue, mappedValue);
     assertTrue("jsonObj is not empty", jsonObj.isEmpty());
   }
+  
+  @Test
+  public void testMapperDate_patternWithoutYear_previousYearLog() throws Exception {
+    LOG.info("testMapperDate_patternWithoutYear_previousYearLog()");
+    String fieldName = "logtime";
+    Calendar currentCalendar = Calendar.getInstance();
+    Map<String, Object> mapConfigs = new HashMap<>();
+    mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+    String srcDatePattern ="MMM dd HH:mm:ss";
+    mapConfigs.put("src_date_pattern", srcDatePattern);
+    MapperDate mapperDate = new MapperDate();
+    assertTrue("Could not initialize!", mapperDate.init(null, fieldName, null, mapConfigs));
+    Map<String, Object> jsonObj = new HashMap<>();
+    Calendar nextMonthCalendar = Calendar.getInstance();
+    
+    nextMonthCalendar.set(Calendar.MONTH, currentCalendar.get(Calendar.MONTH)+1 );
+    String inputDateStr = new SimpleDateFormat("MMM").format(nextMonthCalendar.getTime()) + " 01 12:01:45";
+    Object mappedValue = mapperDate.apply(jsonObj, inputDateStr);
+    Date mappedDateValue = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).parse(mappedValue.toString());
+    String mappedDateValueStr = new SimpleDateFormat(srcDatePattern).format(mappedDateValue);
+    assertEquals(Date.class, mappedDateValue.getClass());
+    
+    int expectedLogYear = currentCalendar.get(Calendar.YEAR)-1;
+    Calendar mapppedValueCalendar = Calendar.getInstance();
+    mapppedValueCalendar.setTime(mappedDateValue);
+    assertEquals("Mapped year wasn't matched properly", expectedLogYear, mapppedValueCalendar.get(Calendar.YEAR));
+    assertEquals("Mapped date wasn't matched properly", inputDateStr, mappedDateValueStr);
+    assertEquals("Value wasn't put into jsonObj",mappedValue, jsonObj.remove(fieldName));
+    assertTrue("jsonObj is not empty", jsonObj.isEmpty());
+  }
+  
+  @Test
+  public void testMapperDate_patternWithoutYear_currentYearLog() throws Exception {
+    LOG.info("testMapperDate_patternWithoutYear_currentYearLog()");
+    String fieldName = "logtime";
+    Calendar currentCalendar = Calendar.getInstance();
+    Map<String, Object> mapConfigs = new HashMap<>();
+    mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+    String srcDatePattern ="MMM dd HH:mm:ss";
+    mapConfigs.put("src_date_pattern", srcDatePattern);
+    MapperDate mapperDate = new MapperDate();
+    assertTrue("Could not initialize!", mapperDate.init(null, fieldName, null, mapConfigs));
+    Map<String, Object> jsonObj = new HashMap<>();
+    String inputDateStr = new SimpleDateFormat("MMM").format(currentCalendar.getTime()) + " 01 12:01:45";
+    Object mappedValue = mapperDate.apply(jsonObj, inputDateStr);
+    Date mappedDateValue = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).parse(mappedValue.toString());
+    String mappedDateValueStr = new SimpleDateFormat(srcDatePattern).format(mappedDateValue);
+    assertEquals(Date.class, mappedDateValue.getClass());
+    int expectedLogYear = currentCalendar.get(Calendar.YEAR);
+    Calendar mapppedValueCalendar = Calendar.getInstance();
+    mapppedValueCalendar.setTime(mappedDateValue);
+    assertEquals("Mapped year wasn't matched properly", expectedLogYear, mapppedValueCalendar.get(Calendar.YEAR));
+    assertEquals("Mapped date wasn't matched properly", inputDateStr, mappedDateValueStr);
+    assertEquals("Value wasn't put into jsonObj",mappedValue, jsonObj.remove(fieldName));
+    assertTrue("jsonObj is not empty", jsonObj.isEmpty());
+  }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json

@@ -470,6 +470,19 @@
       ],
       "dependencies": [
       ]
+    },
+   "System": {
+      "label": "System",
+      "components": [
+        {
+          "name": "system_message"
+        },
+        {
+          "name": "secure_log"
+        }
+      ],
+      "dependencies": [
+      ]
     }
   }
-}
+}

ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties

@@ -19,8 +19,10 @@ logfeeder.config.files=shipper-conf/global.config.json,\
   shipper-conf/output.config.json,\
   shipper-conf/input.config-zookeeper.json,\
   shipper-conf/input.config-logsearch.json,\
-  shipper-conf/input.config-hst.json
+  shipper-conf/input.config-hst.json,\
+  shipper-conf/input.config-system_message.json,\
+  shipper-conf/input.config-secure_log.json
 logfeeder.log.filter.enable=true
 logfeeder.solr.config.interval=5
 logfeeder.solr.core.config.name=history
-logfeeder.solr.zk_connect_string=localhost:9983
+logfeeder.solr.zk_connect_string=localhost:9983

ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-secure_log.json

@@ -0,0 +1,31 @@
+{
+  "input": [
+    {
+      "type": "secure_log",
+      "rowtype": "service",
+      "path": "/root/test-logs/secure_log/secure-log.txt"
+    }
+  ],
+  "filter": [
+    {
+      "filter": "grok",
+      "conditions": {
+        "fields": {
+          "type": [
+            "secure_log"
+          ]
+        }
+      },
+      "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})",
+      "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values": {
+        "logtime": {
+          "map_date": {
+            "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS",
+            "src_date_pattern" :"MMM dd HH:mm:ss"
+          }
+        }
+      }
+    }
+  ]
+}

ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-system_message.json

@@ -0,0 +1,31 @@
+{
+  "input": [
+    {
+      "type": "system_message",
+      "rowtype": "service",
+      "path": "/root/test-logs/system_message/message_logs.txt"
+    }
+  ],
+  "filter": [
+    {
+      "filter": "grok",
+      "conditions": {
+        "fields": {
+          "type": [
+            "system_message"
+          ]
+        }
+      },
+      "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})",
+      "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values": {
+        "logtime": {
+          "map_date": {
+            "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS",
+            "src_date_pattern" :"MMM dd HH:mm:ss"
+          }
+        }
+      }
+    }
+  ]
+}

ambari-logsearch/docker/test-config/logsearch/logsearch.properties

@@ -44,7 +44,7 @@ logsearch.solr.jmx.port=18886
 
 # Logfeeder Settings
 
-logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE
+logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
 
 # logsearch-admin.json
 logsearch.auth.file.enable=true
@@ -54,4 +54,4 @@ logsearch.auth.ldap.enable=false
 logsearch.auth.simple.enable=false
 logsearch.auth.external_auth.enable=false
 
-logsearch.protocol=http
+logsearch.protocol=http

ambari-logsearch/docker/test-logs/secure_log/secure-log.txt

@@ -0,0 +1,11 @@
+Aug  9 11:53:53 logsearch.apache.org su: pam_unix(su-l:session): session opened for user ambari-qa by (uid=0)
+Aug  9 11:53:53 logsearch.apache.org su: pam_unix(su-l:session): session closed for user ambari-qa
+Aug  9 11:53:53 logsearch.apache.org su: pam_unix(su-l:session): session opened for user ambari-qa by (uid=0)
+Aug  9 11:53:53 logsearch.apache.org su: pam_unix(su-l:session): session closed for user ambari-qa
+Aug  9 11:54:19 logsearch.apache.org su: pam_unix(su-l:session): session opened for user ambari-qa by (uid=0)
+Aug  9 11:54:19 logsearch.apache.org su: pam_unix(su-l:session): session closed for user ambari-qa
+Aug  9 11:54:19 logsearch.apache.org su: pam_unix(su-l:session): session opened for user ambari-qa by (uid=0)
+Aug  9 11:54:19 logsearch.apache.org su: pam_unix(su-l:session): session closed for user ambari-qa
+Aug  9 11:54:22 logsearch.apache.org su: pam_unix(su-l:session): session opened for user yarn by (uid=0)
+Aug  9 11:54:22 logsearch.apache.org su: pam_unix(su-l:session): session closed for user yarn
+Aug  9 11:54:22 logsearch.apache.org su: pam_unix(su-l:session): session opened for user yarn by (uid=0)

ambari-logsearch/docker/test-logs/system_message/message_logs.txt

@@ -0,0 +1,17 @@
+Aug 21 03:09:02 logsearch.apache.org rsyslogd: [origin software="rsyslogd" swVersion="5.8.10" x-pid="955" x-info="http://www.rsyslog.com"] rsyslogd was HUPed
+Aug 21 10:53:02 logsearch.apache.org dhclient[891]: DHCPREQUEST on eth0 to 172.22.112.67 port 67 (xid=0x2cb1d8ad)
+Aug 21 10:53:02 logsearch.apache.org dhclient[891]: DHCPACK from 172.22.112.67 (xid=0x2cb1d8ad)
+Aug 21 10:53:03 logsearch.apache.org dhclient[891]: bound to 172.22.89.76 -- renewal in 37432 seconds.
+Aug 21 21:16:55 logsearch.apache.org dhclient[891]: DHCPREQUEST on eth0 to 172.22.112.67 port 67 (xid=0x2cb1d8ad)
+Aug 21 21:16:55 logsearch.apache.org dhclient[891]: DHCPACK from 172.22.112.67 (xid=0x2cb1d8ad)
+Aug 21 21:16:56 logsearch.apache.org dhclient[891]: bound to 172.22.89.76 -- renewal in 35453 seconds.
+Aug 22 06:39:48 logsearch.apache.org rsyslogd-2177: imuxsock lost 369 messages from pid 5254 due to rate-limiting
+Aug 22 07:07:49 logsearch.apache.org dhclient[891]: DHCPREQUEST on eth0 to 172.22.112.67 port 67 (xid=0x2cb1d8ad)
+Aug 22 07:07:49 logsearch.apache.org dhclient[891]: DHCPACK from 172.22.112.67 (xid=0x2cb1d8ad)
+Aug 22 07:07:50 logsearch.apache.org dhclient[891]: bound to 172.22.89.76 -- renewal in 36355 seconds.
+Aug 22 17:13:45 logsearch.apache.org dhclient[891]: DHCPREQUEST on eth0 to 172.22.112.67 port 67 (xid=0x2cb1d8ad)
+Aug 22 17:13:45 logsearch.apache.org dhclient[891]: DHCPACK from 172.22.112.67 (xid=0x2cb1d8ad)
+Aug 22 17:13:46 logsearch.apache.org dhclient[891]: bound to 172.22.89.76 -- renewal in 37482 seconds.
+Aug 23 03:38:28 logsearch.apache.org dhclient[891]: DHCPREQUEST on eth0 to 172.22.112.67 port 67 (xid=0x2cb1d8ad)
+Aug 23 03:38:28 logsearch.apache.org dhclient[891]: DHCPACK from 172.22.112.67 (xid=0x2cb1d8ad)
+Aug 23 03:38:29 logsearch.apache.org dhclient[891]: bound to 172.22.89.76 -- renewal in 31187 seconds.

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-accumulo.json.j2

@@ -62,7 +62,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -91,7 +91,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
           
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2

@@ -78,7 +78,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         },
@@ -110,7 +110,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"dd MMM yyyy HH:mm:ss"
+            "target_date_pattern":"dd MMM yyyy HH:mm:ss"
           }
 
         }
@@ -135,7 +135,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -160,7 +160,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -185,7 +185,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
           }
 
         },
@@ -217,7 +217,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -242,7 +242,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX"
+            "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2

@@ -61,7 +61,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -87,7 +87,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -112,7 +112,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy/MM/dd HH:mm:ss"
+            "target_date_pattern":"yyyy/MM/dd HH:mm:ss"
           }
 
         },
@@ -169,7 +169,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-atlas.json.j2

@@ -41,7 +41,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-falcon.json.j2

@@ -41,7 +41,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2

@@ -42,7 +42,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+            "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2

@@ -53,7 +53,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -77,7 +77,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2

@@ -84,7 +84,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
         }
 
@@ -108,7 +108,7 @@
       "post_map_values":{
         "evtTime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2

@@ -52,7 +52,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -77,7 +77,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+            "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
           }
 
         },

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2

@@ -55,7 +55,7 @@
          "post_map_values":{
            "logtime":{
               "map_date":{
-                "date_pattern":"dd MMM yyyy HH:mm:ss"
+                "target_date_pattern":"dd MMM yyyy HH:mm:ss"
               }
             },
             "level":{
@@ -83,7 +83,7 @@
          "post_map_values":{
            "logtime":{
               "map_date":{
-                 "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+                 "target_date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
                }
 
            },

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-infra.json.j2

@@ -42,7 +42,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-kafka.json.j2

@@ -64,7 +64,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -91,7 +91,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-knox.json.j2

@@ -54,7 +54,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-nifi.json.j2

@@ -60,7 +60,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-oozie.json.j2

@@ -42,7 +42,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ranger.json.j2

@@ -58,7 +58,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -83,7 +83,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }
@@ -108,7 +108,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"dd MMM yyyy HH:mm:ss"
+            "target_date_pattern":"dd MMM yyyy HH:mm:ss"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2

@@ -53,7 +53,7 @@
           "post_map_values":{
             "logtime":{
               "map_date":{
-                "date_pattern":"yy/MM/dd HH:mm:ss"
+                "target_date_pattern":"yy/MM/dd HH:mm:ss"
               }
 
             },

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2

@@ -47,7 +47,7 @@
           "post_map_values":{
             "logtime":{
               "map_date":{
-                "date_pattern":"yy/MM/dd HH:mm:ss"
+                "target_date_pattern":"yy/MM/dd HH:mm:ss"
               }
 
             },

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-storm.json.j2

@@ -72,7 +72,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2

@@ -72,7 +72,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2

@@ -42,7 +42,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zookeeper.json.j2

@@ -42,7 +42,7 @@
       "post_map_values":{
         "logtime":{
           "map_date":{
-            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+            "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
           }
 
         }