AMBARI-17899. Login flow break when only ldap auth is enabled. (Hayat Behlim via oleewere)

ambari-logsearch/ambari-logsearch-appender/README.md

@@ -0,0 +1,57 @@
+
+
+
+# Ambari Logsearch Appender
+
+Ambari Logsearch Appender is a log4j base appender that write logs in json format.
+
+
+
+## Setup Ambari Logsearch Appender
+
+#### Add dependency 
+```xml
+    	<dependency>
+    	  <groupId>org.apache.ambari</groupId>
+    	  <artifactId>ambari-logsearch-appender</artifactId>
+    	  <version>${version}</version>
+    	</dependency>
+```
+####Dependent dependency 
+```xml   
+		 <dependency>
+		  <groupId>log4j</groupId>
+		  <artifactId>log4j</artifactId>
+		  <version>1.2.17</version>
+		</dependency>
+		<dependency>
+		  <groupId>com.google.code.gson</groupId>
+		  <artifactId>gson</artifactId>
+		  <version>2.6.2</version>
+		</dependency>
+```
+
+## Configuration
+####  Sample Configuration for log4j.properties
+```java
+log4j.appender.logsearchJson=org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender
+log4j.appender.logsearchJson.File=path/file_name.json
+log4j.appender.logsearchJson.maxFileSize=10MB
+log4j.appender.logsearchJson.maxBackupIndex=10
+log4j.appender.logsearchJson.Append=true
+log4j.appender.logsearchJson.layout=org.apache.ambari.logsearch.appender.LogsearchConversion
+```
+###                                OR
+#### Sample Configuration for log4j.xml
+```xml
+<appender name="logsearchJson"
+    class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
+    <param name="file" value="path/file_name.json" />
+		<param name="append" value="true" />
+		<param name="maxFileSize" value="10MB" />
+		<param name="maxBackupIndex" value="10" />
+    <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" />
+</appender> 
+```
+
+

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/exception/LogfeederException.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.exception;
+
+public class LogfeederException extends Exception {
+
+  public LogfeederException(String message, Throwable throwable) {
+    super(message, throwable);
+  }
+
+  public LogfeederException(String message) {
+    super(message);
+  }
+}

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java

@@ -31,6 +31,7 @@ import org.apache.ambari.logfeeder.MetricCount;
 import org.apache.ambari.logfeeder.OutputMgr;
 import org.apache.ambari.logfeeder.AliasUtil.ALIAS_PARAM;
 import org.apache.ambari.logfeeder.AliasUtil.ALIAS_TYPE;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.input.Input;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.ambari.logfeeder.mapper.Mapper;
@@ -125,7 +126,7 @@ public abstract class Filter extends ConfigBlock {
   /**
    * Deriving classes should implement this at the minimum
    */
-  public void apply(String inputStr, InputMarker inputMarker) {
+  public void apply(String inputStr, InputMarker inputMarker) throws LogfeederException  {
     // TODO: There is no transformation for string types.
     if (nextFilter != null) {
       nextFilter.apply(inputStr, inputMarker);
@@ -134,7 +135,7 @@ public abstract class Filter extends ConfigBlock {
     }
   }
 
-  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) {
+  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException {
     if (postFieldValueMappers.size() > 0) {
       for (String fieldName : postFieldValueMappers.keySet()) {
         Object value = jsonObj.get(fieldName);

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java

@@ -36,6 +36,7 @@ import oi.thekraken.grok.api.exception.GrokException;
 
 import org.apache.ambari.logfeeder.LogFeederUtil;
 import org.apache.ambari.logfeeder.MetricCount;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
@@ -167,7 +168,7 @@ public class FilterGrok extends Filter {
   }
 
   @Override
-  public void apply(String inputStr, InputMarker inputMarker) {
+  public void apply(String inputStr, InputMarker inputMarker) throws LogfeederException {
     if (grokMessage == null) {
       return;
     }
@@ -205,7 +206,7 @@ public class FilterGrok extends Filter {
   }
 
   @Override
-  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) {
+  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException {
     if (sourceField != null) {
       savedInputMarker = inputMarker;
       applyMessage((String) jsonObj.get(sourceField), jsonObj, null);
@@ -218,9 +219,10 @@ public class FilterGrok extends Filter {
   /**
    * @param inputStr
    * @param jsonObj
+   * @throws LogfeederException 
    */
   private void applyMessage(String inputStr, Map<String, Object> jsonObj,
-                            String multilineJsonStr) {
+                            String multilineJsonStr) throws LogfeederException {
     String jsonStr = grokParse(inputStr);
 
     boolean parseError = false;
@@ -257,7 +259,6 @@ public class FilterGrok extends Filter {
         jsonObj.put("log_message", inputStr);
       }
     }
-
     super.apply(jsonObj, savedInputMarker);
     statMetric.count++;
   }
@@ -286,7 +287,11 @@ public class FilterGrok extends Filter {
     if (strBuff != null) {
       Map<String, Object> jsonObj = Collections
         .synchronizedMap(new HashMap<String, Object>());
-      applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr);
+      try {
+        applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr);
+      } catch (LogfeederException e) {
+        logger.error(e.getLocalizedMessage(), e.getCause());
+      }
       strBuff = null;
       savedInputMarker = null;
     }

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/JSONFilterCode.java → ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java

@@ -21,20 +21,28 @@ package org.apache.ambari.logfeeder.filter;
 import java.util.Map;
 
 import org.apache.ambari.logfeeder.LogFeederUtil;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.log4j.Logger;
 
-public class JSONFilterCode extends Filter {
+public class FilterJSON extends Filter {
+  
+  private static final Logger logger  = Logger.getLogger(FilterJSON.class);
 
   @Override
-  public void apply(String inputStr, InputMarker inputMarker) {
-    Map<String, Object> jsonMap = LogFeederUtil.toJSONObject(inputStr);
-
+  public void apply(String inputStr, InputMarker inputMarker) throws LogfeederException {
+    Map<String, Object> jsonMap = null;
+    try {
+      jsonMap = LogFeederUtil.toJSONObject(inputStr);
+    } catch (Exception e) {
+      logger.error(e.getLocalizedMessage());
+      throw new LogfeederException("Json parsing failed for inputstr = " + inputStr ,e.getCause());
+    }
     Double lineNumberD = (Double) jsonMap.get("line_number");
     if (lineNumberD != null) {
       long lineNumber = lineNumberD.longValue();
       jsonMap.put("line_number", lineNumber);
     }
-
     String timeStampStr = (String) jsonMap.get("logtime");
     if (timeStampStr != null && !timeStampStr.isEmpty()) {
       String logtime = LogFeederUtil.getDate(timeStampStr);

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java

@@ -25,6 +25,7 @@ import java.util.StringTokenizer;
 
 import org.apache.ambari.logfeeder.LogFeederUtil;
 import org.apache.ambari.logfeeder.MetricCount;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Level;
@@ -59,12 +60,12 @@ public class FilterKeyValue extends Filter {
   }
 
   @Override
-  public void apply(String inputStr, InputMarker inputMarker) {
+  public void apply(String inputStr, InputMarker inputMarker) throws LogfeederException {
     apply(LogFeederUtil.toJSONObject(inputStr), inputMarker);
   }
 
   @Override
-  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) {
+  public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException {
     if (sourceField == null) {
       return;
     }

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java

@@ -30,6 +30,7 @@ import org.apache.ambari.logfeeder.ConfigBlock;
 import org.apache.ambari.logfeeder.InputMgr;
 import org.apache.ambari.logfeeder.MetricCount;
 import org.apache.ambari.logfeeder.OutputMgr;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.filter.Filter;
 import org.apache.ambari.logfeeder.output.Output;
 import org.apache.log4j.Logger;
@@ -102,7 +103,11 @@ public abstract class Input extends ConfigBlock implements Runnable {
     readBytesMetric.count += (line.length());
 
     if (firstFilter != null) {
-      firstFilter.apply(line, marker);
+      try {
+        firstFilter.apply(line, marker);
+      } catch (LogfeederException e) {
+        logger.error(e.getLocalizedMessage(),e);
+      }
     } else {
       // TODO: For now, let's make filter mandatory, so that no one
       // accidently forgets to write filter

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java

@@ -29,7 +29,7 @@ import java.util.Random;
 
 import org.apache.ambari.logfeeder.LogFeederUtil;
 import org.apache.ambari.logfeeder.filter.Filter;
-import org.apache.ambari.logfeeder.filter.JSONFilterCode;
+import org.apache.ambari.logfeeder.filter.FilterJSON;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.solr.common.util.Base64;
 
@@ -64,7 +64,7 @@ public class InputSimulate extends Input {
     this.logText = getLogText();
     this.sleepMillis = LogFeederUtil.getIntProperty("logfeeder.simulate.sleep_milliseconds", 10000);
     
-    Filter filter = new JSONFilterCode();
+    Filter filter = new FilterJSON();
     filter.setInput(this);
     setFirstFilter(filter);
   }

ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/alias_config.json

@@ -12,7 +12,7 @@
   },
   "filter": {
     "json": {
-      "klass": "org.apache.ambari.logfeeder.filter.JSONFilterCode"
+      "klass": "org.apache.ambari.logfeeder.filter.FilterJSON"
     },
     "keyvalue": {
       "klass": "org.apache.ambari.logfeeder.filter.FilterKeyValue"

ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java → ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java

@@ -27,6 +27,7 @@ import java.util.TimeZone;
 
 import org.apache.ambari.logfeeder.LogFeederUtil;
 import org.apache.ambari.logfeeder.OutputMgr;
+import org.apache.ambari.logfeeder.exception.LogfeederException;
 import org.apache.ambari.logfeeder.input.InputMarker;
 import org.apache.log4j.Logger;
 import org.easymock.Capture;
@@ -37,11 +38,12 @@ import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-public class JSONFilterCodeTest {
-  private static final Logger LOG = Logger.getLogger(JSONFilterCodeTest.class);
+public class FilterJSONTest {
+  private static final Logger LOG = Logger.getLogger(FilterJSONTest.class);
 
-  private JSONFilterCode jsonFilterCode;
+  private FilterJSON filterJson;
   private OutputMgr mockOutputMgr;
   private Capture<Map<String, Object>> capture;
 
@@ -49,10 +51,10 @@ public class JSONFilterCodeTest {
     mockOutputMgr = EasyMock.strictMock(OutputMgr.class);
     capture = EasyMock.newCapture(CaptureType.LAST);
 
-    jsonFilterCode = new JSONFilterCode();
-    jsonFilterCode.loadConfig(params);
-    jsonFilterCode.setOutputMgr(mockOutputMgr);
-    jsonFilterCode.init();
+    filterJson = new FilterJSON();
+    filterJson.loadConfig(params);
+    filterJson.setOutputMgr(mockOutputMgr);
+    filterJson.init();
   }
 
   @Test
@@ -69,7 +71,7 @@ public class JSONFilterCodeTest {
     DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
     sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
     String dateString = sdf.format(d);
-    jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker());
+    filterJson.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker());
 
     EasyMock.verify(mockOutputMgr);
     Map<String, Object> jsonParams = capture.getValue();
@@ -93,7 +95,7 @@ public class JSONFilterCodeTest {
     DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
     sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
     String dateString = sdf.format(d);
-    jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker());
+    filterJson.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker());
 
     EasyMock.verify(mockOutputMgr);
     Map<String, Object> jsonParams = capture.getValue();
@@ -113,7 +115,7 @@ public class JSONFilterCodeTest {
     EasyMock.expectLastCall();
     EasyMock.replay(mockOutputMgr);
 
-    jsonFilterCode.apply("{ line_number: 100, some_field: 'abc' }", new InputMarker());
+    filterJson.apply("{ line_number: 100, some_field: 'abc' }", new InputMarker());
 
     EasyMock.verify(mockOutputMgr);
     Map<String, Object> jsonParams = capture.getValue();
@@ -122,6 +124,20 @@ public class JSONFilterCodeTest {
     assertEquals("Incorrect decoding: some field", "abc", jsonParams.remove("some_field"));
     assertTrue("jsonParams are not empty!", jsonParams.isEmpty());
   }
+  
+  
+  @Test
+  public void testJSONFilterCode_invalidJson() throws Exception {
+    LOG.info("testJSONFilterCode_invalidJson()");
+    init(new HashMap<String, Object>());
+    String inputStr="invalid json";
+    try{
+    filterJson.apply(inputStr,new InputMarker());
+    fail("Expected LogfeederException was not occured");
+    }catch(LogfeederException logfeederException){
+      assertEquals("Json parsing failed for inputstr = "+inputStr, logfeederException.getLocalizedMessage());
+    }
+  }
 
   @After
   public void cleanUp() {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java

@@ -23,6 +23,7 @@ import java.util.List;
 import org.apache.log4j.Logger;
 import org.springframework.ldap.CommunicationException;
 import org.springframework.ldap.core.support.LdapContextSource;
+import org.springframework.security.authentication.BadCredentialsException;
 import org.springframework.security.core.Authentication;
 import org.springframework.security.core.AuthenticationException;
 import org.springframework.security.core.userdetails.UsernameNotFoundException;
@@ -86,6 +87,7 @@ public class LogsearchLdapAuthenticationProvider extends
       logger.warn("Ldap authentication failed. username="
         + authentication.getName() + ", details="
         + authentication.getDetails());
+      throw new BadCredentialsException("Invalid credentials!!");
     }
     return authentication;
   }