AMBARI-24129. Log Search - Logs histogram query is not filtered by include/exclude fields. (#1562)

* AMBARI-24129. Log Search - Logs histogram query is not filtered by include/exclude fields.

* AMBARI-24129. Fix unit tests

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java

@@ -140,6 +140,23 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
     return query;
   }
 
+  public SolrQuery addIncludeFieldValues(SolrQuery query, String fieldValuesMapStr) {
+    if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
+      List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
+        new TypeToken<List<HashMap<String, String>>>(){}.getType());
+      for (Map<String, String> criteriaMap : criterias) {
+        for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
+          if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
+            addLogMessageFilter(query, fieldEntry.getValue(), false);
+          } else {
+            query.addFilterQuery(String.format("%s:%s", fieldEntry.getKey(), escapeNonLogMessageField(fieldEntry)));
+          }
+        }
+      }
+    }
+    return query;
+  }
+
   public Query addExcludeFieldValues(Query query, String fieldValuesMapStr) {
     if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
       List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
@@ -157,6 +174,23 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
     return query;
   }
 
+  public SolrQuery addExcludeFieldValues(SolrQuery query, String fieldValuesMapStr) {
+    if (StringUtils.isNotEmpty(fieldValuesMapStr)) {
+      List<Map<String, String>> criterias = new Gson().fromJson(fieldValuesMapStr,
+        new TypeToken<List<HashMap<String, String>>>(){}.getType());
+      for (Map<String, String> criteriaMap : criterias) {
+        for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
+          if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
+            addLogMessageFilter(query, fieldEntry.getValue(), true);
+          } else {
+            query.addFilterQuery(String.format("-%s:%s", fieldEntry.getKey(), escapeNonLogMessageField(fieldEntry)));
+          }
+        }
+      }
+    }
+    return query;
+  }
+
   public SolrQuery addListFilterToSolrQuery(SolrQuery solrQuery, String fieldName, String fieldValue) {
     return SolrUtil.addListFilterToSolrQuery(solrQuery, fieldName, fieldValue);
   }
@@ -182,6 +216,26 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
     }
   }
 
+  private void addLogMessageFilter(SolrQuery query, String value, boolean negate) {
+    StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
+    String negateToken = negate ? "-" : "";
+    for (String token : tokenizer.getTokenArray()) {
+      token = token.trim();
+      if (token.contains(" ") || !token.startsWith("*") && !token.endsWith("*")) {
+        query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(token)));
+      } else if (token.startsWith("*") && token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 1, -1);
+        query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+      } else if (token.startsWith("*") && !token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 1);
+        query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+      } else if (!token.startsWith("*") && token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 0, -1);
+        query.addFilterQuery(String.format("%s%s:%s", negateToken, LOG_MESSAGE, SolrUtil.escapeQueryChars(plainToken)));
+      }
+    }
+  }
+
   private String escapeNonLogMessageField(Map.Entry<String, String> fieldEntry) {
     Map<String, String> schemaFieldNameMap = solrSchemaFieldDao.getSchemaFieldNameMap(getLogType());
     Map<String, String> schemaFieldTypeMap = solrSchemaFieldDao.getSchemaFieldTypeMap(getLogType());

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverter.java

@@ -58,6 +58,8 @@ public class AuditBarGraphRequestQueryConverter extends AbstractDateRangeFacetQu
     addInFiltersIfNotNullAndEnabled(query, request.getUserList(),
       SolrConstants.AuditLogConstants.AUDIT_REQUEST_USER,
       StringUtils.isNotBlank(request.getUserList()));
+    addIncludeFieldValues(query, request.getIncludeQuery());
+    addExcludeFieldValues(query, request.getExcludeQuery());
     return query;
   }
 }

ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverter.java

@@ -61,6 +61,8 @@ public class ServiceLogLevelDateRangeRequestQueryConverter extends AbstractDateR
     }
     addListFilterToSolrQuery(solrQuery, CLUSTER, request.getClusters());
     addListFilterToSolrQuery(solrQuery, COMPONENT, request.getMustBe());
+    addIncludeFieldValues(solrQuery, request.getIncludeQuery());
+    addExcludeFieldValues(solrQuery, request.getExcludeQuery());
     return solrQuery;
   }
 

ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/AuditBarGraphRequestQueryConverterTest.java

@@ -45,9 +45,10 @@ public class AuditBarGraphRequestQueryConverterTest extends AbstractRequestConve
     request.setUnit("+1HOUR");
     // THEN
     SolrQuery query = underTest.convert(request);
-    assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1&facet.sort=index" +
-      "&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR" +
-      "&rows=0&start=0&fq=cluster%3Acl1&fq=reqUser%3A%28joe+OR+steven%29",
+    assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Drepo&facet.mincount=1&facet.limit=-1" +
+        "&facet.sort=index&facet.range=%7B%21tag%3Dr1%7DevtTime&f.evtTime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
+        "&f.evtTime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.evtTime.facet.range.gap=%2B1HOUR&rows=0&start=0" +
+        "&fq=cluster%3Acl1&fq=reqUser%3A%28joe+OR+steven%29&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage",
       query.toQueryString());
   }
 

ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelDateRangeRequestQueryConverterTest.java

@@ -47,8 +47,7 @@ public class ServiceLogLevelDateRangeRequestQueryConverterTest extends AbstractR
     // THEN
     assertEquals("?q=*%3A*&facet=true&facet.pivot=%7B%21range%3Dr1%7Dlevel&facet.mincount=1&facet.limit=-1" +
       "&facet.sort=index&facet.range=%7B%21tag%3Dr1%7Dlogtime&f.logtime.facet.range.start=2016-09-13T22%3A00%3A01.000Z" +
-      "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0" +
-      "&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29&fq=cluster%3Acl1&fq=type%3A%28logsearch_app+OR+secure_log%29", query.toQueryString());
+      "&f.logtime.facet.range.end=2016-09-14T22%3A00%3A01.000Z&f.logtime.facet.range.gap=%2B1HOUR&rows=0&start=0&fq=level%3A%28WARN+OR+ERROR+OR+FATAL%29&fq=cluster%3Acl1&fq=type%3A%28logsearch_app+OR+secure_log%29&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage", query.toQueryString());
   }
 
   @Test