AMBARI-18660. Log Search schema meta data loader fix (Miklos Gergely via oleewere)

Change-Id: Ibbf3f15984b3439e25cb2b7f06be1830aa389240

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java

@@ -19,8 +19,6 @@
 package org.apache.ambari.logsearch.conf;
 
 import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder;
 import org.apache.commons.lang.StringUtils;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
@@ -32,11 +30,13 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.DependsOn;
 import org.springframework.data.solr.core.SolrTemplate;
 import org.springframework.data.solr.repository.config.EnableSolrRepositories;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 import javax.inject.Inject;
 
 @Configuration
 @EnableSolrRepositories
+@EnableScheduling
 public class SolrConfig {
 
   private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class);
@@ -80,18 +80,9 @@ public class SolrConfig {
       solrUserConfigPropsConfig.getCollection()));
   }
 
-  @Bean(name = "serviceSolrFieldDao")
-  public SolrSchemaFieldDao serviceSolrFieldDao() {
-    return new SolrSchemaFieldDao();
-  }
-
-  @Bean(name = "auditSolrFieldDao")
-  public SolrSchemaFieldDao auditSolrFieldDao() {
-    return new SolrSchemaFieldDao();
-  }
-
-  @Bean(name = "userConfigSolrFieldDao")
-  public SolrSchemaFieldDao userConfigSolrFieldDao() {
+  @Bean
+  @DependsOn({"serviceSolrTemplate", "auditSolrTemplate"})
+  public SolrSchemaFieldDao solrSchemaFieldDao() {
     return new SolrSchemaFieldDao();
   }
 

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java

@@ -46,5 +46,4 @@ public interface SolrPropsConfig {
   String getSplitInterval();
 
   void setSplitInterval(String splitInterval);
-
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java

@@ -44,6 +44,9 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig {
   @Value("#{'${logsearch.logfeeder.include.default.level:FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}'.split(',')}")
   private List<String> logLevels;
 
+  @Value("${logsearch.schema.fields.populate.interval.mins:1}")
+  private Integer populateIntervalMins;
+  
   @Override
   public String getCollection() {
     return collection;
@@ -101,4 +104,13 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig {
   public void setLogLevels(List<String> logLevels) {
     this.logLevels = logLevels;
   }
+  
+
+  public Integer getPopulateIntervalMins() {
+    return populateIntervalMins;
+  }
+  
+  void setPopulateIntervalMins(Integer populateIntervalMins) {
+    this.populateIntervalMins = populateIntervalMins;
+  }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java

@@ -26,13 +26,13 @@ import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
 import org.apache.ambari.logsearch.util.SolrUtil;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrTokenizer;
 import org.springframework.data.solr.core.query.Criteria;
 import org.springframework.data.solr.core.query.Query;
 import org.springframework.data.solr.core.query.SimpleFilterQuery;
 import org.springframework.data.solr.core.query.SimpleStringCriteria;
 
 import javax.inject.Inject;
-import javax.inject.Named;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,12 +43,7 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
   extends AbstractConverterAware<REQUEST_TYPE, QUERY_TYPE> {
 
   @Inject
-  @Named("serviceSolrFieldDao")
-  private SolrSchemaFieldDao serviceSolrSchemaFieldDao;
-
-  @Inject
-  @Named("auditSolrFieldDao")
-  private SolrSchemaFieldDao auditSolrSchemaFieldDao;
+  private SolrSchemaFieldDao solrSchemaFieldDao;
 
   public List<String> splitValueAsList(String value, String separator) {
     return StringUtils.isNotEmpty(value) ? Splitter.on(separator).omitEmptyStrings().splitToList(value) : null;
@@ -111,11 +106,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
         new TypeToken<List<HashMap<String, String>>>(){}.getType());
       for (Map<String, String> criteriaMap : criterias) {
         for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
-          escapeFieldValueByType(fieldEntry);
           if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
-            addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), false);
+            addLogMessageFilter(query, fieldEntry.getValue(), false);
           } else {
-            addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), false);
+            addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), false);
           }
         }
       }
@@ -129,11 +123,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
         new TypeToken<List<HashMap<String, String>>>(){}.getType());
       for (Map<String, String> criteriaMap : criterias) {
         for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) {
-          escapeFieldValueByType(fieldEntry);
           if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
-            addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), true);
+            addLogMessageFilter(query, fieldEntry.getValue(), true);
           } else {
-            addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), true);
+            addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), true);
           }
         }
       }
@@ -143,18 +136,31 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE
 
   public abstract LogType getLogType();
 
-  private String escapeFieldValueByType(Map.Entry<String, String> fieldEntry) {
-    String escapedFieldValue;
-    if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) {
-      escapedFieldValue = SolrUtil.escapeForLogMessage(fieldEntry.getValue());
-    } else {
-      escapedFieldValue = SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldEntry.getKey(), getSchemaFieldsTypeMapByLogType(getLogType()));
+  private void addLogMessageFilter(Query query, String value, boolean negate) {
+    StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"');
+    for (String token : tokenizer.getTokenArray()) {
+      token = token.trim();
+      if (token.contains(" ") || !token.startsWith("*") && !token.endsWith("*")) {
+        addFilterQuery(query, new Criteria(LOG_MESSAGE).is(SolrUtil.escapeQueryChars(token)), negate);
+      } else if (token.startsWith("*") && token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 1, -1);
+        addFilterQuery(query, new Criteria(LOG_MESSAGE).contains(SolrUtil.escapeQueryChars(plainToken)), negate);
+      } else if (token.startsWith("*") && !token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 1);
+        addFilterQuery(query, new Criteria(LOG_MESSAGE).endsWith(SolrUtil.escapeQueryChars(plainToken)), negate);
+      } else if (!token.startsWith("*") && token.endsWith("*")) {
+        String plainToken = StringUtils.substring(token, 0, -1);
+        addFilterQuery(query, new Criteria(LOG_MESSAGE).startsWith(SolrUtil.escapeQueryChars(plainToken)), negate);
+      }
     }
-    return escapedFieldValue;
   }
 
-  private Map<String, String> getSchemaFieldsTypeMapByLogType(LogType logType) {
-    return LogType.AUDIT.equals(logType) ? auditSolrSchemaFieldDao.getSchemaFieldTypeMap() : serviceSolrSchemaFieldDao.getSchemaFieldTypeMap();
+  private String escapeNonLogMessageField(Map.Entry<String, String> fieldEntry) {
+    Map<String, String> schemaFieldNameMap = solrSchemaFieldDao.getSchemaFieldNameMap(getLogType());
+    Map<String, String> schemaFieldTypeMap = solrSchemaFieldDao.getSchemaFieldTypeMap(getLogType());
+    String fieldType = schemaFieldNameMap.get(fieldEntry.getKey());
+    String fieldTypeMetaData = schemaFieldTypeMap.get(fieldType);
+    return SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldType, fieldTypeMetaData);
   }
 
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java

@@ -18,7 +18,6 @@
  */
 package org.apache.ambari.logsearch.converter;
 
-import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
 import org.apache.ambari.logsearch.model.request.impl.CommonSearchRequest;
 import org.apache.commons.lang.StringUtils;
 import org.springframework.data.domain.PageRequest;
@@ -27,9 +26,6 @@ import org.springframework.data.solr.core.query.Criteria;
 import org.springframework.data.solr.core.query.Query;
 import org.springframework.data.solr.core.query.SimpleStringCriteria;
 
-import javax.inject.Inject;
-import javax.inject.Named;
-
 public abstract class AbstractSearchRequestQueryConverter<REQUEST_TYPE extends CommonSearchRequest, QUERY_TYPE extends Query>
   extends AbstractOperationHolderConverter<REQUEST_TYPE, QUERY_TYPE> {
 

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java

@@ -48,7 +48,6 @@ public class AuditSolrDao extends SolrDaoBase {
   private SolrCollectionDao solrCollectionDao;
 
   @Inject
-  @Named("auditSolrFieldDao")
   private SolrSchemaFieldDao solrSchemaFieldDao;
 
   public AuditSolrDao() {
@@ -70,10 +69,9 @@ public class AuditSolrDao extends SolrDaoBase {
       boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection));
       solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig);
       if (createAlias) {
-        solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig);
-      } else {
-        solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig);
+        solrAliasDao.setupAlias(getSolrClient(), solrAuditLogPropsConfig);
       }
+      solrSchemaFieldDao.auditCollectionSetUp();
     } catch (Exception e) {
       LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" +
           solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e);

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java

@@ -44,7 +44,6 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
   private SolrTemplate serviceSolrTemplate;
 
   @Inject
-  @Named("serviceSolrFieldDao")
   private SolrSchemaFieldDao solrSchemaFieldDao;
 
   public ServiceLogsSolrDao() {
@@ -62,7 +61,7 @@ public class ServiceLogsSolrDao extends SolrDaoBase {
     try {
       solrCollectionDao.checkSolrStatus(getSolrClient());
       solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig);
-      solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig);
+      solrSchemaFieldDao.serviceCollectionSetUp();
     } catch (Exception e) {
       LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl()
         + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString()

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java

@@ -32,11 +32,10 @@ import javax.inject.Named;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
 
 @Named
-public class SolrAliasDao {
+class SolrAliasDao {
 
   private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class);
 
@@ -45,9 +44,9 @@ public class SolrAliasDao {
   @Inject
   private SolrCollectionDao solrCollectionDao;
 
-  protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient,
-                            final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
-    final Collection<String> collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
+  void setupAlias(final CloudSolrClient solrClient, final SolrAuditLogPropsConfig solrPropsConfig) throws Exception {
+    final Collection<String> collectionListIn =
+        Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim());
 
     if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) {
       LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " +
@@ -71,7 +70,6 @@ public class SolrAliasDao {
               if (count == collectionListIn.size()) {
                 LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " +
                   "Collections=" + collectionListIn);
-                solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig);
                 break;
               }
             } else {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java

@@ -40,7 +40,7 @@ import java.util.HashSet;
 import java.util.List;
 
 @Named
-public class SolrCollectionDao {
+class SolrCollectionDao {
 
   private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class);
 
@@ -50,7 +50,7 @@ public class SolrCollectionDao {
    * This will try to get the collections from the Solr. Ping doesn't work if
    * collection is not given
    */
-  public boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
+  boolean checkSolrStatus(CloudSolrClient cloudSolrClient) {
     int waitDurationMS = 3 * 60 * 1000;
     boolean status = false;
     try {
@@ -86,7 +86,7 @@ public class SolrCollectionDao {
     return status;
   }
 
-  public void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
+  void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception {
     boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig);
     LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus);
     if (!setupStatus) {
@@ -136,7 +136,7 @@ public class SolrCollectionDao {
   }
 
   @SuppressWarnings("unchecked")
-  public List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
+  List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException,
     IOException {
     try {
       CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java

@@ -19,123 +19,178 @@
 package org.apache.ambari.logsearch.dao;
 
 import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.LogType;
 import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.conf.SolrPropsConfig;
 import org.apache.ambari.logsearch.conf.SolrUserPropsConfig;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.LukeRequest;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
 import org.apache.solr.client.solrj.request.schema.SchemaRequest;
 import org.apache.solr.client.solrj.response.LukeResponse;
 import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo;
 import org.apache.solr.client.solrj.response.schema.SchemaResponse;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
 import org.codehaus.jettison.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.data.solr.core.SolrTemplate;
+import org.springframework.scheduling.annotation.Scheduled;
 
-import javax.inject.Inject;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+import javax.inject.Named;
+
 public class SolrSchemaFieldDao {
 
   private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class);
 
-  private static final int SETUP_RETRY_SECOND = 30;
-  private static final int SETUP_UPDATE_SECOND = 1 * 60; // 1 min
+  private static final int RETRY_SECOND = 30;
   
-  private boolean populateFieldsThreadActive = false;
-
-  private Map<String, String> schemaFieldNameMap = new HashMap<>();
-  private Map<String, String> schemaFieldTypeMap = new HashMap<>();
+  @Inject
+  @Named("serviceSolrTemplate")
+  private SolrTemplate serviceSolrTemplate;
 
   @Inject
-  private SolrUserPropsConfig solrUserPropsConfig;
+  @Named("auditSolrTemplate")
+  private SolrTemplate auditSolrTemplate;
+  
+  @Inject
+  private SolrUserPropsConfig solrUserConfigPropsConfig;
+  
+  private CloudSolrClient serviceSolrClient;
+  private CloudSolrClient auditSolrClient;
+  
+  private int retryCount;
+  private int skipCount;
 
-  public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) {
-    if (!populateFieldsThreadActive) {
-      populateFieldsThreadActive = true;
-      LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection());
-      Thread fieldPopulationThread = new Thread("populated_fields_" + solrPropsConfig.getCollection()) {
-        @Override
-        public void run() {
-          LOG.info("Started thread to get fields for collection=" + solrPropsConfig.getCollection());
-          int retryCount = 0;
-          while (true) {
-            try {
-              Thread.sleep(SETUP_RETRY_SECOND * 1000);
-              retryCount++;
-              boolean _result = _populateSchemaFields(solrClient, solrPropsConfig);
-              if (_result) {
-                LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " +
-                  SETUP_UPDATE_SECOND + " sec");
-                Thread.sleep(SETUP_UPDATE_SECOND * 1000);
-              }
-            } catch (InterruptedException sleepInterrupted) {
-              LOG.info("Sleep interrupted while populating fields for collection " + solrPropsConfig.getCollection());
-              break;
-            } catch (Exception ex) {
-              LOG.error("Error while populating fields for collection " + solrPropsConfig.getCollection() + ", retryCount=" + retryCount);
-            }
-          }
-          populateFieldsThreadActive = false;
-          LOG.info("Exiting thread for populating fields. collection=" + solrPropsConfig.getCollection());
-        }
-      };
-      fieldPopulationThread.setDaemon(true);
-      fieldPopulationThread.start();
+  private boolean serviceCollectionSetUp = false;
+  private boolean auditCollectionSetUp = false;
+  
+  private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>();
+  private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>();
+  private Map<String, String> auditSchemaFieldNameMap = new HashMap<>();
+  private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>();
+
+  @PostConstruct
+  public void init() {
+    this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient();
+    this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient();
+  }
+  
+  void serviceCollectionSetUp() {
+    this.serviceCollectionSetUp = true;
+  }
+  
+  void auditCollectionSetUp() {
+    this.auditCollectionSetUp = true;
+  }
+  
+  @Scheduled(fixedDelay = RETRY_SECOND * 1000)
+  public void populateAllSchemaFields() {
+    if (skipCount > 0) {
+      skipCount--;
+      return;
+    }
+    
+    if (serviceCollectionSetUp) {
+      populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap);
+    }
+    if (auditCollectionSetUp) {
+      populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap);
     }
   }
 
-  /**
-   * Called from the thread. Don't call this directly
-   */
-  private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) {
-    String historyCollection = solrUserPropsConfig.getCollection();
-    if (solrClient != null && !solrPropsConfig.getCollection().equals(historyCollection)) {
-      LukeResponse lukeResponse = null;
+  private void populateSchemaFields(CloudSolrClient solrClient, Map<String, String> schemaFieldNameMap,
+      Map<String, String> schemaFieldTypeMap) {
+    if (solrClient != null) {
+      LOG.debug("Started thread to get fields for collection=" + solrClient.getDefaultCollection());
+      List<LukeResponse> lukeResponses = null;
       SchemaResponse schemaResponse = null;
       try {
-        LukeRequest lukeRequest = new LukeRequest();
-        lukeRequest.setNumTerms(0);
-        lukeResponse = lukeRequest.process(solrClient);
-        
+        lukeResponses = getLukeResponsesForCores(solrClient);
+
         SolrRequest<SchemaResponse> schemaRequest = new SchemaRequest();
         schemaRequest.setMethod(SolrRequest.METHOD.GET);
         schemaRequest.setPath("/schema");
         schemaResponse = schemaRequest.process(solrClient);
         
-        LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", luke=" + lukeResponse +
+        LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses +
             ", schema= " + schemaResponse);
       } catch (SolrException | SolrServerException | IOException e) {
-        LOG.error("Error occured while popuplating field. collection=" + solrPropsConfig.getCollection(), e);
+        LOG.error("Error occured while popuplating field. collection=" + solrClient.getDefaultCollection(), e);
       }
 
-      if (lukeResponse != null && schemaResponse != null) {
-        extractSchemaFieldsName(lukeResponse, schemaResponse);
-        return true;
+      if (schemaResponse != null) {
+        extractSchemaFieldsName(lukeResponses, schemaResponse, schemaFieldNameMap, schemaFieldTypeMap);
+        LOG.debug("Populate fields for collection " + solrClient.getDefaultCollection()+ " was successful, next update it after " +
+            solrUserConfigPropsConfig.getPopulateIntervalMins() + " minutes");
+        retryCount = 0;
+        skipCount = (solrUserConfigPropsConfig.getPopulateIntervalMins() * 60) / RETRY_SECOND - 1;
+      }
+      else {
+        retryCount++;
+        LOG.error("Error while populating fields for collection " + solrClient.getDefaultCollection() + ", retryCount=" + retryCount);
       }
     }
-    return false;
+  }
+  
+  private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2";
+  
+  @SuppressWarnings("unchecked")
+  private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) {
+    ZkStateReader zkStateReader = solrClient.getZkStateReader();
+    Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection());
+    
+    List<LukeResponse> lukeResponses = new ArrayList<>();
+    for (Slice slice : activeSlices) {
+      for (Replica replica : slice.getReplicas()) {
+        try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) {
+          HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX);
+          HttpResponse response = httpClient.execute(request);
+          NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent());
+          LukeResponse lukeResponse = new LukeResponse();
+          lukeResponse.setResponse(lukeData);
+          lukeResponses.add(lukeResponse);
+        } catch (IOException e) {
+          LOG.error("Exception during getting luke responses", e);
+        }
+      }
+    }
+    return lukeResponses;
   }
 
-  private void extractSchemaFieldsName(LukeResponse lukeResponse, SchemaResponse schemaResponse) {
+  private void extractSchemaFieldsName(List<LukeResponse> lukeResponses, SchemaResponse schemaResponse,
+      Map<String, String> schemaFieldNameMap, Map<String, String> schemaFieldTypeMap) {
     try {
       HashMap<String, String> _schemaFieldNameMap = new HashMap<>();
       HashMap<String, String> _schemaFieldTypeMap = new HashMap<>();
       
-      for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
-        String name = e.getKey();
-        String type = e.getValue().getType();
-        if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") &&
-          !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) {
-          _schemaFieldNameMap.put(name, type);
+      for (LukeResponse lukeResponse : lukeResponses) {
+        for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) {
+          String name = e.getKey();
+          String type = e.getValue().getType();
+          if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") &&
+              !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) {
+            _schemaFieldNameMap.put(name, type);
+          }
         }
       }
       
@@ -162,19 +217,21 @@ public class SolrSchemaFieldDao {
       }
       
       synchronized (this) {
-        schemaFieldNameMap = _schemaFieldNameMap;
-        schemaFieldTypeMap = _schemaFieldTypeMap;
+        schemaFieldNameMap.clear();
+        schemaFieldNameMap.putAll(_schemaFieldNameMap);
+        schemaFieldTypeMap.clear();
+        schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
       }
     } catch (Exception e) {
       LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM);
     }
   }
 
-  public synchronized Map<String, String> getSchemaFieldNameMap() {
-    return schemaFieldNameMap;
+  public Map<String, String> getSchemaFieldNameMap(LogType logType) {
+    return LogType.AUDIT == logType ? auditSchemaFieldNameMap : serviceSchemaFieldNameMap;
   }
 
-  public synchronized Map<String, String> getSchemaFieldTypeMap() {
-    return schemaFieldTypeMap;
+  public Map<String, String> getSchemaFieldTypeMap(LogType logType) {
+    return LogType.AUDIT == logType ? auditSchemaFieldTypeMap : serviceSchemaFieldTypeMap;
   }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java

@@ -68,10 +68,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
   @Inject
   private SolrCollectionDao solrCollectionDao;
 
-  @Inject
-  @Named("userConfigSolrFieldDao")
-  private SolrSchemaFieldDao solrSchemaFieldDao;
-
   @Inject
   @Named("userConfigSolrTemplate")
   private SolrTemplate userConfigSolrTemplate;
@@ -94,7 +90,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
     try {
       solrCollectionDao.checkSolrStatus(getSolrClient());
       solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig);
-      solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig);
       intializeLogFeederFilter();
 
     } catch (Exception e) {
@@ -185,6 +180,6 @@ public class UserConfigSolrDao extends SolrDaoBase {
 
   @Override
   public SolrSchemaFieldDao getSolrSchemaFieldDao() {
-    return solrSchemaFieldDao;
+    throw new UnsupportedOperationException();
   }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java

@@ -90,6 +90,7 @@ public class DocConstants {
     public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields";
     public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch";
     public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data";
+    public static final String REQUEST_CANCEL = "Cancel an ongoing solr request";
   }
 
   public class PublicOperationDescriptions {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java

@@ -36,8 +36,11 @@ import java.util.Map;
 import freemarker.template.Configuration;
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
+
+import org.apache.ambari.logsearch.common.LogType;
 import org.apache.ambari.logsearch.common.MessageEnums;
 import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
 import org.apache.ambari.logsearch.graph.GraphDataGenerator;
 import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest;
 import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest;
@@ -79,6 +82,8 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
   private ConversionService conversionService;
   @Inject
   private Configuration freemarkerConfiguration;
+  @Inject
+  private SolrSchemaFieldDao solrSchemaFieldDao;
 
   public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) {
     return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs");
@@ -118,7 +123,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
   }
 
   public String getAuditLogsSchemaFieldsName() {
-    return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap());
+    return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.AUDIT));
   }
 
   public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java

@@ -28,7 +28,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Scanner;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -42,8 +41,10 @@ import freemarker.template.TemplateException;
 
 import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper;
 import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.common.LogType;
 import org.apache.ambari.logsearch.common.MessageEnums;
 import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
+import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao;
 import org.apache.ambari.logsearch.graph.GraphDataGenerator;
 import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest;
 import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest;
@@ -71,7 +72,6 @@ import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
 import org.apache.ambari.logsearch.solr.model.SolrHostLogData;
 import org.apache.ambari.logsearch.solr.model.SolrServiceLogData;
 import org.apache.ambari.logsearch.util.DownloadUtil;
-import org.apache.ambari.logsearch.util.JSONUtil;
 import org.apache.ambari.logsearch.util.DateUtil;
 import org.apache.ambari.logsearch.util.RESTErrorUtil;
 import org.apache.ambari.logsearch.util.SolrUtil;
@@ -116,6 +116,8 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
   private ConversionService conversionService;
   @Inject
   private Configuration freemarkerConfiguration;
+  @Inject
+  private SolrSchemaFieldDao solrSchemaFieldDao;
 
   public ServiceLogResponse searchLogs(ServiceLogRequest request) {
     String event = "/service/logs";
@@ -418,7 +420,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
   }
 
   public String getServiceLogsSchemaFieldsName() {
-    return convertObjToString(serviceLogsSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap());
+    return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.SERVICE));
   }
 
   public BarGraphDataListResponse getAnyGraphCountData(ServiceAnyGraphRequest request) {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java

@@ -21,6 +21,8 @@ package org.apache.ambari.logsearch.model.request.impl;
 import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.model.request.CommonSearchParamDefinition;
 import org.apache.ambari.logsearch.model.request.SearchRequest;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import javax.ws.rs.QueryParam;
 
@@ -116,4 +118,9 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi
   public void setEndTime(String endTime) {
     this.endTime = endTime;
   }
+  
+  @Override
+  public String toString() {
+    return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
+  }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java

@@ -184,4 +184,13 @@ public class ServiceLogsResource {
   public String getHadoopServiceConfigJSON() {
     return serviceLogsManager.getHadoopServiceConfigJSON();
   }
+
+  @GET
+  @Path("/request/cancel")
+  @Produces({"application/json"})
+  @ApiOperation(REQUEST_CANCEL)
+  public String cancelRequest() {
+    // TODO: create function that cancels an ongoing solr request
+    return "{\"endpoint status\": \"not supported yet\"}";
+  }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java

@@ -35,7 +35,7 @@ import org.apache.solr.schema.TrieFloatField;
 import org.apache.solr.schema.TrieIntField;
 import org.apache.solr.schema.TrieLongField;
 import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 
 public class SolrUtil {
   private SolrUtil() {
@@ -74,50 +74,14 @@ public class SolrUtil {
     if (search == null) {
       return null;
     }
-    String newString = search.trim();
-    String newSearch = escapeQueryChars(newString);
-    boolean isSingleWord = true;
-    for (int i = 0; i < search.length(); i++) {
-      if (Character.isWhitespace(search.charAt(i))) {
-        isSingleWord = false;
-      }
-    }
-    if (!isSingleWord) {
+    String newSearch = escapeQueryChars(search.trim());
+    if (StringUtils.containsWhitespace(newSearch)) {
       newSearch = "\"" + newSearch + "\"";
     }
 
     return newSearch;
   }
 
-  private static String escapeForKeyTokenizer(String search) {
-    if (search.startsWith("*") && search.endsWith("*") && StringUtils.isNotBlank(search)) {
-      // Remove the * from both the sides
-      if (search.length() > 1) {
-        search = search.substring(1, search.length() - 1);
-      } else {
-        //search string have only * 
-        search="";
-      }
-    }
-    search = escapeQueryChars(search);
-
-    return "*" + search + "*";
-  }
-
-  /**
-   * This is a special case scenario to handle log_message for wild card
-   * scenarios
-   */
-  public static String escapeForLogMessage(String search) {
-    if (search.startsWith("*") && search.endsWith("*")) {
-      search = escapeForKeyTokenizer(search);
-    } else {
-      // Use whitespace index
-      search = escapeForStandardTokenizer(search);
-    }
-    return search;
-  }
-
   private static String makeSolrSearchStringWithoutAsterisk(String search) {
     String newString = search.trim();
     String newSearch = newString.replaceAll("(?=[]\\[+&|!(){}^\"~=/$@%?:.\\\\])", "\\\\");
@@ -152,48 +116,45 @@ public class SolrUtil {
   }
   
 
-  private static boolean isSolrFieldNumber(String fieldType, Map<String, String> schemaFieldsMap) {
-    if (StringUtils.isBlank(fieldType)) {
+  private static boolean isSolrFieldNumber(Map<String, Object> fieldTypeInfoMap) {
+    if (MapUtils.isEmpty(fieldTypeInfoMap)) {
       return false;
-    } else {
-      HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType, schemaFieldsMap);
-      if (MapUtils.isEmpty(typeInfoMap)) {
-        return false;
-      }
-      String fieldTypeClassName = (String) typeInfoMap.get("class");
-      return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) ||
-             fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) ||
-             fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) ||
-             fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName());
     }
+    String fieldTypeClassName = (String) fieldTypeInfoMap.get("class");
+    return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) ||
+           fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) ||
+           fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) ||
+           fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName());
   }
 
-  public static String putWildCardByType(String str, String key, Map<String, String> schemaFieldsMap) {
-    String fieldType = schemaFieldsMap.get(key);
+  public static String putWildCardByType(String str, String fieldType, String fieldTypeMetaData) {
+    Map<String, Object> fieldTypeInfoMap = getFieldTypeInfoMap(fieldTypeMetaData);
     if (StringUtils.isNotBlank(fieldType)) {
-      if (isSolrFieldNumber(fieldType, schemaFieldsMap)) {
-        String value = putEscapeCharacterForNumber(str, fieldType, schemaFieldsMap);
+      if (isSolrFieldNumber(fieldTypeInfoMap)) {
+        String value = putEscapeCharacterForNumber(str, fieldTypeInfoMap);
         if (StringUtils.isNotBlank(value)) {
           return value;
         } else {
           return null;
         }
-      } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class, schemaFieldsMap)) {
+      } else if (checkTokenizer(StandardTokenizerFactory.class, fieldTypeInfoMap)) {
         return escapeForStandardTokenizer(str);
-      } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class, schemaFieldsMap)|| "string".equalsIgnoreCase(fieldType)) {
+      } else if (checkTokenizer(KeywordTokenizerFactory.class, fieldTypeInfoMap) || "string".equalsIgnoreCase(fieldType)) {
         return makeSolrSearchStringWithoutAsterisk(str);
-      } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class, schemaFieldsMap)) {
+      } else if (checkTokenizer(PathHierarchyTokenizerFactory.class, fieldTypeInfoMap)) {
         return str;
+      } else {
+        return escapeQueryChars(str);
       }
     }
     return str;
   }
 
-  private static String putEscapeCharacterForNumber(String str,String fieldType, Map<String, String> schemaFieldsMap) {
+  private static String putEscapeCharacterForNumber(String str, Map<String, Object> fieldTypeInfoMap) {
     if (StringUtils.isNotEmpty(str)) {
       str = str.replace("*", "");
     }
-    String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType, schemaFieldsMap);
+    String escapeCharSting = parseInputValueAsPerFieldType(str, fieldTypeInfoMap);
     if (escapeCharSting == null || escapeCharSting.isEmpty()) {
       return null;
     }
@@ -201,9 +162,8 @@ public class SolrUtil {
     return escapeCharSting;
   }
 
-  private static String parseInputValueAsPerFieldType(String str,String fieldType, Map<String, String> schemaFieldsMap) {
+  private static String parseInputValueAsPerFieldType(String str, Map<String, Object> fieldTypeInfoMap) {
     try {
-      HashMap<String, Object> fieldTypeInfoMap = SolrUtil.getFieldTypeInfoMap(fieldType, schemaFieldsMap);
       String className = (String) fieldTypeInfoMap.get("class");
       if (className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())) {
         return "" + Double.parseDouble(str);
@@ -219,8 +179,7 @@ public class SolrUtil {
     }
   }
   
-  private static HashMap<String, Object> getFieldTypeInfoMap(String fieldType, Map<String, String> schemaFieldsTypeMap) {
-    String fieldTypeMetaData = schemaFieldsTypeMap.get(fieldType);
+  private static Map<String, Object> getFieldTypeInfoMap(String fieldTypeMetaData) {
     HashMap<String, Object> fieldTypeMap = JSONUtil.jsonToMapObject(fieldTypeMetaData);
     if (fieldTypeMap == null) {
       return new HashMap<>();
@@ -269,10 +228,8 @@ public class SolrUtil {
   }
 
   @SuppressWarnings("unchecked")
-  private static boolean checkTokenizer(String fieldType, Class<? extends TokenizerFactory> tokenizerFactoryClass,
-      Map<String, String> schemaFieldsMap) {
-    HashMap<String, Object> fieldTypeMap = SolrUtil.getFieldTypeInfoMap(fieldType ,schemaFieldsMap);
-    HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap.get("analyzer");
+  private static boolean checkTokenizer(Class<? extends TokenizerFactory> tokenizerFactoryClass, Map<String, Object> fieldTypeInfoMap) {
+    HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeInfoMap.get("analyzer");
     HashMap<String, Object> tokenizerMap = (HashMap<String, Object>)MapUtils.getObject(analyzer, "tokenizer");
     if (tokenizerMap != null) {
       String tokenizerClass = (String) tokenizerMap.get("class");

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js

@@ -596,12 +596,13 @@ define(['require',
                         this.ui.find.trigger("keyup");
                         this.ui.find.focus();
                     }else if(type === "IA" || type === "EA"){
-    					this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns",
-    						key:Globals.serviceLogsColumns["log_message"],value:"*"+this.selectionText+"*"});
-    				}
+                        this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns",
+                          key:Globals.defaultServiceLogMappings["log_message"],value:"*"+this.selectionText+"*"});
+                    }
                     else {
                         //this.vent.trigger("add:include:exclude",{type:type,value:this.selectionText});
-                        this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns", key: Globals.serviceLogsColumns["log_message"], value: this.selectionText });
+                        this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns",
+                          key: Globals.defaultServiceLogMappings["log_message"], value: this.selectionText });
                     }
                     this.ui.contextMenu.hide();
                 }

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json

@@ -46,7 +46,7 @@
     "logType": "Log Type",
     "logfile_line_number": "Logfile Line Number",
     "logger_name": "Logger Name",
-    "logtime": "logtime",
+    "logtime": "Log Time",
     "path": "Path",
     "perm": "Perm",
     "policy": "Policy",
@@ -76,11 +76,13 @@
     "ws_repo_id": "Repo Id",
     "ws_repo_version": "Repo Version",
     "ws_repositories": "Repositories",
+    "ws_request_id": "Request Id",
     "ws_result_status": "Result Status",
     "ws_roles": "Roles",
     "ws_stack_version": "Stack Version",
     "ws_stack": "Stack",
     "ws_status": "Status",
+    "ws_task_id": "Task Id",
     "ws_version_note": "Version Note",
     "ws_version_number": "Version Number"
   },

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java

@@ -46,7 +46,7 @@ public class AuditLogRequestConverterTest extends AbstractRequestConverterTest {
     SolrQuery queryResult = new DefaultQueryParser().doConstructSolrQuery(simpleQuery);
     // THEN
     assertEquals("?q=*%3A*&start=0&rows=25&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=evtTime+desc%2Cseq_num+desc",
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=evtTime+desc%2Cseq_num+desc",
       queryResult.toQueryString());
   }
 

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java

@@ -43,8 +43,8 @@ public class AuditServiceLoadRequestQueryConverterTest extends AbstractRequestCo
     // WHEN
     SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
-    assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*&" +
-      "fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
+    assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage&" +
+      "fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" +
       "&facet=true&facet.mincount=1&facet.limit=10&facet.field=repo", solrQuery.toQueryString());
   }
 

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java

@@ -53,7 +53,7 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv
     SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery);
     // THEN
     assertEquals("?q=*%3A*&start=0&rows=25&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=host%3Alogsearch.com" +
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=host%3Alogsearch.com" +
       "&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" +
       "&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc",
       solrQuery.toQueryString());

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java

@@ -46,7 +46,7 @@ public class FieldAuditLogRequestQueryConverterTest extends AbstractRequestConve
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
     assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&" +
-      "fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&" +
+      "fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&" +
       "fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.pivot=myfield%2Crepo",
       query.toQueryString());
   }

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java

@@ -44,8 +44,8 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve
     // WHEN
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
-    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
-      "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+      "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
       "&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString());
   }
 

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java

@@ -45,8 +45,8 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR
     // WHEN
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
-    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
-      "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
+    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+      "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
       "&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel",
       query.toQueryString());
   }

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java

@@ -46,7 +46,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
     assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" +
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
       "&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1" +
       "&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel",
       query.toQueryString());

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java

@@ -44,7 +44,7 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
     assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" +
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" +
       "&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level",
       query.toQueryString());
   }

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java

@@ -44,8 +44,8 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques
     // WHEN
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
-    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" +
-      "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" +
+    assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" +
+      "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" +
       "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel",
       query.toQueryString());
   }

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java

@@ -47,7 +47,7 @@ public class ServiceLogTruncatedRequestQueryConverterTest extends AbstractReques
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
     assertEquals("?q=*%3A*&start=0&rows=10&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=logtime+desc%2Cseq_num+desc",
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=logtime+desc%2Cseq_num+desc",
       query.toQueryString());
   }
 

ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java

@@ -44,7 +44,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte
     SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request));
     // THEN
     assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" +
-      "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29" +
+      "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29" +
       "&fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo",
       query.toQueryString());
   }

ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json

@@ -282,16 +282,6 @@
             "new_fieldname":"cliType"
           }
         },
-        "RequestId":{
-          "map_fieldname":{
-            "new_fieldname":"request_id"
-          }
-        },
-        "TaskId":{
-          "map_fieldname":{
-            "new_fieldname":"task_id"
-          }
-        },
         "Operation":{
           "map_fieldname":{
             "new_fieldname":"action"
@@ -374,6 +364,11 @@
             "new_fieldname":"ws_repositories"
           }
         },
+        "RequestId":{
+          "map_fieldname":{
+            "new_fieldname":"ws_request_id"
+          }
+        },
         "Roles":{
           "map_fieldname":{
             "new_fieldname":"ws_roles"
@@ -389,6 +384,11 @@
             "new_fieldname":"ws_stack_version"
           }
         },
+        "TaskId":{
+          "map_fieldname":{
+            "new_fieldname":"ws_task_id"
+          }
+        },
         "VersionNote":{
           "map_fieldvalue":{
             "pre_value":"null",