AMBARI-18225. Fixed search issue for number field in Logsearch portal (Hayat Behlim via oleewere)

ambari-logsearch/ambari-logsearch-portal/pom.xml

@@ -535,6 +535,27 @@
       <artifactId>solr-solrj</artifactId>
       <version>${solr.version}</version>
     </dependency>
+    <dependency>
+    <groupId>org.apache.solr</groupId>
+    <artifactId>solr-core</artifactId>
+    <version>${solr.version}</version>
+    <exclusions>
+      <exclusion>
+        <groupId>*</groupId>
+        <artifactId>*</artifactId>
+      </exclusion>
+    </exclusions>
+  </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-core</artifactId>
+      <version>${solr.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.lucene</groupId>
+      <artifactId>lucene-analyzers-common</artifactId>
+      <version>${solr.version}</version>
+    </dependency>
     <!-- Hadoop -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java

@@ -91,8 +91,6 @@ public class LogSearchConstants {
   public static final String UI_SUFFIX = "@UI@";
   public static final String SOLR_SUFFIX = "@Solr@";
   public static final String NGRAM_SUFFIX = "ngram_";
-  public static final String DEFAULT_SERVICE_COLUMN_SUFFIX = "service"; 
-  public static final String DEFAULT_AUDIT_COLUMN_SUFFIX = "audit";
 
   //Date Format for SOLR
   public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java

@@ -22,10 +22,10 @@ package org.apache.ambari.logsearch.dao;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
-
 import org.apache.ambari.logsearch.common.LogsearchContextUtil;
 import org.apache.ambari.logsearch.common.MessageEnums;
 import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
@@ -60,7 +60,10 @@ import org.springframework.beans.factory.annotation.Autowired;
 
 public abstract class SolrDaoBase {
   static private Logger logger = Logger.getLogger(SolrDaoBase.class);
-
+  
+  public HashMap<String, String> schemaFieldsNameMap = new HashMap<String, String>();
+  public HashMap<String, String> schemaFieldTypeMap = new HashMap<String, String>();
+  
   private static Logger logPerformance = Logger
     .getLogger("org.apache.ambari.logsearch.performance");
 
@@ -95,6 +98,7 @@ public abstract class SolrDaoBase {
   private boolean populateFieldsThreadActive = false;
 
   int SETUP_RETRY_SECOND = 30;
+  int SETUP_UPDATE_SECOND = 10*60; //10 min
   int ALIAS_SETUP_RETRY_SECOND = 30*60; //30 minutes
   
   private boolean isZkConnectString=false;//by default its false
@@ -599,17 +603,16 @@ public abstract class SolrDaoBase {
   }
 
   private void populateSchemaFields() {
-    boolean result = _populateSchemaFields();
-    if (!result && !populateFieldsThreadActive) {
+    if (!populateFieldsThreadActive) {
       populateFieldsThreadActive = true;
       logger.info("Creating thread to populated fields for collection="
-        + collectionName);
+          + collectionName);
       Thread fieldPopulationThread = new Thread("populated_fields_"
-        + collectionName) {
+          + collectionName) {
         @Override
         public void run() {
           logger.info("Started thread to get fields for collection="
-            + collectionName);
+              + collectionName);
           int retryCount = 0;
           while (true) {
             try {
@@ -617,27 +620,25 @@ public abstract class SolrDaoBase {
               retryCount++;
               boolean _result = _populateSchemaFields();
               if (_result) {
-                logger.info("Populate fields for collection "
-                  + collectionName + " is success");
-                break;
+                logger.info("Populate fields for collection " + collectionName
+                    + " is success, Update it after " + SETUP_UPDATE_SECOND
+                    + " sec");
+                Thread.sleep(SETUP_UPDATE_SECOND * 1000);
               }
             } catch (InterruptedException sleepInterrupted) {
-              logger.info("Sleep interrupted while populating fields for collection "
-                + collectionName);
+              logger
+                  .info("Sleep interrupted while populating fields for collection "
+                      + collectionName);
               break;
             } catch (Exception ex) {
               logger.error("Error while populating fields for collection "
-                + collectionName
-                + ", retryCount="
-                + retryCount);
-            } finally {
-              populateFieldsThreadActive = false;
+                  + collectionName + ", retryCount=" + retryCount);
             }
           }
+          populateFieldsThreadActive = false;
           logger.info("Exiting thread for populating fields. collection="
-            + collectionName);
+              + collectionName);
         }
-
       };
       fieldPopulationThread.setDaemon(true);
       fieldPopulationThread.start();
@@ -650,7 +651,7 @@ public abstract class SolrDaoBase {
   private boolean _populateSchemaFields() {
     SolrRequest<SchemaResponse> request = new SchemaRequest();
     request.setMethod(METHOD.GET);
-    request.setPath("/schema/fields");
+    request.setPath("/schema");
     String historyCollection = PropertiesUtil.getProperty("logsearch.solr.collection.history","history");
     if (solrClient != null && !collectionName.equals(historyCollection)) {
       NamedList<Object> namedList = null;
@@ -659,14 +660,13 @@ public abstract class SolrDaoBase {
         logger.info("populateSchemaFields() collection="
           + collectionName + ", fields=" + namedList);
       } catch (SolrException | SolrServerException | IOException e) {
-        
         logger.error(
           "Error occured while popuplating field. collection="
             + collectionName, e);
       }
       if (namedList != null) {
         ConfigUtil.extractSchemaFieldsName(namedList.toString(),
-          collectionName);
+          schemaFieldsNameMap,schemaFieldTypeMap);
         return true;
       }
     }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java

@@ -192,6 +192,9 @@ public class UserDao {
    * @return
    */
   public String encryptPassword(String username, String password) {
+    if (!stringUtil.isEmpty(username)) {
+      username = username.toLowerCase();
+    }
     String saltEncodedpasswd = md5Encoder
       .encodePassword(password, username);
     return saltEncodedpasswd;

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java

@@ -32,6 +32,7 @@ import org.apache.ambari.logsearch.query.QueryGeneration;
 import org.apache.ambari.logsearch.util.ConfigUtil;
 import org.apache.ambari.logsearch.util.DateUtil;
 import org.apache.ambari.logsearch.util.RESTErrorUtil;
+import org.apache.ambari.logsearch.util.SolrUtil;
 import org.apache.ambari.logsearch.util.StringUtil;
 import org.apache.ambari.logsearch.view.VBarDataList;
 import org.apache.ambari.logsearch.view.VBarGraphData;
@@ -45,9 +46,12 @@ import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.client.solrj.response.RangeFacet;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.TextField;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
+
+
 @Component
 public class GraphDataGenerator extends GraphDataGeneratorBase {
 
@@ -62,6 +66,9 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
 
   @Autowired
   DateUtil dateUtil;
+  
+  @Autowired
+  SolrUtil solrUtil;
 
   private static Logger logger = Logger.getLogger(GraphDataGenerator.class);
 
@@ -80,8 +87,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
     String from = (String) searchCriteria.getParamValue("from");
     String to = (String) searchCriteria.getParamValue("to");
     String unit = (String) searchCriteria.getParamValue("unit");
-    String suffix = (String) searchCriteria.getParamValue("suffix");
-    String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix);
+    String typeXAxis = solrDaoBase.schemaFieldsNameMap.get(xAxisField);
     typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis;
 
     // Y axis credentials
@@ -102,8 +108,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
       return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to,
           solrDaoBase, typeXAxis, fieldTime, solrQuery);
     case RANGE_STACK_GRAPH:
-      return rangeStackGraph(xAxisField, yAxisField, stackField, from, to,
-          unit, solrDaoBase, typeXAxis, fieldTime, solrQuery);
+      return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery);
     default:
       logger.warn("Invalid graph type :" + garphType.name());
       return null;
@@ -255,7 +260,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
     queryGenerator.setMainQuery(solrQuery, mainQuery);
     queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX);
     String jsonQuery = "";
-    if (isTypeNumber(typeXAxis)) {
+    if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
       String function = (yAxisField.contains("count")) ? "sum" : yAxisField;
       jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField,
           xAxisField, function);
@@ -323,7 +328,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
     VBarGraphData vBarGraphData = new VBarGraphData();
     Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
     queryGenerator.setMainQuery(solrQuery, null);
-    if (isTypeNumber(typeXAxis)) {
+    if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) {
       queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to);
       return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase,
           typeXAxis, fieldTime, solrQuery);
@@ -363,9 +368,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
   }
 
   @SuppressWarnings("unchecked")
-  private VBarDataList rangeStackGraph(String xAxisField, String yAxisField,
-      String stackField, String from, String to, String unit,
-      SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime,
+  private VBarDataList rangeStackGraph(String xAxisField, String stackField,
+      String from, String to, String unit, SolrDaoBase solrDaoBase,
       SolrQuery solrQuery) {
     VBarDataList dataList = new VBarDataList();
     List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java

@@ -468,12 +468,11 @@ public class AuditMgr extends MgrBase {
   }
 
   public String getAuditLogsSchemaFieldsName() {
-    String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
     String excludeArray[] = PropertiesUtil
         .getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist");
     List<String> fieldNames = new ArrayList<String>();
     HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>();
-    ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+    ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,auditSolrDao);
 
     for (String fieldName : fieldNames) {
       String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName
@@ -492,15 +491,12 @@ public class AuditMgr extends MgrBase {
 
   public String getAnyGraphData(SearchCriteria searchCriteria) {
     searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME);
-    String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
-    searchCriteria.addParam("suffix", suffix);
     SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
     VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
         auditSolrDao, solrQuery);
     if (result == null) {
       result = new VBarDataList();
     }
-
     return convertObjToString(result);
 
   }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java

@@ -1626,12 +1626,11 @@ public class LogsMgr extends MgrBase {
   public String getServiceLogsSchemaFieldsName() {
 
     List<String> fieldNames = new ArrayList<String>();
-    String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
     String excludeArray[] = PropertiesUtil
         .getPropertyStringList("logsearch.solr.service.logs.exclude.columnlist");
 
     HashMap<String, String> uiFieldColumnMapping = new LinkedHashMap<String, String>();
-    ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames);
+    ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,serviceLogsSolrDao);
 
     for (String fieldName : fieldNames) {
       String uiField = ConfigUtil.serviceLogsColumnMapping.get(fieldName
@@ -1695,8 +1694,6 @@ public class LogsMgr extends MgrBase {
 
   public String getAnyGraphData(SearchCriteria searchCriteria) {
     searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME);
-    String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
-    searchCriteria.addParam("suffix", suffix);
     SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
     VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria,
         serviceLogsSolrDao, solrQuery);

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java

@@ -27,11 +27,19 @@ import java.util.regex.Pattern;
 
 import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
 import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE;
 import org.apache.ambari.logsearch.util.ConfigUtil;
 import org.apache.ambari.logsearch.util.PropertiesUtil;
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
+import org.apache.lucene.analysis.core.KeywordTokenizerFactory;
+import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory;
+import org.apache.lucene.analysis.standard.StandardTokenizerFactory;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieLongField;
 import org.springframework.stereotype.Component;
 
 @Component
@@ -245,6 +253,10 @@ public class QueryGeneration extends QueryGenerationBase {
               String value = getOriginalValue(originalKey,
                   "" + columnListMap.get(key));
               orQuery = putWildCardByType(value, originalKey, logType);
+              if (stringUtil.isEmpty(orQuery)) {
+                logger.debug("Removing invalid filter for key :"+originalKey +" and value :" +value );
+                continue;
+              }
               boolean isSame = false;
               if (elments.contains(key)) {
                 isSame = true;
@@ -337,48 +349,72 @@ public class QueryGeneration extends QueryGenerationBase {
   }
 
   private String putWildCardByType(String str, String key, LOG_TYPE logType) {
-    String type;
+    String fieldType;
+    SolrDaoBase solrDaoBase = null;
     switch (logType) {
     case AUDIT:
-      String auditSuffix = PropertiesUtil
-          .getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX);
-      type = ConfigUtil.schemaFieldsName.get(key + auditSuffix);
+      fieldType = auditSolrDao.schemaFieldsNameMap.get(key);
+      solrDaoBase = auditSolrDao;
       break;
     case SERVICE:
-      String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX);
-      type = ConfigUtil.schemaFieldsName.get(key + serviceLogs);
+      fieldType = serviceLogsSolrDao.schemaFieldsNameMap.get(key);
+      solrDaoBase = serviceLogsSolrDao;
       if (key.equalsIgnoreCase(LogSearchConstants.SOLR_LOG_MESSAGE)) {
         return solrUtil.escapeForLogMessage(key, str);
       }
       break;
     default:
       // set as null
-      type = null;
+      logger.error("Invalid logtype :" + logType);
+      fieldType = null;
     }
-    if (type == null) {
-      return key + ":" + "*" + str + "*";
-    } else if ("text_std_token_lower_case".equalsIgnoreCase(type)) {
-      return key + ":" + solrUtil.escapeForStandardTokenizer(str);
-    } else if ("key_lower_case".equalsIgnoreCase(type)
-        || "string".equalsIgnoreCase(type)) {
-      return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
-    } else if ("ip_address".equalsIgnoreCase(type)) {
-      return key + ":" + str;
+    if (!stringUtil.isEmpty(fieldType)) {
+      if (solrUtil.isSolrFieldNumber(fieldType, solrDaoBase)) {
+        String value = putEscapeCharacterForNumber(str, fieldType,solrDaoBase);
+        if (!stringUtil.isEmpty(value)) {
+          return key + ":" + value;
+        } else {
+          return null;
+        }
+      } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class,solrDaoBase)) {
+        return key + ":" + solrUtil.escapeForStandardTokenizer(str);
+      } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class,solrDaoBase)|| "string".equalsIgnoreCase(fieldType)) {
+        return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str);
+      } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class,solrDaoBase)) {
+        return key + ":" + str;
+      }
     }
-    return key + ":" + putEscapeCharacterForNumber(str);
+   return key + ":" + "*" + str + "*";
   }
 
-  private String putEscapeCharacterForNumber(String str) {
-    String escapeCharSting = "" + returnDefaultIfValueNotNumber(str);
-    escapeCharSting = str.replace("-", "\\-");
+  private String putEscapeCharacterForNumber(String str,String fieldType,SolrDaoBase solrDaoBase) {
+    if (!stringUtil.isEmpty(str)) {
+      str = str.replace("*", "");
+    }
+    String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType,solrDaoBase);
+    if (escapeCharSting == null || escapeCharSting.isEmpty()) {
+      return null;
+    }
+    escapeCharSting = escapeCharSting.replace("-", "\\-");
     return escapeCharSting;
   }
 
-  private String returnDefaultIfValueNotNumber(String str) {
+  private String parseInputValueAsPerFieldType(String str,String fieldType,SolrDaoBase solrDaoBase ) {
     try {
-      return "" + Integer.parseInt(str);
+      HashMap<String, Object> fieldTypeInfoMap= solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+      String className = (String) fieldTypeInfoMap.get("class");
+      if( className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())){
+        return ""+ Double.parseDouble(str);
+      }else if(className.equalsIgnoreCase(TrieFloatField.class.getSimpleName())){
+        return ""+ Float.parseFloat(str);
+      }else if(className.equalsIgnoreCase(TrieLongField.class.getSimpleName())){
+        return ""+ Long.parseLong(str);
+      }else {
+        return "" + Integer.parseInt(str);
+      }
     } catch (Exception e) {
-      return "0";
+      logger.debug("Invaid input str: " + str + " For fieldType :" + fieldType);
+      return null;
     }
   }
 
@@ -436,4 +472,25 @@ public class QueryGeneration extends QueryGenerationBase {
     }
     return originalKey;
   }
+  
+  public boolean checkTokenizer(String fieldType,Class tokenizerFactoryClass,SolrDaoBase solrDaoBase) {
+    HashMap<String, Object> fieldTypeMap = solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase);
+    HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap
+        .get("analyzer");
+    if (analyzer != null) {
+      HashMap<String, Object> tokenizerMap = (HashMap<String, Object>) analyzer
+          .get("tokenizer");
+      if (tokenizerMap != null) {
+        String tokenizerClass = (String) tokenizerMap.get("class");
+        if (!StringUtils.isEmpty(tokenizerClass)) {
+          tokenizerClass =tokenizerClass.replace("solr.", "");
+          if (tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass
+              .getSimpleName())) {
+            return true;
+          }
+        }
+      }
+    }
+    return false;
+  }
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java

@@ -24,6 +24,8 @@ import java.util.List;
 
 import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.common.SearchCriteria;
+import org.apache.ambari.logsearch.dao.AuditSolrDao;
+import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
 import org.apache.ambari.logsearch.util.JSONUtil;
 import org.apache.ambari.logsearch.util.QueryBase;
 import org.apache.ambari.logsearch.util.SolrUtil;
@@ -34,6 +36,7 @@ import org.apache.solr.client.solrj.SolrQuery.ORDER;
 import org.springframework.beans.factory.annotation.Autowired;
 
 import com.google.gson.Gson;
+import com.sun.jersey.api.spring.Autowire;
 
 public abstract class QueryGenerationBase extends QueryBase {
 
@@ -47,6 +50,14 @@ public abstract class QueryGenerationBase extends QueryBase {
 
   @Autowired
   JSONUtil jsonUtil;
+  
+  @Autowired
+  AuditSolrDao auditSolrDao;
+  
+  @Autowired
+  ServiceLogsSolrDao serviceLogsSolrDao;
+  
+  
 
   public static enum CONDITION {
     OR, AND

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java

@@ -23,9 +23,9 @@ import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-
 import org.apache.ambari.logsearch.common.LogSearchConstants;
 import org.apache.ambari.logsearch.common.MessageEnums;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
 import org.apache.ambari.logsearch.manager.MgrBase;
 import org.apache.log4j.Logger;
 import org.codehaus.jettison.json.JSONArray;
@@ -38,7 +38,6 @@ public class ConfigUtil {
 
   public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>();
 
-  public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>();
 
   public static void initializeApplicationConfig() {
     initializeColumnMapping();
@@ -78,57 +77,63 @@ public class ConfigUtil {
       auditLogsColumnMapping);
   }
 
+  
   public static void extractSchemaFieldsName(String responseString,
-                                             String suffix) {
+      HashMap<String, String> schemaFieldsNameMap,
+      HashMap<String, String> schemaFieldTypeMap) {
     try {
       JSONObject jsonObject = new JSONObject(responseString);
-      JSONArray jsonArrayList = jsonObject.getJSONArray("fields");
-      
-      if(jsonArrayList == null){
+      JSONObject schemajsonObject = jsonObject.getJSONObject("schema");
+      JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields");
+      JSONArray fieldTypeJsonArray = schemajsonObject
+          .getJSONArray("fieldTypes");
+      if (jsonArrayList == null) {
+        return;
+      }
+      if (fieldTypeJsonArray == null) {
         return;
       }
+      HashMap<String, String> _schemaFieldTypeMap = new HashMap<String, String>();
+      HashMap<String, String> _schemaFieldsNameMap = new HashMap<String, String>();
+      for (int i = 0; i < fieldTypeJsonArray.length(); i++) {
+        JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i);
+        String name = typeObject.getString("name");
+        String fieldTypeJson = typeObject.toString();
+        _schemaFieldTypeMap.put(name, fieldTypeJson);
+      }
 
       for (int i = 0; i < jsonArrayList.length(); i++) {
         JSONObject explrObject = jsonArrayList.getJSONObject(i);
         String name = explrObject.getString("name");
         String type = explrObject.getString("type");
-
         if (!name.contains("@") && !name.startsWith("_")
-          && !name.contains("_md5") && !name.contains("_ms")
-          && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
-          && !name.contains("tags") && !name.contains("_str")) {
-          schemaFieldsName.put(name + suffix, type);
+            && !name.contains("_md5") && !name.contains("_ms")
+            && !name.contains(LogSearchConstants.NGRAM_SUFFIX)
+            && !name.contains("tags") && !name.contains("_str")) {
+          _schemaFieldsNameMap.put(name, type);
         }
       }
-
+      schemaFieldsNameMap.clear();
+      schemaFieldTypeMap.clear();
+      schemaFieldsNameMap.putAll(_schemaFieldsNameMap);
+      schemaFieldTypeMap.putAll(_schemaFieldTypeMap);
     } catch (Exception e) {
-
       logger.error(e + "Credentials not specified in logsearch.properties "
-        + MessageEnums.ERROR_SYSTEM);
-
+          + MessageEnums.ERROR_SYSTEM);
     }
-
   }
 
   @SuppressWarnings("rawtypes")
-  public static void getSchemaFieldsName(String suffix, String excludeArray[],
-                                         List<String> fieldNames) {
-    if (!schemaFieldsName.isEmpty()) {
-      Iterator iteratorSechmaFieldsName = schemaFieldsName.entrySet()
-        .iterator();
-
+  public static void getSchemaFieldsName(String excludeArray[],
+      List<String> fieldNames, SolrDaoBase solrDaoBase) {
+    if (!solrDaoBase.schemaFieldsNameMap.isEmpty()) {
+      Iterator iteratorSechmaFieldsName = solrDaoBase.schemaFieldsNameMap
+          .entrySet().iterator();
       while (iteratorSechmaFieldsName.hasNext()) {
-
-        Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName
-          .next();
+        Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName.next();
         String field = "" + fieldName.getKey();
-
-        if (field.contains(suffix)) {
-          field = field.replace(suffix, "");
-          if (!isExclude(field, excludeArray)) {
-            fieldNames.add(field);
-          }
-
+        if (!isExclude(field, excludeArray)) {
+          fieldNames.add(field);
         }
       }
     }
@@ -144,5 +149,4 @@ public class ConfigUtil {
     }
     return false;
   }
-
 }

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java

@@ -20,10 +20,16 @@
 package org.apache.ambari.logsearch.util;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.Locale;
 
 import org.apache.ambari.logsearch.common.LogSearchConstants;
+import org.apache.ambari.logsearch.dao.SolrDaoBase;
 import org.apache.log4j.Logger;
+import org.apache.solr.schema.TrieDoubleField;
+import org.apache.solr.schema.TrieFloatField;
+import org.apache.solr.schema.TrieIntField;
+import org.apache.solr.schema.TrieLongField;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -34,6 +40,9 @@ public class SolrUtil {
   @Autowired
   StringUtil stringUtil;
 
+  @Autowired
+  JSONUtil jsonUtil;
+
   public String setField(String fieldName, String value) {
     if (value == null || value.trim().length() == 0) {
       return "";
@@ -296,4 +305,49 @@ public class SolrUtil {
 
     return newSearch.replace(" ", "\\ ");
   }
+  
+
+  public boolean isSolrFieldNumber(String fieldType,SolrDaoBase solrDaoBase) {
+    if (stringUtil.isEmpty(fieldType)) {
+      return false;
+    } else {
+      HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType,solrDaoBase);
+      if (typeInfoMap == null || typeInfoMap.isEmpty()) {
+        return false;
+      }
+      String fieldTypeClassName = (String) typeInfoMap.get("class");
+      if (fieldTypeClassName.equalsIgnoreCase(TrieIntField.class
+          .getSimpleName())) {
+        return true;
+      }
+      if (fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class
+          .getSimpleName())) {
+        return true;
+      }
+      if (fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class
+          .getSimpleName())) {
+        return true;
+      }
+      if (fieldTypeClassName.equalsIgnoreCase(TrieLongField.class
+          .getSimpleName())) {
+        return true;
+      }
+      return false;
+    }
+  }
+  
+  public HashMap<String, Object> getFieldTypeInfoMap(String fieldType,SolrDaoBase solrDaoBase) {
+    String fieldTypeMetaData = solrDaoBase.schemaFieldTypeMap.get(fieldType);
+    HashMap<String, Object> fieldTypeMap = jsonUtil
+        .jsonToMapObject(fieldTypeMetaData);
+    if (fieldTypeMap == null) {
+      return new HashMap<String, Object>();
+    }
+    String classname = (String) fieldTypeMap.get("class");
+    if (!stringUtil.isEmpty(classname)) {
+      classname = classname.replace("solr.", "");
+      fieldTypeMap.put("class", classname);
+    }
+    return fieldTypeMap;
+  }
 }