AMBARI-18161. Support UNKNOWN log level for Log Search (Dharmesh Makwana via oleewere)

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/OutputMgr.java

@@ -29,6 +29,7 @@ import java.util.UUID;
 
 import org.apache.ambari.logfeeder.input.Input;
 import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
 import org.apache.ambari.logfeeder.logconfig.filter.FilterLogData;
 import org.apache.ambari.logfeeder.output.Output;
 import org.apache.log4j.Level;
@@ -86,7 +87,11 @@ public class OutputMgr {
     if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
       jsonObj.put("ip", LogFeederUtil.ipAddress);
     }
-
+    
+    //Add level
+    if (jsonObj.get("level") == null) {
+      jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
+    }
     if (input.isUseEventMD5() || input.isGenEventMD5()) {
       String prefix = "";
       Object logtimeObj = jsonObj.get("logtime");

ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java

@@ -22,6 +22,7 @@ public class LogFeederConstants {
 
   public static final String ALL = "all";
   public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
+  public static final String LOG_LEVEL_UNKNOWN = "UNKNOWN";
   // solr fields
   public static final String SOLR_LEVEL = "level";
   public static final String SOLR_COMPONENT = "type";

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java

@@ -27,6 +27,9 @@ public class LogSearchConstants {
   public static final String ERROR = "ERROR";
   public static final String TRACE = "TRACE";
   public static final String FATAL = "FATAL";
+  public static final String UNKNOWN = "UNKNOWN";
+  
+  public static final String[] SUPPORTED_LOG_LEVEL ={FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN};
 
   // Application Constants
   public static final String HOST = "H";

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java

@@ -188,15 +188,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
           }
         }
         if (xAxisField.equalsIgnoreCase(LogSearchConstants.SOLR_LEVEL)) {
-          List<String> logLevels = new ArrayList<String>();
-          logLevels.add(LogSearchConstants.FATAL);
-          logLevels.add(LogSearchConstants.ERROR);
-          logLevels.add(LogSearchConstants.WARN);
-          logLevels.add(LogSearchConstants.INFO);
-          logLevels.add(LogSearchConstants.DEBUG);
-          logLevels.add(LogSearchConstants.TRACE);
           Collection<VNameValue> sortedVNameValues = new ArrayList<VNameValue>();
-          for (String level : logLevels) {
+          for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
             VNameValue value = new VNameValue();
             value.setName(level);
             String val = "0";
@@ -292,17 +285,10 @@ public class GraphDataGenerator extends GraphDataGeneratorBase {
       if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField)
           && LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(xAxisField)) {
         Collection<VBarGraphData> levelVGraphData = dataList.getGraphData();
-        List<String> logLevels = new ArrayList<String>();
-        logLevels.add(LogSearchConstants.FATAL);
-        logLevels.add(LogSearchConstants.ERROR);
-        logLevels.add(LogSearchConstants.WARN);
-        logLevels.add(LogSearchConstants.INFO);
-        logLevels.add(LogSearchConstants.DEBUG);
-        logLevels.add(LogSearchConstants.TRACE);
         for (VBarGraphData garphData : levelVGraphData) {
           Collection<VNameValue> valueList = garphData.getDataCount();
           Collection<VNameValue> valueListSorted = new ArrayList<VNameValue>();
-          for (String level : logLevels) {
+          for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
             String val = "0";
             for (VNameValue value : valueList) {
               if (value.getName().equalsIgnoreCase(level)) {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java

@@ -540,84 +540,25 @@ public class LogsMgr extends MgrBase {
     HashMap<String, String> map = new HashMap<String, String>();
     List<VNameValue> logsCounts = new ArrayList<VNameValue>();
     try {
-
       queryGenerator.setFacetField(query, LogSearchConstants.SOLR_LEVEL);
-
       List<Count> logLevelCounts = getFacetCounts(query,
-        LogSearchConstants.SOLR_LEVEL);
-      if(logLevelCounts == null){
+          LogSearchConstants.SOLR_LEVEL);
+      if (logLevelCounts == null) {
         return logsCounts;
       }
       for (Count count : logLevelCounts) {
         map.put(count.getName().toUpperCase(), "" + count.getCount());
       }
-      String level = LogSearchConstants.FATAL;
-      VNameValue nameValue = null;
-
-      String value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
-      }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
-      level = LogSearchConstants.ERROR;
-
-      value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
-      }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
-      level = LogSearchConstants.WARN;
-
-      value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
-      }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
-      level = LogSearchConstants.INFO;
-
-      value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
-      }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
-      level = LogSearchConstants.DEBUG;
-
-      value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
-      }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
-      level = LogSearchConstants.TRACE;
-
-      value = map.get(level);
-      if (stringUtil.isEmpty(value)){
-        value = defalutValue;
+      for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
+        VNameValue nameValue = new VNameValue();
+        String value = map.get(level);
+        if (stringUtil.isEmpty(value)) {
+          value = defalutValue;
+        }
+        nameValue.setName(level);
+        nameValue.setValue(value);
+        logsCounts.add(nameValue);
       }
-      nameValue = new VNameValue();
-      nameValue.setName(level);
-      nameValue.setValue(value);
-      logsCounts.add(nameValue);
-
     } catch (SolrException | SolrServerException | IOException e) {
       logger.error("Error during solrQuery=" + query, e);
     }
@@ -1260,7 +1201,6 @@ public class LogsMgr extends MgrBase {
     String unit = getUnit((String) searchCriteria.getParamValue("unit"));
 
     List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>();
-    List<String> logLevels = ConfigUtil.logLevels;
 
     String jsonHistogramQuery = queryGenerator
       .buildJSONFacetTermTimeRangeQuery(
@@ -1287,7 +1227,7 @@ public class LogsMgr extends MgrBase {
 
       Collection<VNameValue> vNameValues = new ArrayList<VNameValue>();
       List<VBarGraphData> graphDatas = new ArrayList<VBarGraphData>();
-      for (String level : logLevels) {
+      for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) {
         boolean isLevelPresent = false;
         VBarGraphData vData1 = null;
         for (VBarGraphData vData2 : histogramData) {

ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java

@@ -19,7 +19,6 @@
 
 package org.apache.ambari.logsearch.util;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -35,8 +34,6 @@ import org.codehaus.jettison.json.JSONObject;
 public class ConfigUtil {
   static Logger logger = Logger.getLogger(MgrBase.class);
 
-  public static List<String> logLevels = new ArrayList<String>();
-
   public static HashMap<String, String> serviceLogsColumnMapping = new HashMap<String, String>();
 
   public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>();
@@ -44,7 +41,6 @@ public class ConfigUtil {
   public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>();
 
   public static void initializeApplicationConfig() {
-    intializeLogLevels();
     initializeColumnMapping();
   }
 
@@ -67,16 +63,6 @@ public class ConfigUtil {
       }
     }
   }
-
-  private static void intializeLogLevels() {
-    logLevels.add(LogSearchConstants.TRACE);
-    logLevels.add(LogSearchConstants.DEBUG);
-    logLevels.add(LogSearchConstants.INFO);
-    logLevels.add(LogSearchConstants.WARN);
-    logLevels.add(LogSearchConstants.ERROR);
-    logLevels.add(LogSearchConstants.FATAL);
-  }
-
   private static void initializeColumnMapping() {
     String serviceLogsColumnMappingArray[] = PropertiesUtil
       .getPropertyStringList("logsearch.solr.service.logs.column.mapping");

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/common/Header.js

@@ -172,7 +172,7 @@ define(['require',
                         content: view,
                         viewType: 'Filter',
                         resizable: false,
-                        width: 950,
+                        width: 1000,
                         height: 550,
                         autoFocus1stElement : false,
                         buttons: [{

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelBoxView.js

@@ -44,6 +44,7 @@ define(['require',
 			DEBUG : "[data-id='DEBUG']",
 			FATAL : "[data-id='FATAL']",
 			TRACE : "[data-id='TRACE']",
+			UNKNOWN : "[data-id='UNKNOWN']",
 			loader:".server-info .fa-spin"
 		},
 
@@ -80,7 +81,7 @@ define(['require',
 			}
 		},
 		fetchLogLevelCounts : function(params){
-			$.extend(this.logLevelList.queryParams,params,{level: "FATAL,ERROR,WARN,INFO,DEBUG,TRACE"});
+			$.extend(this.logLevelList.queryParams,params,{level: "FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN"});
 			this.ui.loader.show();
 			this.logLevelList.fetch({reset:true});
 		},

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dashboard/LogLevelView.js

@@ -45,6 +45,7 @@ define(['require',
 			DEBUG : "[data-id='DEBUG']",
 			FATAL : "[data-id='FATAL']",
 			TRACE : "[data-id='TRACE']",
+			UNKNOWN : "[data-id='UNKNOWN']",
 			togglePieViewButton:'#logToggle',
 			pieRegionId:'#r_logLevelPieChart',
 			logTable:'#logTable'

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/filter/CreateLogfeederFilterView.js

@@ -81,7 +81,7 @@ define(['require',
                 this.model = new VUserFilter();
 
                 this.levelCollection = new Backbone.Collection();
-                var levelArr = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"];
+                var levelArr = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE", "UNKNOWN"];
 
                 for (var i in levelArr) {
                     this.levelCollection.add(new Backbone.Model({ type: levelArr[i] }));

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GraphLayoutView.js

@@ -255,7 +255,8 @@ define(['require',
                             (model.get('name') === 'WARN') ? ("#FF8916") :
                             (model.get('name') === 'FATAL') ? ("#830A0A") :
                             (model.get('name') === 'DEBUG') ? ("#65E8FF") :
-                            (model.get('name') === 'TRACE') ? ("#888888") : ("white"));
+                            (model.get('name') === 'TRACE') ? ("#888888") : 
+                            (model.get('name') === 'UNKNOWN') ? ("#bdbdbd") : ("white"));
                     } else {
                         Obj['color'] = color[i];
                     }

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/graphs/GridGraphLayoutView.js

@@ -203,7 +203,8 @@ define(['require',
                                    ( (""+model.get('name')).toUpperCase() === 'WARN') ? ("#FF8916") :
                                    ( (""+model.get('name')).toUpperCase() === 'FATAL') ? ("#830A0A") :
                                    ( (""+model.get('name')).toUpperCase() === 'DEBUG') ? ("#65E8FF") :
-                                   ( (""+model.get('name')).toUpperCase() === 'TRACE') ? ("#888888") : color[i]);
+                                   ( (""+model.get('name')).toUpperCase() === 'TRACE') ? ("#888888") :
+                                   ( (""+model.get('name')).toUpperCase() === 'UNKNOWN') ? ("#bdbdbd") : color[i]);
                            } else {
                                Obj['color'] = color[i];
                            }

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js

@@ -495,7 +495,7 @@ define(['require',
                         direction: "descending",
                         orderable: true,
                         displayOrder: 1,
-                        width: "10",
+                        width: "17",
                         className: "logTime",
                         formatter: _.extend({}, Backgrid.CellFormatter.prototype, {
                             fromRaw: function(rawValue, model) {

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/troubleshoot/TroubleShootLayoutView.js

@@ -369,7 +369,8 @@ define(['backbone',
                                 ( (""+name).toUpperCase() === 'WARN') ? ("#FF8916") :
                                 ( (""+name).toUpperCase() === 'FATAL') ? ("#830A0A") :
                                 ( (""+name).toUpperCase() === 'DEBUG') ? ("#65E8FF") :
-                                ( (""+name).toUpperCase() === 'TRACE') ? ("#888888") : "")
+                                ( (""+name).toUpperCase() === 'TRACE') ? ("#888888") :
+                                ( (""+name).toUpperCase() === 'UNKNOWN') ? ("#bdbdbd") : "")
             			});
             			
             		}

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/styles/style.css

@@ -261,7 +261,9 @@ p.log-line:before {
 .bubbleGraph .label,.node--root,.node--leaf {
   pointer-events: none;
 }
-
+.UNKNOWN {
+  fill: #bdbdbd;
+}
 .TRACE {
   fill: #888888;
 }
@@ -489,6 +491,9 @@ div.columnmanager-visibilitycontrol {
   height: 100%;
   float: left;
 }
+.node.UNKNOWN {
+  background-color: #bdbdbd;
+}
 
 .node.TRACE {
   background-color: #888888;
@@ -514,6 +519,9 @@ div.columnmanager-visibilitycontrol {
   background-color: #830A0A;
 }
 
+.label-UNKNOWN {
+  background-color: #bdbdbd;
+}
 .label-TRACE {
   background-color: #888888;
 }
@@ -589,6 +597,15 @@ div.columnmanager-visibilitycontrol {
   top: -3px;
   right: 0;
 }
+.server-info a.node.UNKNOWN {
+  border-color: #bdbdbd;
+  color: #bdbdbd;
+}
+
+.server-info a.node.UNKNOWN.active {
+  background-color: #bdbdbd;
+  color: #fff;
+}
 
 .server-info a.node.TRACE {
   border-color: #888888;
@@ -649,6 +666,9 @@ div.columnmanager-visibilitycontrol {
   background-color: #830A0A;
   color: #fff;
 }
+.UNKNOWN {
+  color: #bdbdbd;
+}
 
 .TRACE {
   color: #888888;
@@ -2348,19 +2368,19 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
 }
 
 .logFeederTable {
-  width: 906px;
+  width: 965px;
   display: block;
 }
 
 .logFeederTable tbody {
   height: 365px;
   overflow-y: auto;
-  width: 906px;
+  width: 965px;
   display: block;
 }
 
 .logFeederTable th {
-  width: 90px;
+  width: 106px;
   border-left: 1px solid #CCC;
   border-right: 1px solid #CCC;
   border-top: 1px solid #CCC !important;
@@ -2376,7 +2396,7 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
 }
 
 .logFeederTable th:nth-child(2) {
-  width: 190px;
+  width: 165px;
   border-top: 1px solid #CCC !important;
   border-bottom: 1px solid #CCC !important;
 }
@@ -2391,12 +2411,12 @@ input:-webkit-autofill,input:-webkit-autofill:hover,input:-webkit-autofill:focus
 }
 
 .logFeederTable td:first-of-type {
-  width: 269px;
+  width: 240px;
   padding: 0 10px;
 }
 
 .logFeederTable td:nth-child(2) {
-  width: 178px;
+  width: 135px;
   padding: 0 10px;
 }
 
@@ -2929,4 +2949,16 @@ button.defaultCancelBtn{
 }
 .comparisonTab .daterangepicker .ranges{
   float: right !important;
+}
+
+/*------------New Log Levels Tab CSS--------------*/
+.levelBox{
+  width: 13%;
+  margin: 0 .6%;
+  float: left;
+  position: relative;
+  min-height: 1px;
+}
+.levelBox:last-of-type{
+  clear : right;
 }

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/dashboard/LogLevelBoxView_tmpl.html

@@ -16,29 +16,34 @@
 -->
 <div class="server-info">
 	<a href="javascript:void(0);"
-		class="col-md-2 node FATAL text-center active"> <strong
+		class="levelBox node FATAL text-center active"> <strong
 		data-total="FATAL">0</strong><span>Fatal</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a> <a href="javascript:void(0);"
-		class="col-md-2 node ERROR text-center active"> <strong
+		class="levelBox node ERROR text-center active"> <strong
 		data-total="ERROR">0</strong><span>Error</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a> <a href="javascript:void(0);"
-		class="col-md-2 node WARN text-center active"> <strong
+		class="levelBox node WARN text-center active"> <strong
 		data-total="WARN">0</strong><span>Warn</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a> <a href="javascript:void(0);"
-		class="col-md-2 node INFO text-center active"> <strong
+		class="levelBox node INFO text-center active"> <strong
 		data-total="INFO">0</strong><span>Info</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a> <a href="javascript:void(0);"
-		class="col-md-2 node DEBUG text-center active"> <strong
+		class="levelBox node DEBUG text-center active"> <strong
 		data-total="DEBUG">0</strong><span>Debug</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a> <a href="javascript:void(0);"
-		class="col-md-2 node TRACE text-center active"> <strong
+		class="levelBox node TRACE text-center active"> <strong
 		data-total="TRACE">0</strong><span>Trace</span> <i
 		class="fa fa-spinner fa-spin"></i>
 	</a>
+  <a href="javascript:void(0);"
+    class="levelBox node UNKNOWN text-center active"> <strong
+    data-total="UNKNOWN">0</strong><span>Unknown</span> <i
+    class="fa fa-spinner fa-spin"></i>
+  </a>  
 
 </div>

ambari-logsearch/ambari-logsearch-portal/src/main/webapp/templates/filter/CreateLogfeederFilter_tmpl.html

@@ -34,6 +34,7 @@
                                            <th class="INFO  text-center"><span class="pull-left"><input type="checkbox" data-value="INFO"></span><span>INFO</span></th>
                                            <th class="DEBUG  text-center"><span class="pull-left"><input type="checkbox" data-value="DEBUG"></span><span>DEBUG</span></th>
                                            <th class="TRACE  text-center"><span class="pull-left"><input type="checkbox" data-value="TRACE"></span><span>TRACE</span></th>
+                                           <th class="UNKNOWN  text-center"><span class="pull-left"><input type="checkbox" data-value="UNKNOWN"></span><span>UNKNOWN</span></th>
                                         </tr>
                                        </thead>
                                        <tbody id="filterContent"></tbody>