AMBARI-16091. Add 'logfeeder log level include' feature to logsearch stack definition (oleewere)

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-site.xml

@@ -78,4 +78,11 @@
     </description>
   </property>
 
+  <property>
+    <name>logsearch.logfeeder.include.default.level</name>
+    <value>fatal,error,warn</value>
+    <description>Include default Logfeeder log levels for Logsearch</description>
+    <display-name>Logfeeder log levels</display-name>
+  </property>
+
 </configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py

@@ -149,6 +149,7 @@ logsearch_repfactor = str(config['configurations']['logsearch-site']['logsearch.
 
 logsearch_solr_collection_service_logs = default('/configurations/logsearch-site/logsearch.solr.collection.service.logs', 'hadoop_logs')
 logsearch_solr_collection_audit_logs = default('/configurations/logsearch-site/logsearch.solr.collection.audit.logs','audit_logs')
+logsearch_logfeeder_log_level_include = default('/configurations/logsearch-site/logsearch.logfeeder.include.default.level', 'fatal,error,warn')
 
 solr_audit_logs_use_ranger = default('/configurations/logsearch-env/logsearch_solr_audit_logs_use_ranger', False)
 solr_audit_logs_url = ''

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2

@@ -20,17 +20,17 @@
     {
       "type":"logsearch_app",
       "rowtype":"service",
-      "path":"{{logsearch_log_dir}}/logsearch.log"
+      "path":"{{logsearch_log_dir}}/logsearch.json"
     },
     {
       "type":"logsearch_feeder",
       "rowtype":"service",
-      "path":"{{logfeeder_log_dir}}/logfeeder.log"
+      "path":"{{logfeeder_log_dir}}/logfeeder.json"
     },
     {
       "type":"logsearch_perf",
       "rowtype":"service",
-      "path":"{{logsearch_log_dir}}/logsearch-performance.log"
+      "path":"{{logsearch_log_dir}}/logsearch-performance.json"
     }
 
   ],

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2

@@ -38,3 +38,7 @@ solr.audit_logs.replication_factor={{logsearch_repfactor}}
 solr.core.history=history
 solr.history.config_name=history
 solr.history.replication_factor={{logsearch_repfactor}}
+
+#Logfeeder Settings
+
+logfeeder.include.default.level={{logsearch_logfeeder_log_level_include}}

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logfeeder-log4j.xml.j2

@@ -34,24 +34,38 @@ limitations under the License.
     </layout>
   </appender>
 
+  <appender name="rolling_file_json"
+    class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
+    <param name="file" value="logs/logsearch-logfeeder.json" />
+    <param name="append" value="true" />
+    <param name="maxFileSize" value="10MB" />
+    <param name="maxBackupIndex" value="10" />
+    <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" />
+  </appender>
+
+
   <!-- Logs to suppress BEGIN -->
   <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false">
     <priority value="error" />
-    <appender-ref ref="rolling_file" />
+    <appender-ref ref="daily_rolling_file" />
   </category>
 
   <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false">
     <priority value="fatal" />
-    <appender-ref ref="rolling_file" />
+    <appender-ref ref="daily_rolling_file" />
   </category>
+  <!-- Logs to suppress END -->
 
   <category name="org.apache.ambari.logfeeder" additivity="false">
-    <priority value="info"/>
-    <appender-ref ref="rolling_file"/>
+    <priority value="INFO" />
+    <appender-ref ref="console" />
+    <!-- <appender-ref ref="daily_rolling_file" /> -->
+    <appender-ref ref="rolling_file_json"/>
   </category>
 
   <root>
     <priority value="warn"/>
-    <appender-ref ref="rolling_file"/>
+    <!-- <appender-ref ref="rolling_file"/> -->
+    <!-- <appender-ref ref="daily_rolling_file" /> -->
   </root>
 </log4j:configuration>

ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/logsearch-log4j.xml.j2

@@ -24,8 +24,8 @@ limitations under the License.
     </layout>
   </appender>
 
-  <appender name="rolling_file" class="org.apache.log4j.RollingFileAppender">
-    <param name="file" value="{{logsearch_log_dir}}/logsearch.log"/>
+  <appender name="rolling_file_json" class="org.apache.log4j.RollingFileAppender">
+    <param name="file" value="{{logsearch_log_dir}}/logsearch.json"/>
     <param name="append" value="true"/>
     <param name="maxFileSize" value="10MB"/>
     <param name="maxBackupIndex" value="10"/>
@@ -34,8 +34,8 @@ limitations under the License.
     </layout>
   </appender>
 
-  <appender name="audit_rolling_file" class="org.apache.log4j.RollingFileAppender">
-    <param name="file" value="{{logsearch_log_dir}}/logsearch-audit.log"/>
+  <appender name="audit_rolling_file_json" class="org.apache.log4j.RollingFileAppender">
+    <param name="file" value="{{logsearch_log_dir}}/logsearch-audit.json"/>
     <param name="append" value="true"/>
     <param name="maxFileSize" value="10MB"/>
     <param name="maxBackupIndex" value="10"/>
@@ -44,8 +44,8 @@ limitations under the License.
     </layout>
   </appender>
 
-  <appender name="performance_analyzer" class="org.apache.log4j.RollingFileAppender">
-    <param name="file" value="{{logsearch_log_dir}}/logsearch-performance.log"/>
+  <appender name="performance_analyzer_json" class="org.apache.log4j.RollingFileAppender">
+    <param name="file" value="{{logsearch_log_dir}}/logsearch-performance.json"/>
     <param name="Threshold" value="info"/>
     <param name="append" value="true"/>
     <param name="maxFileSize" value="10MB"/>
@@ -56,20 +56,22 @@ limitations under the License.
   </appender>
 
   <logger name="org.apache.ambari.logsearch.audit" additivity="true">
-    <appender-ref ref="audit_rolling_file"/>
+    <appender-ref ref="audit_rolling_file_json"/>
   </logger>
 
   <logger name="org.apache.ambari.logsearch.performance" additivity="false">
-    <appender-ref ref="performance_analyzer"/>
+    <appender-ref ref="performance_analyzer_json"/>
   </logger>
 
   <category name="org.apache.ambari.logsearch" additivity="false">
     <priority value="info"/>
-    <appender-ref ref="rolling_file"/>
+    <appender-ref ref="rolling_file_json"/>
   </category>
 
   <root>
     <priority value="warn"/>
-    <appender-ref ref="rolling_file"/>
+    <!-- <appender-ref ref="console" /> -->
+    <!--<appender-ref ref="rolling_file" /> -->
+    <appender-ref ref="rolling_file_json"/>
   </root>
 </log4j:configuration>

ambari-web/app/data/HDP2/site_properties.js

@@ -2171,6 +2171,13 @@ var hdp2properties = [
     "category": "Advanced logsearch-site",
     "index": 8
   },
+  {
+    "name": "logsearch.logfeeder.include.default.level",
+    "serviceName": "LOGSEARCH",
+    "filename": "logsearch-site.xml",
+    "category": "Advanced logsearch-site",
+    "index": 9
+  },
   /*logsearch-solr-env*/
   {
     "name": "logsearch_solr_datadir",