AMBARI-10249 - Add a Log Appender for Alert State Change Events (jonathanhurley)

ambari-server/conf/log4j.properties

@@ -1,21 +0,0 @@
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-# log4j configuration used during build and unit tests
-
-log4j.rootLogger=INFO,stdout
-log4j.threshhold=ALL
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
-
-log4j.logger.org.apache.ambari=DEBUG

ambari-server/conf/unix/log4j.properties

@@ -18,11 +18,16 @@
 
 # Define some default values that can be overridden by system properties
 # Root logger option
+ambari.log.dir=/var/log/ambari-server
+ambari.log.file=ambari-server.log
+ambari.config-changes.file=ambari-config-changes.log
+ambari.alerts.file=ambari-alerts.log
+
 log4j.rootLogger=INFO,file
 
 # Direct log messages to a log file
 log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.File=/var/log/ambari-server/ambari-server.log
+log4j.appender.file.File=${ambari.log.dir}/${ambari.log.file}
 log4j.appender.file.MaxFileSize=80MB
 log4j.appender.file.MaxBackupIndex=60
 log4j.appender.file.layout=org.apache.log4j.PatternLayout
@@ -32,10 +37,18 @@ log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
 log4j.logger.configchange=INFO,configchange
 log4j.additivity.configchange=false
 log4j.appender.configchange=org.apache.log4j.FileAppender
-log4j.appender.configchange.File=/var/log/ambari-server/ambari-config-changes.log
+log4j.appender.configchange.File=${ambari.log.dir}/${ambari.config-changes.file}
 log4j.appender.configchange.layout=org.apache.log4j.PatternLayout
 log4j.appender.configchange.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
 
+# Log alert state changes
+log4j.logger.alerts=INFO,alerts
+log4j.additivity.alerts=false
+log4j.appender.alerts=org.apache.log4j.FileAppender
+log4j.appender.alerts.File=${ambari.log.dir}/${ambari.alerts.file}
+log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
+log4j.appender.alerts.layout.ConversionPattern=%d{ISO8601} %m%n
+
 log4j.logger.org.apache.hadoop.yarn.client=WARN
 log4j.logger.org.apache.slider.common.tools.SliderUtils=WARN
 log4j.logger.org.apache.ambari.server.security.authorization=WARN

ambari-server/conf/windows/log4j.properties

@@ -21,6 +21,7 @@ ambari.root.logger=INFO,DRFA
 ambari.log.dir=\\var\\log\\ambari-server-1.3.0-SNAPSHOT\\
 ambari.log.file=ambari-server.log
 ambari.config-changes.file=ambari-config-changes.log
+ambari.alerts.file=ambari-alerts.log
 
 
 # Define the root logger to the system property "ambari.root.logger".
@@ -66,3 +67,11 @@ log4j.appender.configchange=org.apache.log4j.FileAppender
 log4j.appender.configchange.File=${ambari.log.dir}\${ambari.config-changes.file}
 log4j.appender.configchange.layout=org.apache.log4j.PatternLayout
 log4j.appender.configchange.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
+
+# Log alert state changes
+log4j.logger.alerts=INFO,alerts
+log4j.additivity.alerts=false
+log4j.appender.alerts=org.apache.log4j.FileAppender
+log4j.appender.alerts.File=${ambari.log.dir}\${ambari.alerts.file}
+log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
+log4j.appender.alerts.layout.ConversionPattern=%d{ISO8601} %m%n

ambari-server/src/main/conf/log4j.properties

@@ -19,8 +19,9 @@
 # Define some default values that can be overridden by system properties
 ambari.root.logger=INFO,DRFA
 ambari.log.dir=/var/log/ambari-server
-ambari.log.file=ambari.log
+ambari.log.file=ambari-server.log
 ambari.config-changes.file=ambari-config-changes.log
+ambari.alerts.file=ambari-alerts.log
 
 
 # Define the root logger to the system property "ambari.root.logger".
@@ -66,3 +67,12 @@ log4j.appender.configchange=org.apache.log4j.FileAppender
 log4j.appender.configchange.File=${ambari.log.dir}/${ambari.config-changes.file}
 log4j.appender.configchange.layout=org.apache.log4j.PatternLayout
 log4j.appender.configchange.layout.ConversionPattern=%d{ISO8601} %5p - %m%n
+
+# Log alert state changes
+log4j.logger.alerts=INFO,alerts
+log4j.additivity.alerts=false
+log4j.appender.alerts=org.apache.log4j.FileAppender
+log4j.appender.alerts.File=${ambari.log.dir}/${ambari.alerts.file}
+log4j.appender.alerts.layout=org.apache.log4j.PatternLayout
+log4j.appender.alerts.layout.ConversionPattern=%d{ISO8601} %m%n
+

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/alerts/AlertStateChangedListener.java

@@ -31,11 +31,12 @@ import org.apache.ambari.server.orm.entities.AlertGroupEntity;
 import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
 import org.apache.ambari.server.orm.entities.AlertNoticeEntity;
 import org.apache.ambari.server.orm.entities.AlertTargetEntity;
+import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.NotificationState;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.eventbus.AllowConcurrentEvents;
 import com.google.common.eventbus.Subscribe;
@@ -54,7 +55,19 @@ public class AlertStateChangedListener {
   /**
    * Logger.
    */
-  private static Log LOG = LogFactory.getLog(AlertStateChangedListener.class);
+  private static Logger LOG = LoggerFactory.getLogger(AlertStateChangedListener.class);
+
+  /**
+   * A logger that is only for logging alert state changes so that there is an
+   * audit trail in the event that definitions/history are removed.
+   */
+  private static final Logger ALERT_LOG = LoggerFactory.getLogger("alerts");
+
+  /**
+   * [CRITICAL] [HDFS] [namenode_hdfs_blocks_health] (NameNode Blocks Health)
+   * Total Blocks:[100], Missing Blocks:[6]
+   */
+  private static final String ALERT_LOG_MESSAGE = "[{}] [{}] [{}] ({}) {}";
 
   /**
    * Used for looking up groups and targets.
@@ -83,7 +96,20 @@ public class AlertStateChangedListener {
   @Subscribe
   @AllowConcurrentEvents
   public void onAlertEvent(AlertStateChangeEvent event) {
-    LOG.debug(event);
+    Alert alert = event.getAlert();
+    AlertHistoryEntity history = event.getNewHistoricalEntry();
+    AlertDefinitionEntity definition = history.getAlertDefinition();
+
+    // log to the alert audit log so there is physical record even if
+    // definitions and historical enties are removed
+    ALERT_LOG.info(ALERT_LOG_MESSAGE, alert.getState(),
+        definition.getServiceName(), definition.getDefinitionName(),
+        definition.getLabel(), alert.getText());
+
+    // normal logging for Ambari
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("An alert has changed state: {}", event);
+    }
 
     // don't create any outbound alert notices if in MM
     AlertCurrentEntity currentAlert = event.getCurrentAlert();
@@ -92,9 +118,6 @@ public class AlertStateChangedListener {
       return;
     }
 
-    AlertHistoryEntity history = event.getNewHistoricalEntry();
-    AlertDefinitionEntity definition = history.getAlertDefinition();
-
     List<AlertGroupEntity> groups = m_alertsDispatchDao.findGroupsByDefinition(definition);
 
     // for each group, determine if there are any targets that need to receive