AMBARI-17135 : added servlet filter and used Logging frameworks MDC feature to insert view related information from URL. Updated view log4j files for the same. (Nitiraj Rathore via dipayanb)

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java

@@ -19,21 +19,16 @@
 package org.apache.ambari.server.controller;
 
 
-import java.io.File;
-import java.io.IOException;
-import java.net.Authenticator;
-import java.net.BindException;
-import java.net.PasswordAuthentication;
-import java.net.URL;
-import java.util.EnumSet;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.LogManager;
-
-import javax.crypto.BadPaddingException;
-import javax.servlet.DispatcherType;
-
+import com.google.common.util.concurrent.ServiceManager;
+import com.google.gson.Gson;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+import com.google.inject.persist.Transactional;
+import com.sun.jersey.spi.container.servlet.ServletContainer;
 import org.apache.ambari.eventdb.webservice.WorkflowJsonService;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.StateRecoveryManager;
@@ -99,9 +94,9 @@ import org.apache.ambari.server.security.CertificateManager;
 import org.apache.ambari.server.security.SecurityFilter;
 import org.apache.ambari.server.security.authentication.AmbariAuthenticationFilter;
 import org.apache.ambari.server.security.authorization.AmbariAuthorizationFilter;
+import org.apache.ambari.server.security.authorization.AmbariLdapAuthenticationProvider;
 import org.apache.ambari.server.security.authorization.AmbariLocalUserProvider;
 import org.apache.ambari.server.security.authorization.AmbariUserAuthorizationFilter;
-import org.apache.ambari.server.security.authorization.AmbariLdapAuthenticationProvider;
 import org.apache.ambari.server.security.authorization.PermissionHelper;
 import org.apache.ambari.server.security.authorization.Users;
 import org.apache.ambari.server.security.authorization.internal.AmbariInternalAuthenticationProvider;
@@ -120,6 +115,7 @@ import org.apache.ambari.server.utils.AmbariPath;
 import org.apache.ambari.server.utils.RetryHelper;
 import org.apache.ambari.server.utils.StageUtils;
 import org.apache.ambari.server.utils.VersionUtils;
+import org.apache.ambari.server.view.AmbariViewsMDCLoggingFilter;
 import org.apache.ambari.server.view.ViewDirectoryWatcher;
 import org.apache.ambari.server.view.ViewRegistry;
 import org.apache.ambari.server.view.ViewThrottleFilter;
@@ -152,19 +148,23 @@ import org.springframework.web.context.request.RequestContextListener;
 import org.springframework.web.context.support.GenericWebApplicationContext;
 import org.springframework.web.filter.DelegatingFilterProxy;
 
-import com.google.common.util.concurrent.ServiceManager;
-import com.google.gson.Gson;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Scopes;
-import com.google.inject.Singleton;
-import com.google.inject.name.Named;
-import com.google.inject.persist.Transactional;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
+import javax.crypto.BadPaddingException;
+import javax.servlet.DispatcherType;
+import java.io.File;
+import java.io.IOException;
+import java.net.Authenticator;
+import java.net.BindException;
+import java.net.PasswordAuthentication;
+import java.net.URL;
+import java.util.EnumSet;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.LogManager;
 
 @Singleton
 public class AmbariServer {
+  public static final String VIEWS_URL_PATTERN = "/api/v1/views/*";
   private static Logger LOG = LoggerFactory.getLogger(AmbariServer.class);
 
   /**
@@ -382,14 +382,18 @@ public class AmbariServer {
 
       // The security header filter - conditionally adds security-related headers to the HTTP response for Ambari Views
       // requests.
-      root.addFilter(new FilterHolder(injector.getInstance(AmbariViewsSecurityHeaderFilter.class)), "/api/v1/views/*",
+      root.addFilter(new FilterHolder(injector.getInstance(AmbariViewsSecurityHeaderFilter.class)), VIEWS_URL_PATTERN,
           DISPATCHER_TYPES);
 
       // since views share the REST API threadpool, a misbehaving view could
       // consume all of the available threads and effectively cause a loss of
       // service for Ambari
       root.addFilter(new FilterHolder(injector.getInstance(ViewThrottleFilter.class)),
-          "/api/v1/views/*", DISPATCHER_TYPES);
+        VIEWS_URL_PATTERN, DISPATCHER_TYPES);
+
+      // adds MDC info for views logging
+      root.addFilter(new FilterHolder(injector.getInstance(AmbariViewsMDCLoggingFilter.class)),
+        VIEWS_URL_PATTERN, DISPATCHER_TYPES);
 
       // session-per-request strategy for api
       root.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/api/*", DISPATCHER_TYPES);

ambari-server/src/main/java/org/apache/ambari/server/view/AmbariViewsMDCLoggingFilter.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.view;
+
+import com.google.inject.Singleton;
+import org.slf4j.MDC;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Inserts view name, view version and view instance name into MDC info for views REST for logging in views.
+ */
+@Singleton
+public class AmbariViewsMDCLoggingFilter implements Filter {
+
+  private final static String patternStr = "/api/v1/views/(.*)/versions/(.*)/instances/([^/]+).*";
+  private final static Pattern pattern = Pattern.compile(patternStr);
+
+  @Override
+  public void init(FilterConfig filterConfig) throws ServletException {
+    // do nothing
+  }
+
+  @Override
+  public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
+
+    buildMDC(servletRequest);
+    try {
+      filterChain.doFilter(servletRequest, servletResponse);
+    } finally {
+      clear();
+    }
+  }
+
+  private void buildMDC(ServletRequest request) {
+    if ((request instanceof HttpServletRequest) && MDC.getMDCAdapter() != null) {
+      String url = ((HttpServletRequest) request).getRequestURI();
+      Matcher matcher = pattern.matcher(url);
+      if (matcher.find()) {
+        MDC.put("viewName", matcher.group(1));
+        MDC.put("viewVersion", matcher.group(2));
+        MDC.put("viewInstanceName", matcher.group(3));
+      }
+    }
+  }
+
+  private void clear() {
+    MDC.clear();
+  }
+
+  @Override
+  public void destroy() {
+    //do nothing
+  }
+}

contrib/views/capacity-scheduler/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.capshedView=org.apache.log4j.RollingFileAppender
-log4j.appender.capshedView.File=/var/log/ambari-server/capshed-view/capshed-view.log
+log4j.appender.capshedView.File=${ambari.log.dir}/capshed-view/capshed-view.log
 log4j.appender.capshedView.MaxFileSize=80MB
 log4j.appender.capshedView.MaxBackupIndex=60
 log4j.appender.capshedView.layout=org.apache.log4j.PatternLayout
-log4j.appender.capshedView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.capshedView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.capacityscheduler=INFO,capshedView
 log4j.additivity.org.apache.ambari.view.capacityscheduler = false

contrib/views/files/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.filesView=org.apache.log4j.RollingFileAppender
-log4j.appender.filesView.File=/var/log/ambari-server/files-view/files-view.log
+log4j.appender.filesView.File=${ambari.log.dir}/files-view/files-view.log
 log4j.appender.filesView.MaxFileSize=80MB
 log4j.appender.filesView.MaxBackupIndex=60
 log4j.appender.filesView.layout=org.apache.log4j.PatternLayout
-log4j.appender.filesView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.filesView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.filebrowser=INFO,filesView
 log4j.additivity.org.apache.ambari.view.filebrowser= false

contrib/views/hive-next/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.hiveNextView=org.apache.log4j.RollingFileAppender
-log4j.appender.hiveNextView.File=/var/log/ambari-server/hive-next-view/hive-view.log
+log4j.appender.hiveNextView.File=${ambari.log.dir}/hive-next-view/hive-view.log
 log4j.appender.hiveNextView.MaxFileSize=80MB
 log4j.appender.hiveNextView.MaxBackupIndex=60
 log4j.appender.hiveNextView.layout=org.apache.log4j.PatternLayout
-log4j.appender.hiveNextView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.hiveNextView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.hive2=INFO,hiveNextView
 log4j.logger.org.apache.hive.jdbc=INFO,hiveNextView

contrib/views/hive/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.hiveView=org.apache.log4j.RollingFileAppender
-log4j.appender.hiveView.File=/var/log/ambari-server/hive-view/hive-view.log
+log4j.appender.hiveView.File=${ambari.log.dir}/hive-view/hive-view.log
 log4j.appender.hiveView.MaxFileSize=80MB
 log4j.appender.hiveView.MaxBackupIndex=60
 log4j.appender.hiveView.layout=org.apache.log4j.PatternLayout
-log4j.appender.hiveView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.hiveView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.hive=INFO,hiveView
 log4j.additivity.org.apache.ambari.view.hive = false

contrib/views/hueambarimigration/src/main/resources/view.log4j.properties

@@ -16,16 +16,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-log4j.logger.org.apache.ambari.view.huetoambarimigration=DEBUG, hueambarimigration
-log4j.additivity.org.apache.ambari.view.huetoambarimigration=false
-
-
 
 # Redirect log messages to a log file, support file rolling.
 log4j.appender.hueambarimigration=org.apache.log4j.RollingFileAppender
-log4j.appender.hueambarimigration.File=/var/log/ambari-server/huetoambarimigration-view/huetoambarimigration.log
-log4j.appender.hueambarimigration.MaxFileSize=5MB
+log4j.appender.hueambarimigration.File=${ambari.log.dir}/huetoambarimigration-view/huetoambarimigration.log
+log4j.appender.hueambarimigration.MaxFileSize=80MB
 log4j.appender.hueambarimigration.MaxBackupIndex=10
 log4j.appender.hueambarimigration.layout=org.apache.log4j.PatternLayout
-log4j.appender.hueambarimigration.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
+log4j.appender.hueambarimigration.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
+log4j.logger.org.apache.ambari.view.huetoambarimigration=INFO, hueambarimigration
+log4j.additivity.org.apache.ambari.view.huetoambarimigration=false

contrib/views/jobs/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.jobsView=org.apache.log4j.RollingFileAppender
-log4j.appender.jobsView.File=/var/log/ambari-server/jobs-view/jobs-view.log
+log4j.appender.jobsView.File=${ambari.log.dir}/jobs-view/jobs-view.log
 log4j.appender.jobsView.MaxFileSize=80MB
 log4j.appender.jobsView.MaxBackupIndex=60
 log4j.appender.jobsView.layout=org.apache.log4j.PatternLayout
-log4j.appender.jobsView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.jobsView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 
 log4j.logger.org.apache.ambari.view.jobs=INFO,jobsView

contrib/views/pig/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.pigView=org.apache.log4j.RollingFileAppender
-log4j.appender.pigView.File=/var/log/ambari-server/pig-view/pig-view.log
+log4j.appender.pigView.File=${ambari.log.dir}/pig-view/pig-view.log
 log4j.appender.pigView.MaxFileSize=80MB
 log4j.appender.pigView.MaxBackupIndex=60
 log4j.appender.pigView.layout=org.apache.log4j.PatternLayout
-log4j.appender.pigView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.pigView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.pig=INFO,pigView
 log4j.additivity.org.apache.ambari.view.pig=false

contrib/views/slider/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.sliderView=org.apache.log4j.RollingFileAppender
-log4j.appender.sliderView.File=/var/log/ambari-server/slider-view/slider-view.log
+log4j.appender.sliderView.File=${ambari.log.dir}/slider-view/slider-view.log
 log4j.appender.sliderView.MaxFileSize=80MB
 log4j.appender.sliderView.MaxBackupIndex=60
 log4j.appender.sliderView.layout=org.apache.log4j.PatternLayout
-log4j.appender.sliderView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.sliderView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.slider=INFO,sliderView
 log4j.additivity.org.apache.ambari.view.slider=false

contrib/views/storm/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.stormView=org.apache.log4j.RollingFileAppender
-log4j.appender.stormView.File=/var/log/ambari-server/storm-view/storm-view.log
+log4j.appender.stormView.File=${ambari.log.dir}/storm-view/storm-view.log
 log4j.appender.stormView.MaxFileSize=80MB
 log4j.appender.stormView.MaxBackupIndex=60
 log4j.appender.stormView.layout=org.apache.log4j.PatternLayout
-log4j.appender.stormView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.stormView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.storm=INFO,stormView
 log4j.additivity.org.apache.ambari.storm=false

contrib/views/tez/src/main/resources/view.log4j.properties

@@ -17,11 +17,11 @@
 # limitations under the License.
 
 log4j.appender.tezView=org.apache.log4j.RollingFileAppender
-log4j.appender.tezView.File=/var/log/ambari-server/tez-view/tez-view.log
+log4j.appender.tezView.File=${ambari.log.dir}/tez-view/tez-view.log
 log4j.appender.tezView.MaxFileSize=80MB
 log4j.appender.tezView.MaxBackupIndex=60
 log4j.appender.tezView.layout=org.apache.log4j.PatternLayout
-log4j.appender.tezView.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n
+log4j.appender.tezView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
 
 log4j.logger.org.apache.ambari.view.tez=INFO,tezView
 log4j.additivity.org.apache.ambari.view.tez=false

contrib/views/zeppelin/src/main/resources/view.log4j.properties

@@ -0,0 +1,27 @@
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.appender.zeppelinView=org.apache.log4j.RollingFileAppender
+log4j.appender.zeppelinView.File=${ambari.log.dir}/zeppelin-view/zeppelin-view.log
+log4j.appender.zeppelinView.MaxFileSize=80MB
+log4j.appender.zeppelinView.MaxBackupIndex=60
+log4j.appender.zeppelinView.layout=org.apache.log4j.PatternLayout
+log4j.appender.zeppelinView.layout.ConversionPattern=%d{DATE} %5p [%t] [%X{viewName} %X{viewVersion} %X{viewInstanceName}] %c{1}:%L - %m%n
+
+log4j.logger.org.apache.ambari.view.zeppelin=INFO,zeppelinView
+log4j.additivity.org.apache.ambari.view.zeppelin=false