AMBARI-7221 - Ambari Server REST API Memory Leak (jonathanhurley)

ambari-server/conf/unix/ambari.properties

@@ -53,3 +53,6 @@ agent.threadpool.size.max=25
 
 # linux open-file limit
 ulimit.open.files=10000
+
+# Server HTTP settings
+server.http.session.inactive_timeout=1800

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -261,12 +261,11 @@ public class Configuration {
   private static final String RESOURCES_DIR_DEFAULT =
       "/var/lib/ambari-server/resources/";
   private static final String ANONYMOUS_AUDIT_NAME_KEY = "anonymous.audit.name";
-  private static final String CLIENT_SECURITY_DEFAULT = "local";
+
   private static final int CLIENT_API_PORT_DEFAULT = 8080;
   private static final int CLIENT_API_SSL_PORT_DEFAULT = 8443;
-  private static final String USER_ROLE_NAME_DEFAULT = "user";
-  private static final String ADMIN_ROLE_NAME_DEFAULT = "admin";
   private static final String LDAP_BIND_ANONYMOUSLY_DEFAULT = "true";
+
   //TODO For embedded server only - should be removed later
   private static final String LDAP_PRIMARY_URL_DEFAULT = "localhost:33389";
   private static final String LDAP_BASE_DN_DEFAULT = "dc=ambari,dc=apache,dc=org";
@@ -310,6 +309,8 @@ public class Configuration {
   private static final String VIEW_EXTRACTION_THREADPOOL_TIMEOUT_KEY = "view.extraction.threadpool.timeout";
   private static final long VIEW_EXTRACTION_THREADPOOL_TIMEOUT_DEFAULT = 100000L;
 
+  private static final String SERVER_HTTP_SESSION_INACTIVE_TIMEOUT = "server.http.session.inactive_timeout";
+
   private static final Logger LOG = LoggerFactory.getLogger(
       Configuration.class);
   private Properties properties;
@@ -404,7 +405,7 @@ public class Configuration {
     }
     configsMap.put(SRVR_CRT_PASS_KEY, password);
 
-    if (this.getApiSSLAuthentication()) {
+    if (getApiSSLAuthentication()) {
       LOG.info("API SSL Authentication is turned on.");
       File httpsPassFile = new File(configsMap.get(CLIENT_API_SSL_KSTR_DIR_NAME_KEY)
         + File.separator + configsMap.get(CLIENT_API_SSL_CRT_PASS_FILE_NAME_KEY));
@@ -467,14 +468,14 @@ public class Configuration {
   private synchronized void loadCredentialProvider() {
     if (!credentialProviderInitialized) {
       try {
-        this.credentialProvider = new CredentialProvider(null,
+        credentialProvider = new CredentialProvider(null,
           getMasterKeyLocation(), isMasterKeyPersisted());
       } catch (Exception e) {
         LOG.info("Credential provider creation failed. Reason: " + e.getMessage());
         if (LOG.isDebugEnabled()) {
           e.printStackTrace();
         }
-        this.credentialProvider = null;
+        credentialProvider = null;
       }
       credentialProviderInitialized = true;
     }
@@ -490,8 +491,9 @@ public class Configuration {
     //Get property file stream from classpath
     InputStream inputStream = Configuration.class.getClassLoader().getResourceAsStream(CONFIG_FILE);
 
-    if (inputStream == null)
+    if (inputStream == null) {
       throw new RuntimeException(CONFIG_FILE + " not found in classpath");
+    }
 
     // load the properties
     try {
@@ -534,8 +536,9 @@ public class Configuration {
   public String getBootSetupAgentPassword() {
     String pass = configsMap.get(PASSPHRASE_KEY);
 
-    if (null != pass)
+    if (null != pass) {
       return pass;
+    }
 
     // fallback
     return properties.getProperty(BOOTSTRAP_SETUP_AGENT_PASSWORD, "password");
@@ -688,8 +691,9 @@ public class Configuration {
 
   public String getLocalDatabaseUrl() {
     String dbName = properties.getProperty(SERVER_DB_NAME_KEY);
-    if(dbName == null || dbName.isEmpty())
+    if(dbName == null || dbName.isEmpty()) {
       throw new RuntimeException("Server DB Name is not configured!");
+    }
 
     return JDBC_LOCAL_URL + dbName;
   }
@@ -705,10 +709,11 @@ public class Configuration {
       dbpasswd = readPasswordFromStore(passwdProp);
     }
 
-    if (dbpasswd != null)
+    if (dbpasswd != null) {
       return dbpasswd;
-    else
+    } else {
       return readPasswordFromFile(passwdProp, SERVER_JDBC_USER_PASSWD_DEFAULT);
+    }
   }
 
   public String getRcaDatabaseDriver() {
@@ -727,8 +732,9 @@ public class Configuration {
     String passwdProp = properties.getProperty(SERVER_JDBC_RCA_USER_PASSWD_KEY);
     if (passwdProp != null) {
       String dbpasswd = readPasswordFromStore(passwdProp);
-      if (dbpasswd != null)
+      if (dbpasswd != null) {
         return dbpasswd;
+      }
     }
     return readPasswordFromFile(passwdProp, SERVER_JDBC_RCA_USER_PASSWD_DEFAULT);
   }
@@ -1084,4 +1090,18 @@ public class Configuration {
     return Long.parseLong(properties.getProperty(
         VIEW_EXTRACTION_THREADPOOL_TIMEOUT_KEY, String.valueOf(VIEW_EXTRACTION_THREADPOOL_TIMEOUT_DEFAULT)));
   }
+
+  /**
+   * Gets the inactivity timeout value, in seconds, for sessions created in
+   * Jetty by Spring Security. Without this timeout value, each request to the
+   * REST APIs will create new sessions that are never reaped since their
+   * default time is -1.
+   *
+   * @return the time value or {@code 1800} seconds for default.
+   */
+  public int getHttpSessionInactiveTimeout() {
+    return Integer.parseInt(properties.getProperty(
+        SERVER_HTTP_SESSION_INACTIVE_TIMEOUT,
+        "1800"));
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariServer.java

@@ -230,8 +230,18 @@ public class AmbariServer {
       root.setErrorHandler(injector.getInstance(AmbariErrorHandler.class));
       root.getSessionHandler().setSessionManager(sessionManager);
 
-      //Changing session cookie name to avoid conflicts
-      root.getSessionHandler().getSessionManager().setSessionCookie("AMBARISESSIONID");
+      SessionManager jettySessionManager = root.getSessionHandler().getSessionManager();
+
+      // use AMBARISESSIONID instead of JSESSIONID to avoid conflicts with
+      // other services (like HDFS) that run on the same context but a different
+      // port
+      jettySessionManager.setSessionCookie("AMBARISESSIONID");
+
+      // each request that does not use AMBARISESSIONID will create a new
+      // HashedSession in Jetty; these MUST be reaped after inactivity in order
+      // to prevent a memory leak
+      int sessionInactivityTimeout = configs.getHttpSessionInactiveTimeout();
+      jettySessionManager.setMaxInactiveInterval(sessionInactivityTimeout);
 
       GenericWebApplicationContext springWebAppContext = new GenericWebApplicationContext();
       springWebAppContext.setServletContext(root.getServletContext());
@@ -246,8 +256,10 @@ public class AmbariServer {
 
       certMan.initRootCert();
 
-      ServletContextHandler agentroot = new ServletContextHandler(serverForAgent,
-          "/", ServletContextHandler.SESSIONS );
+      // the agent communication (heartbeats, registration, etc) is stateless
+      // and does not use sessions.
+      ServletContextHandler agentroot = new ServletContextHandler(
+          serverForAgent, "/", ServletContextHandler.NO_SESSIONS);
 
       ServletHolder rootServlet = root.addServlet(DefaultServlet.class, "/");
       rootServlet.setInitParameter("dirAllowed", "false");
@@ -262,8 +274,8 @@ public class AmbariServer {
       root.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/proxy/*", 1);
       root.addFilter(new FilterHolder(new MethodOverrideFilter()), "/api/*", 1);
       root.addFilter(new FilterHolder(new MethodOverrideFilter()), "/proxy/*", 1);
-      agentroot.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/agent/*", 1);
 
+      agentroot.addFilter(new FilterHolder(injector.getInstance(AmbariPersistFilter.class)), "/agent/*", 1);
       agentroot.addFilter(SecurityFilter.class, "/*", 1);
 
       if (configs.getApiAuthentication()) {