|
|
@@ -0,0 +1,173 @@
|
|
|
+/**
|
|
|
+ * Licensed to the Apache Software Foundation (ASF) under one
|
|
|
+ * or more contributor license agreements. See the NOTICE file
|
|
|
+ * distributed with this work for additional information
|
|
|
+ * regarding copyright ownership. The ASF licenses this file
|
|
|
+ * to you under the Apache License, Version 2.0 (the
|
|
|
+ * "License"); you may not use this file except in compliance
|
|
|
+ * with the License. You may obtain a copy of the License at
|
|
|
+ *
|
|
|
+ * http://www.apache.org/licenses/LICENSE-2.0
|
|
|
+ *
|
|
|
+ * Unless required by applicable law or agreed to in writing, software
|
|
|
+ * distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
+ * See the License for the specific language governing permissions and
|
|
|
+ * limitations under the License.
|
|
|
+ */
|
|
|
+package org.apache.ambari.server.view;
|
|
|
+
|
|
|
+import java.io.IOException;
|
|
|
+import java.util.concurrent.Semaphore;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+
|
|
|
+import javax.servlet.Filter;
|
|
|
+import javax.servlet.FilterChain;
|
|
|
+import javax.servlet.FilterConfig;
|
|
|
+import javax.servlet.ServletException;
|
|
|
+import javax.servlet.ServletRequest;
|
|
|
+import javax.servlet.ServletResponse;
|
|
|
+import javax.servlet.http.HttpServletRequest;
|
|
|
+import javax.servlet.http.HttpServletResponse;
|
|
|
+
|
|
|
+import org.apache.ambari.server.configuration.Configuration;
|
|
|
+import org.eclipse.jetty.continuation.Continuation;
|
|
|
+import org.slf4j.Logger;
|
|
|
+import org.slf4j.LoggerFactory;
|
|
|
+
|
|
|
+import com.google.inject.Inject;
|
|
|
+import com.google.inject.Singleton;
|
|
|
+
|
|
|
+/**
|
|
|
+ * The {@link ViewThrottleFilter} is used to ensure that views which misbehave
|
|
|
+ * do not cause a loss of service for Ambari. The underlying problem is that
|
|
|
+ * views are accessed off of the REST endpoint (/api/v1/views). This means that
|
|
|
+ * the Ambari REST API connector is going to handle the request from its own
|
|
|
+ * threadpool. There is no way to configure Jetty to use a different threadpool
|
|
|
+ * for the same connector. As a result, if a request to load a view holds the
|
|
|
+ * Jetty thread hostage, eventually we will see thread starvation and loss of
|
|
|
+ * service.
|
|
|
+ * <p/>
|
|
|
+ * An example of this situation is a view which makes an innocent request to a
|
|
|
+ * remote resource. If the view's request has a timeout of 60 seconds, then the
|
|
|
+ * Jetty thread is going to be held for that amount of time. With concurrent
|
|
|
+ * users and multiple instances of that view deployed, the Jetty threadpool can
|
|
|
+ * becomes exhausted quickly.
|
|
|
+ * <p/>
|
|
|
+ * Although there are more graceful ways of handling this situation, they mostly
|
|
|
+ * involve substantial re-architecture and design.
|
|
|
+ * <ul>
|
|
|
+ * <li>The use of a new connector and threadpool would require binding to
|
|
|
+ * another port for view requests. This will cause problems with "local" views
|
|
|
+ * and their assumption that if they run on the Ambari server they can share the
|
|
|
+ * same session.
|
|
|
+ * <li>The use of a {@link Continuation} in Jetty which can suspend the incoming
|
|
|
+ * request. We would need the ability for views to signal that they have
|
|
|
+ * completed their work in order to proceed with the suspended request.
|
|
|
+ * </ul>
|
|
|
+ */
|
|
|
+@Singleton
|
|
|
+public class ViewThrottleFilter implements Filter {
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Logger.
|
|
|
+ */
|
|
|
+ private static final Logger LOG = LoggerFactory.getLogger(ViewThrottleFilter.class);
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Used to determine the correct number of threads to allocate to view
|
|
|
+ * requests.
|
|
|
+ */
|
|
|
+ @Inject
|
|
|
+ private Configuration m_configuration;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * Used to restrict how many REST API threads can be utilizied concurrently by
|
|
|
+ * view requests.
|
|
|
+ */
|
|
|
+ private Semaphore m_semaphore;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * A timeout that a blocked view request should wait for an available thread
|
|
|
+ * before returning an error.
|
|
|
+ */
|
|
|
+ private int m_timeout;
|
|
|
+
|
|
|
+ /**
|
|
|
+ * {@inheritDoc}
|
|
|
+ */
|
|
|
+ @Override
|
|
|
+ public void init(FilterConfig filterConfig) throws ServletException {
|
|
|
+ m_timeout = m_configuration.getViewRequestThreadPoolTimeout();
|
|
|
+
|
|
|
+ int clientThreadPoolSize = m_configuration.getClientThreadPoolSize();
|
|
|
+ int viewThreadPoolSize = m_configuration.getViewRequestThreadPoolMaxSize();
|
|
|
+
|
|
|
+ // start out using 1/2 of the available REST API request threads
|
|
|
+ int viewSemaphoreCount = clientThreadPoolSize / 2;
|
|
|
+
|
|
|
+ // if the size is specified, see if it's valid
|
|
|
+ if (viewThreadPoolSize > 0) {
|
|
|
+ viewSemaphoreCount = viewThreadPoolSize;
|
|
|
+
|
|
|
+ if (viewThreadPoolSize > clientThreadPoolSize) {
|
|
|
+ LOG.warn(
|
|
|
+ "The number of view processing threads ({}) cannot be greater than the REST API client threads {{})",
|
|
|
+ viewThreadPoolSize, clientThreadPoolSize);
|
|
|
+
|
|
|
+ viewSemaphoreCount = clientThreadPoolSize;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // log that we are restricting it
|
|
|
+ LOG.info("Ambari Views will be able to utilize {} concurrent REST API threads",
|
|
|
+ viewSemaphoreCount);
|
|
|
+
|
|
|
+ m_semaphore = new Semaphore(viewSemaphoreCount);
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * {@inheritDoc}
|
|
|
+ */
|
|
|
+ @Override
|
|
|
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
|
|
|
+ throws IOException, ServletException {
|
|
|
+
|
|
|
+ // do nothing if this is not an http request
|
|
|
+ if (!(request instanceof HttpServletRequest)) {
|
|
|
+ chain.doFilter(request, response);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ HttpServletResponse httpResponse = (HttpServletResponse) response;
|
|
|
+ boolean acquired = false;
|
|
|
+
|
|
|
+ try {
|
|
|
+ acquired = m_semaphore.tryAcquire(m_timeout, TimeUnit.MILLISECONDS);
|
|
|
+ } catch (InterruptedException interruptedException) {
|
|
|
+ LOG.warn("While waiting for an available thread, the view request was interrupted");
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!acquired) {
|
|
|
+ httpResponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,
|
|
|
+ "There are no available threads to handle view requests");
|
|
|
+
|
|
|
+ // return to prevent the view's request from making it down any farther
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ // let the request go through
|
|
|
+ try {
|
|
|
+ chain.doFilter(request, response);
|
|
|
+ } finally {
|
|
|
+ m_semaphore.release();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * {@inheritDoc}
|
|
|
+ */
|
|
|
+ @Override
|
|
|
+ public void destroy() {
|
|
|
+ }
|
|
|
+}
|
Jonathan Hurley