AMBARI-14866. Increase timeout for server-side tasks (rlevas)

ambari-server/conf/unix/ambari.properties

@@ -76,6 +76,9 @@ agent.task.timeout=900
 # Default timeout in seconds before package installation task is killed
 agent.package.install.task.timeout=1800
 
+# Default timeout in seconds before a server-side task is killed
+server.task.timeout=1200
+
 # thread pool maximums
 client.threadpool.size.max=25
 agent.threadpool.size.max=25

ambari-server/conf/windows/ambari.properties

@@ -65,6 +65,9 @@ agent.task.timeout=600
 # Default timeout in seconds before package installation task is killed
 agent.package.install.task.timeout=1800
 
+# Default timeout in seconds before a server-side task is killed
+server.task.timeout=1200
+
 # thread pool maximums
 client.threadpool.size.max=25
 agent.threadpool.size.max=25

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -357,6 +357,12 @@ public class Configuration {
   public static final String AGENT_TASK_TIMEOUT_DEFAULT = "900";
   public static final String AGENT_PACKAGE_INSTALL_TASK_TIMEOUT_DEFAULT = "1800";
 
+  /**
+   * Server side task (default) timeout value
+   */
+  public static final String SERVER_TASK_TIMEOUT_KEY = "server.task.timeout";
+  public static final String SERVER_TASK_TIMEOUT_DEFAULT = "1200";
+
   public static final String CUSTOM_ACTION_DEFINITION_KEY = "custom.action.definitions";
   public static final String SHARED_RESOURCES_DIR_KEY = "shared.resources.dir";
   private static final String CUSTOM_ACTION_DEFINITION_DEF_VALUE = "/var/lib/ambari-server/resources/custom_action_definitions";
@@ -1879,6 +1885,20 @@ public class Configuration {
     }
   }
 
+  /**
+   * @return default server-side task timeout in seconds.
+   */
+  public Integer getDefaultServerTaskTimeout() {
+    String value = properties.getProperty(SERVER_TASK_TIMEOUT_KEY, SERVER_TASK_TIMEOUT_DEFAULT);
+    if (StringUtils.isNumeric(value)) {
+      return Integer.parseInt(value);
+    } else {
+      LOG.warn("Value of {} ({}) should be a number, falling back to default value ({})",
+          SERVER_TASK_TIMEOUT_KEY, value, SERVER_TASK_TIMEOUT_DEFAULT);
+      return Integer.parseInt(SERVER_TASK_TIMEOUT_DEFAULT);
+    }
+  }
+
   public String getResourceDirPath() {
     return properties.getProperty(RESOURCES_DIR_KEY, RESOURCES_DIR_DEFAULT);
   }

ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java

@@ -2418,7 +2418,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Preparing Operations",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2441,7 +2441,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Preparing Operations",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2464,7 +2464,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Preparing Operations",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2487,7 +2487,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Create Principals",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2510,7 +2510,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Destroy Principals",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2533,7 +2533,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Create Keytabs",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2665,7 +2665,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Update Service Configurations",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);
@@ -2718,7 +2718,7 @@ public class KerberosHelperImpl implements KerberosHelper {
           event,
           commandParameters,
           "Kerberization Clean Up",
-          1200);
+          configuration.getDefaultServerTaskTimeout());
 
       RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder);
       roleGraph.build(stage);

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -1513,7 +1513,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
         getManagementController().getAuthName(), Role.AMBARI_SERVER_ACTION, RoleCommand.EXECUTE,
         cluster.getClusterName(),
         new ServiceComponentHostServerActionEvent(null, System.currentTimeMillis()), commandParams,
-        itemDetail, null, Integer.valueOf(1200), allowRetry,
+        itemDetail, null, s_configuration.getDefaultServerTaskTimeout(), allowRetry,
         context.isComponentFailureAutoSkipped());
 
     request.addStages(Collections.singletonList(stage));

ambari-server/src/main/java/org/apache/ambari/server/serveraction/ServerActionExecutor.java

@@ -35,6 +35,7 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.actionmanager.Request;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.controller.internal.CalculatedStatus;
 import org.apache.ambari.server.security.authorization.internal.InternalAuthenticationToken;
 import org.apache.ambari.server.utils.StageUtils;
@@ -57,7 +58,7 @@ import org.springframework.security.core.context.SecurityContextHolder;
 public class ServerActionExecutor {
 
   private final static Logger LOG = LoggerFactory.getLogger(ServerActionExecutor.class);
-  private final static Long EXECUTION_TIMEOUT_MS = 1000L * 60 * 5;
+  private final static Long DEFAULT_EXECUTION_TIMEOUT_MS = 1000L * 60 * 5;
   private final static Long POLLING_TIMEOUT_MS = 1000L * 5;
 
   /**
@@ -66,6 +67,13 @@ public class ServerActionExecutor {
   @Inject
   private static Injector injector;
 
+  /**
+   * The Ambari configuration used to obtain configured details such as the default server-side action
+   * timeout.
+   */
+  @Inject
+  private static Configuration configuration;
+
   /**
    * Maps request IDs to "blackboards" of shared data.
    * <p/>
@@ -354,13 +362,24 @@ public class ServerActionExecutor {
     try {
       timeout = (paramsTimeout == null)
           ? null
-          : (Long.parseLong(paramsTimeout) * 1000); // Convert seconds to milliseconds
+          : (Long.parseLong(paramsTimeout) * 1000L); // Convert seconds to milliseconds
     } catch (NumberFormatException e) {
       timeout = null;
     }
 
+    // If a configured timeout value is not set for the command, attempt to get the configured
+    // default
+    if (timeout == null) {
+      Integer defaultTimeoutSeconds = configuration.getDefaultServerTaskTimeout();
+      if (defaultTimeoutSeconds != null) {
+        timeout = defaultTimeoutSeconds * 1000L; // convert seconds to milliseconds
+      }
+    }
+
+    // If a timeout value was not determined, return the hard-coded timeout value, else return the
+    // determined timeout value.
     return (timeout == null)
-        ? EXECUTION_TIMEOUT_MS
+        ? DEFAULT_EXECUTION_TIMEOUT_MS
         : ((timeout < 0) ? 0 : timeout);
   }
 

ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java

@@ -318,6 +318,21 @@ public class ConfigurationTest {
   }
 
 
+  @Test
+  public void testGetDefaultServerTaskTimeout() {
+    Properties ambariProperties = new Properties();
+    Configuration conf = new Configuration(ambariProperties);
+
+    Assert.assertEquals(Integer.valueOf(1200), conf.getDefaultServerTaskTimeout());
+
+    ambariProperties = new Properties();
+    ambariProperties.setProperty(Configuration.SERVER_TASK_TIMEOUT_KEY, "3600");
+
+    conf = new Configuration(ambariProperties);
+
+    Assert.assertEquals(Integer.valueOf(3600), conf.getDefaultServerTaskTimeout());
+  }
+
   @Test
   public void testGetLdapServerProperties_WrongManagerPassword() throws Exception {
     final Properties ambariProperties = new Properties();