AMBARI-12202 - RU: YARN daemons started by Ambari have invalid classpath entries (jonathanhurley)

ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py

@@ -148,7 +148,7 @@ def get_role_component_current_hdp_version():
   return current_hdp_version
 
 
-def get_hadoop_dir(target):
+def get_hadoop_dir(target, force_latest_on_upgrade=False):
   """
   Return the hadoop shared directory in the following override order
   1. Use default for 2.1 and lower
@@ -157,6 +157,8 @@ def get_hadoop_dir(target):
   However, if the upgrade has not yet invoked hdp-select, return the current
   version of the component.
   :target: the target directory
+  :force_latest_on_upgrade: if True, then this will return the "current" directory
+  without the HDP version built into the path, such as /usr/hdp/current/hadoop-client
   """
 
   if not target in HADOOP_DIR_DEFAULTS:
@@ -171,22 +173,25 @@ def get_hadoop_dir(target):
     else:
       hadoop_dir = HADOOP_DIR_TEMPLATE.format("current", "hadoop-client", target)
 
-    stack_info = _get_upgrade_stack()
-
-    if stack_info is not None:
-      stack_version = stack_info[1]
-
-      # determine if hdp-select has been run and if not, then use the current
-      # hdp version until this component is upgraded
-      current_hdp_version = get_role_component_current_hdp_version()
-      if current_hdp_version is not None and stack_version != current_hdp_version:
-        stack_version = current_hdp_version
-
-      if target == "home":
-        # home uses a different template
-        hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_version, "hadoop")
-      else:
-        hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_version, "hadoop", target)
+    # if we are not forcing "current" for HDP 2.2, then attempt to determine
+    # if the exact version needs to be returned in the directory
+    if not force_latest_on_upgrade:
+      stack_info = _get_upgrade_stack()
+
+      if stack_info is not None:
+        stack_version = stack_info[1]
+
+        # determine if hdp-select has been run and if not, then use the current
+        # hdp version until this component is upgraded
+        current_hdp_version = get_role_component_current_hdp_version()
+        if current_hdp_version is not None and stack_version != current_hdp_version:
+          stack_version = current_hdp_version
+
+        if target == "home":
+          # home uses a different template
+          hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_version, "hadoop")
+        else:
+          hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_version, "hadoop", target)
 
   return hadoop_dir
 

ambari-server/src/main/java/org/apache/ambari/server/checks/AbstractCheckDescriptor.java

@@ -17,8 +17,7 @@
  */
 package org.apache.ambari.server.checks;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.LinkedHashSet;
 import java.util.Map;
 
 import org.apache.ambari.server.AmbariException;
@@ -226,7 +225,7 @@ public abstract class AbstractCheckDescriptor {
     }
 
     if (fail.contains("{{fails}}")) {
-      List<String> names = prerequisiteCheck.getFailedOn();
+      LinkedHashSet<String> names = prerequisiteCheck.getFailedOn();
 
       // If Type=PrereqCheckType.HOST, names list is already populated
       if (getDescription().getType() == PrereqCheckType.SERVICE) {
@@ -239,7 +238,7 @@ public abstract class AbstractCheckDescriptor {
               c.getDesiredStackVersion().getStackName(),
               c.getDesiredStackVersion().getStackVersion());
 
-          List<String> displays = new ArrayList<String>();
+          LinkedHashSet<String> displays = new LinkedHashSet<String>();
           for (String name : names) {
             if (services.containsKey(name)) {
               displays.add(services.get(name).getDisplayName());
@@ -270,10 +269,11 @@ public abstract class AbstractCheckDescriptor {
    * @param entities list of entities to format
    * @return formatted entity list
    */
-  protected String formatEntityList(List<String> entities) {
+  protected String formatEntityList(LinkedHashSet<String> entities) {
     if (entities == null || entities.isEmpty()) {
       return "";
     }
+
     final StringBuilder formatted = new StringBuilder(StringUtils.join(entities, ", "));
     if (entities.size() > 1) {
       formatted.replace(formatted.lastIndexOf(","), formatted.lastIndexOf(",") + 1, " and");
@@ -281,5 +281,4 @@ public abstract class AbstractCheckDescriptor {
 
     return formatted.toString();
   }
-
 }

ambari-server/src/main/java/org/apache/ambari/server/checks/ServicesUpCheck.java

@@ -20,14 +20,13 @@ package org.apache.ambari.server.checks;
 import java.text.MessageFormat;
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import com.google.inject.Inject;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.controller.PrereqCheckRequest;
-import org.apache.ambari.server.orm.dao.HostDAO;
 import org.apache.ambari.server.orm.models.HostComponentSummary;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Service;
@@ -35,9 +34,9 @@ import org.apache.ambari.server.state.ServiceComponent;
 import org.apache.ambari.server.state.State;
 import org.apache.ambari.server.state.stack.PrereqCheckStatus;
 import org.apache.ambari.server.state.stack.PrerequisiteCheck;
+import org.apache.commons.lang.StringUtils;
 
 import com.google.inject.Singleton;
-import org.apache.commons.lang.StringUtils;
 
 /**
  * Checks that services are up.
@@ -103,7 +102,7 @@ public class ServicesUpCheck extends AbstractCheckDescriptor {
     }
 
     if (!errorMessages.isEmpty()) {
-      prerequisiteCheck.setFailedOn(new ArrayList<String>(failedServiceNames));
+      prerequisiteCheck.setFailedOn(new LinkedHashSet<String>(failedServiceNames));
       prerequisiteCheck.setStatus(PrereqCheckStatus.FAIL);
       prerequisiteCheck.setFailReason("The following Service Components should be in a started state.  Please invoke a service Stop and full Start and try again. " + StringUtils.join(errorMessages, ", "));
     }

ambari-server/src/main/java/org/apache/ambari/server/checks/UpgradeCheckGroup.java

@@ -59,6 +59,11 @@ public enum UpgradeCheckGroup {
    */
   MULTIPLE_COMPONENT_WARNING(6.0f),
 
+  /**
+   * A general group for warning about configuration properties.
+   */
+  CONFIGURATION_WARNING(7.0f),
+
   /**
    * All other checks.
    */

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java

@@ -51,9 +51,9 @@ import org.apache.ambari.server.controller.AmbariSessionManager;
 import org.apache.ambari.server.controller.ClusterResponse;
 import org.apache.ambari.server.controller.ConfigurationResponse;
 import org.apache.ambari.server.controller.MaintenanceStateHelper;
+import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.controller.ServiceConfigVersionResponse;
 import org.apache.ambari.server.orm.RequiresSession;
-import org.apache.ambari.server.orm.cache.ConfigGroupHostMapping;
 import org.apache.ambari.server.orm.cache.HostConfigMapping;
 import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
 import org.apache.ambari.server.orm.dao.AlertDispatchDAO;
@@ -397,7 +397,6 @@ public class ClusterImpl implements Cluster {
                     "Can not get service info: stackName=%s, stackVersion=%s, serviceName=%s",
                     stackId.getStackName(), stackId.getStackVersion(),
                     serviceEntity.getServiceName()));
-                e.printStackTrace();
               }
             }
           }
@@ -2488,39 +2487,50 @@ public class ClusterImpl implements Cluster {
       for (Entry<String, ServiceComponentHostEvent> entry : eventMap.entries()) {
         String serviceName = entry.getKey();
         ServiceComponentHostEvent event = entry.getValue();
+        String serviceComponentName = event.getServiceComponentName();
+
+        // server-side events either don't have a service name or are AMBARI;
+        // either way they are not handled by this method since it expects a
+        // real service and component
+        if (StringUtils.isBlank(serviceName) || Services.AMBARI.name().equals(serviceName)) {
+          continue;
+        }
+
+        if (StringUtils.isBlank(serviceComponentName)) {
+          continue;
+        }
+
         try {
           Service service = getService(serviceName);
-          ServiceComponent serviceComponent = service.getServiceComponent(event.getServiceComponentName());
+          ServiceComponent serviceComponent = service.getServiceComponent(serviceComponentName);
           ServiceComponentHost serviceComponentHost = serviceComponent.getServiceComponentHost(event.getHostName());
           serviceComponentHost.handleEvent(event);
         } catch (ServiceNotFoundException e) {
           String message = String.format("ServiceComponentHost lookup exception. Service not found for Service: %s. Error: %s",
                   serviceName, e.getMessage());
           LOG.error(message);
-          e.printStackTrace();
           failedEvents.put(event, message);
         } catch (ServiceComponentNotFoundException e) {
           String message = String.format("ServiceComponentHost lookup exception. Service Component not found for Service: %s, Component: %s. Error: %s",
-                  serviceName, event.getServiceComponentName(), e.getMessage());
+              serviceName, serviceComponentName, e.getMessage());
           LOG.error(message);
-          e.printStackTrace();
           failedEvents.put(event, message);
         } catch (ServiceComponentHostNotFoundException e) {
           String message = String.format("ServiceComponentHost lookup exception. Service Component Host not found for Service: %s, Component: %s, Host: %s. Error: %s",
-                  serviceName, event.getServiceComponentName(), event.getHostName(), e.getMessage());
+              serviceName, serviceComponentName, event.getHostName(), e.getMessage());
           LOG.error(message);
-          e.printStackTrace();
           failedEvents.put(event, message);
         } catch (AmbariException e) {
           String message = String.format("ServiceComponentHost lookup exception %s", e.getMessage());
           LOG.error(message);
-          e.printStackTrace();
           failedEvents.put(event, message);
         } catch (InvalidStateTransitionException e) {
           LOG.error("Invalid transition ", e);
           if ((e.getEvent() == ServiceComponentHostEventType.HOST_SVCCOMP_START) &&
             (e.getCurrentState() == State.STARTED)) {
-            LOG.warn("Component request for component = " + event.getServiceComponentName() + " to start is invalid, since component is already started. Ignoring this request.");
+            LOG.warn("Component request for component = "
+                + serviceComponentName
+                + " to start is invalid, since component is already started. Ignoring this request.");
             // skip adding this as a failed event, to work around stack ordering issues with Hive
           } else {
             failedEvents.put(event, String.format("Invalid transition. %s", e.getMessage()));

ambari-server/src/main/java/org/apache/ambari/server/state/stack/PrerequisiteCheck.java

@@ -18,6 +18,7 @@
 package org.apache.ambari.server.state.stack;
 
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 
 import org.apache.ambari.server.checks.CheckDescription;
@@ -31,7 +32,7 @@ public class PrerequisiteCheck {
   private final String m_clusterName;
   private PrereqCheckStatus m_status = PrereqCheckStatus.PASS;
   private String m_failReason = "";
-  private List<String> m_failedOn = new ArrayList<String>();
+  private LinkedHashSet<String> m_failedOn = new LinkedHashSet<String>();
   private List<Object> m_failedDetail = new ArrayList<Object>();
 
   public PrerequisiteCheck(CheckDescription description, String clusterName) {
@@ -63,7 +64,7 @@ public class PrerequisiteCheck {
     m_failReason = failReason;
   }
 
-  public List<String> getFailedOn() {
+  public LinkedHashSet<String> getFailedOn() {
     return m_failedOn;
   }
 
@@ -71,8 +72,7 @@ public class PrerequisiteCheck {
     return m_failedDetail;
   }
 
-
-  public void setFailedOn(List<String> failedOn) {
+  public void setFailedOn(LinkedHashSet<String> failedOn) {
     m_failedOn = failedOn;
   }
 

ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ClusterGrouping.java

@@ -106,7 +106,7 @@ public class ClusterGrouping extends Grouping {
 
     @Override
     public List<StageWrapper> build(UpgradeContext ctx) {
-      if (null == ClusterGrouping.this.executionStages) {
+      if (null == executionStages) {
         return Collections.emptyList();
       }
 
@@ -128,6 +128,7 @@ public class ClusterGrouping extends Grouping {
               wrapper = getManualStageWrapper(ctx, execution);
               break;
 
+            case CONFIGURE:
             case SERVER_ACTION:
               wrapper = new StageWrapper(
                   StageWrapper.Type.SERVER_SIDE_ACTION,

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py

@@ -328,10 +328,6 @@ if not lzo_enabled:
   
 name_node_params = default("/commandParams/namenode", None)
 
-#hadoop params
-hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
-
-#hadoop-env.sh
 java_home = config['hostLevelParams']['java_home']
 java_version = int(config['hostLevelParams']['java_version'])
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -79,13 +79,18 @@ def is_secure_port(port):
 
 # hadoop default params
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
-hadoop_home = hdp_select.get_hadoop_dir("home")
+
+# upgrades would cause these directories to have a version instead of "current"
+# which would cause a lot of problems when writing out hadoop-env.sh; instead
+# force the use of "current" in the hook
+hadoop_home = hdp_select.get_hadoop_dir("home", force_latest_on_upgrade=True)
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
+hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True)
+
+hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
 hadoop_secure_dn_user = hdfs_user
 hadoop_dir = "/etc/hadoop"
 versioned_hdp_root = '/usr/hdp/current'
-hadoop_conf_dir = conf_select.get_hadoop_conf_dir(force_latest_on_upgrade=True)
-hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec")
-hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
 
 # HDP 2.2+ params
 if Script.is_hdp_stack_greater_or_equal("2.2"):

ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml

@@ -23,7 +23,7 @@
   <order>
     <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
         <task xsi:type="execute" hosts="master">
           <script>scripts/namenode.py</script>
@@ -63,6 +63,16 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Modify hadoop-env.sh">
+        <task xsi:type="configure">
+          <type>hadoop-env</type>
+          <replace key="content" find="# Add libraries required by nodemanager" replace-with="" />
+          <replace key="content" find="MAPREDUCE_LIBS={{mapreduce_libs_path}}" replace-with="" />
+          <replace key="content" find=":${MAPREDUCE_LIBS}" replace-with="" />
+          <replace key="content" find=":/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/" replace-with="" />
+          <replace key="content" find=":/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/" replace-with="" />
+        </task>
+      </execute-stage>
     </group>
 
     <group name="ZOOKEEPER" title="ZooKeeper">

ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/configuration/hadoop-env.xml

@@ -129,28 +129,18 @@ export HADOOP_IDENT_STRING=$USER
 
 # Use libraries from standard classpath
 JAVA_JDBC_LIBS=""
+
 #Add libraries required by mysql connector
 for jarFile in `ls /usr/share/java/*mysql* 2>/dev/null`
 do
   JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
 done
+
 # Add libraries required by oracle connector
 for jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`
 do
   JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile
 done
-# Add libraries required by nodemanager
-MAPREDUCE_LIBS={{mapreduce_libs_path}}
-export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}
-
-# added to the HADOOP_CLASSPATH
-if [ -d "/usr/hdp/current/tez-client" ]; then
-  if [ -d "/usr/hdp/current/tez-client/conf/" ]; then
-    # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.
-    export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/usr/hdp/current/tez-client/conf/
-  fi
-fi
-
 
 # Setting path to hdfs command line
 export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}

ambari-server/src/test/java/org/apache/ambari/server/checks/AbstractCheckDescriptorTest.java

@@ -17,8 +17,7 @@
  */
 package org.apache.ambari.server.checks;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.LinkedHashSet;
 
 import junit.framework.Assert;
 
@@ -57,7 +56,7 @@ public class AbstractCheckDescriptorTest {
 
     Assert.assertEquals("", check.formatEntityList(null));
 
-    final List<String> failedOn = new ArrayList<String>();
+    final LinkedHashSet<String> failedOn = new LinkedHashSet<String>();
     Assert.assertEquals("", check.formatEntityList(failedOn));
 
     failedOn.add("host1");