AMBARI-9054. Rolling Upgrade Bootstrap - Allow Ambari upgrade from 1.7.0 - 2.0.0 to populate new tables (Yurii Shylov via ncole)

ambari-common/src/main/python/resource_management/libraries/script/script.py

@@ -37,6 +37,7 @@ from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus
 from resource_management.core.resources.packaging import Package
 from resource_management.libraries.functions.version_select_util import get_component_version
 from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
 
 if OSCheck.is_windows_family():
@@ -115,23 +116,45 @@ class Script(object):
     except IOError, err:
       Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
 
-  def save_component_version_to_structured_out(self, stack_name):
+  def save_component_version_to_structured_out(self):
     """
     :param stack_name: One of HDP, HDPWIN, PHD, BIGTOP.
     :return: Append the version number to the structured out.
     """
-    import params
-    component_version = None
-
+    from resource_management.libraries.functions.default import default
+    stack_name = default("/hostLevelParams/stack_name", None)
     stack_to_component = self.get_stack_to_component()
-    if stack_to_component:
-      if stack_name == "HDP" and params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
-        component_name = stack_to_component[stack_name] if stack_name in stack_to_component else None
-        component_version = get_component_version(stack_name, component_name)
+    if stack_to_component and stack_name:
+      component_name = stack_to_component[stack_name] if stack_name in stack_to_component else None
+      component_version = get_component_version(stack_name, component_name)
 
       if component_version:
         self.put_structured_out({"version": component_version})
 
+  def should_expose_component_version(self, command_name):
+    """
+    Analyzes config and given command to determine if stack version should be written
+    to structured out. Currently only HDP stack versions >= 2.2 are supported.
+    :param command_name: command name
+    :return: True or False
+    """
+    from resource_management.libraries.functions.default import default
+    stack_version_unformatted = str(default("/hostLevelParams/stack_version", ""))
+    hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+    if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+      if command_name.lower() == "status":
+        request_version = default("/commandParams/request_version", None)
+        if request_version is not None:
+          return True
+        else:
+          if "version" in Script.structuredOut:
+            del Script.structuredOut["version"]
+            Script.structuredOut.update({})
+      else:
+        # Populate version only on base commands
+        return command_name.lower() == "start" or command_name.lower() == "install" or command_name.lower() == "restart"
+    return False
+
   def execute(self):
     """
     Sets up logging;
@@ -183,16 +206,6 @@ class Script(object):
       with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
         env.config.download_path = Script.tmp_dir
         method(env)
-
-        # For start actions, try to advertise the component's version
-        if command_name.lower() == "start" or command_name.lower() == "install":
-          try:
-            import params
-            # This is to support older stacks
-            if hasattr(params, "stack_name"):
-              self.save_component_version_to_structured_out(params.stack_name)
-          except ImportError:
-            logger.error("Executing command %s could not import params" % str(command_name))
     except ClientComponentHasNoStatus or ComponentIsNotRunning:
       # Support of component status checks.
       # Non-zero exit code is interpreted as an INSTALLED status of a component
@@ -200,7 +213,9 @@ class Script(object):
     except Fail:
       logger.exception("Error while executing command '{0}':".format(command_name))
       sys.exit(1)
-
+    finally:
+      if self.should_expose_component_version(command_name):
+        self.save_component_version_to_structured_out()
 
   def choose_method_to_execute(self, command_name):
     """
@@ -350,13 +365,8 @@ class Script(object):
       if rolling_restart:
         self.post_rolling_restart(env)
 
-    try:
-      import params
-      if hasattr(params, "stack_name"):
-        self.save_component_version_to_structured_out(params.stack_name)
-    except ImportError:
-      logger.error("Restart command could not import params")
-
+    if self.should_expose_component_version("restart"):
+      self.save_component_version_to_structured_out()
 
   def post_rolling_restart(self, env):
     """

ambari-server/src/main/java/org/apache/ambari/server/agent/ComponentStatus.java

@@ -116,14 +116,14 @@ public class ComponentStatus {
   public void setClusterName(String clusterName) {
     this.clusterName = clusterName;
   }
-  
+
   /**
    * @param tags the config tags that match this status
    */
   public void setConfigTags(Map<String, Map<String,String>> tags) {
     configurationTags = tags;
   }
-  
+
   /**
    * @return the config tags that match this command, or <code>null</code>
    * if none are present
@@ -131,31 +131,27 @@ public class ComponentStatus {
   public Map<String, Map<String,String>> getConfigTags() {
     return configurationTags;
   }
-  
+
   /**
-   * Sets extra information coming from the status. 
+   * Sets extra information coming from the status.
    */
   public void setExtra(Map<String, Object> info) {
     extra = info;
   }
 
   /**
-   * Gets extra information coming from the status. 
+   * Gets extra information coming from the status.
    */
   public Map<String, Object> getExtra() {
     return extra;
   }
-  
+
   @Override
   public String toString() {
-    return "ComponentStatus{" +
-            "componentName='" + componentName + '\'' +
-            ", msg='" + msg + '\'' +
-            ", status='" + status + '\'' +
-            ", securityState='" + securityState + '\'' +
-            ", serviceName='" + serviceName + '\'' +
-            ", clusterName='" + clusterName + '\'' +
-            ", stackVersion='" + stackVersion + '\'' +
-            '}';
+    return "ComponentStatus [componentName=" + componentName + ", msg=" + msg
+        + ", status=" + status + ", securityState=" + securityState
+        + ", serviceName=" + serviceName + ", clusterName=" + clusterName
+        + ", stackVersion=" + stackVersion + ", configurationTags="
+        + configurationTags + ", extra=" + extra + "]";
   }
 }

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -66,7 +66,6 @@ import org.apache.ambari.server.state.HostHealthStatus;
 import org.apache.ambari.server.state.HostHealthStatus.HealthStatus;
 import org.apache.ambari.server.state.HostState;
 import org.apache.ambari.server.state.MaintenanceState;
-import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.SecurityState;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
@@ -515,16 +514,11 @@ public class HeartBeatHandler {
                 //do nothing, pass this data further for processing
               }
               if (structuredOutput != null && StringUtils.isNotBlank(structuredOutput.getVersion())) {
-                final String previousVersion = scHost.getVersion();
-                if (!StringUtils.equals(previousVersion, structuredOutput.getVersion())) {
-                  scHost.setVersion(structuredOutput.getVersion());
-                  if (previousVersion != null && !previousVersion.equalsIgnoreCase(State.UNKNOWN.toString())) {
-                    scHost.setUpgradeState(UpgradeState.COMPLETE);
-                  }
-                }
+                handleComponentVersionReceived(scHost, structuredOutput.getVersion());
               }
               // Safer to recalculate the version even if we don't detect a difference in the value.
               // This is useful in case that a manual database edit is done while ambari-server is stopped.
+              // TODO should be included into handleComponentVersionReceived() after RU becomes stable
               HostComponentVersionEvent event = new HostComponentVersionEvent(cl, scHost);
               ambariEventPublisher.publish(event);
             }
@@ -671,6 +665,14 @@ public class HeartBeatHandler {
                     List<Map<String, String>> list = (List<Map<String, String>>) extra.get("processes");
                     scHost.setProcesses(list);
                   }
+                  if (extra.containsKey("version")) {
+                    boolean versionWasUpdated = handleComponentVersionReceived(scHost, extra.get("version").toString());
+                    if (versionWasUpdated) {
+                      // TODO should be included into handleComponentVersionReceived() after RU becomes stable
+                      HostComponentVersionEvent event = new HostComponentVersionEvent(cl, scHost);
+                      ambariEventPublisher.publish(event);
+                    }
+                  }
 
                 } catch (Exception e) {
                   LOG.error("Could not access extra JSON for " +
@@ -721,6 +723,26 @@ public class HeartBeatHandler {
     }
   }
 
+  /**
+   * Updates version of service component and sets upgrade state if needed.
+   *
+   * @param scHost service component host
+   * @param newVersion new version of service component
+   *
+   * @return true if component version was updated to new one
+   */
+  private boolean handleComponentVersionReceived(ServiceComponentHost scHost, String newVersion) {
+    final String previousVersion = scHost.getVersion();
+    if (!StringUtils.equals(previousVersion, newVersion)) {
+      scHost.setVersion(newVersion);
+      if (previousVersion != null && !previousVersion.equalsIgnoreCase(State.UNKNOWN.toString())) {
+        scHost.setUpgradeState(UpgradeState.COMPLETE);
+      }
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Adds commands from action queue to a heartbeat responce
    */
@@ -859,6 +881,11 @@ public class HeartBeatHandler {
     // Get status of service components
     List<StatusCommand> cmds = heartbeatMonitor.generateStatusCommands(hostname);
 
+    // Add request for component version
+    for (StatusCommand command: cmds) {
+      command.getCommandParams().put("request_version", String.valueOf(true));
+    }
+
     // Save the prefix of the log file paths
     hostObject.setPrefix(register.getPrefix());
 

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java

@@ -224,7 +224,6 @@ public class HeartbeatMonitor implements Runnable {
                                ServiceComponentHost sch) throws AmbariException {
     String serviceName = sch.getServiceName();
     String componentName = sch.getServiceComponentName();
-    Service service = cluster.getService(sch.getServiceName());
     StackId stackId = cluster.getDesiredStackVersion();
     ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(),
         stackId.getStackVersion(), serviceName);