AMBARI-9281. RU fails without build # in version (ncole)

ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java

@@ -19,10 +19,10 @@
 
 package org.apache.ambari.server.bootstrap;
 
-import com.google.gson.annotations.SerializedName;
-
 import java.util.List;
 
+import com.google.gson.annotations.SerializedName;
+
 /**
  * This class is used for mapping json of structured output for
  * "Distribute repositories/install packages" action.
@@ -47,7 +47,20 @@ public class DistributeRepositoriesStructuredOutput {
   @SerializedName("package_installation_result")
   private String packageInstallationResult;
 
+  /**
+   * The actual version returned
+   */
+  @SerializedName("actual_version")
+  private String actualVersion;
+
+  /**
+   * The stack id used to look up version
+   */
+  @SerializedName("stack_id")
+  private String stackId;
+
   public String getInstalledRepositoryVersion() {
+
     return installedRepositoryVersion;
   }
 
@@ -58,4 +71,12 @@ public class DistributeRepositoriesStructuredOutput {
   public String getPackageInstallationResult() {
     return packageInstallationResult;
   }
+
+  public String getActualVersion() {
+    return actualVersion;
+  }
+
+  public String getStackId() {
+    return stackId;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java

@@ -29,8 +29,6 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.AmbariException;
-import com.google.gson.Gson;
-import com.google.inject.Provider;
 import org.apache.ambari.server.StaticallyInject;
 import org.apache.ambari.server.actionmanager.ActionManager;
 import org.apache.ambari.server.actionmanager.RequestFactory;
@@ -66,10 +64,12 @@ import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.ServiceOsSpecific;
+import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.utils.StageUtils;
 
+import com.google.gson.Gson;
 import com.google.inject.Inject;
-import org.apache.ambari.server.state.StackId;
+import com.google.inject.Provider;
 
 /**
  * Resource provider for cluster stack versions resources.
@@ -263,7 +263,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
       throw new NoSuchParentResourceException(e.getMessage(), e);
     }
 
-    String stackId;
+    final String stackId;
     if (propertyMap.containsKey(CLUSTER_STACK_VERSION_STACK_PROPERTY_ID) &&
             propertyMap.containsKey(CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID)) {
       stackName = (String) propertyMap.get(CLUSTER_STACK_VERSION_STACK_PROPERTY_ID);
@@ -347,6 +347,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou
       final String repoList = gson.toJson(repoInfo);
 
       Map<String, String> params = new HashMap<String, String>() {{
+        put("stack_id", stackId);
         put("repository_version", desiredRepoVersion);
         put("base_urls", repoList);
         put("package_list", packageList);

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java

@@ -17,46 +17,30 @@
  */
 package org.apache.ambari.server.events.listeners.upgrade;
 
-import com.google.common.eventbus.AllowConcurrentEvents;
-import com.google.common.eventbus.Subscribe;
-import com.google.gson.JsonSyntaxException;
-import com.google.inject.Inject;
-import com.google.inject.Provider;
-import com.google.inject.Singleton;
+import java.util.List;
+
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.EagerSingleton;
 import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.bootstrap.DistributeRepositoriesStructuredOutput;
-import org.apache.ambari.server.controller.RootServiceResponseFactory.Services;
 import org.apache.ambari.server.events.ActionFinalReportReceivedEvent;
-import org.apache.ambari.server.events.AlertReceivedEvent;
-import org.apache.ambari.server.events.AlertStateChangeEvent;
-import org.apache.ambari.server.events.publishers.AlertEventPublisher;
 import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
-import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
-import org.apache.ambari.server.orm.dao.AlertsDAO;
-import org.apache.ambari.server.orm.dao.ClusterVersionDAO;
 import org.apache.ambari.server.orm.dao.HostVersionDAO;
-import org.apache.ambari.server.orm.entities.AlertCurrentEntity;
-import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
-import org.apache.ambari.server.orm.entities.AlertHistoryEntity;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
 import org.apache.ambari.server.orm.entities.HostVersionEntity;
-import org.apache.ambari.server.state.Alert;
-import org.apache.ambari.server.state.AlertState;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
-import org.apache.ambari.server.state.Host;
-import org.apache.ambari.server.state.MaintenanceState;
 import org.apache.ambari.server.state.RepositoryVersionState;
-import org.apache.ambari.server.state.Service;
-import org.apache.ambari.server.state.ServiceComponentHost;
 import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.common.eventbus.Subscribe;
+import com.google.gson.JsonSyntaxException;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
 
 /**
  * The {@link org.apache.ambari.server.events.listeners.upgrade.DistributeRepositoriesActionListener} class
@@ -80,9 +64,7 @@ public class DistributeRepositoriesActionListener {
   private Provider<Clusters> clusters;
 
   @Inject
-  private Provider<ClusterVersionDAO> clusterVersionDAO;
-
-  private AmbariEventPublisher publisher;
+  private RepositoryVersionDAO repoVersionDAO;
 
 
   /**
@@ -92,7 +74,6 @@ public class DistributeRepositoriesActionListener {
    */
   @Inject
   public DistributeRepositoriesActionListener(AmbariEventPublisher publisher) {
-    this.publisher = publisher;
     publisher.register(this);
   }
 
@@ -115,9 +96,8 @@ public class DistributeRepositoriesActionListener {
       return;
     }
 
-
     String repositoryVersion = null;
-    
+
     if (event.getCommandReport() == null) {
       LOG.error("Command report is null, marking action as INSTALL_FAILED");
     } else {
@@ -126,10 +106,47 @@ public class DistributeRepositoriesActionListener {
         DistributeRepositoriesStructuredOutput structuredOutput = StageUtils.getGson().fromJson(
                 event.getCommandReport().getStructuredOut(),
                 DistributeRepositoriesStructuredOutput.class);
+
+        repositoryVersion = structuredOutput.getInstalledRepositoryVersion();
+
         if (event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString())) {
           newHostState = RepositoryVersionState.INSTALLED;
+
+          if (null != structuredOutput.getActualVersion() &&
+              null != structuredOutput.getInstalledRepositoryVersion() &&
+              null != structuredOutput.getStackId() &&
+              !structuredOutput.getActualVersion().equals(structuredOutput.getInstalledRepositoryVersion())) {
+
+            // !!! getInstalledRepositoryVersion() from the agent is the one
+            // entered in the UI.  getActualVersion() is computed.
+
+            RepositoryVersionEntity version = repoVersionDAO.findByStackAndVersion(
+                structuredOutput.getStackId(), structuredOutput.getInstalledRepositoryVersion());
+
+            if (null != version) {
+              LOG.info("Repository version {} was found, but {} is the actual value",
+                  structuredOutput.getInstalledRepositoryVersion(),
+                  structuredOutput.getActualVersion());
+              // !!! the entered version is not correct
+              version.setVersion(structuredOutput.getActualVersion());
+              repoVersionDAO.merge(version);
+              repositoryVersion = structuredOutput.getActualVersion();
+            } else {
+              // !!! extra check that the actual version is correct
+              version = repoVersionDAO.findByStackAndVersion(
+                  structuredOutput.getStackId(), structuredOutput.getActualVersion());
+
+              LOG.debug("Repository version {} was not found, check for {}.  Found={}",
+                  structuredOutput.getInstalledRepositoryVersion(),
+                  structuredOutput.getActualVersion(),
+                  Boolean.valueOf(null != version));
+
+              if (null != version) {
+                repositoryVersion = structuredOutput.getActualVersion();
+              }
+            }
+          }
         }
-        repositoryVersion = structuredOutput.getInstalledRepositoryVersion();
       } catch (JsonSyntaxException e) {
         LOG.error("Can not parse structured output %s", e);
       }

ambari-server/src/main/resources/custom_actions/scripts/install_packages.py

@@ -28,6 +28,7 @@ from resource_management import *
 from resource_management.libraries.functions.list_ambari_managed_repos import list_ambari_managed_repos
 from ambari_commons.os_check import OSCheck, OSConst
 from resource_management.libraries.functions.packages_analyzer import allInstalledPackages
+from resource_management.core.shell import call
 
 
 class InstallPackages(Script):
@@ -53,16 +54,20 @@ class InstallPackages(Script):
       repository_version = config['roleParams']['repository_version']
       base_urls = json.loads(config['roleParams']['base_urls'])
       package_list = json.loads(config['roleParams']['package_list'])
+      stack_id = config['roleParams']['stack_id']
     except KeyError:
       # Last try
       repository_version = config['commandParams']['repository_version']
       base_urls = json.loads(config['commandParams']['base_urls'])
       package_list = json.loads(config['commandParams']['package_list'])
+      stack_id = config['commandParams']['stack_id']
 
     # Install/update repositories
     installed_repositories = []
     current_repositories = ['base']  # Some our packages are installed from the base repo
     current_repo_files = set(['base'])
+    old_versions = self.hdp_versions()
+    
     try:
       append_to_file = False
       for url_info in base_urls:
@@ -108,8 +113,16 @@ class InstallPackages(Script):
     structured_output = {
       'ambari_repositories': installed_repositories,
       'installed_repository_version': repository_version,
+      'stack_id': stack_id,
       'package_installation_result': 'SUCCESS' if package_install_result else 'FAIL'
     }
+
+    if package_install_result:
+      new_versions = self.hdp_versions()
+      deltas = set(new_versions) - set(old_versions)
+      if 1 == len(deltas):
+        structured_output['actual_version'] = next(iter(deltas))
+
     self.put_structured_out(structured_output)
 
     # Provide correct exit code
@@ -161,6 +174,17 @@ class InstallPackages(Script):
     else:
       return package_name
 
+  def hdp_versions(self):
+    code, out = call("hdp-select versions")
+    if 0 == code:
+      versions = []
+      for line in out.splitlines():
+        versions.append(line.rstrip('\n'))
+      return versions
+    else:
+      return []
+
+
 
 if __name__ == "__main__":
   InstallPackages().execute()

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java

@@ -79,6 +79,8 @@ import org.apache.ambari.server.events.publishers.AmbariEventPublisher;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
 import org.apache.ambari.server.orm.OrmTestHelper;
+import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
+import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.state.Alert;
 import org.apache.ambari.server.state.AlertState;
 import org.apache.ambari.server.state.Cluster;
@@ -105,6 +107,7 @@ import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.gson.JsonObject;
 import com.google.inject.Guice;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
@@ -2199,8 +2202,8 @@ public class TestHeartbeatHandler {
     StackId stackId = new StackId(DummyStackId);
     cluster.setDesiredStackVersion(stackId);
     cluster.setCurrentStackVersion(stackId);
-    helper.getOrCreateRepositoryVersion(stackId.getStackName(), stackId.getStackVersion());
-    cluster.createClusterVersion(stackId.getStackName(), stackId.getStackVersion(), "admin", RepositoryVersionState.UPGRADING);
+    helper.getOrCreateRepositoryVersion(stackId.getStackId(), stackId.getStackVersion());
+    cluster.createClusterVersion(stackId.getStackId(), stackId.getStackVersion(), "admin", RepositoryVersionState.UPGRADING);
     return cluster;
   }
 
@@ -2405,4 +2408,66 @@ public class TestHeartbeatHandler {
     // should NOT throw AmbariException from alerts.
     handler.handleHeartBeat(hb);
   }
+
+  @Test
+  public void testInstallPackagesWithVersion() throws Exception {
+
+    final HostRoleCommand command = new HostRoleCommand(DummyHostname1,
+        Role.DATANODE, null, null);
+
+    ActionManager am = getMockActionManager();
+    expect(am.getTasks(anyObject(List.class))).andReturn(
+        Collections.singletonList(command)).anyTimes();
+    replay(am);
+
+    Cluster cluster = getDummyCluster();
+
+    @SuppressWarnings("serial")
+    Set<String> hostNames = new HashSet<String>() {{
+      add(DummyHostname1);
+    }};
+    clusters.mapHostsToCluster(hostNames, DummyCluster);
+
+
+    HeartBeatHandler handler = getHeartBeatHandler(am, new ActionQueue());
+    HeartBeat hb = new HeartBeat();
+
+    JsonObject json = new JsonObject();
+    json.addProperty("actual_version", "2.2.1.0-2222");
+    json.addProperty("package_installation_result", "SUCCESS");
+    json.addProperty("installed_repository_version", "0.1");
+    json.addProperty("stack_id", cluster.getDesiredStackVersion().getStackId());
+
+
+    CommandReport cmdReport = new CommandReport();
+    cmdReport.setActionId(StageUtils.getActionId(requestId, stageId));
+    cmdReport.setTaskId(1);
+    cmdReport.setCustomCommand("install_packages");
+    cmdReport.setStructuredOut(json.toString());
+    cmdReport.setRoleCommand(RoleCommand.ACTIONEXECUTE.name());
+    cmdReport.setStatus(HostRoleStatus.COMPLETED.name());
+    cmdReport.setRole("install_packages");
+    cmdReport.setClusterName(DummyCluster);
+
+    hb.setReports(Collections.singletonList(cmdReport));
+    hb.setTimestamp(0L);
+    hb.setResponseId(0);
+    hb.setNodeStatus(new HostStatus(Status.HEALTHY, DummyHostStatus));
+    hb.setHostname(DummyHostname1);
+    hb.setComponentStatus(new ArrayList<ComponentStatus>());
+
+
+    RepositoryVersionDAO dao = injector.getInstance(RepositoryVersionDAO.class);
+    RepositoryVersionEntity entity = dao.findByStackAndVersion("HDP-0.1", "0.1");
+    Assert.assertNotNull(entity);
+
+    handler.handleHeartBeat(hb);
+
+    entity = dao.findByStackAndVersion("HDP-0.1", "0.1");
+    Assert.assertNull(entity);
+
+    entity = dao.findByStackAndVersion("HDP-0.1", "2.2.1.0-2222");
+    Assert.assertNotNull(entity);
+
+  }
 }

ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -51,6 +51,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertEquals(put_structured_out.call_args[0][0],
                       {'package_installation_result': 'SUCCESS',
                        'installed_repository_version': u'2.2.0.1-885',
+                       'stack_id': 'HDP-2.2',
                        'ambari_repositories': []})
     self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
                               base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
@@ -97,6 +98,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertEquals(put_structured_out.call_args[0][0],
                       {'package_installation_result': 'SUCCESS',
                        'installed_repository_version': u'2.2.0.1-885',
+                       'stack_id': 'HDP-2.2',
                        'ambari_repositories': ["HDP-UTILS-2.2.0.1-885"]})
     self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
                               base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
@@ -160,6 +162,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertEquals(put_structured_out.call_args[0][0],
                       {'package_installation_result': 'FAIL',
                        'installed_repository_version': u'2.2.0.1-885',
+                       'stack_id': 'HDP-2.2',
                        'ambari_repositories': []})
     self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
                               base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',
@@ -201,6 +204,7 @@ class TestInstallPackages(RMFTestCase):
     self.assertEquals(put_structured_out.call_args[0][0],
                       {'package_installation_result': 'SUCCESS',
                        'installed_repository_version': u'2.2.0.1-885',
+                       'stack_id': 'HDP-2.2',
                        'ambari_repositories': []})
     self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885',
                               base_url=u'http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0',

ambari-server/src/test/python/custom_actions/configs/install_packages_config.json

@@ -22,6 +22,7 @@
     }, 
     "commandType": "EXECUTION_COMMAND", 
     "roleParams": {
+        "stack_id": "HDP-2.2",
         "repository_version": "2.2.0.1-885",
         "base_urls": "[{\"name\":\"HDP-UTILS\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-UTILS-1.1.0.20\"},{\"name\":\"HDP\",\"baseUrl\":\"http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos5/2.x/updates/2.2.0.0\",\"repositoryId\":\"HDP-2.2\"}]",
         "package_list": "[{\"name\":\"hadoop_2_2_*\"},{\"name\":\"snappy\"},{\"name\":\"snappy-devel\"},{\"name\":\"lzo\"},{\"name\":\"hadooplzo_2_2_*\"},{\"name\":\"hadoop_2_2_*-libhdfs\"},{\"name\":\"ambari-log4j\"}]"
@@ -76,4 +77,4 @@
             "0b3.vm"
         ]
     }
-}
+}