AMBARI-21653. Create VDF and xsd to support MAINT upgrades (ncole)

ambari-server/src/main/java/org/apache/ambari/server/state/RepositoryType.java

@@ -24,21 +24,25 @@ public enum RepositoryType {
 
   /**
    * Repository should be considered to have all components for a cluster
-   * deployment
+   * deployment.
    */
   STANDARD,
 
   /**
    * Repository may have only minimum components and is used for patching
-   * purposes
+   * purposes.
    */
   PATCH,
 
   /**
-   * Repository is used to update services
+   * Repository is used as Maintenance release, which could be several patches rolled up in one.
+   * Orchestration should treat Maintenance just as it does for Patch..
    */
-  SERVICE
-
+  MAINT,
 
+  /**
+   * Repository is used to update services.
+   */
+  SERVICE
 
 }

ambari-server/src/main/java/org/apache/ambari/server/state/repository/AvailableVersion.java

@@ -33,6 +33,10 @@ public class AvailableVersion {
   @JsonProperty("version")
   private String version;
 
+  @JsonProperty("release_version")
+  @JsonSerialize(include=Inclusion.NON_NULL)
+  private String releaseVersion;
+
   @JsonProperty("version_id")
   @JsonSerialize(include=Inclusion.NON_NULL)
   private String versionId;
@@ -40,12 +44,27 @@ public class AvailableVersion {
   @JsonProperty
   private Set<Component> components;
 
-  AvailableVersion(String version, String versionId, Set<Component> components) {
+  AvailableVersion(String version, String versionId, String releaseVersion, Set<Component> components) {
     this.version = version;
     this.versionId = versionId;
+    this.releaseVersion = releaseVersion;
     this.components = components;
   }
 
+  /**
+   * @return the binary version of the available service.
+   */
+  public String getVersion() {
+    return version;
+  }
+
+  /**
+   * @return the release version of the available service.
+   */
+  public String getReleaseVersion() {
+    return releaseVersion;
+  }
+
   static class Component {
     @JsonProperty("name")
     private String name;

ambari-server/src/main/java/org/apache/ambari/server/state/repository/ManifestService.java

@@ -38,7 +38,7 @@ public class ManifestService {
   public String serviceName;
 
   /**
-   * Version of the service.  This is the publicly available version.
+   * Version of the service.  This is the publicly available version of the binary.
    */
   @XmlAttribute(name="version")
   public String version;
@@ -48,4 +48,12 @@ public class ManifestService {
    */
   @XmlAttribute(name="version-id")
   public String versionId;
+
+  /**
+   * The release version of the service.  This is not the same as {@link #version}; that is a binary version.
+   * This version is a build specific string that is independent of the binary and roughly matches
+   * {@link Release#version} for upgrade orchestration.
+   */
+  @XmlAttribute(name="release-version")
+  public String releaseVersion;
 }

ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java

@@ -43,7 +43,7 @@ public class Release {
   public String stackId;
 
   /**
-   * The stack version.
+   * The stack-based release version.
    */
   @XmlElement(name="version")
   public String version;

ambari-server/src/main/java/org/apache/ambari/server/state/repository/VersionDefinitionXml.java

@@ -126,10 +126,10 @@ public class VersionDefinitionXml {
       m_availableMap = new HashMap<>();
 
       if (availableServices.isEmpty()) {
+        // !!! populate available services from the manifest
         for (ManifestService ms : manifests.values()) {
           addToAvailable(ms, stack, Collections.<String>emptySet());
         }
-
       } else {
         for (AvailableServiceReference ref : availableServices) {
           ManifestService ms = manifests.get(ref.serviceIdReference);
@@ -229,21 +229,24 @@ public class VersionDefinitionXml {
 
   /**
    * Helper method to use a {@link ManifestService} to generate the available services structure
-   * @param ms          the ManifestService instance
+   * @param manifestService          the ManifestService instance
    * @param stack       the stack object
    * @param components  the set of components for the service
    */
-  private void addToAvailable(ManifestService ms, StackInfo stack, Set<String> components) {
-    ServiceInfo service = stack.getService(ms.serviceName);
+  private void addToAvailable(ManifestService manifestService, StackInfo stack, Set<String> components) {
+    ServiceInfo service = stack.getService(manifestService.serviceName);
 
-    if (!m_availableMap.containsKey(ms.serviceName)) {
-      String display = (null == service) ? ms.serviceName: service.getDisplayName();
+    if (!m_availableMap.containsKey(manifestService.serviceName)) {
+      String display = (null == service) ? manifestService.serviceName: service.getDisplayName();
 
-      m_availableMap.put(ms.serviceName, new AvailableService(ms.serviceName, display));
+      AvailableService available = new AvailableService(manifestService.serviceName, display);
+      m_availableMap.put(manifestService.serviceName, available);
     }
 
-    AvailableService as = m_availableMap.get(ms.serviceName);
-    as.getVersions().add(new AvailableVersion(ms.version, ms.versionId,
+    AvailableService as = m_availableMap.get(manifestService.serviceName);
+    as.getVersions().add(new AvailableVersion(manifestService.version,
+        manifestService.versionId,
+        manifestService.releaseVersion,
         buildComponents(service, components)));
   }
 

ambari-server/src/main/resources/version_definition.xsd

@@ -44,6 +44,7 @@
       <xs:enumeration value="STANDARD" />
       <xs:enumeration value="SERVICE" />
       <xs:enumeration value="PATCH" />
+      <xs:enumeration value="MAINT" />
     </xs:restriction>
   </xs:simpleType>
 
@@ -82,6 +83,7 @@
           <xs:attribute name="version" type="xs:string" use="required" />
           <xs:attribute name="id" type="xs:string" use="required" />
           <xs:attribute name="version-id" type="xs:string" />
+          <xs:attribute name="release-version" type="xs:string" />
         </xs:complexType>
       </xs:element>
     </xs:sequence>

ambari-server/src/test/java/org/apache/ambari/server/state/repository/VersionDefinitionTest.java

@@ -29,6 +29,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.ambari.server.state.ComponentInfo;
 import org.apache.ambari.server.state.RepositoryType;
 import org.apache.ambari.server.state.ServiceInfo;
 import org.apache.ambari.server.state.StackInfo;
@@ -308,7 +309,74 @@ public class VersionDefinitionTest {
     assertNotNull(xml.getPackageVersion("redhat6"));
     assertEquals("2_3_4_0_3396", xml.getPackageVersion("redhat6"));
     assertNull(xml.getPackageVersion("suse11"));
+  }
+
+  @Test
+  public void testMaintVersion() throws Exception {
+    File f = new File("src/test/resources/version_definition_test_maint.xml");
+
+    VersionDefinitionXml xml = VersionDefinitionXml.load(f.toURI().toURL());
+
+    String xmlString = xml.toXml();
+
+    xml = VersionDefinitionXml.load(xmlString);
+
+    assertEquals(RepositoryType.MAINT, xml.release.repositoryType);
+    assertEquals("2.3.4.1", xml.release.version);
+    assertEquals("1234", xml.release.build);
+    assertEquals("redhat6", xml.repositoryInfo.getOses().get(0).getFamily());
+
+
+
+    List<AvailableServiceReference> availableServices = xml.availableServices;
+    assertEquals(3, availableServices.size());
+
+    List<ManifestService> manifestServices = xml.manifestServices;
+    assertEquals(4, manifestServices.size());
+
+    ManifestService hdfs = null;
+    ManifestService hive = null;
+    for (ManifestService as : manifestServices) {
+      if (as.serviceId.equals("HDFS-271")) {
+        hdfs = as;
+      } else if (as.serviceId.equals("HIVE-200")) {
+        hive = as;
+      }
+    }
+
+    assertNotNull(hdfs);
+    assertNotNull(hive);
+
+    assertEquals("2.3.3", hdfs.releaseVersion);
+    assertNull(hive.releaseVersion);
+
+    StackInfo stack = new StackInfo() {
+      @Override
+      public ServiceInfo getService(String name) {
+        return makeService("HIVE", "HIVE_METASTORE");
+      }
+    };
 
+    Collection<AvailableService> availables = xml.getAvailableServices(stack);
+
+    assertEquals(2, availables.size());
+
+    boolean found = false;
+    for (AvailableService available : availables) {
+      if (available.getName().equals("HIVE")) {
+        found = true;
+        assertEquals(2, available.getVersions().size());
+        for (AvailableVersion version : available.getVersions()) {
+          if (version.getVersion().equals("1.1.0")) {
+            assertEquals("1.0.9", version.getReleaseVersion());
+          } else {
+            assertNull(version.getReleaseVersion());
+          }
+        }
+      }
+    }
+
+    assertTrue("Found available version for HIVE", found);
   }
 
 
@@ -334,4 +402,31 @@ public class VersionDefinitionTest {
     };
   }
 
+  private static ServiceInfo makeService(final String name, final String component) {
+    return new ServiceInfo() {
+      @Override
+      public String getName() {
+        return name;
+      }
+      @Override
+      public String getDisplayName() {
+        return name + " Display";
+      }
+      @Override
+      public String getVersion() {
+        return "1.1.1";
+      }
+      @Override
+      public String getComment() {
+        return name + " Comment";
+      }
+
+      @Override
+      public ComponentInfo getComponentByName(String name) {
+        return null;
+      }
+
+    };
+  }
+
 }

ambari-server/src/test/resources/version_definition_test_maint.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<repository-version xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:noNamespaceSchemaLocation="version_definition.xsd">
+  
+  <release>
+    <type>MAINT</type>
+    <stack-id>HDP-2.3</stack-id>
+    <version>2.3.4.1</version>
+    <build>1234</build>
+    <compatible-with>2.3.4.[1-9]</compatible-with>
+    <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes>
+  </release>
+  
+  <manifest>
+    <service id="HDFS-271" name="HDFS" version="2.7.1" version-id="10" release-version="2.3.3" />
+    <service id="HIVE-200" name="HIVE" version="2.0.0" />
+    <service id="HIVE-110" name="HIVE" version="1.1.0" release-version="1.0.9" />
+    <service id="HBASE-899" name="HBASE" version="8.9.9" />
+  </manifest>
+  
+  <available-services>
+    <service idref="HDFS-271" />
+    <service idref="HIVE-200" />
+    <service idref="HIVE-110">
+      <component>HIVE_METASTORE</component>
+    </service>
+  </available-services>
+  
+  <repository-info>
+    <os family="redhat6">
+      <repo>
+        <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl>
+        <repoid>HDP-2.3</repoid>
+        <reponame>HDP</reponame>
+        <unique>true</unique>
+      </repo>
+      <repo>
+        <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
+        <repoid>HDP-UTILS-1.1.0.20</repoid>
+        <reponame>HDP-UTILS</reponame>
+        <unique>false</unique>
+      </repo>
+    </os>
+  </repository-info>
+</repository-version>

contrib/version-builder/example.py

@@ -27,6 +27,7 @@ def main(args):
   vb.set_os("redhat6", package_version="2_4_2_0_12345")
 
   vb.add_manifest("HDFS-271", "HDFS", "2.7.1.2.4.0")
+  vb.add_manifest("YARN-271", "HDFS", "2.7.1.2.4.0", version_id = "1", release_version = "2.4.1.0")
 
   vb.add_repo("redhat6", "HDP-2.4", "HDP", "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.4.2.0", "true")
   vb.add_repo("redhat6", "HDP-UTILS-1.1.0.20", "HDP-UTILS", "http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6", "false")

contrib/version-builder/example.sh

@@ -30,11 +30,12 @@ python version_builder.py --file $filename --release-display HDP-2.4.1.1-1234
 python version_builder.py --file $filename --release-compatible 2.4.[0-1].0
 
 # call any number of times for each service in the repo
-python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4
+python version_builder.py --file $filename --manifest --manifest-id HDFS-271 --manifest-service HDFS --manifest-version 2.7.1.2.4 --manifest-release-version 2.4.0.0
 python version_builder.py --file $filename --manifest --manifest-id HBASE-132 --manifest-service HBASE --manifest-version 1.3.2.4.3
 
 #call any number of times for the target services to upgrade
 python version_builder.py --file $filename --available --manifest-id HDFS-271
+python version_builder.py --file $filename --available --manifest-id HBASE-132 --release-version 2.4.0
 
 # must be before repo calls
 python version_builder.py --file $filename --os --os-family redhat6 --os-package-version 2_4_1_1_12345

contrib/version-builder/version_builder.py

@@ -122,7 +122,7 @@ class VersionBuilder:
       pv_element.text = package_version
 
 
-  def add_manifest(self, id, service_name, version, version_id = None):
+  def add_manifest(self, id, service_name, version, version_id=None, release_version=None):
     """
     Add a manifest service.  A manifest lists all services in a repo, whether they are to be
     upgraded or not.
@@ -143,6 +143,9 @@ class VersionBuilder:
     if version_id:
       service_element.set('version-id', version_id)
 
+    if release_version:
+      service_element.set('release-version', release_version)
+
   def add_available(self, manifest_id, available_components=None):
     """
     Adds services available to upgrade for patches
@@ -296,7 +299,8 @@ def process_manifest(vb, options):
   if not options.manifest:
     return
 
-  vb.add_manifest(options.manifest_id, options.manifest_service, options.manifest_version, options.manifest_version_id)
+  vb.add_manifest(options.manifest_id, options.manifest_service, options.manifest_version, options.manifest_version_id,
+    options.manifest_release_version)
 
 def process_available(vb, options):
   """
@@ -411,15 +415,16 @@ def main(argv):
     help="Identifier to use when installing packages, generally a part of the package name")
 
   parser.add_option('--manifest', action='store_true', dest='manifest',
-    help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id")
+    help="Add a manifest service with other options: --manifest-id, --manifest-service, --manifest-version, --manifest-version-id, --manifest-release-version")
   parser.add_option('--manifest-id', dest='manifest_id',
     help="Unique ID for a service in a manifest.  Required when specifying --manifest and --available")
   parser.add_option('--manifest-service', dest='manifest_service')
   parser.add_option('--manifest-version', dest='manifest_version')
   parser.add_option('--manifest-version-id', dest='manifest_version_id')
+  parser.add_option('--manifest-release-version', dest='manifest_release_version')
 
   parser.add_option('--available', action='store_true', dest='available',
-    help="Add an available service with other options: --manifest-id, --available-components")
+    help="Add an available service with other options: --manifest-id, --available-components --service-release-version")
   parser.add_option('--available-components', dest='available_components',
     help="A CSV of service components that are intended to be upgraded via patch. \
       Omitting this implies the entire service should be upgraded")