AMBARI-21440. Cross-stack migration, package supplemental stack in RPM (Alejandro Fernandez via jonathanhurley)

ambari-agent/pom.xml

@@ -486,6 +486,7 @@
                     <include>common-services/**</include>
                     <include>stacks/stack_advisor.py</include>
                     <include>stacks/${stack.distribution}/**/*</include>
+                    <include>stacks/${supplemental.distribution}/**/*</include>
                   </includes>
                   <filtering>false</filtering>
                 </resource>
@@ -661,6 +662,7 @@
       <dirsep>/</dirsep>
       <pathsep>:</pathsep>
       <stack.distribution>HDP</stack.distribution>
+      <supplemental.distribution>BigInsights</supplemental.distribution>
       <executable.python>${project.basedir}/../ambari-common/src/main/unix/ambari-python-wrap</executable.python>
       <executable.shell>sh</executable.shell>
       <fileextension.shell>sh</fileextension.shell>

ambari-server/pom.xml

@@ -42,6 +42,10 @@
     <ambariProperties>conf/unix/ambari.properties</ambariProperties>
     <commonServicesSrcLocation>src/main/resources/common-services</commonServicesSrcLocation>
     <stacksSrcLocation>src/main/resources/stacks/${stack.distribution}</stacksSrcLocation>
+
+    <!-- Additional stack to include in the server and agent paths. -->
+    <supplementalDistribution>src/main/resources/stacks/${supplemental.distribution}</supplementalDistribution>
+
     <tarballResourcesFolder>src/main/resources</tarballResourcesFolder>
     <skipPythonTests>false</skipPythonTests>
     <hadoop.version>2.7.2</hadoop.version>

ambari-server/src/main/assemblies/server.xml

@@ -188,6 +188,16 @@
         <include>/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar</include>
 	  </includes>
     </fileSet>
+    <!-- Used to also the source stack during a stack migration. -->
+    <fileSet>
+      <fileMode>755</fileMode>
+      <directory>${supplementalDistribution}</directory>
+      <outputDirectory>/var/lib/ambari-server/resources/stacks/${supplemental.distribution}</outputDirectory>
+      <excludes>
+        <exclude>/4.2/hooks/before-START/files/fast-hdfs-resource.jar</exclude>
+        <exclude>/4.2.5/hooks/before-START/files/fast-hdfs-resource.jar</exclude>
+      </excludes>
+    </fileSet>
     <fileSet>
       <fileMode>755</fileMode>
       <directory>src/main/python/ambari_server</directory>

dev-support/docker/docker/bin/ambaribuild.py

@@ -35,13 +35,15 @@ def ambariUnitTest():
 			cwd="/tmp/ambari")
 	return proc.wait()
 
-def buildAmbari(stack_distribution):
-	stack_distribution_param = ""
-	if stack_distribution is not None:
-		stack_distribution_param = "-Dstack.distribution=" + stack_distribution
+def buildAmbari(stack_distribution, supplemental_distribution=None):
+	stack_distribution_param = ("-Dstack.distribution=" + stack_distribution) if stack_distribution is not None else ""
+	supplemental_distribution_param = ("-Dsupplemental.distribution=" + supplemental_distribution) if supplemental_distribution is not None else ""
+
 	proc = subprocess.Popen("mvn -B clean install package rpm:rpm -Dmaven.clover.skip=true -Dfindbugs.skip=true "
 						+ SKIP_TEST + " "
-						+ stack_distribution_param + " -Dpython.ver=\"python >= 2.6\"",
+						+ stack_distribution_param + " "
+					  + supplemental_distribution_param + " "
+						+ " -Dpython.ver=\"python >= 2.6\"",
 			shell=True,
 			cwd="/tmp/ambari")
 	return proc.wait()
@@ -143,6 +145,7 @@ class ParseResult:
 	is_deep_clean = False
 	is_rebuild = False
 	stack_distribution = None
+	supplemental_distribution = None
 	is_test = False
 	is_install_server = False
 	is_install_agent = False
@@ -169,6 +172,10 @@ def parse(argv):
 				dest="stack_distribution",
 				help="set a stack distribution. [HDP|PHD|BIGTOP]. Make sure -b is also set when you set a stack distribution")
 
+		parser.add_option("-x", "--supplemental_distribution",
+											dest="supplemental_distribution",
+											help="set a supplement stack distribution in addition to the primary one. [BigInsights]. Make sure -b is also set when you set a supplement stack distribution")
+
 		parser.add_option("-d", "--server_debug",
 				dest="is_server_debug",
 				action="store_true",
@@ -182,6 +189,8 @@ def parse(argv):
 			result.is_rebuild = True
 		if options.stack_distribution:
 			result.stack_distribution = options.stack_distribution
+		if options.supplemental_distribution:
+			result.supplemental_distribution = options.supplemental_distribution
 		if options.is_server_debug:
 			result.is_server_debug = True
 
@@ -232,7 +241,7 @@ if __name__ == "__main__":
 		sys.exit(retcode)
 
 	if parsed_args.is_rebuild:
-		retcode = buildAmbari(parsed_args.stack_distribution)
+		retcode = buildAmbari(parsed_args.stack_distribution, supplemental_distribution=parsed_args.supplemental_distribution)
 		if retcode != 0: sys.exit(retcode)
 
 	if parsed_args.is_install_server:

dev-support/docker/docker/bin/test/ambaribuild_test.py

@@ -22,6 +22,7 @@ def unittest():
 	assert result.is_test == True
 	assert result.is_rebuild == False
 	assert result.stack_distribution == None
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == False
 	assert result.is_install_agent == False
 	assert result.is_deploy == False
@@ -32,6 +33,7 @@ def unittest():
 	assert result.is_test == False
 	assert result.is_rebuild == False
 	assert result.stack_distribution == None
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == True
 	assert result.is_install_agent == False
 	assert result.is_deploy == False
@@ -42,6 +44,7 @@ def unittest():
 	assert result.is_test == False
 	assert result.is_rebuild == False
 	assert result.stack_distribution == None
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == True
 	assert result.is_install_agent == True
 	assert result.is_deploy == False
@@ -52,6 +55,7 @@ def unittest():
 	assert result.is_test == False
 	assert result.is_rebuild == True
 	assert result.stack_distribution == None
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == True
 	assert result.is_install_agent == True
 	assert result.is_deploy == False
@@ -62,6 +66,7 @@ def unittest():
 	assert result.is_test == False
 	assert result.is_rebuild == False
 	assert result.stack_distribution == None
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == True
 	assert result.is_install_agent == True
 	assert result.is_deploy == True
@@ -72,11 +77,23 @@ def unittest():
 	assert result.is_test == False
 	assert result.is_rebuild == True
 	assert result.stack_distribution == "BIGTOP"
+	assert result.supplemental_distribution == None
 	assert result.is_install_server == True
 	assert result.is_install_agent == True
 	assert result.is_deploy == True
 	assert result.is_server_debug == True
 
+	result = ambaribuild.parse(["deploy", "-b", "-s", "HDP", "-x", "BigInsights", "-c"])
+	assert result.is_deep_clean == True
+	assert result.is_test == False
+	assert result.is_rebuild == True
+	assert result.stack_distribution == "HDP"
+	assert result.supplemental_distribution == "BigInsights"
+	assert result.is_install_server == True
+	assert result.is_install_agent == True
+	assert result.is_deploy == True
+	assert result.is_server_debug == False
+
 if __name__ == "__main__":
 	unittest()
 

pom.xml

@@ -52,6 +52,10 @@
     <deb.architecture>i386 amd64</deb.architecture>
     <deb.priority>extra</deb.priority>
     <stack.distribution>HDP</stack.distribution>
+
+    <!-- Default supplement distribution is BigInsights for Ambari 2.5.2 and higher. -->
+    <supplemental.distribution>BigInsights</supplemental.distribution>
+
     <ambari.dir>${project.basedir}</ambari.dir>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <distMgmtSnapshotsId>apache.snapshots.https</distMgmtSnapshotsId>