AMBARI-7875 Docker build on builds.a.o and support JDK1.6 and python2.6 (jaoki)

dev-support/docker/README.md

@@ -0,0 +1,51 @@
+<!--
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+
+how to build
+--------------------
+
+```
+docker build -t ambari/build ./docker
+```
+
+how to run
+--------------------
+
+```
+# bash
+docker run --privileged -t -i -p 80:80 -p 5005:5005 -p 8080:8080 -h node1.mydomain.com --name ambari1 -v ${AMBARI_SRC:-$(pwd)}:/tmp/ambari ambari/build bash
+# where 5005 is java debug port and 8080 is the default http port, if no --privileged ambari-server start fails due to access to /proc/??/exe
+# -t is required otherwise, sudo commands do not run
+
+# build, install ambari and deploy hadoop in container
+cd {ambari src}
+docker rm ambari1
+docker run --privileged -t -p 80:80 -p 5005:5005 -p 8080:8080 -h node1.mydomain.com --name ambari1 -v ${AMBARI_SRC:-$(pwd)}:/tmp/ambari ambari/build /tmp/ambari-build-docker/bin/ambaribuild.py [test|server|agent|deploy] [-b] [-s [HDP|BIGTOP|PHD]]
+where
+test: mvn test
+server: install and run ambari-server
+agent: install and run ambari-server and ambari-agent
+deploy: install and run ambari-server and ambari-agent, and deploy a hadoop
+-b option to rebuild ambari
+```
+
+how to run unit test
+--------------------
+```
+cd docker
+python -m bin.test.ambaribuild_test
+
+```
+

dev-support/docker/docker/Dockerfile

@@ -0,0 +1,79 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+FROM centos:centos6
+
+RUN echo root:changeme | chpasswd
+
+## Install some basic utilities that aren't in the default image
+RUN yum -y install vim wget rpm-build sudo which telnet tar openssh-server openssh-clients ntp git python-setuptools httpd
+# phantomjs dependency
+RUN yum -y install fontconfig freetype libfreetype.so.6 libfontconfig.so.1 libstdc++.so.6
+RUN rpm -e --nodeps --justdb glibc-common
+RUN yum -y install glibc-common
+
+ENV HOME /root
+
+#Install JAVA
+# RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u55-b13/jdk-7u55-linux-x64.rpm -O jdk-7u55-linux-x64.rpm
+# RUN yum -y install jdk-7u55-linux-x64.rpm
+RUN wget --no-check-certificate --no-cookies --header "Cookie:oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/6u45-b06/jdk-6u45-linux-x64-rpm.bin -O jdk-6u45-linux-x64-rpm.bin
+RUN chmod +x jdk-6u45-linux-x64-rpm.bin
+RUN ./jdk-6u45-linux-x64-rpm.bin
+ENV JAVA_HOME /usr/java/default/
+
+#Install Maven
+RUN mkdir -p /opt/maven
+WORKDIR /opt/maven
+RUN wget http://apache.cs.utah.edu/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz
+RUN tar -xvzf /opt/maven/apache-maven-3.0.5-bin.tar.gz
+RUN rm -rf /opt/maven/apache-maven-3.0.5-bin.tar.gz
+
+ENV M2_HOME /opt/maven/apache-maven-3.0.5
+ENV MAVEN_OPTS -Xmx2048m -XX:MaxPermSize=256m
+ENV PATH $PATH:$JAVA_HOME/bin:$M2_HOME/bin
+
+
+# SSH key
+RUN ssh-keygen -f /root/.ssh/id_rsa -t rsa -N ''
+RUN cat /root/.ssh/id_rsa.pub > /root/.ssh/authorized_keys
+RUN chmod 600 /root/.ssh/authorized_keys
+RUN sed -ri 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config
+
+# Install python, nodejs and npm
+RUN yum -y install http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+RUN yum -y install nodejs npm --enablerepo=epel
+RUN npm install -g brunch@1.7.13
+
+# Once run some mvn commands to cache .m2/repository
+WORKDIR /tmp
+RUN git clone https://github.com/apache/ambari.git
+WORKDIR /tmp/ambari
+RUN mvn versions:set -DnewVersion=1.6.1.0
+RUN mvn -B clean install package rpm:rpm -DskipTests -DnewVersion=1.6.1.0 -Dpython.ver="python >= 2.6" -Preplaceurl
+
+# also build ambari-log4j and install
+WORKDIR /tmp/ambari/contrib/ambari-log4j
+RUN mvn package rpm:rpm
+RUN yum install -y  target/rpm/ambari-log4j/RPMS/noarch/ambari-log4j-1.2.1-*.noarch.rpm
+
+# clean git code because I want to use the one on local filesystem.
+WORKDIR /tmp
+RUN rm -rf /tmp/ambari
+
+RUN mkdir -p /tmp/ambari-build-docker/blueprints
+ADD ./blueprints /tmp/ambari-build-docker/blueprints
+RUN mkdir -p /tmp/ambari-build-docker/bin
+ADD ./bin /tmp/ambari-build-docker/bin
+
+WORKDIR /tmp
+

dev-support/docker/docker/bin/__init__.py

@@ -0,0 +1,12 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+

dev-support/docker/docker/bin/ambaribuild.py

@@ -0,0 +1,233 @@
+#!/usr/bin/python
+# coding: utf-8
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import subprocess, time, sys
+import json
+from optparse import OptionParser
+
+SKIP_TEST="-DskipTests"
+AMBARI_AUTH_HEADERS = "--header 'Authorization:Basic YWRtaW46YWRtaW4=' --header 'X-Requested-By: PIVOTAL'"
+AMBARI_BUILD_DOCKER_ROOT = "/tmp/ambari-build-docker"
+NO_EXIT_SLEEP_TIME=60
+RETRY_MAX=20
+
+def git_deep_cleaning():
+	proc = subprocess.Popen("git clean -xdf",
+			shell=True,
+			cwd="/tmp/ambari")
+	return proc.wait()
+
+def ambariUnitTest():
+	git_deep_cleaning()
+	proc = subprocess.Popen("mvn -fae clean install",
+			shell=True,
+			cwd="/tmp/ambari")
+	return proc.wait()
+
+def buildAmbari(stack_distribution):
+	git_deep_cleaning()
+	stack_distribution_param = ""
+	if stack_distribution is not None:
+		stack_distribution_param = "-Dstack.distribution=" + stack_distribution
+	proc = subprocess.Popen("mvn -B clean install package rpm:rpm -Dmaven.clover.skip=true -Dfindbugs.skip=true "
+						+ SKIP_TEST + " "
+						+ stack_distribution_param + " -Dpython.ver=\"python >= 2.6\"",
+			shell=True,
+			cwd="/tmp/ambari")
+	return proc.wait()
+
+def install_ambari_server():
+	proc = subprocess.Popen("sudo yum install -y ambari-server-*.noarch.rpm",
+			shell=True,
+			cwd="/tmp/ambari/ambari-server/target/rpm/ambari-server/RPMS/noarch")
+	return proc.wait()
+
+def install_ambari_agent():
+	proc = subprocess.Popen("sudo yum install -y ambari-agent-*.x86_64.rpm",
+			shell=True,
+			cwd="/tmp/ambari/ambari-agent/target/rpm/ambari-agent/RPMS/x86_64")
+	return proc.wait()
+
+def setup_ambari_server():
+	proc = subprocess.Popen("echo -e '\n\n\n\n' | sudo ambari-server setup",
+			shell=True)
+	return proc.wait()
+
+def start_ambari_server(debug=False):
+	proc = subprocess.Popen("sudo ambari-server start" + (" --debug" if debug else ""),
+			shell=True)
+	return proc.wait()
+
+def start_dependant_services():
+	retcode = 0
+	proc = subprocess.Popen("sudo service sshd start", shell=True)
+	retcode += proc.wait()
+	proc = subprocess.Popen("sudo service ntpd start", shell=True)
+	retcode += proc.wait()
+	return retcode
+
+def configure_ambari_agent():
+	proc = subprocess.Popen("hostname -f", stdout=subprocess.PIPE, shell=True)
+	hostname = proc.stdout.read().rstrip()
+	proc = subprocess.Popen("sudo sed -i 's/hostname=localhost/hostname=" + hostname + "/g' /etc/ambari-agent/conf/ambari-agent.ini",
+			shell=True)
+	return proc.wait()
+
+def start_ambari_agent(wait_until_registered = True):
+	retcode = 0
+	proc = subprocess.Popen("service ambari-agent start",
+			shell=True)
+	retcode += proc.wait()
+	if wait_until_registered:
+		if not wait_until_ambari_agent_registered():
+			print "ERROR: ambari-agent was not registered."
+			sys.exit(1)
+
+	return retcode
+
+def wait_until_ambari_agent_registered():
+	'''
+	return True if ambari agent is found registered.
+	return False if timeout
+	'''
+	count = 0
+	while count < RETRY_MAX:
+		count += 1
+		proc = subprocess.Popen("curl " +
+				"http://localhost:8080/api/v1/hosts " +
+				AMBARI_AUTH_HEADERS,
+				stdout=subprocess.PIPE,
+				shell=True)
+		hosts_result_string = proc.stdout.read()
+		hosts_result_json = json.loads(hosts_result_string)
+		if len(hosts_result_json["items"]) != 0:
+			return True
+		time.sleep(5)
+	return False
+
+def post_blueprint():
+	proc = subprocess.Popen("curl -X POST -D - " +
+			"-d @single-node-HDP-2.1-blueprint1.json http://localhost:8080/api/v1/blueprints/myblueprint1 " +
+			AMBARI_AUTH_HEADERS ,
+			cwd=AMBARI_BUILD_DOCKER_ROOT + "/blueprints",
+			shell=True)
+	return proc.wait()
+
+def create_cluster():
+	proc = subprocess.Popen("curl -X POST -D - " +
+			"-d @single-node-hostmapping1.json http://localhost:8080/api/v1/clusters/mycluster1 " +
+			AMBARI_AUTH_HEADERS ,
+			cwd=AMBARI_BUILD_DOCKER_ROOT + "/blueprints",
+			shell=True)
+	return proc.wait()
+
+# Loop to not to exit Docker container
+def no_exit():
+	print "loop to not to exit docker container..."
+	while True:
+		time.sleep(NO_EXIT_SLEEP_TIME)
+
+class ParseResult:
+	is_rebuild = False
+	stack_distribution = None
+	is_test = False
+	is_install_server = False
+	is_install_agent = False
+	is_deploy = False
+
+def parse(argv):
+	result = ParseResult()
+	if len(argv) >=2:
+		parser = OptionParser()
+		parser.add_option("-b", "--rebuild",
+				dest="is_rebuild",
+				action="store_true",
+				default=False,
+				help="set this flag if you want to rebuild Ambari code")
+		parser.add_option("-s", "--stack_distribution",
+				dest="stack_distribution",
+				help="set a stack distribution. [HDP|PHD|BIGTOP]. Make sure -b is also set when you set a stack distribution")
+		(options, args) = parser.parse_args(argv[1:])
+		if options.is_rebuild:
+			result.is_rebuild = True
+		if options.stack_distribution:
+			result.stack_distribution = options.stack_distribution
+
+	if argv[0] == "test":
+		result.is_test = True
+
+	if argv[0] == "server":
+		result.is_install_server = True
+
+	if argv[0] == "agent":
+		result.is_install_server = True
+		result.is_install_agent = True
+
+	if argv[0] == "deploy":
+		result.is_install_server = True
+		result.is_install_agent = True
+		result.is_deploy = True
+
+	return result
+
+if __name__ == "__main__":
+
+	if len(sys.argv) == 1:
+		print "specify one of test, server, agent or deploy"
+		sys.exit(1)
+
+	# test: execute unit test
+	# server: install ambari-server
+	#    with or without rebuild
+	# agent: install ambari-server and ambari-agent
+	#    with or without rebuild
+	# deploy: install ambari-server, ambari-agent and deploy Hadoop
+	#    with or without rebuild
+
+	parsed_args = parse(sys.argv[1:])
+
+	if parsed_args.is_test:
+		retcode = ambariUnitTest()
+		sys.exit(retcode)
+
+	if parsed_args.is_rebuild:
+		retcode = buildAmbari(parsed_args.stack_distribution)
+		if retcode != 0: sys.exit(retcode)
+
+	if parsed_args.is_install_server:
+		retcode = install_ambari_server()
+		if retcode != 0: sys.exit(retcode)
+		retcode = setup_ambari_server()
+		if retcode != 0: sys.exit(retcode)
+		retcode = start_ambari_server()
+		if retcode != 0: sys.exit(retcode)
+		retcode = start_dependant_services()
+		if retcode != 0: sys.exit(retcode)
+
+	if parsed_args.is_install_agent:
+		retcode = install_ambari_agent()
+		if retcode != 0: sys.exit(retcode)
+		retcode = configure_ambari_agent()
+		if retcode != 0: sys.exit(retcode)
+		retcode = start_ambari_agent()
+		if retcode != 0: sys.exit(retcode)
+
+	if parsed_args.is_deploy:
+		retcode = post_blueprint()
+		if retcode != 0: sys.exit(retcode)
+		retcode = create_cluster()
+		if retcode != 0: sys.exit(retcode)
+
+	no_exit()
+

dev-support/docker/docker/bin/test/__init__.py

@@ -0,0 +1,12 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+

dev-support/docker/docker/bin/test/ambaribuild_test.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# coding: utf-8
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+from bin import ambaribuild
+
+# TODO move this to a proper location
+def unittest():
+	# parse
+	result = ambaribuild.parse(["test"])
+	assert result.is_test == True
+	assert result.is_rebuild == False
+	assert result.stack_distribution == None
+	assert result.is_install_server == False
+	assert result.is_install_agent == False
+	assert result.is_deploy == False
+
+	result = ambaribuild.parse(["server"])
+	assert result.is_test == False
+	assert result.is_rebuild == False
+	assert result.stack_distribution == None
+	assert result.is_install_server == True
+	assert result.is_install_agent == False
+	assert result.is_deploy == False
+
+	result = ambaribuild.parse(["agent"])
+	assert result.is_test == False
+	assert result.is_rebuild == False
+	assert result.stack_distribution == None
+	assert result.is_install_server == True
+	assert result.is_install_agent == True
+	assert result.is_deploy == False
+
+	result = ambaribuild.parse(["agent", "-b"])
+	assert result.is_test == False
+	assert result.is_rebuild == True
+	assert result.stack_distribution == None
+	assert result.is_install_server == True
+	assert result.is_install_agent == True
+	assert result.is_deploy == False
+
+	result = ambaribuild.parse(["deploy"])
+	assert result.is_test == False
+	assert result.is_rebuild == False
+	assert result.stack_distribution == None
+	assert result.is_install_server == True
+	assert result.is_install_agent == True
+	assert result.is_deploy == True
+
+	result = ambaribuild.parse(["deploy", "-b", "-s", "BIGTOP"])
+	assert result.is_test == False
+	assert result.is_rebuild == True
+	assert result.stack_distribution == "BIGTOP"
+	assert result.is_install_server == True
+	assert result.is_install_agent == True
+	assert result.is_deploy == True
+
+if __name__ == "__main__":
+	unittest()
+

dev-support/docker/docker/blueprints/single-node-HDP-2.1-blueprint1.json

@@ -0,0 +1,64 @@
+{
+	"configurations" : [
+		{
+			"nagios-env" : {
+				"nagios_contact": "me@my-awesome-domain.example"
+			}
+		}
+	],
+  "host_groups" : [
+    {
+      "name" : "host_group_1",
+      "components" : [
+        {
+          "name" : "ZOOKEEPER_SERVER"
+        },
+        {
+          "name" : "ZOOKEEPER_CLIENT"
+        },
+        {
+          "name" : "NAMENODE"
+        },
+        {
+          "name" : "SECONDARY_NAMENODE"
+        },
+        {
+          "name" : "DATANODE"
+        },
+        {
+          "name" : "HDFS_CLIENT"
+        },
+        {
+          "name" : "NODEMANAGER"
+        },
+        {
+          "name" : "RESOURCEMANAGER"
+        },
+        {
+          "name" : "HISTORYSERVER"
+        },
+        {
+          "name" : "APP_TIMELINE_SERVER"
+        },
+        {
+          "name" : "YARN_CLIENT"
+        },
+        {
+          "name" : "MAPREDUCE2_CLIENT"
+        },
+        {
+          "name" : "GANGLIA_SERVER"
+        },
+        {
+          "name" : "NAGIOS_SERVER"
+        }
+      ],
+      "cardinality" : "1"
+    }
+  ],
+  "Blueprints" : {
+    "blueprint_name" : "single-node-blueprint1",
+    "stack_name" : "HDP",
+    "stack_version" : "2.1"
+  }
+}

dev-support/docker/docker/blueprints/single-node-hostmapping1.json

@@ -0,0 +1,14 @@
+{
+    "blueprint": "single-node-blueprint1",
+    "default_password": "changeme",
+    "host_groups": [
+        {
+            "hosts": [
+                {
+                    "fqdn": "node1.mydomain.com"
+                }
+            ],
+            "name": "host_group_1"
+        }
+    ]
+}