AMBARI-12367. Automate creating multiple ambari-agents using Docker on GCE (Pengcheng Xu via alejandro)

contrib/agent-simulator/Docker/Dockerfile

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Set the bas image to CentOS 6.x
+FROM centos:7
+
+# Author
+MAINTAINER Pengcheng_Xu
+
+# Copy the files into Docker: Agent package file
+ADD agent.rpm /agent.rpm
+# Copy the files into Docker: launcher_agent.py
+ADD launcher_agent.py /launcher_agent.py
+# Copy the files into Docker: ambari_agent_start.sh
+ADD ambari_agent_start.sh /ambari_agent_start.sh
+# Copy hostname file into Docker: hosts
+ADD hosts /hosts
+
+RUN chmod 755 /ambari_agent_start.sh
+
+# Install ambari-agent
+# RUN yum install -y /agent.rpm
+RUN yum install -y wget
+RUN wget -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.0/ambaribn.repo
+RUN yum install -y epel-release
+RUN yum install -y ambari-agent

contrib/agent-simulator/Docker/__init__.py

@@ -0,0 +1,17 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''

contrib/agent-simulator/Docker/ambari_agent_start.sh

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#!/bin/bash
+ambari-agent start

contrib/agent-simulator/Docker/launcher_agent.py

@@ -0,0 +1,70 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import sys
+import subprocess
+import time
+
+def replace_conf(server_ip):
+    f = open("/etc/ambari-agent/conf/ambari-agent.ini")
+    lines = f.readlines()
+    f.close()
+
+    f = open("/etc/ambari-agent/conf/ambari-agent.ini", "w+")
+    for line in lines:
+        line = line.replace("hostname=localhost", "hostname=" + server_ip)
+        f.write(line)
+    f.close()
+
+def run_ambari_agent():
+    # command = ["sudo", "ambari-agent", "start"]
+    # subprocess.call(command)
+    subprocess.call("./ambari_agent_start.sh")
+
+# add all the hostnames of other containers to /etc/hosts
+def add_hostnames():
+    etc_hosts = open("/etc/hosts", "a")
+    etc_hosts.write("\n")
+
+    docker_hosts = open("/hosts")
+    for line in docker_hosts.readlines():
+        etc_hosts.write(line)
+    docker_hosts.close()
+
+    etc_hosts.close()
+
+def remove_default_hostname(hostname):
+    etc_hosts = open("/etc/hosts")
+    all_resolution = etc_hosts.readlines()
+    etc_hosts.close()
+
+    etc_hosts = open("/etc/hosts", "w")
+    for line in all_resolution:
+        if hostname not in line:
+            etc_hosts.write(line)
+        else:
+            etc_hosts.write("#")
+            etc_hosts.write(line)
+    etc_hosts.close()
+
+ambari_server_ip = sys.argv[1]
+my_hostname = sys.argv[2]
+replace_conf(ambari_server_ip)
+remove_default_hostname(my_hostname)
+add_hostnames()
+run_ambari_agent()

contrib/agent-simulator/Linux/CentOS7/docker_install.sh

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo curl -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum localinstall -y --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum update -y device-mapper
+sudo service docker start

contrib/agent-simulator/Linux/CentOS7/weave_install.sh

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo curl -L git.io/weave -o /usr/bin/weave
+sudo chmod a+x /usr/bin/weave

contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh

@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+wget -qO- https://get.docker.com/ | sh

contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+sudo apt-get install -y curl
+sudo curl -L git.io/weave -o /usr/bin/weave
+sudo chmod a+x /usr/bin/weave

contrib/agent-simulator/cluster.py

@@ -0,0 +1,175 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import subprocess
+import time
+from config import Config
+from docker import Docker
+from vm import VM
+
+
+
+class Cluster:
+    def __init__(self):
+        self.cluster_name = ""
+        self.VMs_num = 0
+        self.VM_list = []
+
+    # read cluster info from a file
+    def load_cluster_info(self, filename):
+        file = open(filename)
+
+        self.cluster_name = file.next().split()[1]
+        self.VMs_num = int(file.next().split()[1])
+        for VM_index in range(0, self.VMs_num):
+            vm = VM(file.next().split()[1])
+            docker_num = int(file.next().split()[1])
+            for Docker_index in range(0, docker_num):
+                line = file.next()
+                IP = line.split()[0].split("/")[0]
+                mask = line.split()[0].split("/")[1]
+                hostname = line.split()[1]
+                docker = Docker(IP, mask, hostname)
+                vm.add_docker(docker)
+            self.VM_list.append(vm)
+
+        file.close()
+
+    def __extract_VM_IP__(self, GCE_info_file_name):
+        f = open(GCE_info_file_name)
+        lines = f.readlines()
+        f.close()
+
+        ip_list = []
+        for line in lines:
+            tokens = line.split()
+            ip_list.append(tokens[1])
+        return ip_list[1:]
+
+    # request a new cluster
+    def request_GCE_cluster(self, vms_num, docker_num, cluster_name):
+        # reload configuration file
+        config = Config()
+        config.load()
+        # request cluster
+        gce_key = config.ATTRIBUTES["GCE_controller_key_file"]
+        gce_login = config.ATTRIBUTES["GCE_controller_user"] + "@" + config.ATTRIBUTES["GCE_controller_IP"]
+        gce_up_cmd = "gce up " + cluster_name + " " + str(vms_num) + " " + config.ATTRIBUTES["GCE_VM_type"] + \
+            " " + config.ATTRIBUTES["GCE_VM_OS"]
+        subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_up_cmd])
+
+        print "cluster launched successufully, wait 5 seconds for cluster info ... ..."
+        time.sleep(5)
+
+        # request cluster info
+        gce_info_output_file = open(config.ATTRIBUTES["GCE_info_output"], "w")
+        gce_info_cmd = "gce info " + cluster_name
+        subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd], \
+                        stdout=gce_info_output_file)
+        gce_info_output_file.close()
+        print "cluster info is saved to file " + config.ATTRIBUTES["GCE_info_output"]
+
+        # prepare all attributes of the cluster, write to a file
+        VM_IP_list = self.__extract_VM_IP__(config.ATTRIBUTES["GCE_info_output"])
+        self.generate_cluster_info(VM_IP_list, cluster_name, docker_num)
+        self.overwrite_to_file(config.ATTRIBUTES["cluster_info_file"])
+        # server need this file to resolve the host names of the agents
+        self.export_hostnames(config.ATTRIBUTES["Docker_hostname_info"])
+
+    # save info to file
+    def overwrite_to_file(self, filename):
+        file = open(filename, "w")
+        file.write("cluster_name: " + self.cluster_name + "\n")
+        file.write("VMs_num: " + str(self.VMs_num) + "\n")
+
+        for vm in self.VM_list:
+            file.write("\t\t")
+            file.write("VM_IP: " + vm.external_ip + "\n")
+            file.write("\t\t")
+            file.write("Docker_num: " + str(len(vm.docker_list)) + "\n")
+            for docker in vm.docker_list:
+                file.write("\t\t\t\t")
+                file.write(docker.IP + "/" + docker.mask + " " + docker.hostname + "\n")
+
+        file.close()
+
+    def __increase_IP__(self, base_IP, increase):
+        IP = [int(base_IP[0]), int(base_IP[1]), int(base_IP[2]), int(base_IP[3])]
+        IP[3] = IP[3] + increase
+        for index in reversed(range(0, 4)):
+            if IP[index] > 255:
+                IP[index - 1] = IP[index - 1] + IP[index] / 256
+                IP[index] = IP[index] % 256
+        return IP
+
+    # generate VM and docker info for this cluster
+    # set up parameter as this info
+    def generate_cluster_info(self, VM_IP_list, cluster_name, docker_num):
+        config = Config()
+        config.load()
+        Docker_IP_base = config.ATTRIBUTES["Docker_IP_base"].split(".")
+        Docker_IP_mask = config.ATTRIBUTES["Docker_IP_mask"]
+
+        VM_index = 0
+        for VM_IP in VM_IP_list:
+            vm = VM(VM_IP)
+
+            for Docker_index in range(0, docker_num):
+                total_Docker_index = VM_index * docker_num + Docker_index
+                docker_IP = self.__increase_IP__(Docker_IP_base, total_Docker_index)
+                docker_IP_str = str(docker_IP[0]) + "." + str(docker_IP[1]) + "." + \
+                                str(docker_IP[2]) + "." + str(docker_IP[3])
+                docker_hostname = cluster_name + "-" + str(VM_index) + "-" + str(Docker_index)
+                docker = Docker(docker_IP_str, str(Docker_IP_mask), docker_hostname)
+                # print docker
+                vm.add_docker(docker)
+            VM_index = VM_index + 1
+            self.VM_list.append(vm)
+
+        self.VMs_num = len(VM_IP_list)
+        self.cluster_name = cluster_name
+
+    # run all dockers for all the VMs in the cluster
+    # upload necessary file to each machine in cluster, run launcher_docker.py in each machine with parameter
+    def run_docker_on_cluster(self, server_external_IP, server_Weave_IP):
+        config = Config()
+        config.load()
+
+        for vm in self.VM_list:
+            # upload necessary file to each machine in cluster
+            VM_external_IP = vm.external_ip
+            VM_directory = "root@" + VM_external_IP + ":" + config.ATTRIBUTES["VM_code_directory"]
+            VM_key = config.ATTRIBUTES["GCE_VM_key_file"]
+            subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i", VM_key, "-r", ".", VM_directory])
+
+            # run launcher_docker.py in each machine with parameters
+            subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", VM_key, \
+                             "root@" + VM_external_IP, \
+                             "cd " + config.ATTRIBUTES["VM_code_directory"] + "; python launcher_docker.py" + \
+                             " " + VM_external_IP + " " + server_Weave_IP + " " + server_external_IP])
+
+    # export host names to a file
+    def export_hostnames(self, filename):
+        hostname_file = open(filename, "w")
+        for vm in self.VM_list:
+            for docker in vm.docker_list:
+                hostname_file.write(docker.IP)
+                hostname_file.write(" ")
+                hostname_file.write(docker.hostname)
+                hostname_file.write("\n")
+        hostname_file.close()

contrib/agent-simulator/config.py

@@ -0,0 +1,36 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import ConfigParser
+
+class Config:
+    ATTRIBUTES = {}
+
+    @staticmethod
+    def load():
+        config = ConfigParser.RawConfigParser()
+        # keep file case sensitive
+        config.optionxform = str
+        config.read("config/config.ini")
+        for section in config.sections():
+            for key in config.options(section):
+                Config.ATTRIBUTES[key] = config.get(section, key)
+
+
+# Config.load()
+# print Config.ATTRIBUTES

contrib/agent-simulator/config/config.ini

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This configuration file is case SENSITIVE
+# For more information about this configuration file, check https://docs.python.org/2/library/configparser.html
+
+[GCE]
+GCE_controller_IP = 104.196.89.197
+GCE_controller_user = root
+GCE_controller_key_file = ~/gce-key
+GCE_VM_type = --xlarge
+# only support centos 7
+GCE_VM_OS = --centos7
+# the result of command "gce info clustername" on the GCE controller
+GCE_info_output = output/gce_info_output.txt
+# the same as the GCE controller for GCE
+GCE_VM_key_file = ~/gce-key
+
+
+[Cluster]
+cluster_info_file = config/cluster.txt
+VM_code_directory = /simulator-script
+VM_hostname_file = /etc/hosts
+Docker_hostname_info = config/hosts.txt
+
+[Weave]
+# Docker_IP_base is the starting point of IP address. with mask 16, ideally:
+# the IP of docker will be 192.168.[1-255].[1-255]/16
+# if there are more dockers, which means there is not enough IPs within this mask for all dockers,
+# the IP will continue to increase to be 192.169.*.*, dockers might not be able to talk to each other,
+# because the mask is 16
+Docker_IP_base = 192.168.1.2
+Docker_IP_mask = 16
+
+
+[Docker]
+Docker_image_name = ambari/agent-sim

contrib/agent-simulator/docker.py

@@ -0,0 +1,27 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+class Docker:
+    def __init__(self, IP, mask, hostname):
+        self.IP = IP
+        self.mask = mask
+        self.hostname = hostname
+
+    def __str__(self):
+        return str(self.IP) + "/" + str(self.mask) + " " + self.hostname

contrib/agent-simulator/launcher_cluster.py

@@ -0,0 +1,136 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import time
+import sys
+from cluster import Cluster
+from config import Config
+
+def all(argv):
+    if len(argv) < 7:
+        print_help()
+        exit(1)
+
+    cluster_name = argv[2]
+    VMs_num = int(argv[3])
+    Docker_num_each_VM = int(argv[4])
+    server_Weave_IP = argv[5]
+    server_external_IP = argv[6]
+
+    cluster = Cluster()
+    cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
+    print "wait 50 seconds for the cluster to boot ... ..."
+    time.sleep(50)
+    cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
+
+def request_cluster(argv):
+    if len(argv) < 5:
+        print_help()
+        exit(1)
+
+    cluster_name = argv[2]
+    VMs_num = int(argv[3])
+    Docker_num_each_VM = int(argv[4])
+
+    cluster = Cluster()
+    cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
+    print "Before run Docker on the Cluster, wait at least 50 seconds for the cluster to boot"
+
+def run_cluster(argv):
+    if len(argv) < 4:
+        print_help()
+        exit(1)
+
+    server_Weave_IP = argv[2]
+    server_external_IP = argv[3]
+
+    config = Config
+    config.load()
+
+    cluster = Cluster()
+    cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
+    cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
+
+def terminate():
+    print "Log into GCE controller to terminate your cluster manually"
+
+def print_help():
+    print "usage:"
+    print
+
+    print "all", "  ", "request a GCE cluster and run Docker containers with Ambari-agent in all VMs"
+    print "\t\t", "<the name of the cluster>"
+    print "\t\t", "<number of VMs>"
+    print "\t\t", "<number of dockers each VMs>"
+    print "\t\t", "<Weave IP of Ambari-server>"
+    print "\t\t", "<external IP of Ambari-server>"
+    print
+
+    print "request", "  ", "request a cluster from GCE, generate the configuration for the cluster"
+    print "\t\t", "<the name of the cluster>"
+    print "\t\t", "<number of VMs>"
+    print "\t\t", "<number of dockers each VMs>"
+    print
+
+    print "run", "  ", "run Docker containers with Ambari-agent in all VMs of the cluster"
+    print "\t\t", "<Weave IP of Ambari-server>"
+    print "\t\t", "<external IP of Ambari-server>"
+    print
+
+    print "terminate", "  ", "terminate the cluster"
+    print
+
+    print "help", "  ", "help info"
+    print
+
+    print "more options in configuration file config/config.ini"
+    print "see more instructions in tips.txt"
+    print
+
+def main(argv):
+    # the first argument is the python file name
+    if len(argv) < 2:
+        print_help()
+        exit(1)
+
+    command = argv[1]
+    if command == "all":
+        all(argv)
+
+    elif command == "request":
+        request_cluster(argv)
+
+    elif command == "run":
+        run_cluster(argv)
+
+    elif command == "terminate":
+        terminate()
+
+    elif command == "help":
+        print_help()
+
+    else:
+        print_help()
+
+main(sys.argv)
+
+# python launcher_cluster.py all test-cluster 2 3 192.168.10.10 104.196.84.248
+
+
+

contrib/agent-simulator/launcher_docker.py

@@ -0,0 +1,41 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import sys
+from config import Config
+from cluster import Cluster
+
+def get_VM(VM_IP, cluster):
+    for vm in cluster.VM_list:
+        if vm.external_ip == VM_IP:
+            return vm
+
+config = Config()
+config.load()
+
+cluster = Cluster()
+cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
+
+my_external_IP = sys.argv[1]
+server_weave_IP = sys.argv[2]
+server_external_IP = sys.argv[3]
+
+vm = get_VM(my_external_IP, cluster)
+vm.run_docker(server_weave_IP, server_external_IP, cluster)
+

contrib/agent-simulator/server_setup.sh

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/bin/bash
+# run this script with root
+# $1 <Weave internal IP with mask>
+# $2 <hostname files with all agents>
+
+if [ $# -lt 2 ]; then
+    echo "usage: ./sever_setup.sh <Weave internal IP with mask> <hostname file with all agents>"
+    echo "example: ./server_setup.sh 192.168.10.10/16 /user/simulator-script/hosts.txt"
+    echo "note: the hostname file is generated automatically when you request a cluster"
+    exit 1
+fi
+
+# install weave
+chmod 755 ./Linux/CentOS7/weave_install.sh
+./Linux/CentOS7/weave_install.sh
+
+# reset weave
+weave reset
+
+# launch weave
+weave launch
+
+# expose IP
+weave expose $1
+
+# add hosname of all agents
+cat $2 >> /etc/hosts3

contrib/agent-simulator/tips.txt

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+1. Ambari-agent and Ambari-server have to be the same version to successfully register. The command used to install Ambari-agent is in the Dockerfile
+2. Use CTRL + P, then CTRL + Q to exit Docker container without terminate the container.

contrib/agent-simulator/vm.py

@@ -0,0 +1,81 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+
+import subprocess
+from config import Config
+
+class VM:
+    def __init__(self, external_ip):
+        self.external_ip = external_ip
+        self.docker_list = []
+
+    def add_docker(self, docker):
+        self.docker_list.append(docker)
+
+    # install Weave on this VM
+    def __centos7_weave_install__(self):
+        subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/weave_install.sh"])
+        subprocess.call("./Linux/CentOS7/weave_install.sh")
+
+    # launch Weave, make this VM connect with other VM
+    def __set_weave_network__(self, VMs_external_IP_list, server_external_ip):
+        # add other VMs and the ambari-server to set up connections
+        command = ["sudo", "weave", "launch"]
+        command.extend(VMs_external_IP_list)
+        command.append(server_external_ip)
+        subprocess.call(command)
+
+    # install Docker on this VM
+    def __centos7_docker_install__(self):
+        subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/docker_install.sh"])
+        subprocess.call("./Linux/CentOS7/docker_install.sh")
+
+    # build docker image
+    def __build_docker_image__(self, image_name):
+        subprocess.call(["sudo", "docker", "build", "-t", image_name, "Docker/"])
+
+    # launch Docker containers, issue the script to install, configure and launch Agent inside Docker.
+    def __launch_containers__(self, docker_image, server_weave_ip):
+        # print docker_ip_list
+        for docker in self.docker_list:
+            docker_IP = docker.IP
+            docker_mask = docker.mask
+            docker_hostname = docker.hostname
+
+            cmd = "python /launcher_agent.py " + server_weave_ip + " " + docker_hostname + "; /bin/bash"
+            command = ["sudo", "weave", "run", docker_IP + "/" + docker_mask, "-d", "-it", "-h", docker_hostname, \
+                       docker_image, "bash", "-c", cmd]
+            print command
+            subprocess.call(command)
+
+    def run_docker(self, server_weave_IP, server_external_IP, cluster):
+        config = Config()
+        config.load()
+
+        VMs_IP_list = []
+        for vm in cluster.VM_list:
+            VMs_IP_list.append(vm.external_ip)
+
+        cluster.export_hostnames("./Docker/hosts")
+
+        self.__centos7_docker_install__()
+        self.__centos7_weave_install__()
+        self.__build_docker_image__(config.ATTRIBUTES["Docker_image_name"])
+        self.__set_weave_network__(VMs_IP_list, server_external_IP)
+        self.__launch_containers__(config.ATTRIBUTES["Docker_image_name"], server_weave_IP)