AMBARI-12483. Simulate Ambari Agents to test 3k Node Cluster (Pengcheng Xu via alejandro)

contrib/agent-simulator/Docker/Dockerfile

@@ -1,39 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Set the bas image to CentOS 6.x
-FROM centos:7
-
-# Author
-MAINTAINER Pengcheng_Xu
-
-# Copy the files into Docker: Agent package file
-ADD agent.rpm /agent.rpm
-# Copy the files into Docker: launcher_agent.py
-ADD launcher_agent.py /launcher_agent.py
-# Copy the files into Docker: ambari_agent_start.sh
-ADD ambari_agent_start.sh /ambari_agent_start.sh
-# Copy hostname file into Docker: hosts
-ADD hosts /hosts
-
-RUN chmod 755 /ambari_agent_start.sh
-
-# Install ambari-agent
-# RUN yum install -y /agent.rpm
-RUN yum install -y wget
-RUN wget -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.0/ambaribn.repo
-RUN yum install -y epel-release
-RUN yum install -y ambari-agent

contrib/agent-simulator/Docker/launcher_agent.py

@@ -1,70 +0,0 @@
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import sys
-import subprocess
-import time
-
-def replace_conf(server_ip):
-    f = open("/etc/ambari-agent/conf/ambari-agent.ini")
-    lines = f.readlines()
-    f.close()
-
-    f = open("/etc/ambari-agent/conf/ambari-agent.ini", "w+")
-    for line in lines:
-        line = line.replace("hostname=localhost", "hostname=" + server_ip)
-        f.write(line)
-    f.close()
-
-def run_ambari_agent():
-    # command = ["sudo", "ambari-agent", "start"]
-    # subprocess.call(command)
-    subprocess.call("./ambari_agent_start.sh")
-
-# add all the hostnames of other containers to /etc/hosts
-def add_hostnames():
-    etc_hosts = open("/etc/hosts", "a")
-    etc_hosts.write("\n")
-
-    docker_hosts = open("/hosts")
-    for line in docker_hosts.readlines():
-        etc_hosts.write(line)
-    docker_hosts.close()
-
-    etc_hosts.close()
-
-def remove_default_hostname(hostname):
-    etc_hosts = open("/etc/hosts")
-    all_resolution = etc_hosts.readlines()
-    etc_hosts.close()
-
-    etc_hosts = open("/etc/hosts", "w")
-    for line in all_resolution:
-        if hostname not in line:
-            etc_hosts.write(line)
-        else:
-            etc_hosts.write("#")
-            etc_hosts.write(line)
-    etc_hosts.close()
-
-ambari_server_ip = sys.argv[1]
-my_hostname = sys.argv[2]
-replace_conf(ambari_server_ip)
-remove_default_hostname(my_hostname)
-add_hostnames()
-run_ambari_agent()

contrib/agent-simulator/Linux/CentOS7/docker_install.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,8 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-sudo curl -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
-sudo yum localinstall -y --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
-sudo yum update -y device-mapper
+# This script will install and start the Docker service on a CentOS 7 machine
+
+sudo curl -s -O -sSL https://get.docker.com/rpm/1.7.0/centos-7/RPMS/x86_64/docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum localinstall -y -q --nogpgcheck docker-engine-1.7.0-1.el7.centos.x86_64.rpm
+sudo yum update -y -q device-mapper
 sudo service docker start

contrib/agent-simulator/Linux/CentOS7/set_docker_partition.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script will set the docker use a new partition as its storage
+
+# $1 mount point to other part
+
+mount_point=$1
+docker_dir=/var/lib/docker
+
+while [ ! -d "$docker_dir" ]; do
+    echo "$docker_dir does not exist, wait for docker service to create the directory"
+    sleep 5
+done
+
+echo "move $docker_dir to partition $mount_point"
+
+service docker stop
+cp -r ${docker_dir}/* $mount_point
+rm -rf $docker_dir
+ln -s $mount_point $docker_dir
+service docker start

contrib/agent-simulator/Linux/CentOS7/weave_install.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-sudo curl -L git.io/weave -o /usr/bin/weave
+# This script will install Weave on a CentOS 7 machine
+
+sudo curl -s -L git.io/weave -o /usr/bin/weave
 sudo chmod a+x /usr/bin/weave

contrib/agent-simulator/README.md

@@ -0,0 +1,211 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Ambari-agent Simulator
+============
+This project provides a tool to create a large Ambari-agent cluster in a convenient way.
+
+## Usage:
+Run python launcher_cluster.py to see usage
+
+python launcher_cluster.py request    
+
+    request a cluster from GCE, generate the configuration for the cluster. Parameters:
+	<the name of the cluster>, suggestion: {yourname}-group-a)
+	<number of VMs>
+	<number of dockers each VMs>
+	<number of service servers inside VM>
+	<number of ambari-server>, either 0 or 1
+		
+python launcher_cluster.py up
+        
+    run one cluster, and add to another cluster. Parameters:
+	<the name of the cluster>
+		
+python launcher_cluster.py merge    
+
+    run Docker containers with Ambari-agent in all VMs of the cluster. Parameters:
+	<the name of the cluster to be merged>
+	<the name of the cluster to be extended>
+
+python launcher_cluster.py merge 
+
+    run Docker containers with Ambari-agent in all VMs of the cluster. Parameters:
+    <the name of the cluster to be merged>
+    <Weave IP of the Ambari-server>
+    <External IP of the Ambari-server>
+
+python launcher_cluster.py list    
+        
+    list all the cluster
+    
+python launcher_cluster.py show
+    
+    show cluster information. Parameters:
+    <the name of the cluster>
+    
+python launcher_cluster.py help    
+        
+    show help info
+
+## Introduction to Weave Networking
+[Weave](https://github.com/weaveworks/weave) is a tool to connect Docker containers distributed across different hosts. 
+This project use Weave to assign each Docker container (with Ambari-agent) a unique internal IP and a domain name.
+In each VM, a special Docker container will be launched by Weave to act as a Router, 
+and to connect all the Docker containers in this VM with other Docker containers in other VMs.
+Also, another special Docker container will be launched by Weave to act as a DNS for this VM.
+Each Docker container can connect with each other by using the internal IP, host name or domain name.
+
+All the Weave internal IP should be configured by the user. 
+In this following document, we use subnet 192.168.#.#/16, and use the IP within this subnet to configure Weave. 
+Actually, You can use any IP as you wish, even public IP, in which case, 
+it will replace the connection to the real outside connection, and redirect the connection to the internal Docker container.
+
+## Quick Start
+With the following 6 steps, you can create a cluster with Ambari-agents, and connect them to your Ambari-server. 
+Among all the steps, the step 3 is to configure this program, the step 4 is the one which really matters, 
+and other steps act as a one-time configuration or suggestion for your Ambari-server.
+
+You can start with this guide by downloading the code to your own computer or any computers which can access the GCE controller.
+
+* Step 1: Mark down IP of the GCE VM which installed Ambari-server, Ambari_Server_IP=104.196.81.81
+* Step 2: Copy example/config.ini to config/config.ini
+* Step 3: Modify the following values in config/config.ini
+    * Output_folder
+    * GCE_controller_key_file
+    * VM_key_file
+    * cluster_info_file, use {yourname}-group-a
+* Step 4: Use the command line to run 3 VMs, each VM has 5 Ambari-agents.
+
+        python launcher_cluster.py all {yourname}-group-a 3 5 192.168.255.1 Ambari_Server_IP
+
+* Step 5: Log into your Ambari-server machine, run the following command line to set up Weave internal network 
+
+        cd agent-simulator/network
+        ./set_ambari_server_network.sh 192.168.255.1 192.168.255.2 16
+
+* Step 6: Operate on Ambari-server web GUI
+    * Add all agents: docker-[0-14]-{yourname}-group-a.weave.local
+    * Choose manual registration
+    * Choose to install HDFS, YARN, HBase, Zookeeper, Ambari-Metrics
+    
+
+#### More on Quick Start
+* Step 7: Add one more cluster to your Ambari-server
+    * Modify cluster_info_file in config/config.ini, use {yourname}-group-b
+    * Modify Docker_IP_base in config.config.ini, use 192.168.2.1
+    * Use the command line to run 2 VMs, each VM has 10 Ambari-agents:
+
+                python launcher_cluster.py all {yourname}-group-b 2 10 192.168.255.1 Ambari_Server_IP
+
+    * On Ambari-server web GUI, add all agents: docker-[0-19]-{yourname}-group-b.weave.local
+* Step 8: Add one VM with Ambari-agent installed to your Ambari-server
+    * Log into your VM, set Ambari_Server_IP as the value of server hostname in the file /etc/ambari-agent/conf/ambari-agent.ini
+    * On the VM, Run the following command set up Weave internal network
+
+                cd agent-simulator/network
+                ./set_host_network.sh 192.168.254.1 192.168.254.2 16 Ambari_Server_IP
+
+
+## Detail Work Flow:
+* Step 1: Install Ambari-server
+    * Use existing Ambari-server, or, install and launch a new one
+    * agent-simulator/server/ambari_server_install.sh: this shell might help you install the Ambari-server
+    * agent-simulator/server/ambari_server-_reset_data.sh: this shell might help you set Ambari-server to initial state
+        
+* Step 2: Decide IP in your mind
+    * Mark down the IP of the Ambari-server, say {IP of Ambari-server = 104.196.81.81}
+    * Come up a subnet say, {subnet = 192.168.#.#/16} {Docker_IP_mask = 16}
+    * Pick one address closer to the END of the subnet as the Weave INTERNAL IP of Ambari-server, 
+    and another one as the Weave DNS IP of Ambari-server, 
+    say {Weave IP of Ambari-server = 192.168.255.1} {Weave DNS IP of Ambari-server = 192.168.255.2}
+    * Pick one address closer to the START of the subnet as the Weave INTERNAL IP of the FIRST Ambari-agent, 
+    say {Docker_IP_base = 192.168.1.1}
+    * Other Weave INTERNAL IP of Amari-agent will be automatically assigned based on the FIRST one (increasingly).
+    
+* Step 3: First time set up Ambari-server       
+    * Copy all the agent-simulator code base to Ambari-server
+    * cd agent-simulator/network
+    * Run set_ambari_server_network.sh
+    * In this example, use parameters: {Weave IP of Ambari-server = 192.168.255.1} {Weave DNS IP of Ambari-server = 192.168.255.2} {Docker_IP_mask = 16}
+    
+* Step 4: Modify config.ini
+    * Modify attributes: Output_folder, GCE_controller_key_file, GCE_VM_key_file, cluster_info_file
+    * Change Docker_IP_base and Docker_IP_mask, in this example {Docker_IP_base = 192.168.1.1} {Docker_IP_mask = 16}
+    
+* Step 5: Request Ambari-agent cluster
+    * Run python launcher_cluster.py request
+    * Use {your name}-group-a as the cluster name. In case you wanna add more cluster to your Ambari-server, change the last letter
+    
+* Step 6: Modify Cluster Information File
+    * A TXT file will appear under directory ./config within 1 minutes, which has the information about the cluster
+    * Typically, you would like NameNode, RegionServer, ResourceManager, etc.. to dominate one VM. 
+    * In this example, change the configuration of the first and the second VM, make each of them only have one Docker. 
+    You can install different server services only into these two Docker containers later on the Ambari-server web GUI.
+
+* Step 7: Run Ambari-agent Cluster
+    Run python launcher_cluster.py run
+    In this exmaple, use parameters: {Weave IP of Ambari-server = 192.168.255.1} {IP of Ambari-server = 104.196.81.81}
+    
+* Step 8: Operate on Ambari-server web GUI
+
+
+## Expand Cluster With This Script
+Be careful if you wanna use this script to add more Ambari-agents AGAIN to your Ambari-server
+
+* Use different Cluster Name when providing parameters to launcher_cluster.py
+* In config.ini, use the same Docker_IP_mask, make sure the same subnet
+* Change config.ini to use different Docker_IP_base, make sure that all new IPs never overlap with the existing IPs
+* Change config.ini to use different cluster_info_file, make sure the existing cluster information is not overwritten
+   
+## Expand Cluster By Adding other Hosts/VMs
+   
+## Naming Convention
+Cluster Name: the name of the cluster must be unique to make sure every VM on GCE has its unique name. The suggestion is using {your name}-group-{a-z}
+
+VM Name: each VM has a domain name assigned by GCE, its host name is {cluster name}-{index}
+
+Docker Container Name: the domain name of Docker container is docker-{index}-{cluster name}.weave.local, 
+the prefix "docker" can be configured by value Container_hostname_fix in config/config.ini, 
+you can find out which VM has which Docker container in the cluster information file.
+
+
+## Image for Docker Container
+
+## Use Different Partition for Docker Container
+
+## The IP assign mechanism.
+Basically, you don't have to worry about IP. The maximum number of IP is limited by weave_ip_base and weave_ip_mask.
+By fault, the subnet is 192.168.#.#/16. Once you have already created 256*256 agents (the real number is smaller, 
+since the DNS on the VM also uses IP address), some address might fall out side of subnet and some fall inside,
+which causes a connection issue. The function related is cluster._increase_ip(). You might want to wrap the IP around, 
+but corner cases are always there, there is no silver bullet.
+
+
+## Issues
+* This tool do NOT support parallel usage
+* If GCE has no enough resource, the cluster returned to you will have a smaller number of VM
+* Don't merge your cluster into someone else's cluster. Actually you can do it, but you have to dig into the network, and
+make sure the IP configuration is right.
+
+## Suggestions:
+* Make sure your cluster name is unique, or you might cause trouble to other people's VM
+* Use CTRL + P, then CTRL + Q to exit Docker container. Use "exit" will terminate the container.
+* Remove ~/.ssh/know_hosts files, especially if you run a large cluster. 
+You might get a warning from SSH, because the new GCE VM assigned to you might have the same IP with the VMs you saved in know_hosts file. 
+Remove .ssh/know_hosts before run this script.
+* Ambari-agent and Ambari-server have to be the same version to successfully register. 
+The command used to install Ambari-agent is in the Dockerfile
+    

contrib/agent-simulator/cluster.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,162 +14,572 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import subprocess
-import time
+import datetime
 from config import Config
 from docker import Docker
 from vm import VM
-
+import os
+import time
+from log import Log
+from data import Data
 
 
 class Cluster:
+    """
+    The Cluster instance holds a list of VMs,
+    it has the methods to request cluster, generate information and run all Ambari-agent and Ambari-server
+    """
+
+    # The constants represent the state of the cluster
+    # A newly requested cluster is in FREE state
+    STATE_FREE = "FREE"
+    # A cluster with running Ambari-server and Ambari-agents is in RUNNING state
+    STATE_RUNNING = "RUNNING"
+    # A cluster is merged into another cluster and running, is in MERGE state
+    # the name of the extended cluster is directly following the state String in JSON
+    STATE_MERGE = "MERGE"
+
     def __init__(self):
         self.cluster_name = ""
-        self.VMs_num = 0
-        self.VM_list = []
-
-    # read cluster info from a file
-    def load_cluster_info(self, filename):
-        file = open(filename)
-
-        self.cluster_name = file.next().split()[1]
-        self.VMs_num = int(file.next().split()[1])
-        for VM_index in range(0, self.VMs_num):
-            vm = VM(file.next().split()[1])
-            docker_num = int(file.next().split()[1])
-            for Docker_index in range(0, docker_num):
-                line = file.next()
-                IP = line.split()[0].split("/")[0]
-                mask = line.split()[0].split("/")[1]
-                hostname = line.split()[1]
-                docker = Docker(IP, mask, hostname)
-                vm.add_docker(docker)
-            self.VM_list.append(vm)
+        self.state = ""
+        self.create_time = ""
+        # The list should only has one or zero VM, which holds the Ambari-server
+        self.ambari_server_vm = []
+        # The list of VMs, with Ambari-agents directly inside (not in Docker)
+        self.service_server_vm_list = []
+        # The list of VMs, each will hold multiple Docker containers with Ambari-agent inside
+        self.ambari_agent_vm_list = []
+
+    def _get_int_interval(self, int_list):
+        """
+        get the interval of the integer list
+        example: input[4,5,6,1,2,3], output [(1,3),(4,6)]
+        example: input[4,5,6,100,2,3], output [(2,3),(4,6),(100,100)]
+        :param int_list: the list of integer
+        :return: a tuple, each tuple has 2 integer, representing one interval
+        """
+        interval_list = []
+        int_list.sort()
+
+        begin = None
+        end = None
+        for integer in int_list:
+            if begin is None:
+                begin = integer
+                end = integer
+            else:
+                if integer == end + 1:
+                    end = integer
+                else:
+                    interval_list.append((begin, end))
+                    begin = integer
+                    end = integer
+
+        if begin is not None:
+            interval_list.append((begin, end))
+
+        return interval_list
+
+    def print_description(self):
+        print "cluster name: ", self.cluster_name
+        print "create time: ", self.create_time
+        print "state: ", self.state
+        print
+
+        print "Ambari Server: "
+        ambari_server_vm = self.get_ambari_server_vm()
+        if ambari_server_vm is None:
+            print "None"
+        else:
+            print ambari_server_vm.domain_name, " ", ambari_server_vm.external_ip, " ",\
+                ambari_server_vm.weave_internal_ip
+        print
+
+        print "Service Server with Ambari Agent directly installed: "
+        if len(self.service_server_vm_list) == 0:
+            print "None"
+        for vm in self.service_server_vm_list:
+            print vm.weave_domain_name, " ", vm.external_ip, " ", vm.weave_internal_ip
+        print
+
+        print "Ambari Agent in Docker Container: "
+        int_list = []
+        for vm in self.ambari_agent_vm_list:
+            for docker in vm.docker_list:
+                int_list.append(int(docker.get_index()))
+        interval_list = self._get_int_interval(int_list)
+        for interval in interval_list:
+            interval_str = ""
+            if interval[0] == interval[1]:
+                interval_str = str(interval(0))
+            else:
+                interval_str = "[{0}-{1}]".format(interval[0], interval[1])
+            print Docker.get_pattern_presentation(self.cluster_name, interval_str)
+        print
+
+    def get_agent_vm(self, vm_ip):
+        """
+        get the VM instance which holds Docker containers from the cluster instance
+        :param vm_ip: the external IP of the target VM
+        :return: the VM instance with the specified iP
+        """
+        for vm in self.ambari_agent_vm_list:
+            if vm.external_ip == vm_ip:
+                return vm
+
+    def get_ambari_server_vm(self):
+        """
+        get the VM instance which hold the Ambari-server
+        :return: the VM instance hold the Ambari-server, or None if no Ambari-server in this cluster
+        """
+        for vm in self.ambari_server_vm:
+            return vm
+        return None
+
+    def get_service_server_vm(self, vm_ip):
+        """
+        get the VM instance which directly hold the Ambari-agent
+        :param vm_ip: the external IP of the target VM
+        :return:
+        """
+        for vm in self.service_server_vm_list:
+            if vm.external_ip == vm_ip:
+                return vm
+
+    def to_json(self):
+        """
+        create a map to hold the information of the Cluster instance
+        :return: A map, which is JSON format object.
+        """
+        cluster = {}
+        cluster["cluster_name"] = self.cluster_name
+        cluster["create_time"] = self.create_time
+        cluster["state"] = self.state
+
+        cluster["ambari_server_vm"] = []
+        for vm in self.ambari_server_vm:
+            cluster["ambari_server_vm"].append(vm.to_json())
 
-        file.close()
+        cluster["service_server_vm_list"] = []
+        for vm in self.service_server_vm_list:
+            cluster["service_server_vm_list"].append(vm.to_json())
 
-    def __extract_VM_IP__(self, GCE_info_file_name):
-        f = open(GCE_info_file_name)
-        lines = f.readlines()
-        f.close()
+        cluster["ambari_agent_vm_list"] = []
+        for vm in self.ambari_agent_vm_list:
+            cluster["ambari_agent_vm_list"].append(vm.to_json())
 
-        ip_list = []
-        for line in lines:
+        return cluster
+
+    @staticmethod
+    def load_from_json(cluster_name):
+        """
+        load the cluster information from json file
+        :param cluster_name: the name of the cluster
+        :return: a Cluster instance or None if no such cluster
+        """
+        data = Data()
+        json_data = data.read_cluster_json(cluster_name)
+        if json_data is None:
+            return None
+
+        ambari_server_vm = []
+        service_server_vm_list = []
+        ambari_agent_vm_list = []
+
+        for vm_json in json_data["ambari_server_vm"]:
+            ambari_server_vm.append(VM.load_from_json(vm_json))
+
+        for vm_json in json_data["service_server_vm_list"]:
+            service_server_vm_list.append(VM.load_from_json(vm_json))
+
+        for vm_json in json_data["ambari_agent_vm_list"]:
+            ambari_agent_vm_list.append(VM.load_from_json(vm_json))
+
+        cluster = Cluster()
+        cluster.cluster_name = cluster_name
+        cluster.state = json_data["state"]
+        cluster.create_time = json_data["create_time"]
+        cluster.ambari_server_vm = ambari_server_vm
+        cluster.service_server_vm_list = service_server_vm_list
+        cluster.ambari_agent_vm_list = ambari_agent_vm_list
+        return cluster
+
+    def _extract_vm_fqdn_ip(self, gce_info_file_name):
+        """
+        exatract domain name and IP address of VMs from the output file of GCE
+        :param gce_info_file_name: output file of "GCE info" command
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        lines = []
+        with open(gce_info_file_name) as f:
+            lines = f.readlines()
+
+        vm_list = []
+        # the first line in the output file is title
+        for line in lines[1:]:
             tokens = line.split()
-            ip_list.append(tokens[1])
-        return ip_list[1:]
-
-    # request a new cluster
-    def request_GCE_cluster(self, vms_num, docker_num, cluster_name):
-        # reload configuration file
-        config = Config()
-        config.load()
-        # request cluster
-        gce_key = config.ATTRIBUTES["GCE_controller_key_file"]
-        gce_login = config.ATTRIBUTES["GCE_controller_user"] + "@" + config.ATTRIBUTES["GCE_controller_IP"]
-        gce_up_cmd = "gce up " + cluster_name + " " + str(vms_num) + " " + config.ATTRIBUTES["GCE_VM_type"] + \
-            " " + config.ATTRIBUTES["GCE_VM_OS"]
+            fqdn_ip = (tokens[0], tokens[1])
+            vm_list.append(fqdn_ip)
+        return vm_list
+
+    def request_vm(self, name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd):
+        """
+        Request VMs from GCE
+        :param name: the name prefix of all requesting VMs
+        :param vm_num: the number of VM
+        :param gce_vm_type: the type of VM
+        :param gce_vm_os: the OS of VM
+        :param gce_extra_cmd: extra command for requesting the VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_key = Config.ATTRIBUTES["gce_controller_key_file"]
+        gce_login = "{0}@{1}".format(Config.ATTRIBUTES["gce_controller_user"], Config.ATTRIBUTES["gce_controller_ip"])
+        gce_up_cmd = "gce up {0} {1} {2} {3} {4}".format(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
         subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_up_cmd])
 
-        print "cluster launched successufully, wait 5 seconds for cluster info ... ..."
-        time.sleep(5)
+        Log.write("cluster launched, wait for cluster info ... ...")
+
+        fqdn_ip_pairs = []
+        # wait for long enough. the more VM, more time it takes.
+        for retry in range(max(6, vm_num)):
+            time.sleep(10)
+
+            # request cluster info
+            with open(Config.ATTRIBUTES["gce_info_output"], "w") as gce_info_output_file:
+                gce_info_cmd = "gce info {0}".format(name)
+                subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd],
+                                stdout=gce_info_output_file)
+
+            fqdn_ip_pairs = self._extract_vm_fqdn_ip(Config.ATTRIBUTES["gce_info_output"])
+
+            if len(fqdn_ip_pairs) == vm_num:
+                Log.write("Get info for all ", str(len(fqdn_ip_pairs)), " VMs successfully")
+                break
+            Log.write("Only get info for ", str(len(fqdn_ip_pairs)), " VMs, retry ... ...")
+        return fqdn_ip_pairs
+
+    def request_ambari_server_vm(self, name):
+        """
+        request a VM for holding Ambari-server
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        # only 1 ambari server
+        vm_num = 1
+        gce_vm_type = Config.ATTRIBUTES["ambari_server_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["ambari_server_vm_os"]
+
+        gce_extra_cmd = ""
+        if "ambari_server_vm_extra" in Config.ATTRIBUTES:
+            gce_extra_cmd = Config.ATTRIBUTES["ambari_server_vm_extra"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
+        return fqdn_ip_pairs
+
+    def reqeust_service_server_vm(self, vm_num, name):
+        """
+        Request VMs to directly hold Ambari-agent (not inside Docker)
+        :param vm_num: the number of VM to request
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_vm_type = Config.ATTRIBUTES["service_server_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["service_server_vm_os"]
+
+        gce_extra_cmd = ""
+        if "service_server_vm_extra" in Config.ATTRIBUTES:
+            gce_extra_cmd = Config.ATTRIBUTES["service_server_vm_extra"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_cmd)
+        return fqdn_ip_pairs
 
-        # request cluster info
-        gce_info_output_file = open(config.ATTRIBUTES["GCE_info_output"], "w")
-        gce_info_cmd = "gce info " + cluster_name
-        subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-i", gce_key, gce_login, gce_info_cmd], \
-                        stdout=gce_info_output_file)
-        gce_info_output_file.close()
-        print "cluster info is saved to file " + config.ATTRIBUTES["GCE_info_output"]
+    def request_agent_vm(self, vm_num, name):
+        """
+        Request VMs to hold Docker containers, each with Ambari-agent inside
+        :param vm_num: the number of VM to request
+        :param name: the name prefix of all requesting VMs
+        :return: A list of tuple, each tuple has domain name and IP of a VM
+        """
+        gce_vm_type = Config.ATTRIBUTES["ambari_agent_vm_type"]
+        gce_vm_os = Config.ATTRIBUTES["ambari_agent_vm_os"]
+        gce_extra_disk = ""
+        if "ambari_agent_vm_extra_disk" in Config.ATTRIBUTES:
+            gce_extra_disk = Config.ATTRIBUTES["ambari_agent_vm_extra_disk"]
+
+        fqdn_ip_pairs = self.request_vm(name, vm_num, gce_vm_type, gce_vm_os, gce_extra_disk)
+        return fqdn_ip_pairs
+
+    def request_gce_cluster(self, ambari_agent_vm_num, docker_num,
+                            service_server_num, with_ambari_server, cluster_name):
+        """
+        Request a cluster from GCE
+        :param ambari_agent_vm_num: number of VMs to hold Docker containers
+        :param docker_num: number of Docker containers inside each VM
+        :param service_server_num: number of VMs which has Ambari-agent directly installed (not in Docker)
+        :param with_ambari_server: True or False, whether to request a VM to hold Ambari-server
+        :param cluster_name: the name of the cluster
+        :return: None
+        """
+        ambari_server_fqdn_ip_pairs = []
+        if with_ambari_server is True:
+            ambari_server_fqdn_ip_pairs = self.request_ambari_server_vm(VM.get_ambari_server_vm_name(cluster_name))
+        service_server_fqdn_ip_pairs = self.reqeust_service_server_vm(service_server_num,
+                                                                      VM.get_service_server_vm_name(cluster_name))
+        ambari_agent_fqdn_ip_pairs = self.request_agent_vm(ambari_agent_vm_num,
+                                                           VM.get_ambari_agent_vm_name(cluster_name))
 
         # prepare all attributes of the cluster, write to a file
-        VM_IP_list = self.__extract_VM_IP__(config.ATTRIBUTES["GCE_info_output"])
-        self.generate_cluster_info(VM_IP_list, cluster_name, docker_num)
-        self.overwrite_to_file(config.ATTRIBUTES["cluster_info_file"])
-        # server need this file to resolve the host names of the agents
-        self.export_hostnames(config.ATTRIBUTES["Docker_hostname_info"])
-
-    # save info to file
-    def overwrite_to_file(self, filename):
-        file = open(filename, "w")
-        file.write("cluster_name: " + self.cluster_name + "\n")
-        file.write("VMs_num: " + str(self.VMs_num) + "\n")
-
-        for vm in self.VM_list:
-            file.write("\t\t")
-            file.write("VM_IP: " + vm.external_ip + "\n")
-            file.write("\t\t")
-            file.write("Docker_num: " + str(len(vm.docker_list)) + "\n")
-            for docker in vm.docker_list:
-                file.write("\t\t\t\t")
-                file.write(docker.IP + "/" + docker.mask + " " + docker.hostname + "\n")
+        self.generate_cluster_info(cluster_name, ambari_server_fqdn_ip_pairs, service_server_fqdn_ip_pairs,
+                                   ambari_agent_fqdn_ip_pairs, docker_num)
 
-        file.close()
+    def generate_cluster_info(self, cluster_name, ambari_server_fqdn_ip_pairs, service_server_fqdn_ip_pairs,
+                              ambari_agent_fqdn_ip_pairs, docker_num):
+        """
+        generate VM and docker info for this cluster
+        set up parameter of the class instance as this info
+        :param cluster_name: the name of the cluster
+        :param ambari_server_fqdn_ip_pairs: the domain name and IP pairs for Ambari-server
+        :param service_server_fqdn_ip_pairs: the domain name and IP pairs for VMs with Ambari-agent installed
+        :param ambari_agent_fqdn_ip_pairs: the domain name and IP pairs for VM with Docker containers
+        :param docker_num: the number of Dockers inside each VMs
+        :return: None
+        """
+        weave_ip_base = Config.ATTRIBUTES["weave_ip_base"]
+        weave_ip_mask = Config.ATTRIBUTES["weave_ip_mask"]
+        current_ip = weave_ip_base
 
-    def __increase_IP__(self, base_IP, increase):
-        IP = [int(base_IP[0]), int(base_IP[1]), int(base_IP[2]), int(base_IP[3])]
-        IP[3] = IP[3] + increase
-        for index in reversed(range(0, 4)):
-            if IP[index] > 255:
-                IP[index - 1] = IP[index - 1] + IP[index] / 256
-                IP[index] = IP[index] % 256
-        return IP
-
-    # generate VM and docker info for this cluster
-    # set up parameter as this info
-    def generate_cluster_info(self, VM_IP_list, cluster_name, docker_num):
-        config = Config()
-        config.load()
-        Docker_IP_base = config.ATTRIBUTES["Docker_IP_base"].split(".")
-        Docker_IP_mask = config.ATTRIBUTES["Docker_IP_mask"]
-
-        VM_index = 0
-        for VM_IP in VM_IP_list:
-            vm = VM(VM_IP)
-
-            for Docker_index in range(0, docker_num):
-                total_Docker_index = VM_index * docker_num + Docker_index
-                docker_IP = self.__increase_IP__(Docker_IP_base, total_Docker_index)
-                docker_IP_str = str(docker_IP[0]) + "." + str(docker_IP[1]) + "." + \
-                                str(docker_IP[2]) + "." + str(docker_IP[3])
-                docker_hostname = cluster_name + "-" + str(VM_index) + "-" + str(Docker_index)
-                docker = Docker(docker_IP_str, str(Docker_IP_mask), docker_hostname)
-                # print docker
+        for vm_domain_name, vm_ip in ambari_server_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+            current_ip = self._increase_ip(current_ip, 1)
+            vm.weave_internal_ip = current_ip
+            self.ambari_server_vm.append(vm)
+
+        for vm_domain_name, vm_ip in service_server_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+            current_ip = self._increase_ip(current_ip, 1)
+            vm.weave_internal_ip = current_ip
+            self.service_server_vm_list.append(vm)
+
+        vm_index = 0
+        for vm_domain_name, vm_ip in ambari_agent_fqdn_ip_pairs:
+            current_ip = self._increase_ip(current_ip, 1)
+            weave_dns_ip = current_ip
+            vm = VM(vm_ip, vm_domain_name, weave_dns_ip, weave_ip_mask)
+
+            for docker_index in range(0, docker_num):
+                current_ip = self._increase_ip(current_ip, 1)
+                docker_ip_str = current_ip
+
+                total_docker_index = vm_index * docker_num + docker_index
+                docker_domain_name = Docker.get_weave_domain_name(cluster_name, total_docker_index)
+
+                docker = Docker(docker_ip_str, str(weave_ip_mask), docker_domain_name)
                 vm.add_docker(docker)
-            VM_index = VM_index + 1
-            self.VM_list.append(vm)
 
-        self.VMs_num = len(VM_IP_list)
+            vm_index += 1
+            self.ambari_agent_vm_list.append(vm)
+
         self.cluster_name = cluster_name
+        self.create_time = str(datetime.datetime.now())
+        self.state = Cluster.STATE_FREE
 
-    # run all dockers for all the VMs in the cluster
-    # upload necessary file to each machine in cluster, run launcher_docker.py in each machine with parameter
-    def run_docker_on_cluster(self, server_external_IP, server_Weave_IP):
-        config = Config()
-        config.load()
-
-        for vm in self.VM_list:
-            # upload necessary file to each machine in cluster
-            VM_external_IP = vm.external_ip
-            VM_directory = "root@" + VM_external_IP + ":" + config.ATTRIBUTES["VM_code_directory"]
-            VM_key = config.ATTRIBUTES["GCE_VM_key_file"]
-            subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i", VM_key, "-r", ".", VM_directory])
-
-            # run launcher_docker.py in each machine with parameters
-            subprocess.call(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", VM_key, \
-                             "root@" + VM_external_IP, \
-                             "cd " + config.ATTRIBUTES["VM_code_directory"] + "; python launcher_docker.py" + \
-                             " " + VM_external_IP + " " + server_Weave_IP + " " + server_external_IP])
-
-    # export host names to a file
-    def export_hostnames(self, filename):
-        hostname_file = open(filename, "w")
-        for vm in self.VM_list:
-            for docker in vm.docker_list:
-                hostname_file.write(docker.IP)
-                hostname_file.write(" ")
-                hostname_file.write(docker.hostname)
-                hostname_file.write("\n")
-        hostname_file.close()
+        # update config file.
+        # This step makes the user avoid reconfiguring the IP for next cluster creation
+        Config.update("weave", "weave_ip_base", current_ip)
+
+    def _increase_ip(self, base_ip_str, increase):
+        """
+        increase the IP address.
+        example: 192.168.1.1, increased by 1: 192.168.1.2
+        example: 192.168.1.254, increased by 2: 192.168.2.1
+        :param base_ip_str: the IP to be increased
+        :param increase: the amount of increase
+        :return: the new IP address, in String
+        """
+        base_ip = base_ip_str.split(".")
+        new_ip = [int(base_ip[0]), int(base_ip[1]), int(base_ip[2]), int(base_ip[3])]
+        new_ip[3] = new_ip[3] + increase
+        for index in reversed(range(0, 4)):
+            if new_ip[index] > 255:
+                new_ip[index - 1] += (new_ip[index] / 256)
+                new_ip[index] %= 256
+        return "{0}.{1}.{2}.{3}".format(new_ip[0], new_ip[1], new_ip[2], new_ip[3])
+
+    def _scp_upload(self, vm_external_ip):
+        """
+        upload all the code in a VM
+        :param vm_external_ip: the external IP of the VM
+        :return: None
+        """
+        # upload necessary file to VM
+        vm_directory = "{0}@{1}:{2}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip,
+                                            Config.ATTRIBUTES["vm_code_directory"])
+        vm_key = Config.ATTRIBUTES["vm_key_file"]
+
+        upload_return_code = 0
+        with open(os.devnull, 'w') as shutup:
+            upload_return_code = subprocess.call(["scp", "-o", "StrictHostKeyChecking=no", "-i",
+                                                  vm_key, "-r", ".", vm_directory],
+                                                 stdout=shutup, stderr=shutup)
+        if upload_return_code == 0:
+            Log.write("VM ", vm_external_ip, " file upload succeed")
+        else:
+            Log.write("VM ", vm_external_ip, " file upload fail")
+
+    def run_cluster(self, server_weave_ip, server_external_ip):
+        """
+        Run all Ambari-agents and Ambari-server in the cluster in parallel
+        Wait until all processes finish
+        :param server_weave_ip: the Weave IP of Ambari-server
+        :param server_external_ip: the external IP of Ambari-server
+        :return: None
+        """
+        process_list = {}
+        process_list.update(self.run_ambari_server_asyn())
+        process_list.update(self.run_service_server_asyn(server_weave_ip, server_external_ip))
+        process_list.update(self.run_docker_on_cluster_asyn(server_weave_ip, server_external_ip))
+
+        terminate_state_list = {}
+        for hostname in process_list:
+            terminate_state_list[hostname] = False
+
+        Log.write("Wait for all VMs to finish configuration ... ...")
+
+        # Wait for all configuration subprocesses
+        while True:
+            all_finished = True
+            for hostname in process_list:
+                output_file, output_file_path, process = process_list[hostname]
+                if terminate_state_list[hostname] is False:
+                    all_finished = False
+                    returncode = process.poll()
+                    if returncode is None:
+                        continue
+                    else:
+                        Log.write("VM ", hostname, " configuration completed, return code: ", str(returncode) \
+                                  , ", output file path: ", output_file_path)
+                        terminate_state_list[hostname] = True
+                        output_file.close()
+                else:
+                    pass
+            if all_finished:
+                break
+            time.sleep(5)
+
+        Log.write("All VM configuration completed.")
+
+    def run_ambari_server_asyn(self):
+        """
+        Run configuration for Ambari-server in this cluster
+        Set up Ambari-server and Weave network
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.ambari_server_vm:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            # ssh install server
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_ambari_server.py {0}".format(self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                       vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+        return process_list
+
+    def run_service_server_asyn(self, server_weave_ip, server_external_ip):
+        """
+        Run configuration, set up Ambari-agent in this VM, and the Weave network
+        :param server_weave_ip: the Weave IP of the Ambari-server
+        :param server_external_ip: the external IP of the Ambari-server
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.service_server_vm_list:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            # ssh install server
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_service_server.py {0} {1} {2} {3}".format(
+                vm_external_ip, server_weave_ip, server_external_ip, self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                        vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+        return process_list
+
+    def run_docker_on_cluster_asyn(self, server_weave_ip, server_external_ip):
+        """
+        Run configuration, set up Docker and Weave network
+        run all containers, each with Ambari-agent inside
+        :param server_weave_ip: the Weave IP of the Ambari-server
+        :param server_external_ip: the external IP of the Ambari-server
+        The method is NON-BLOCK
+        :return: a map of tuple, the key of the map is the host name of the VM,
+                the tuple has 3 elements: the file handler of the output of the VM,
+                the file path of the output of the VM,
+                and the process object of configuration for the VM
+        """
+        process_list = {}
+
+        for vm in self.ambari_agent_vm_list:
+            vm_external_ip = vm.external_ip
+            self._scp_upload(vm_external_ip)
+
+            vm_output_file_path = vm.get_ssh_output_file_path()
+            vm_output_file = open(vm_output_file_path, "w")
+
+            vm_ssh_login = "{0}@{1}".format(Config.ATTRIBUTES["vm_user"], vm_external_ip)
+            vm_ssh_cd_cmd = "cd {0}".format(Config.ATTRIBUTES["vm_code_directory"])
+            vm_ssh_python_cmd = "python launcher_docker.py {0} {1} {2} {3}".format(
+                vm_external_ip, server_weave_ip, server_external_ip, self.cluster_name)
+            vm_ssh_cmd = "{0};{1}".format(vm_ssh_cd_cmd, vm_ssh_python_cmd)
+            vm_key = Config.ATTRIBUTES["vm_key_file"]
+            Log.write(vm_ssh_python_cmd)
+
+            process = subprocess.Popen(["ssh", "-o", "StrictHostKeyChecking=no", "-t", "-i", vm_key,
+                                        vm_ssh_login, vm_ssh_cmd],
+                                       stdout=vm_output_file, stderr=vm_output_file)
+
+            process_list[vm.hostname] = (vm_output_file, vm_output_file_path, process)
+            Log.write("Configuring VM ", vm.hostname, " ... ...")
+
+        return process_list

contrib/agent-simulator/config.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,23 +14,58 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 import ConfigParser
+import os
+
 
 class Config:
     ATTRIBUTES = {}
 
+    AMBARI_AGENT_VM = "ambari_agent_vm"
+    AMBARI_SERVER_VM = "ambari_server_vm"
+    SERVICE_SERVER_VM = "service_server_vm"
+
+    RELATIVE_CONFIG_FILE_PATH = "config/config.ini"
+
+    def __init__(self):
+        pass
+
     @staticmethod
     def load():
+        """
+        load configuration from file, add all configuration to the map ATTRIBUTES
+        :return: None
+        """
         config = ConfigParser.RawConfigParser()
         # keep file case sensitive
         config.optionxform = str
-        config.read("config/config.ini")
+        config.read(Config.RELATIVE_CONFIG_FILE_PATH)
         for section in config.sections():
             for key in config.options(section):
                 Config.ATTRIBUTES[key] = config.get(section, key)
 
+        # set output file path
+        for key in config.options("output"):
+            if key == "output_folder":
+                # create the folder
+                if not os.path.exists(Config.ATTRIBUTES["output_folder"]):
+                    os.makedirs(Config.ATTRIBUTES["output_folder"])
+            else:
+                Config.ATTRIBUTES[key] = Config.ATTRIBUTES["output_folder"] + "/" + Config.ATTRIBUTES[key]
 
-# Config.load()
-# print Config.ATTRIBUTES
+    @staticmethod
+    def update(section, key, value):
+        """
+        Update the key value of the configuration file
+        :param section: section is inside []
+        :param key: the key
+        :param value: the value
+        :return: None
+        """
+        config = ConfigParser.RawConfigParser()
+        config.read(Config.RELATIVE_CONFIG_FILE_PATH)
+        config.set(section, key, value)
+        with open(Config.RELATIVE_CONFIG_FILE_PATH, 'wb') as configfile:
+                config.write(configfile)

contrib/agent-simulator/config/config.ini

@@ -1,49 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information rega4rding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This configuration file is case SENSITIVE
-# For more information about this configuration file, check https://docs.python.org/2/library/configparser.html
-
-[GCE]
-GCE_controller_IP = 104.196.89.197
-GCE_controller_user = root
-GCE_controller_key_file = ~/gce-key
-GCE_VM_type = --xlarge
-# only support centos 7
-GCE_VM_OS = --centos7
-# the result of command "gce info clustername" on the GCE controller
-GCE_info_output = output/gce_info_output.txt
-# the same as the GCE controller for GCE
-GCE_VM_key_file = ~/gce-key
-
-
-[Cluster]
-cluster_info_file = config/cluster.txt
-VM_code_directory = /simulator-script
-VM_hostname_file = /etc/hosts
-Docker_hostname_info = config/hosts.txt
-
-[Weave]
-# Docker_IP_base is the starting point of IP address. with mask 16, ideally:
-# the IP of docker will be 192.168.[1-255].[1-255]/16
-# if there are more dockers, which means there is not enough IPs within this mask for all dockers,
-# the IP will continue to increase to be 192.169.*.*, dockers might not be able to talk to each other,
-# because the mask is 16
-Docker_IP_base = 192.168.1.2
-Docker_IP_mask = 16
-
-
-[Docker]
-Docker_image_name = ambari/agent-sim

contrib/agent-simulator/data.py

@@ -0,0 +1,104 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os.path
+import json
+from config import Config
+
+
+class Data:
+    def __init__(self):
+        self.data_filename = Config.ATTRIBUTES["cluster_info_file"]
+
+    def _load_data(self):
+        """
+        load all data from JSON file
+        :return: a map, which is a JSON format object
+        """
+        json_data = {"clusters": []}
+        if os.path.isfile(self.data_filename):
+            with open(self.data_filename) as f:
+                json_data = json.load(f)
+        return json_data
+
+    def _save_data(self, json_data):
+        """
+        save the JSON object into a file
+        :param json_data: a map, which is a JSON format object
+        :return: None
+        """
+        with open(self.data_filename, "w") as f:
+            json.dump(json_data, f, indent=4, separators=(',', ': '))
+
+    def add_new_cluster(self, cluster):
+        """
+        add a new cluster into the JSON file
+        :param cluster: the cluster instance
+        :return: None
+        """
+        json_data = self._load_data()
+        new_cluster_json = cluster.to_json()
+        json_data["clusters"].insert(0, new_cluster_json)
+        self._save_data(json_data)
+
+    def set_cluster_state(self, cluster_name, state):
+        """
+        set the state of a cluster into JSON file
+        :param cluster_name: the name of the cluster
+        :param state: the name of the state
+        :return: None
+        """
+        json_data = self._load_data()
+        for cluster in json_data["clusters"]:
+            if cluster["cluster_name"] == cluster_name:
+                cluster["state"] = state
+                break
+        self._save_data(json_data)
+
+    def read_cluster_json(self, cluster_name):
+        """
+        get the JSON object for the cluster
+        :param cluster_name: the name of cluster
+        :return: a map which is a JSON object or None if the cluster is not found
+        """
+        json_data = self._load_data()
+        for cluster_json in json_data["clusters"]:
+            if cluster_json["cluster_name"] == cluster_name:
+                return cluster_json
+        return None
+
+    def print_cluster_summary_list(self):
+        """
+        get a brief description of all the cluster from the JSON file
+        :return: a list of tuple. The elements of the tuple are:
+                 cluster_name, state, agent_number,
+                 service_server_num, ambari_server_num, create_time
+        """
+        print "(cluster_name, state, agent_number, service_server_num, ambari_server_num, create_time)"
+
+        json_data = self._load_data()
+        for cluster in json_data["clusters"]:
+            cluster_name = cluster["cluster_name"]
+            state = cluster["state"]
+            create_time = cluster["create_time"]
+            agent_number = 0
+            for agent_vm in cluster["ambari_agent_vm_list"]:
+                agent_number += len(agent_vm["docker_list"])
+            service_server_num = len(cluster["service_server_vm_list"])
+            ambari_server_num = len(cluster["ambari_server_vm"])
+            print cluster_name, state, agent_number, service_server_num, ambari_server_num, create_time

contrib/agent-simulator/docker.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,14 +14,72 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
+
+from config import Config
 
 
 class Docker:
-    def __init__(self, IP, mask, hostname):
-        self.IP = IP
+    """
+    Docker represents a Docker container, each with its IP and domain name
+    """
+    def __init__(self, ip, mask, weave_domain_name):
+        self.ip = ip
         self.mask = mask
-        self.hostname = hostname
+        self.weave_domain_name = weave_domain_name
+
+    def to_json(self):
+        """
+        create a map to hold the information of the Docker instance
+        :return: A map, which is JSON format object.
+        """
+        docker_json = {}
+        docker_json["weave_ip"] = "{0}/{1}".format(self.ip, self.mask)
+        docker_json["weave_domain_name"] = self.weave_domain_name
+        return docker_json
+
+    @staticmethod
+    def load_from_json(json_data):
+        """
+        load the docker information from a JSON object
+        :param json_data: a map, which is a JSON object
+        :return: a Docker object
+        """
+        ip = json_data["weave_ip"].split("/")[0]
+        mask = json_data["weave_ip"].split("/")[1]
+        weave_domain_name = json_data["weave_domain_name"]
+        return Docker(ip, mask, weave_domain_name)
 
     def __str__(self):
-        return str(self.IP) + "/" + str(self.mask) + " " + self.hostname
+        return str(self.ip) + "/" + str(self.mask) + " " + self.weave_domain_name
+
+    @staticmethod
+    def get_weave_domain_name(cluster_name, index):
+        """
+        given the index and the name of cluster, generate the  Weave domain name for the docker
+        :param cluster_name: the name of the cluster
+        :param index: a number
+        :return: Weave domain name of the docker container
+        """
+        return "{0}-{1}-{2}.{3}".format(Config.ATTRIBUTES["container_hostname_fix"],
+                                        index, cluster_name, "weave.local")
+
+    @staticmethod
+    def get_pattern_presentation(cluster_name, range_str):
+        return Docker.get_weave_domain_name(cluster_name, range_str)
+
+    def get_index(self):
+        """
+        extract the index of the docker within the cluster
+        :return: the index
+        """
+        return self.weave_domain_name.split("-")[1]
+
+    def get_container_name(self):
+        """
+        :return: the name of the container
+        """
+        return self.get_hostname()
+
+    def get_hostname(self):
+        return self.weave_domain_name.split(".")[0]

contrib/agent-simulator/docker_image/Yum_Dockerfile

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set the base image to CentOS 7
+FROM centos:7
+
+#====================================set SSH service====================================================================
+RUN yum -y -q install openssh-server epel-release openssh-clients&& \
+    yum -y -q install pwgen && \
+    rm -f /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_rsa_key && \
+    ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_ecdsa_key && \
+    ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
+    sed -i "s/#UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
+
+ADD ssh_service/set_root_pw.sh /set_root_pw.sh
+ADD ssh_service/run.sh /run_ssh.sh
+RUN chmod +x /*.sh
+
+ENV AUTHORIZED_KEYS **None**
+ENV ROOT_PASS ambariagent
+#====================================set SSH service====================================================================
+
+
+# Copy the files into Docker
+ADD launcher_agent.py /launcher_agent.py
+ADD ambari_agent_start.sh /ambari_agent_start.sh
+ADD package_list.txt /package_list.txt
+ADD ambari_agent_install.sh /ambari_agent_install.sh
+
+RUN chmod 755 /ambari_agent_start.sh && \
+    chmod 755 /ambari_agent_install.sh
+
+# Install ambari-agent
+RUN /ambari_agent_install.sh
+
+# Set Hadoop Repo
+RUN wget -O /etc/yum.repos.d/hdp.repo http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos7/2.x/updates/2.3.0.0/hdp.repo
+
+# Install all package
+RUN yum install -y -q $(cat package_list.txt)
+RUN yum update -y -q
+
+# Clean yum download cache
+RUN yum clean -y all

contrib/agent-simulator/Docker/__init__.py → contrib/agent-simulator/docker_image/__init__.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,4 +14,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""

contrib/agent-simulator/tips.txt → contrib/agent-simulator/docker_image/ambari_agent_install.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,6 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# this script will install Ambari agent on Centos 7
 
-1. Ambari-agent and Ambari-server have to be the same version to successfully register. The command used to install Ambari-agent is in the Dockerfile
-2. Use CTRL + P, then CTRL + Q to exit Docker container without terminate the container.
+yum install -y -q wget
+wget -q -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.1/ambaribn.repo
+yum install -y -q epel-release
+yum install -y -q ambari-agent

contrib/agent-simulator/Docker/ambari_agent_start.sh → contrib/agent-simulator/docker_image/ambari_agent_start.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,6 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# this script will start Ambari agent on Centos 7
 
-#!/bin/bash
 ambari-agent start

contrib/agent-simulator/docker_image/launcher_agent.py

@@ -0,0 +1,86 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+
+import sys
+import subprocess
+
+
+def replace_conf(server_ip):
+    """
+    replace the server host IP in the Ambari-agent configuration file
+    :param server_ip: internal Weave IP address of Ambari-server
+    :return: None
+    """
+    lines = []
+    with open("/etc/ambari-agent/conf/ambari-agent.ini") as f:
+        lines = f.readlines()
+
+    with open("/etc/ambari-agent/conf/ambari-agent.ini", "w+") as f:
+        for line in lines:
+            line = line.replace("hostname=localhost", "hostname=" + server_ip)
+            f.write(line)
+
+
+def run_ssh():
+    """
+    run SSH service on this Docker Container
+    :return: None
+    """
+    subprocess.call("/run_ssh.sh")
+
+
+def run_ambari_agent():
+    """
+    command line to run Ambari-agent
+    :return: None
+    """
+    subprocess.call("/ambari_agent_start.sh")
+
+
+def set_weave_ip(weave_ip):
+    """
+    set the IP and hostname mapping for this Container
+    Docker will assign an IP to each Container, and map it to hostname, which is not we want
+    We want our Weave IP to be mapped to hostname
+    :param weave_ip:
+    :return: None
+    """
+    with open("/etc/hosts") as etc_hosts:
+        all_resolution = etc_hosts.readlines()
+
+    with open("/etc/hosts", "w") as etc_hosts:
+        for index in range(len(all_resolution)):
+            if index == 0:
+                token = all_resolution[index].split()
+                etc_hosts.write("{0} {1} {2}\n".format(weave_ip, token[1], token[2]))
+            else:
+                etc_hosts.write(all_resolution[index])
+
+
+def main():
+    ambari_server_ip = sys.argv[1]
+    my_weave_ip = sys.argv[2]
+    replace_conf(ambari_server_ip)
+    set_weave_ip(my_weave_ip)
+    run_ambari_agent()
+    run_ssh()
+
+
+if __name__ == "__main__":
+    main()

contrib/agent-simulator/docker_image/package_list.txt

@@ -0,0 +1,235 @@
+alsa-lib.x86_64
+at.x86_64
+atk.x86_64
+avahi-libs.x86_64
+bc.x86_64
+cairo.x86_64
+cpp.x86_64
+cronie.x86_64
+cronie-anacron.x86_64
+crontabs.noarch
+desktop-file-utils.x86_64
+ed.x86_64
+emacs-filesystem.noarch
+fontconfig.x86_64
+fontpackages-filesystem.noarch
+foomatic-filters.x86_64
+fuse.x86_64
+fuse-libs.x86_64
+gcc.x86_64
+gdbm-devel.x86_64
+gdk-pixbuf2.x86_64
+gettext.x86_64
+gettext-libs.x86_64
+ghostscript.x86_64
+ghostscript-fonts.noarch
+glibc-devel.x86_64
+glibc-headers.x86_64
+graphite2.x86_64
+gtk2.x86_64
+harfbuzz.x86_64
+hicolor-icon-theme.noarch
+initscripts.x86_64
+jasper-libs.x86_64
+jbigkit-libs.x86_64
+kmod.x86_64
+kmod-libs.x86_64
+lcms2.x86_64
+libICE.x86_64
+libSM.x86_64
+libX11.x86_64
+libX11-common.noarch
+libXau.x86_64
+libXcomposite.x86_64
+libXcursor.x86_64
+libXdamage.x86_64
+libXext.x86_64
+libXfixes.x86_64
+libXfont.x86_64
+libXft.x86_64
+libXi.x86_64
+libXinerama.x86_64
+libXrandr.x86_64
+libXrender.x86_64
+libXt.x86_64
+libXtst.x86_64
+libXxf86vm.x86_64
+libdb-devel.x86_64
+libdrm.x86_64
+libfontenc.x86_64
+libjpeg-turbo.x86_64
+libmng.x86_64
+libmpc.x86_64
+libpciaccess.x86_64
+libpipeline.x86_64
+libpng.x86_64
+libpng12.x86_64
+libthai.x86_64
+libtiff.x86_64
+libtirpc.x86_64
+libxcb.x86_64
+m4.x86_64
+mailcap.noarch
+mailx.x86_64
+make.x86_64
+man-db.x86_64
+mesa-libEGL.x86_64
+mesa-libGL.x86_64
+mesa-libGLU.x86_64
+mesa-libgbm.x86_64
+mesa-libglapi.x86_64
+mpfr.x86_64
+nmap-ncat.x86_64
+pango.x86_64
+passwd.x86_64
+patch.x86_64
+pciutils.x86_64
+pciutils-libs.x86_64
+perl.x86_64
+perl-B-Lint.noarch
+perl-Business-ISBN.noarch
+perl-Business-ISBN-Data.noarch
+perl-CGI.noarch
+perl-CPAN.noarch
+perl-Carp.noarch
+perl-Class-ISA.noarch
+perl-Compress-Raw-Bzip2.x86_64
+perl-Compress-Raw-Zlib.x86_64
+perl-Data-Dumper.x86_64
+perl-Digest.noarch
+perl-Digest-MD5.x86_64
+perl-Digest-SHA.x86_64
+perl-Encode.x86_64
+perl-Encode-Locale.noarch
+perl-Env.noarch
+perl-Exporter.noarch
+perl-ExtUtils-Install.noarch
+perl-ExtUtils-MakeMaker.noarch
+perl-ExtUtils-Manifest.noarch
+perl-ExtUtils-ParseXS.noarch
+perl-FCGI.x86_64
+perl-File-CheckTree.noarch
+perl-File-Listing.noarch
+perl-File-Path.noarch
+perl-File-Temp.noarch
+perl-Filter.x86_64
+perl-Getopt-Long.noarch
+perl-HTML-Parser.x86_64
+perl-HTML-Tagset.noarch
+perl-HTTP-Cookies.noarch
+perl-HTTP-Daemon.noarch
+perl-HTTP-Date.noarch
+perl-HTTP-Message.noarch
+perl-HTTP-Negotiate.noarch
+perl-HTTP-Tiny.noarch
+perl-IO-Compress.noarch
+perl-IO-HTML.noarch
+perl-IO-Socket-IP.noarch
+perl-IO-Socket-SSL.noarch
+perl-LWP-MediaTypes.noarch
+perl-Locale-Codes.noarch
+perl-Locale-Maketext.noarch
+perl-Module-Pluggable.noarch
+perl-Net-HTTP.noarch
+perl-Net-LibIDN.x86_64
+perl-Net-SSLeay.x86_64
+perl-PathTools.x86_64
+perl-Pod-Checker.noarch
+perl-Pod-Escapes.noarch
+perl-Pod-LaTeX.noarch
+perl-Pod-Parser.noarch
+perl-Pod-Perldoc.noarch
+perl-Pod-Plainer.noarch
+perl-Pod-Simple.noarch
+perl-Pod-Usage.noarch
+perl-Scalar-List-Utils.x86_64
+perl-Socket.x86_64
+perl-Storable.x86_64
+perl-Sys-Syslog.x86_64
+perl-Test-Harness.noarch
+perl-Test-Simple.noarch
+perl-Text-ParseWords.noarch
+perl-Text-Soundex.x86_64
+perl-Text-Unidecode.noarch
+perl-Time-HiRes.x86_64
+perl-Time-Local.noarch
+perl-TimeDate.noarch
+perl-URI.noarch
+perl-WWW-RobotRules.noarch
+perl-XML-LibXML.x86_64
+perl-XML-NamespaceSupport.noarch
+perl-XML-SAX.noarch
+perl-XML-SAX-Base.noarch
+perl-autodie.noarch
+perl-constant.noarch
+perl-devel.x86_64
+perl-libs.x86_64
+perl-libwww-perl.noarch
+perl-local-lib.noarch
+perl-macros.x86_64
+perl-parent.noarch
+perl-podlators.noarch
+perl-threads.x86_64
+perl-threads-shared.x86_64
+pixman.x86_64
+poppler-data.noarch
+psmisc.x86_64
+pyparsing.noarch
+qt.x86_64
+qt-settings.noarch
+qt-x11.x86_64
+qt3.x86_64
+redhat-lsb.x86_64
+redhat-lsb-core.x86_64
+redhat-lsb-cxx.x86_64
+redhat-lsb-desktop.x86_64
+redhat-lsb-languages.x86_64
+redhat-lsb-printing.x86_64
+redhat-lsb-submod-multimedia.x86_64
+redhat-lsb-submod-security.x86_64
+rpcbind.x86_64
+snappy-devel.x86_64
+spax.x86_64
+sysvinit-tools.x86_64
+tcp_wrappers-libs.x86_64
+time.x86_64
+urw-fonts.noarch
+wget.x86_64
+xorg-x11-font-utils.x86_64
+bigtop-jsvc.x86_64
+bigtop-tomcat.noarch
+hadoop_2_3_0_0_2557.x86_64
+hadoop_2_3_0_0_2557-client.x86_64
+hadoop_2_3_0_0_2557-conf-pseudo.x86_64
+hadoop_2_3_0_0_2557-doc.x86_64
+hadoop_2_3_0_0_2557-hdfs.x86_64
+hadoop_2_3_0_0_2557-hdfs-datanode.x86_64
+hadoop_2_3_0_0_2557-hdfs-fuse.x86_64
+hadoop_2_3_0_0_2557-hdfs-journalnode.x86_64
+hadoop_2_3_0_0_2557-hdfs-namenode.x86_64
+hadoop_2_3_0_0_2557-hdfs-zkfc.x86_64
+hadoop_2_3_0_0_2557-httpfs.x86_64
+hadoop_2_3_0_0_2557-httpfs-server.x86_64
+hadoop_2_3_0_0_2557-libhdfs.x86_64
+hadoop_2_3_0_0_2557-mapreduce.x86_64
+hadoop_2_3_0_0_2557-source.x86_64
+hadoop_2_3_0_0_2557-yarn.x86_64
+hadoop_2_3_0_0_2557-yarn-nodemanager.x86_64
+hadoop_2_3_0_0_2557-yarn-proxyserver.x86_64
+hadoop_2_3_0_0_2557-yarn-resourcemanager.x86_64
+hadoop_2_3_0_0_2557-yarn-timelineserver.x86_64
+hbase_2_3_0_0_2557.noarch
+hbase_2_3_0_0_2557-doc.noarch
+hbase_2_3_0_0_2557-master.noarch
+hbase_2_3_0_0_2557-regionserver.noarch
+hbase_2_3_0_0_2557-rest.noarch
+hbase_2_3_0_0_2557-thrift.noarch
+hbase_2_3_0_0_2557-thrift2.noarch
+hdp-select.noarch
+phoenix_2_3_0_0_2557.noarch
+ranger_2_3_0_0_2557-hbase-plugin.x86_64
+ranger_2_3_0_0_2557-hdfs-plugin.x86_64
+ranger_2_3_0_0_2557-yarn-plugin.x86_64
+zookeeper_2_3_0_0_2557.noarch
+ambari-metrics-monitor.x86_64
+ambari-metrics-hadoop-sink.x86_64

contrib/agent-simulator/docker_image/ssh_service/run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "${AUTHORIZED_KEYS}" != "**None**" ]; then
+    echo "=> Found authorized keys"
+    mkdir -p /root/.ssh
+    chmod 700 /root/.ssh
+    touch /root/.ssh/authorized_keys
+    chmod 600 /root/.ssh/authorized_keys
+    IFS=$'\n'
+    arr=$(echo ${AUTHORIZED_KEYS} | tr "," "\n")
+    for x in $arr
+    do
+        x=$(echo $x |sed -e 's/^ *//' -e 's/ *$//')
+        cat /root/.ssh/authorized_keys | grep "$x" >/dev/null 2>&1
+        if [ $? -ne 0 ]; then
+            echo "=> Adding public key to /root/.ssh/authorized_keys: $x"
+            echo "$x" >> /root/.ssh/authorized_keys
+        fi
+    done
+fi
+
+if [ ! -f /.root_pw_set ]; then
+	/set_root_pw.sh
+fi
+exec /usr/sbin/sshd -D

contrib/agent-simulator/docker_image/ssh_service/set_root_pw.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ -f /.root_pw_set ]; then
+	echo "Root password already set!"
+	exit 0
+fi
+
+PASS=${ROOT_PASS:-$(pwgen -s 12 1)}
+_word=$( [ ${ROOT_PASS} ] && echo "preset" || echo "random" )
+echo "=> Setting a ${_word} password to the root user"
+echo "root:$PASS" | chpasswd
+
+echo "=> Done!"
+touch /.root_pw_set

contrib/agent-simulator/example/config.ini

@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This configuration file is case SENSITIVE
+# This configuration file will be updated by the script. After updating, all comments will be gone
+# For more information about this configuration file, check https://docs.python.org/2/library/configparser.html
+
+# the folder will be added to the file name automatically.
+[output]
+output_folder = /your_path/output
+# the result of command "gce info clustername" on the GCE controller
+gce_info_output = gce_info_output.txt
+log_file = log.txt
+
+[gce]
+gce_controller_ip = 104.196.89.197
+gce_controller_user = root
+gce_controller_key_file = /your_path/gce-key
+# time in seconds waiting for GCE VM to boot
+gce_boot_time = 50
+
+[cluster]
+vm_user = root
+# the same as the GCE controller for GCE
+vm_key_file = /your_path/gce-key
+vm_code_directory = /agent-simulator
+cluster_info_file = config/cluster.txt
+
+[ambari_agent_vm]
+# ambari_agent_vm_type = --base
+# 2 cores, 7.5GB RAM
+# ambari_agent_vm_type = --medium
+# 4 cores, 15GB RAM
+# ambari_agent_vm_type = --large
+# 8 cores, 30GB RAM
+# ambari_agent_vm_type = --xxlarge
+ambari_agent_vm_type = --xlarge
+# only support centos 7
+ambari_agent_vm_os = --centos7
+#---------------------------------------------
+# use other partition or directory as the storage of docker. comment this line to disable
+# /grid/0/ requires --extradisk to be present for this VM
+use_partition = /grid/0/
+# use extra disk. comment this to disable
+ambari_agent_vm_extra_disk = --extradisk
+#---------------------------------------------
+
+[docker]
+# The docker hostname would be {Container_hostname_fix}-{index}-{cluster name}
+# The docker domain name would be {Container_hostname_fix}-{index}-{cluster name}.weave.local
+container_hostname_fix = docker
+
+#==============================================================================================================
+# pull this image from Docker hub instead of build from Docker file. yes to confirm, other string to disable
+# pull_docker_hub = yes
+# docker_image_name = pengchexuacnt/hadp.2.3:1
+#++++++++++++++++++++++++++++++++++++++++++++++++++++
+pull_docker_hub = no
+docker_image_name = ambari/agent-sim
+# The name of the Dockerfile under Docker/
+dockerfile_name = Yum_Dockerfile
+#dockerfile_name = Raw_Dockerfile
+#==============================================================================================================
+
+
+[ambari_server_vm]
+ambari_server_vm_type = --xlarge
+ambari_server_vm_os = --centos7
+#ambari_server_vm = --extradisk
+
+[service_server_vm]
+service_server_vm_type = --xlarge
+service_server_vm_os = --centos7
+#service_server_vm_extra = --extradisk
+
+[weave]
+# Docker_IP_base is the starting point of IP address. with mask 16, ideally:
+# the IP of docker will be 192.168.[1-255].[1-255]/16
+# if there are more dockers, which means there is not enough IPs within this mask for all dockers,
+# the IP will continue to increase to be 192.169.*.*, dockers might not be able to talk to each other,
+# because the mask is 16
+weave_ip_base = 192.168.1.1
+weave_ip_mask = 16
+
+
+
+

contrib/agent-simulator/launcher_ambari_server.py

@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# configure the VM to run Ambari-server and set Weave network
+# argv 1: the name of cluster
+
+from config import Config
+from cluster import Cluster
+import sys
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print "configure the VM to run Ambari-server and set Weave network"
+        print "Arg: <the name of cluster>"
+        exit(1)
+
+    Config.load()
+
+    cluster_name = sys.argv[1]
+    cluster = Cluster.load_from_json(cluster_name)
+
+    vm = cluster.get_ambari_server_vm()
+    vm.run_ambari_server()

contrib/agent-simulator/launcher_cluster.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,95 +14,225 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
-
+"""
 
 import time
 import sys
 from cluster import Cluster
 from config import Config
+from data import Data
+import subprocess
+
 
-def all(argv):
+def request_cluster(argv):
+    """
+    only request cluster on GCE, and output all configuration information
+    :param argv: sys.argv
+    :return: None
+    """
     if len(argv) < 7:
         print_help()
         exit(1)
 
     cluster_name = argv[2]
-    VMs_num = int(argv[3])
-    Docker_num_each_VM = int(argv[4])
-    server_Weave_IP = argv[5]
-    server_external_IP = argv[6]
+    ambari_agent_vm_num = int(argv[3])
+    docker_num = int(argv[4])
+    service_server_num = int(argv[5])
+    with_ambari_server = False
+    ambari_server_num = int(argv[6])
+    if ambari_server_num > 0:
+        with_ambari_server = True
 
     cluster = Cluster()
-    cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
-    print "wait 50 seconds for the cluster to boot ... ..."
-    time.sleep(50)
-    cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
-
-def request_cluster(argv):
-    if len(argv) < 5:
+    cluster.request_gce_cluster(ambari_agent_vm_num, docker_num, service_server_num,
+                                with_ambari_server, cluster_name)
+
+    data = Data()
+    data.add_new_cluster(cluster)
+
+    time_to_wait = Config.ATTRIBUTES["gce_boot_time"]
+    print "wait ", str(time_to_wait), " seconds for the cluster to boot ... ..."
+    time.sleep(int(time_to_wait))
+    print "complete"
+
+
+def up_cluster(argv):
+    """
+    run all Ambari-agents in Docker container and VMs,
+    run Ambari-server if there is according to the configuration file
+    :param argv: sys.argv
+    :return: None
+    """
+    if len(argv) < 3:
         print_help()
         exit(1)
 
     cluster_name = argv[2]
-    VMs_num = int(argv[3])
-    Docker_num_each_VM = int(argv[4])
+    cluster = Cluster.load_from_json(cluster_name)
 
-    cluster = Cluster()
-    cluster.request_GCE_cluster(VMs_num, Docker_num_each_VM, cluster_name)
-    print "Before run Docker on the Cluster, wait at least 50 seconds for the cluster to boot"
+    if cluster is None:
+        print cluster_name, " cluster not found"
+        exit(1)
+
+    if cluster.state != Cluster.STATE_FREE:
+        print cluster_name, " cluster is already running"
+        exit(1)
+
+    ambari_server = cluster.get_ambari_server_vm()
+    if ambari_server is None:
+        print "Unable to run cluster", cluster_name,\
+            " no Ambari-server in this cluster, you can only merge this cluster into another one"
+        exit(1)
+
+    print "Configuring cluster"
+    print "Check output folder: ", Config.ATTRIBUTES["output_folder"]
+
+    cluster.run_cluster(ambari_server.weave_internal_ip, ambari_server.external_ip)
+    data = Data()
+    data.set_cluster_state(cluster_name, Cluster.STATE_RUNNING)
 
-def run_cluster(argv):
+    # reset terminal. The SSH subprocess call of the program cause the terminal display to be abnormal.
+    # This is an unsolved minor issue.
+    subprocess.call(["reset"])
+
+    print "Complete"
+
+
+def merge_cluster(argv):
+    """
+    Merge the cluster to another running cluster
+    :param argv: sys.argv
+    :return: None
+    """
     if len(argv) < 4:
         print_help()
         exit(1)
 
-    server_Weave_IP = argv[2]
-    server_external_IP = argv[3]
+    merged_cluster_name = argv[2]
+    merged_cluster = Cluster.load_from_json(merged_cluster_name)
+    if merged_cluster is None:
+        print merged_cluster_name, " cluster not found"
+        exit(1)
 
-    config = Config
-    config.load()
+    if merged_cluster.state != Cluster.STATE_FREE:
+        print merged_cluster_name, " cluster is already running"
+        exit(1)
 
-    cluster = Cluster()
-    cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
-    cluster.run_docker_on_cluster(server_external_IP, server_Weave_IP)
+    weave_ip = ""
+    external_ip = ""
+    extended_cluster_name = ""
+    if len(argv) == 4:
+        extended_cluster_name = argv[3]
+        extended_cluster = Cluster.load_from_json(extended_cluster_name)
+        if extended_cluster is None:
+            print extended_cluster_name, " cluster not found"
+            exit(1)
+
+        if extended_cluster.state != Cluster.STATE_RUNNING:
+            if extended_cluster.state == Cluster.STATE_FREE:
+                print extended_cluster_name, " cluster is not running, can't be extended"
+            elif extended_cluster.state.startswith(Cluster.STATE_MERGE):
+                print extended_cluster_name, " cluster is merged to another cluster, can't be extended"
+            exit(1)
+
+        ambari_server = extended_cluster.get_ambari_server_vm()
+        weave_ip = ambari_server.weave_internal_ip
+        external_ip = ambari_server.external_ip
+
+    elif len(argv) == 5:
+        weave_ip = argv[3]
+        external_ip = argv[4]
+
+    else:
+        print_help()
+        exit(1)
+
+    if merged_cluster.get_ambari_server_vm() is not None:
+        print merged_cluster, " cluster has one VM to install Ambari-server, which will NOT be merged"
+
+    print "Configuring cluster"
+    print "Check output folder: ", Config.ATTRIBUTES["output_folder"]
+    merged_cluster.run_cluster(weave_ip, external_ip)
+
+    data = Data()
+    data.set_cluster_state(merged_cluster_name, "{0} to {1}".format(Cluster.STATE_MERGE, extended_cluster_name))
+
+    # reset terminal. The SSH subprocess call of the program cause the terminal display to be abnormal.
+    # This is an unsolved minor issue.
+    subprocess.call(["reset"])
+    print "Complete"
+
+
+def list_cluster():
+    """
+    list the cluster creation history
+    :return: None
+    """
+    data = Data()
+    data.print_cluster_summary_list()
+
+
+def show_cluster(argv):
+    """
+    show detail information about a cluster
+    :param argv: sys.argv
+    :return: None
+    """
+    if len(argv) < 3:
+        print_help()
+        exit(1)
+    cluster_name = argv[2]
+    cluster = Cluster.load_from_json(cluster_name)
+
+    if cluster is None:
+        print cluster_name, " cluster not found"
+        exit(1)
+
+    cluster.print_description()
 
-def terminate():
-    print "Log into GCE controller to terminate your cluster manually"
 
 def print_help():
+    """
+    print help information
+    :return: None
+    """
     print "usage:"
     print
 
-    print "all", "  ", "request a GCE cluster and run Docker containers with Ambari-agent in all VMs"
+    print "request", "  ", "--request a cluster from GCE, generate the configuration for the cluster"
     print "\t\t", "<the name of the cluster>"
     print "\t\t", "<number of VMs>"
-    print "\t\t", "<number of dockers each VMs>"
-    print "\t\t", "<Weave IP of Ambari-server>"
-    print "\t\t", "<external IP of Ambari-server>"
+    print "\t\t", "<number of dockers each VM>"
+    print "\t\t", "<number of service servers inside VM>"
+    print "\t\t", "<number of ambari-server>, either 0 or 1"
     print
 
-    print "request", "  ", "request a cluster from GCE, generate the configuration for the cluster"
+    print "up", "  ", "--run all Ambari-agents and Ambari-server of the cluster"
     print "\t\t", "<the name of the cluster>"
-    print "\t\t", "<number of VMs>"
-    print "\t\t", "<number of dockers each VMs>"
     print
 
-    print "run", "  ", "run Docker containers with Ambari-agent in all VMs of the cluster"
-    print "\t\t", "<Weave IP of Ambari-server>"
-    print "\t\t", "<external IP of Ambari-server>"
+    print "merge", "  ", "--run one cluster, and add to another cluster"
+    print "\t\t", "<the name of the cluster to be merged>"
+    print "\t\t", "<the name of the cluster to be extended>"
     print
 
-    print "terminate", "  ", "terminate the cluster"
+    print "merge", "  ", "--run one cluster, and add to another cluster"
+    print "\t\t", "<the name of the cluster to be merged>"
+    print "\t\t", "<Weave IP of the Ambari-server>"
+    print "\t\t", "<External IP of the Ambari-server>"
     print
 
-    print "help", "  ", "help info"
+    print "list", "  ", "--list all the cluster"
+    print
+
+    print "show", "  ", "--show cluster information"
+    print "\t\t", "<the name of the cluster>"
     print
 
-    print "more options in configuration file config/config.ini"
-    print "see more instructions in tips.txt"
+    print "help", "  ", "help info"
     print
 
+
 def main(argv):
     # the first argument is the python file name
     if len(argv) < 2:
@@ -110,17 +240,21 @@ def main(argv):
         exit(1)
 
     command = argv[1]
-    if command == "all":
-        all(argv)
 
-    elif command == "request":
+    if command == "request":
         request_cluster(argv)
 
-    elif command == "run":
-        run_cluster(argv)
+    elif command == "up":
+        up_cluster(argv)
+
+    elif command == "merge":
+        merge_cluster(argv)
+
+    elif command == "list":
+        list_cluster()
 
-    elif command == "terminate":
-        terminate()
+    elif command == "show":
+        show_cluster(argv)
 
     elif command == "help":
         print_help()
@@ -128,9 +262,7 @@ def main(argv):
     else:
         print_help()
 
-main(sys.argv)
-
-# python launcher_cluster.py all test-cluster 2 3 192.168.10.10 104.196.84.248
-
-
 
+if __name__ == "__main__":
+    Config.load()
+    main(sys.argv)

contrib/agent-simulator/launcher_docker.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,28 +14,43 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
+# configure the VM to set Docker, Weave network, run Ambari-agent inside the Docker
+# argv 1: the external IP of this VM
+# argv 2: the Weave IP of the Ambari-server
+# argv 3: the external IP of the Ambari-server
+# argv 4: the name of the cluster
 
 import sys
 from config import Config
 from cluster import Cluster
 
-def get_VM(VM_IP, cluster):
-    for vm in cluster.VM_list:
-        if vm.external_ip == VM_IP:
-            return vm
 
-config = Config()
-config.load()
+if __name__ == "__main__":
+    if len(sys.argv) < 5:
+        print "configure the VM to set Docker, Weave network, run Ambari-agent inside the Docker"
+        print "Args: <the external IP of this VM>"
+        print "     <the Weave IP of the Ambari-server>"
+        print "     <the external IP of the Ambari-server>"
+        print "     <the name of the cluster>"
+        exit(1)
 
-cluster = Cluster()
-cluster.load_cluster_info(config.ATTRIBUTES["cluster_info_file"])
+    Config.load()
 
-my_external_IP = sys.argv[1]
-server_weave_IP = sys.argv[2]
-server_external_IP = sys.argv[3]
+    my_external_ip = sys.argv[1]
+    server_weave_ip = sys.argv[2]
+    server_external_ip = sys.argv[3]
+    cluster_name = sys.argv[4]
 
-vm = get_VM(my_external_IP, cluster)
-vm.run_docker(server_weave_IP, server_external_IP, cluster)
+    cluster = Cluster.load_from_json(cluster_name)
 
+    vm_ip_list = []
+    vm_ip_list.append(server_external_ip)
+    for vm in cluster.ambari_agent_vm_list:
+        vm_ip_list.append(vm.external_ip)
+    for vm in cluster.service_server_vm_list:
+        vm_ip_list.append(vm.external_ip)
+
+    vm = cluster.get_agent_vm(my_external_ip)
+    vm.run_docker(server_weave_ip, vm_ip_list)

contrib/agent-simulator/launcher_service_server.py

@@ -0,0 +1,49 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# configure the VM to set Weave network, run Ambari-agent directly inside VM
+# argv 1: the external IP of this VM
+# argv 2: the Weave IP of the Ambari-server
+# argv 3: the external IP of the Ambari-server
+# argv 4: the name of the cluster
+
+from config import Config
+from cluster import Cluster
+import sys
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 5:
+        print "configure the VM to set Docker, Weave network, run Ambari-agent inside the Docker"
+        print "Args: <the external IP of this VM>"
+        print "     <the Weave IP of the Ambari-server>"
+        print "     <the external IP of the Ambari-server>"
+        print "     <the name of the cluster>"
+        exit(1)
+
+    Config.load()
+
+    my_external_ip = sys.argv[1]
+    server_weave_ip = sys.argv[2]
+    server_external_ip = sys.argv[3]
+    cluster_name = sys.argv[4]
+
+    cluster = Cluster.load_from_json(cluster_name)
+
+    vm = cluster.get_service_server_vm(my_external_ip)
+    vm.run_service_server(server_weave_ip, server_external_ip)

contrib/agent-simulator/log.py

@@ -0,0 +1,35 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from config import Config
+
+
+class Log:
+    """
+    A simple class to write log
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def write(*arg):
+        with open(Config.ATTRIBUTES["log_file"], "a") as log_file:
+            for text in arg:
+                log_file.write(text)
+            log_file.write("\n")

contrib/agent-simulator/network/DNS_editor.py

@@ -0,0 +1,53 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# This script will modify the /etc/resolv.conf
+# A new DNS is added as the first entry of "nameserver"
+# the domain is set to be Weave local domain, which is "weave.local"
+# set the Weave local domain as the first entry of "search"
+
+import sys
+
+local_Weave_DNS_IP = sys.argv[1]
+nameserver_file_name = "/etc/resolv.conf"
+
+lines = []
+with open(nameserver_file_name) as f_read:
+    lines = f_read.readlines()
+
+add_nameserver = False
+with open(nameserver_file_name, "w") as f:
+    for line in lines:
+        if "search" in line:
+            tokens = line.split()
+            f.write("search weave.local")
+            for token in tokens[1:]:
+                f.write(" ")
+                f.write(token)
+            f.write("\n")
+        elif "nameserver" in line:
+            if add_nameserver == False:
+                f.write("nameserver ")
+                f.write(local_Weave_DNS_IP)
+                f.write("\n")
+                add_nameserver = True
+            f.write(line)
+        elif "domain" in line:
+            f.write("domain weave.local\n")
+        else:
+            f.write(line)

contrib/agent-simulator/server_setup.sh → contrib/agent-simulator/network/set_ambari_server_network.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,21 +14,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
+# This script will configure the ambari-server
+# including, set Weave network and add agent hostname resolution
+
 # run this script with root
 # $1 <Weave internal IP with mask>
 # $2 <hostname files with all agents>
+# $3 <Subnet mask of Weave network>
 
-if [ $# -lt 2 ]; then
-    echo "usage: ./sever_setup.sh <Weave internal IP with mask> <hostname file with all agents>"
-    echo "example: ./server_setup.sh 192.168.10.10/16 /user/simulator-script/hosts.txt"
-    echo "note: the hostname file is generated automatically when you request a cluster"
+if [ $# -lt 3 ]; then
+    echo "usage: ./set_ambari_server_network.sh <Weave internal IP> <Weave DNS IP> <Mask>"
+    echo "example: ./set_ambari_server_network.sh 192.168.255.1 192.168.255.2 16"
     exit 1
 fi
 
+weave_internal_ip=$1
+weave_dns_ip=$2
+mask=$3
+
 # install weave
-chmod 755 ./Linux/CentOS7/weave_install.sh
-./Linux/CentOS7/weave_install.sh
+chmod 755 ../Linux/CentOS7/weave_install.sh
+../Linux/CentOS7/weave_install.sh
+
+# install docker
+chmod 755 ../Linux/CentOS7/docker_install.sh
+../Linux/CentOS7/docker_install.sh
 
 # reset weave
 weave reset
@@ -35,8 +46,11 @@ weave reset
 # launch weave
 weave launch
 
+# launch Weave DNS
+weave launch-dns ${weave_dns_ip}/${mask}
+
 # expose IP
-weave expose $1
+weave expose ${weave_internal_ip}/${mask}
 
-# add hosname of all agents
-cat $2 >> /etc/hosts3
+# set domain name resolution
+python dns_editor.py $weave_dns_ip

contrib/agent-simulator/network/set_host_network.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to add a Host/VM to the Docker network
+# for example, an Ambari-agent within a VM (not in a Docker container)
+
+# run this script with root
+# $1 <Weave internal IP of the VM>
+# $2 <Weave DNS IP of the VM>
+# $3 <Subnet mask of Weave network>
+# $4 <The host name of this VM inside the Weave network>
+# $5 <The domain name of this VM inside the Weave network>, which is supposed to be end with "weave.local"
+# $6 <External IP address of Ambari server>
+
+if [ $# -lt 6 ]; then
+    echo "usage: ./set_ambari_server_network.sh <Weave internal IP> <Weave DNS IP> <Mask> <Weave host name> <Weave domain name> <Ambari server IP>"
+    echo "example: ./set_ambari_server_network.sh 192.168.254.1 192.168.254.2 16 yourname-group-c-service-server-1 yourname-group-c-service-server-1.weave.local 104.196.91.170"
+    exit 1
+fi
+
+weave_internal_ip=$1
+weave_dns_ip=$2
+mask=$3
+weave_host_name=$4
+weave_domain_name=$5
+ambari_server_ip=$6
+
+# install weave
+chmod 755 ../Linux/CentOS7/weave_install.sh
+../Linux/CentOS7/weave_install.sh
+
+# install docker
+chmod 755 ../Linux/CentOS7/docker_install.sh
+../Linux/CentOS7/docker_install.sh
+
+# reset weave
+weave reset
+
+# launch weave
+weave launch $ambari_server_ip
+
+# launch Weave DNS
+weave launch-dns ${weave_dns_ip}/${mask}
+
+# expose IP and a new domain name.
+# MUST use one expose command to set both ip and domain name, OR the domain name will not be bind to this weave internal IP
+weave expose ${weave_internal_ip}/${mask} -h $weave_domain_name
+
+# set domain name resolution
+python dns_editor.py $weave_dns_ip
+
+# add Weave local IP, host name, domain name mapping
+content="$(cat /etc/hosts)"
+echo "${weave_internal_ip} ${weave_domain_name} ${weave_host_name}" > /etc/hosts
+echo "$content" >> /etc/hosts

contrib/agent-simulator/server/ambari_server_install.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script will install and start the ambari-server on a CentOS 7 machine
+
+yum install -y -q wget
+wget -q -O /etc/yum.repos.d/ambari.repo http://s3.amazonaws.com/dev.hortonworks.com/ambari/centos7/2.x/latest/2.1.1/ambaribn.repo
+yum install -y -q ambari-server
+sudo ambari-server setup -s

contrib/agent-simulator/Linux/Ubuntu12/weave_install.sh → contrib/agent-simulator/server/ambari_server_reset_data.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,7 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-sudo apt-get install -y curl
-sudo curl -L git.io/weave -o /usr/bin/weave
-sudo chmod a+x /usr/bin/weave
+# This script will clear all the data in the ambari-server
+
+ambari-server stop
+sudo -u postgres psql -c "drop database ambari;"
+sudo -u postgres psql -c "drop database ambarirca;"
+ambari-server setup << _EOF_
+y
+n
+n
+n
+_EOF_
+ambari-server start

contrib/agent-simulator/Linux/Ubuntu12/docker_install.sh → contrib/agent-simulator/server/ambari_server_start.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information rega4rding copyright ownership.
@@ -13,5 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/bin/bash
-wget -qO- https://get.docker.com/ | sh
+# This script will install and start the ambari-server on a CentOS 7 machine
+
+sudo ambari-server start
+echo "ambari-server start"

contrib/agent-simulator/vm.py

@@ -1,4 +1,4 @@
-'''
+"""
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -14,68 +14,262 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-'''
+"""
 
 
 import subprocess
+import shutil
 from config import Config
+import os
+from docker import Docker
+from docker_image.launcher_agent import replace_conf
+
 
 class VM:
-    def __init__(self, external_ip):
+    """
+    This class represents VM, including its network setting and the possible Docker instance list
+    """
+    def __init__(self, external_ip, domain_name, weave_dns_ip, weave_ip_mask):
         self.external_ip = external_ip
+        self.domain_name = domain_name
+        self.hostname = self._gce_get_hostname(domain_name)
+        self.weave_domain_name = self._get_weave_domain_name(self.hostname)
+        self.weave_dns_ip = weave_dns_ip
+        self.weave_internal_ip = ""
+        self.weave_ip_mask = weave_ip_mask
         self.docker_list = []
 
+    def to_json(self):
+        """
+        create a map to hold the information of the VM instance
+        :return: A map, which is JSON format object.
+        """
+        vm_json = {}
+        vm_json["external_ip"] = self.external_ip
+        vm_json["domain_name"] = self.domain_name
+        vm_json["weave_dns_ip"] = self.weave_dns_ip
+        vm_json["weave_internal_ip"] = self.weave_internal_ip
+        vm_json["weave_domain_name"] = self.weave_domain_name
+        vm_json["weave_ip_mask"] = self.weave_ip_mask
+        vm_json["docker_list"] = []
+        for docker in self.docker_list:
+            vm_json["docker_list"].append(docker.to_json())
+        return vm_json
+
+    @staticmethod
+    def load_from_json(json_data):
+        """
+        load the VM information from a JSON object
+        :param json_data: a map, which is a JSON object
+        :return: a VM object
+        """
+        external_ip = json_data["external_ip"]
+        domain_name = json_data["domain_name"]
+        weave_dns_ip = json_data["weave_dns_ip"]
+        weave_internal_ip = json_data["weave_internal_ip"]
+        weave_domain_name = json_data["weave_domain_name"]
+        weave_ip_mask = json_data["weave_ip_mask"]
+        docker_list = []
+        for json_docker in json_data["docker_list"]:
+            docker_list.append(Docker.load_from_json(json_docker))
+
+        vm = VM(external_ip, domain_name, weave_dns_ip, weave_ip_mask)
+        vm.docker_list = docker_list
+        vm.weave_internal_ip = weave_internal_ip
+        vm.weave_domain_name = weave_domain_name
+        return vm
+
+    def _get_weave_domain_name(self, hostname):
+        """
+        get the Weave domain name of the VM
+        :param hostname: the hostname of the VM
+        :return:the Weave domain name
+        """
+        return "{0}.weave.local".format(hostname)
+
+    def _gce_get_hostname(self, domain_name):
+        """
+        The hostname of GCE VM is the first part of the internal domain name
+        :param domain_name: the internal domain name of GCE VM
+        :return: the hostname of GCE VM
+        """
+        return domain_name.split(".")[0]
+
+    def get_ssh_output_file_path(self):
+        """
+        get the file name to hold the SSH output of the VM
+        :return: a file name
+        """
+        vm_output_file_path = "{0}/vm-{1}-{2}".format(Config.ATTRIBUTES["output_folder"],
+                                                      self.hostname, self.external_ip)
+        return vm_output_file_path
+
     def add_docker(self, docker):
+        """
+        add a Docker instance to the VM instance
+        :param docker: the docker instance
+        :return: None
+        """
         self.docker_list.append(docker)
 
-    # install Weave on this VM
-    def __centos7_weave_install__(self):
+    def _centos7_weave_install(self):
+        """
+        install Weave on this VM
+        :return: None
+        """
         subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/weave_install.sh"])
         subprocess.call("./Linux/CentOS7/weave_install.sh")
 
-    # launch Weave, make this VM connect with other VM
-    def __set_weave_network__(self, VMs_external_IP_list, server_external_ip):
+    def _set_weave_network(self, vm_external_ip_list, weave_dns_ip):
+        """
+        launch Weave, make this VM connect with other VM
+        :param vm_external_ip_list: external IP list of all VMs
+        :param weave_dns_ip: the IP of DNS in this VM
+        :return: None
+        """
         # add other VMs and the ambari-server to set up connections
-        command = ["sudo", "weave", "launch"]
-        command.extend(VMs_external_IP_list)
-        command.append(server_external_ip)
-        subprocess.call(command)
+        weave_launch_command = ["sudo", "weave", "launch"]
+        weave_launch_command.extend(vm_external_ip_list)
+
+        print weave_launch_command
 
-    # install Docker on this VM
-    def __centos7_docker_install__(self):
+        with open(os.devnull, 'w') as shutup:
+            subprocess.call(weave_launch_command, stdout=shutup)
+
+        # establish DNS server
+        weave_dns_ip_with_mask = "{0}/{1}".format(weave_dns_ip, Config.ATTRIBUTES["weave_ip_mask"])
+        weave_launch_dns_command = ["sudo", "weave", "launch-dns", weave_dns_ip_with_mask]
+        subprocess.call(weave_launch_dns_command)
+
+    def _centos7_docker_install(self):
+        """
+        install Docker on this VM
+        :return: None
+        """
         subprocess.call(["sudo", "chmod", "755", "Linux/CentOS7/docker_install.sh"])
         subprocess.call("./Linux/CentOS7/docker_install.sh")
 
-    # build docker image
-    def __build_docker_image__(self, image_name):
-        subprocess.call(["sudo", "docker", "build", "-t", image_name, "Docker/"])
+    def _build_docker_image(self, image_name):
+        """
+        build docker image
+        :param image_name: the name of the Docker image
+        :return: None
+        """
+        # choose the right Dockerfile
+        target_dockerfile_name = "docker_image/{0}".format(Config.ATTRIBUTES["dockerfile_name"])
+        standard_dockerfile_name = "docker_image/Dockerfile"
+        shutil.copyfile(target_dockerfile_name, standard_dockerfile_name)
+        with open(os.devnull, 'w') as shutup:
+            subprocess.call(["sudo", "docker", "build", "-t", image_name, "docker_image/"])
+            # subprocess.call(["sudo", "docker", "build", "-q", "-t", image_name, "docker_image/"], stdout=shutup)
+        os.remove(standard_dockerfile_name)
+
+    def _pull_docker_image(self, image_name):
+        with open(os.devnull, 'w') as shutup:
+            subprocess.call(["sudo", "docker", "pull", image_name], stdout=shutup)
 
-    # launch Docker containers, issue the script to install, configure and launch Agent inside Docker.
-    def __launch_containers__(self, docker_image, server_weave_ip):
-        # print docker_ip_list
+    def _launch_containers(self, docker_image, server_weave_ip):
+        """
+        launch Docker containers, issue the script to install,
+        configure and launch Ambari-gent inside Docker.
+        :param docker_image: the name of the Docker image
+        :param server_weave_ip: Weave internal IP of Ambari-server
+        :return: None
+        """
         for docker in self.docker_list:
-            docker_IP = docker.IP
-            docker_mask = docker.mask
-            docker_hostname = docker.hostname
+            docker_ip_with_mask = "{0}/{1}".format(docker.ip, docker.mask)
+            cmd = "python /launcher_agent.py {0} {1}; /bin/bash".format(server_weave_ip, docker.ip)
 
-            cmd = "python /launcher_agent.py " + server_weave_ip + " " + docker_hostname + "; /bin/bash"
-            command = ["sudo", "weave", "run", docker_IP + "/" + docker_mask, "-d", "-it", "-h", docker_hostname, \
+            command = ["sudo", "weave", "run", docker_ip_with_mask, "-d", "-it",
+                       "-h", docker.weave_domain_name,
+                       "--name", docker.get_container_name(),
                        docker_image, "bash", "-c", cmd]
             print command
             subprocess.call(command)
 
-    def run_docker(self, server_weave_IP, server_external_IP, cluster):
-        config = Config()
-        config.load()
+    def _set_docker_partition(self, mount_point):
+        """
+        set docker container to use the disk storage of other partitions.
+        :param mount_point: the mount point of the partition to be used
+        :return: None
+        """
+        subprocess.call(["sudo", "chmod", "755", "./Linux/CentOS7/set_docker_partition.sh"])
+        subprocess.call(["./Linux/CentOS7/set_docker_partition.sh", mount_point])
+
+    def run_ambari_server(self):
+        """
+        set up Weave network, run Ambari-server in this VM
+        :return: None
+        """
+        # set up network, run script inside the network directory
+        os.chdir("network")
+        subprocess.call(["sudo", "chmod", "755", "set_ambari_server_network.sh"])
+        subprocess.call(["./set_ambari_server_network.sh", self.weave_internal_ip,
+                         self.weave_dns_ip, self.weave_ip_mask])
+        os.chdir("..")
+
+        # install ambari server and start service
+        subprocess.call(["sudo", "chmod", "755", "./server/ambari_server_install.sh"])
+        subprocess.call(["./server/ambari_server_install.sh"])
+
+        # start service
+        subprocess.call(["sudo", "ambari-server", "start"])
+
+    def run_service_server(self, ambari_server_weave_ip, ambari_server_external_ip):
+        """
+        set up Weave network, run Ambari-agent in this VM
+        :param ambari_server_weave_ip: the Weave IP of Ambari-server
+        :param ambari_server_external_ip: the external IP of Ambari-server
+        :return: None
+        """
+        # set up network, run script inside the network directory
+        os.chdir("network")
+        subprocess.call(["sudo", "chmod", "755", "set_host_network.sh"])
+        subprocess.call(["./set_host_network.sh", self.weave_internal_ip,
+                         self.weave_dns_ip, self.weave_ip_mask, self.hostname,
+                         self.weave_domain_name, ambari_server_external_ip])
+        os.chdir("..")
+
+        # install ambari agent and start service
+        subprocess.call(["sudo", "chmod", "755", "./docker_image/ambari_agent_install.sh"])
+        subprocess.call(["./docker_image/ambari_agent_install.sh"])
+        replace_conf(ambari_server_weave_ip)
+
+        # start service
+        subprocess.call(["sudo", "ambari-agent", "start"])
+
+    def run_docker(self, server_weave_ip, vm_ip_list):
+        """
+        run all Docker containers with Ambari-agent inside
+        :param server_weave_ip: Weave internal IP of Ambari-server
+        :param vm_ip_list: external IP list of all other VMs to be connected
+                each docker vm connect to each other and service VM (and ambari-server)
+        :return: None
+        """
+        self._centos7_docker_install()
+
+        if "use_partition" in Config.ATTRIBUTES:
+            self._set_docker_partition(Config.ATTRIBUTES["use_partition"])
+
+        self._centos7_weave_install()
+
+        image_name = Config.ATTRIBUTES["docker_image_name"]
+        if "pull_docker_hub" in Config.ATTRIBUTES and Config.ATTRIBUTES["pull_docker_hub"] == "yes":
+            self._pull_docker_image(image_name)
+        else:
+            self._build_docker_image(image_name)
+
+        self._set_weave_network(vm_ip_list, self.weave_dns_ip)
+        self._launch_containers(Config.ATTRIBUTES["docker_image_name"], server_weave_ip)
 
-        VMs_IP_list = []
-        for vm in cluster.VM_list:
-            VMs_IP_list.append(vm.external_ip)
+    @staticmethod
+    def get_ambari_agent_vm_name(cluster_name):
+        return "{0}-agent-vm".format(cluster_name)
 
-        cluster.export_hostnames("./Docker/hosts")
+    @staticmethod
+    def get_ambari_server_vm_name(cluster_name):
+        return "{0}-ambari-server".format(cluster_name)
 
-        self.__centos7_docker_install__()
-        self.__centos7_weave_install__()
-        self.__build_docker_image__(config.ATTRIBUTES["Docker_image_name"])
-        self.__set_weave_network__(VMs_IP_list, server_external_IP)
-        self.__launch_containers__(config.ATTRIBUTES["Docker_image_name"], server_weave_IP)
+    @staticmethod
+    def get_service_server_vm_name(cluster_name):
+        return "{0}-service-server".format(cluster_name)

pom.xml

@@ -262,6 +262,7 @@
             <exclude>contrib/addons/test/dataServices/jmx/data/cluster_configuration.json.nohbase</exclude>
             <exclude>contrib/ambari-scom/msi/src/GUI_Ambari.sln</exclude>
             <exclude>contrib/fast-hdfs-resource/dependency-reduced-pom.xml</exclude>
+            <exclude>contrib/agent-simulator/docker_image/package_list.txt</exclude>
             <exclude>version</exclude>
             <!--IDE and GIT files-->
             <exclude>**/.idea/</exclude>