AMBARI-8316. Bootstaraping and starting agents as non-root (aonishuk)

ambari-agent/conf/unix/ambari-agent

@@ -48,6 +48,12 @@ AMBARI_AGENT_PY_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/AmbariAgent
 OK=1
 NOTOK=0
 
+current_user=`awk -v val=$EUID -F ":" '$3==val{print $1}' /etc/passwd`
+# setup necessary ownership
+sudo chown -R $current_user "/var/lib/ambari-agent/ambari-env.sh"
+sudo chown -R $current_user "/var/run/ambari-agent"
+sudo chown -R $current_user "/var/log/ambari-agent"
+sudo chown -R $current_user "/var/lib/ambari-agent/data"
 
 if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
   PYTHON=/usr/bin/python2.7
@@ -106,8 +112,9 @@ check_python_version ()
 
 retcode=0
 
-if [ "$(id -u)" != "0" ]; then
- echo "You can't perform this operation as non-root user. Please, re-login as root user"
+sudo -v
+if [ "$?" != "0" ]; then
+ echo "You can't perform this operation as non-sudoer user. Please, re-login as one"
  exit 0
 fi
 

ambari-agent/conf/unix/ambari-agent.ini

@@ -28,6 +28,7 @@ data_cleanup_max_size_MB = 100
 ping_port=8670
 cache_dir=/var/lib/ambari-agent/cache
 tolerate_download_failures=true
+run_as_user=root
 
 [command]
 maxretries=2

ambari-agent/etc/init.d/ambari-agent

@@ -19,20 +19,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+run_as_user=`cat /etc/ambari-agent/conf/ambari-agent.ini | grep run_as_user | tr -d ' ' | awk -F '=' '{ print $2}'`
+
+if [ "$EUID" != `id -u $run_as_user` ] ; then
+  command_prefx="su - $run_as_user -c"
+else
+  command_prefx="bash -c"
+fi
+
 case "$1" in
   start)
-        /usr/sbin/ambari-agent $@
+        $command_prefx "/usr/sbin/ambari-agent $@"
         ;;
   stop)
-        /usr/sbin/ambari-agent $@
+        $command_prefx "/usr/sbin/ambari-agent $@"
         ;;
   status)
-        /usr/sbin/ambari-agent $@
+        $command_prefx "/usr/sbin/ambari-agent $@"
         exit $?
         ;;
   restart)
-        $0 stop
-        $0 start
+        $command_prefx "$0 stop"
+        $command_prefx "$0 start"
         ;;
   reset)
         /usr/sbin/ambari-agent $@

ambari-agent/src/main/python/ambari_agent/AgentConfig_linux.py

@@ -35,6 +35,7 @@ data_cleanup_max_age=2592000
 data_cleanup_max_size_MB = 100
 ping_port=8670
 cache_dir=/var/lib/ambari-agent/cache
+run_as_user=root
 
 [services]
 

ambari-agent/src/main/python/ambari_agent/main.py

@@ -23,6 +23,7 @@ import signal
 from optparse import OptionParser
 import sys
 import traceback
+import getpass
 import os
 import time
 import platform
@@ -35,6 +36,7 @@ from PingPortListener import PingPortListener
 import hostname
 from DataCleaner import DataCleaner
 import socket
+from ambari_agent import shell
 logger = logging.getLogger()
 
 formatstr = "%(levelname)s %(asctime)s %(filename)s:%(lineno)d - %(message)s"
@@ -153,7 +155,7 @@ def stop_agent():
     pid = f.read()
     pid = int(pid)
     f.close()
-    os.killpg(os.getpgid(pid), signal.SIGTERM)
+    kill_process_group(pid, signal.SIGTERM)
     time.sleep(5)
     if os.path.exists(ProcessHelper.pidfile):
       raise Exception("PID file still exists.")
@@ -162,8 +164,11 @@ def stop_agent():
     if pid == -1:
       print ("Agent process is not running")
     else:
-      os.killpg(os.getpgid(pid), signal.SIGKILL)
+      kill_process_group(pid, signal.SIGKILL) 
     os._exit(1)
+    
+def kill_process_group(pid, sig):
+  shell.shellRunner().run("sudo kill -{0} -$(ps -o pgid= {1} | grep -o [0-9]*)".format(sig, pid))
 
 def reset_agent(options):
   try:
@@ -204,8 +209,10 @@ def main(heartbeat_stop_callback=None):
 
   expected_hostname = options.expected_hostname
 
-  setup_logging(options.verbose)
+  current_user = getpass.getuser()
 
+  setup_logging(options.verbose)
+  
   default_cfg = {'agent': {'prefix': '/home/ambari'}}
   config.load(default_cfg)
 

ambari-agent/src/main/python/ambari_agent/shell.py

@@ -156,8 +156,12 @@ class shellRunnerLinux:
     except Exception:
       logger.warn("can not switch user for RUN_COMMAND.")
     code = 0
-    cmd = " "
-    cmd = cmd.join(script)
+    
+    cmd = script
+    
+    if isinstance(script, list):
+      cmd = " ".join(script)
+
     p = subprocess.Popen(cmd, preexec_fn=_changeUid, stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, shell=True, close_fds=True)
     out, err = p.communicate()

ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java

@@ -150,10 +150,11 @@ class BSRunner extends Thread {
   public void run() {
     String hostString = createHostString(sshHostInfo.getHosts());
     String user = sshHostInfo.getUser();
+    String userRunAs = sshHostInfo.getUserRunAs();
     if (user == null || user.isEmpty()) {
       user = DEFAULT_USER;
     }
-    String commands[] = new String[11];
+    String commands[] = new String[12];
     String shellCommand[] = new String[3];
     BSStat stat = BSStat.RUNNING;
     String scriptlog = "";
@@ -193,13 +194,14 @@ class BSRunner extends Thread {
       commands[7] = this.clusterOsFamily;
       commands[8] = this.projectVersion;
       commands[9] = this.serverPort+"";
+      commands[10] = userRunAs;
       if (this.passwordFile != null) {
-        commands[10] = this.passwordFile.toString();
+        commands[11] = this.passwordFile.toString();
       }
       LOG.info("Host= " + hostString + " bs=" + this.bsScript + " requestDir=" +
           requestIdDir + " user=" + user + " keyfile=" + this.sshKeyFile +
           " passwordFile " + this.passwordFile + " server=" + this.ambariHostname +
-          " version=" + projectVersion + " serverPort=" + this.serverPort);
+          " version=" + projectVersion + " serverPort=" + this.serverPort + " userRunAs="+ userRunAs);
 
       String[] env = new String[] { "AMBARI_PASSPHRASE=" + agentSetupPassword };
       if (this.verbose)

ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java

@@ -50,6 +50,9 @@ public class SshHostInfo {
 
   @XmlElement
   private String password;
+  
+  @XmlElement
+  private String userRunAs;
 
   public String getSshKey() {
     return sshKey;
@@ -90,6 +93,17 @@ public class SshHostInfo {
   public void setPassword(String password) {
     this.password = password;
   }
+  
+  public String getUserRunAs() {
+    // TODO: remove this once UI supports customizing ambari run-as-user
+    if(userRunAs == null)
+      return "root";
+    return userRunAs;
+  }
+
+  public void setUserRunAs(String userRunAs) {
+    this.userRunAs = userRunAs;
+  }
 
   public String hostListAsString() {
     StringBuilder ret = new StringBuilder();

ambari-server/src/main/python/bootstrap.py

@@ -364,11 +364,12 @@ class Bootstrap(threading.Thread):
     setupFile = self.getRemoteName(self.SETUP_SCRIPT_FILENAME)
     passphrase = os.environ[AMBARI_PASSPHRASE_VAR_NAME]
     server = self.shared_state.ambari_server
+    user_run_as = self.shared_state.user_run_as
     version = self.getAmbariVersion()
     port = self.getAmbariPort()
     passwordFile = self.getPasswordFile()
     return "sudo -S python " + str(setupFile) + " " + str(expected_hostname) + \
-           " " + str(passphrase) + " " + str(server) + " " + str(version) + \
+           " " + str(passphrase) + " " + str(server)+ " " + str(user_run_as) + " " + str(version) + \
            " " + str(port) + " < " + str(passwordFile)
 
 
@@ -376,10 +377,11 @@ class Bootstrap(threading.Thread):
     setupFile=self.getRemoteName(self.SETUP_SCRIPT_FILENAME)
     passphrase=os.environ[AMBARI_PASSPHRASE_VAR_NAME]
     server=self.shared_state.ambari_server
+    user_run_as = self.shared_state.user_run_as
     version=self.getAmbariVersion()
     port=self.getAmbariPort()
     return "sudo python " + str(setupFile) + " " + str(expected_hostname) + \
-           " " + str(passphrase) + " " + str(server) + " " + str(version) + \
+           " " + str(passphrase) + " " + str(server)+ " " + str(user_run_as) + " " + str(version) + \
            " " + str(port)
 
 
@@ -620,7 +622,7 @@ class PBootstrap:
 class SharedState:
   def __init__(self, user, sshkey_file, script_dir, boottmpdir, setup_agent_file,
                ambari_server, cluster_os_type, ambari_version, server_port,
-               password_file = None):
+               user_run_as, password_file = None):
     self.hostlist_to_remove_password_file = None
     self.user = user
     self.sshkey_file = sshkey_file
@@ -630,6 +632,7 @@ class SharedState:
     self.ambari_server = ambari_server
     self.cluster_os_type = cluster_os_type
     self.ambari_version = ambari_version
+    self.user_run_as = user_run_as
     self.password_file = password_file
     self.statuses = None
     self.server_port = server_port
@@ -644,9 +647,11 @@ def main(argv=None):
   if len(onlyargs) < 3:
     sys.stderr.write("Usage: <comma separated hosts> "
                      "<tmpdir for storage> <user> <sshkey_file> <agent setup script>"
-                     " <ambari-server name> <cluster os type> <ambari version> <ambari port> <passwordFile>\n")
+                     " <ambari-server name> <cluster os type> <ambari version> <ambari port> <user_run_as> <passwordFile>\n")
     sys.exit(2)
     pass
+  
+
   #Parse the input
   hostList = onlyargs[0].split(",")
   bootdir =  onlyargs[1]
@@ -657,7 +662,8 @@ def main(argv=None):
   cluster_os_type = onlyargs[6]
   ambariVersion = onlyargs[7]
   server_port = onlyargs[8]
-  passwordFile = onlyargs[9]
+  user_run_as = onlyargs[9]
+  passwordFile = onlyargs[10]
 
   # ssh doesn't like open files
   subprocess.Popen(["chmod", "600", sshkey_file], stdout=subprocess.PIPE)
@@ -669,10 +675,10 @@ def main(argv=None):
                " using " + scriptDir + " cluster primary OS: " + cluster_os_type +
                " with user '" + user + "' sshKey File " + sshkey_file + " password File " + passwordFile +\
                " using tmp dir " + bootdir + " ambari: " + ambariServer +"; server_port: " + server_port +\
-               "; ambari version: " + ambariVersion)
+               "; ambari version: " + ambariVersion+"; user_run_as: " + user_run_as)
   sharedState = SharedState(user, sshkey_file, scriptDir, bootdir, setupAgentFile,
                        ambariServer, cluster_os_type, ambariVersion,
-                       server_port, passwordFile)
+                       server_port, user_run_as, passwordFile)
   pbootstrap = PBootstrap(hostList, sharedState)
   pbootstrap.run()
   return 0 # Hack to comply with current usage

ambari-server/src/main/python/setupAgent.py

@@ -62,18 +62,21 @@ def installAgent(projectVersion):
   return execOsCommand(Command, tries=3, try_sleep=10)
 
 
-def configureAgent(server_hostname):
+def configureAgent(server_hostname, user_run_as):
   """ Configure the agent so that it has all the configs knobs properly installed """
   osCommand = ["sed", "-i.bak", "s/hostname=localhost/hostname=" + server_hostname +
                                 "/g", "/etc/ambari-agent/conf/ambari-agent.ini"]
   execOsCommand(osCommand)
+  osCommand = ["sed", "-i.bak", "s/run_as_user=.*$/run_as_user=" + user_run_as +
+                                "/g", "/etc/ambari-agent/conf/ambari-agent.ini"]
+  execOsCommand(osCommand)
   return
 
 
-def runAgent(passPhrase, expected_hostname):
+def runAgent(passPhrase, expected_hostname, user_run_as):
   os.environ[AMBARI_PASSPHRASE_VAR] = passPhrase
-  agent_retcode = subprocess.call("/usr/sbin/ambari-agent restart --expected-hostname=" +
-                                  expected_hostname, shell=True)
+  agent_retcode = subprocess.call("su - {0} -c '/usr/sbin/ambari-agent restart --expected-hostname={1}'".format(user_run_as, expected_hostname)
+                                  , shell=True)
   for i in range(3):
     time.sleep(1)
     ret = execOsCommand(["tail", "-20", "/var/log/ambari-agent/ambari-agent.log"])
@@ -168,8 +171,9 @@ def checkServerReachability(host, port):
 #               0        Expected host name
 #               1        Password
 #               2        Host name
-#      X        3        Project Version (Ambari)
-#      X        4        Server port
+#               3        User to run agent as
+#      X        4        Project Version (Ambari)
+#      X        5        Server port
 
 
 def parseArguments(argv=None):
@@ -182,28 +186,30 @@ def parseArguments(argv=None):
   expected_hostname = args[0]
   passPhrase = args[1]
   hostname = args[2]
+  user_run_as = args[3]
   projectVersion = ""
   server_port = 8080
 
-  if len(args) > 3:
-    projectVersion = args[3]
-
   if len(args) > 4:
+    projectVersion = args[4]
+
+  if len(args) > 5:
     try:
-      server_port = int(args[4])
+      server_port = int(args[5])
     except (Exception):
       server_port = 8080
 
-  return expected_hostname, passPhrase, hostname, projectVersion, server_port
+  return expected_hostname, passPhrase, hostname, user_run_as, projectVersion, server_port
 
 
 def main(argv=None):
   # Parse passed arguments
   expected_hostname, passPhrase, hostname,\
-  projectVersion, server_port = parseArguments(argv)
+  user_run_as, projectVersion, server_port = parseArguments(argv)
 
   checkServerReachability(hostname, server_port)
-
+  
+  projectVersion = "1.7.0"
   if projectVersion == "null" or projectVersion == "{ambariVersion}" or projectVersion == "":
     retcode = getOptimalVersion("")
   else:
@@ -224,9 +230,9 @@ def main(argv=None):
                                         "versions of ambari-agent:"+retcode["log"][0].strip()})
   else:
       sys.exit(retcode)
-
-  configureAgent(hostname)
-  sys.exit(runAgent(passPhrase, expected_hostname))
+  
+  configureAgent(hostname, user_run_as)
+  sys.exit(runAgent(passPhrase, expected_hostname, user_run_as))
 
 if __name__ == '__main__':
   logging.basicConfig(level=logging.DEBUG)