AMBARI-11199. Use Internal calls instead of sudo calls when running as root (aonishuk)

ambari-common/src/main/python/resource_management/core/providers/system.py

@@ -23,9 +23,7 @@ Ambari Agent
 from __future__ import with_statement
 
 import re
-import grp
 import os
-import pwd
 import time
 from resource_management.core import shell
 from resource_management.core import sudo
@@ -33,29 +31,8 @@ from resource_management.core.base import Fail
 from resource_management.core import ExecuteTimeoutException
 from resource_management.core.providers import Provider
 from resource_management.core.logger import Logger
-
-
-def _coerce_uid(user):
-  try:
-    uid = int(user)
-  except ValueError:
-    try:
-      uid = pwd.getpwnam(user).pw_uid
-    except KeyError:
-      raise Fail("User %s doesn't exist." % user)
-  return uid
-
-
-def _coerce_gid(group):
-  try:
-    gid = int(group)
-  except ValueError:
-    try:
-      gid = grp.getgrnam(group).gr_gid
-    except KeyError:
-      raise Fail("Group %s doesn't exist." % group)
-  return gid
-
+from resource_management.core.utils import _coerce_uid
+from resource_management.core.utils import _coerce_gid
 
 def _ensure_metadata(path, user, group, mode=None, cd_access=None):
   stat = sudo.stat(path)

ambari-common/src/main/python/resource_management/core/shell.py

@@ -162,7 +162,7 @@ def _call(command, logoutput=None, throw_on_failure=True,
   # in case logoutput==False, never log.    
   logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
   out = ""
-  read_timeout = .04 # seconds
+  read_timeout = .001 # seconds
 
   try:
     while True:

ambari-common/src/main/python/resource_management/core/source.py

@@ -25,7 +25,6 @@ from resource_management.core.environment import Environment
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
 from resource_management.core.utils import checked_unite
-from resource_management.core import sudo
 
 __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
 
@@ -33,8 +32,6 @@ import os
 import time
 import urllib2
 import urlparse
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from ambari_commons import OSConst
 
 
 class Source(object):
@@ -77,16 +74,10 @@ class StaticFile(Source):
 
     return self.read_file(path)
 
-  @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
   def read_file(self, path):
     from resource_management.core import sudo
     return sudo.read_file(path)
 
-  @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
-  def read_file(self, path):
-    with open(path, "rb") as fp:
-      return fp.read()
-
 
 try:
   from ambari_jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined

ambari-common/src/main/python/resource_management/core/sudo.py

@@ -21,111 +21,191 @@ Ambari Agent
 
 import os
 import tempfile
+import shutil
+import stat
 from resource_management.core import shell
 from resource_management.core.logger import Logger
+from resource_management.core.utils import _coerce_uid
+from resource_management.core.utils import _coerce_gid
+from ambari_commons.os_check import OSCheck
 
-# os.chown replacement
-def chown(path, owner, group):
-  if owner:
-    shell.checked_call(["chown", owner, path], sudo=True)
-  if group:
-    shell.checked_call(["chgrp", group, path], sudo=True)
-    
-# os.chmod replacement
-def chmod(path, mode):
-  shell.checked_call(["chmod", oct(mode), path], sudo=True)
+if os.geteuid() == 0 or OSCheck.is_windows_family():
+  def chown(path, owner, group):
+    return os.chown(path, _coerce_uid(owner) if owner else -1, _coerce_gid(group) if group else -1)
   
-def chmod_extended(path, mode):
-  shell.checked_call(["chmod", mode, path], sudo=True)
+  def chmod(path, mode):
+    return os.chmod(path, mode)
   
-# os.makedirs replacement
-def makedirs(path, mode):
-  shell.checked_call(["mkdir", "-p", path], sudo=True)
-  chmod(path, mode)
-  
-# os.makedir replacement
-def makedir(path, mode):
-  shell.checked_call(["mkdir", path], sudo=True)
-  chmod(path, mode)
-  
-# os.symlink replacement
-def symlink(source, link_name):
-  shell.checked_call(["ln","-sf", source, link_name], sudo=True)
-  
-# os.link replacement
-def link(source, link_name):
-  shell.checked_call(["ln", "-f", source, link_name], sudo=True)
+  mode_to_stat = {"a+x": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, "u+x": stat.S_IXUSR, "g+x": stat.S_IXGRP,  "o+x": stat.S_IXOTH}
+  def chmod_extended(path, mode):
+    if mode in mode_to_stat:
+      st = os.stat(path)
+      os.chmod(path, st.st_mode | mode_to_stat[mode])
+    else:
+      shell.checked_call(["chmod", mode, path])
+    
+  def makedirs(path, mode):
+    os.makedirs(path, mode)
   
-# os unlink
-def unlink(path):
-  shell.checked_call(["rm","-f", path], sudo=True)
+  def makedir(path, mode):
+    os.mkdir(path)
+    
+  def symlink(source, link_name):
+    os.symlink(source, link_name)
+    
+  def link(source, link_name):
+    os.link(source, link_name)
+    
+  def unlink(path):
+    os.unlink(path)
+
+  def rmtree(path):
+    shutil.rmtree(path)
+    
+  def create_file(filename, content, encoding=None):
+    """
+    if content is None, create empty file
+    """
+    with open(filename, "wb") as fp:
+      if content:
+        content = content.encode(encoding) if encoding else content
+        fp.write(content)
+      
+  def read_file(filename, encoding=None):
+    with open(filename, "rb") as fp:
+      content = fp.read()
+        
+    content = content.decode(encoding) if encoding else content
+    return content
+      
+  def path_exists(path):
+    return os.path.exists(path)
   
-# shutil.rmtree
-def rmtree(path):
-  shell.checked_call(["rm","-rf", path], sudo=True)
+  def path_isdir(path):
+    return os.path.isdir(path)
   
-# fp.write replacement
-def create_file(filename, content, encoding=None):
-  """
-  if content is None, create empty file
-  """
-  tmpf = tempfile.NamedTemporaryFile()
+  def path_lexists(path):
+    return os.path.lexists(path)
   
-  if content:
-    content = content.encode(encoding) if encoding else content
-    with open(tmpf.name, "wb") as fp:
-      fp.write(content)
+  def path_isfile(path):
+    return os.path.isfile(path)
+
+  def stat(path):
+    class Stat:
+      def __init__(self, path):
+        stat_val = os.stat(path)
+        self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777
+    return Stat(path)
   
-  with tmpf:    
-    shell.checked_call(["cp", "-f", tmpf.name, filename], sudo=True)
+else:
+
+  # os.chown replacement
+  def chown(path, owner, group):
+    if owner:
+      shell.checked_call(["chown", owner, path], sudo=True)
+    if group:
+      shell.checked_call(["chgrp", group, path], sudo=True)
+      
+  # os.chmod replacement
+  def chmod(path, mode):
+    shell.checked_call(["chmod", oct(mode), path], sudo=True)
     
-  # set default files mode
-  chmod(filename, 0644)
+  def chmod_extended(path, mode):
+    shell.checked_call(["chmod", mode, path], sudo=True)
     
-# fp.read replacement
-def read_file(filename, encoding=None):
-  tmpf = tempfile.NamedTemporaryFile()
-  shell.checked_call(["cp", "-f", filename, tmpf.name], sudo=True)
-  
-  with tmpf:
-    with open(tmpf.name, "rb") as fp:
-      content = fp.read()
-      
-  content = content.decode(encoding) if encoding else content
-  return content
+  # os.makedirs replacement
+  def makedirs(path, mode):
+    shell.checked_call(["mkdir", "-p", path], sudo=True)
+    chmod(path, mode)
     
-# os.path.exists
-def path_exists(path):
-  return (shell.call(["test", "-e", path], sudo=True)[0] == 0)
-
-# os.path.isdir
-def path_isdir(path):
-  return (shell.call(["test", "-d", path], sudo=True)[0] == 0)
-
-# os.path.lexists
-def path_lexists(path):
-  return (shell.call(["test", "-L", path], sudo=True)[0] == 0)
-
-# os.path.isfile
-def path_isfile(path):
-  return (shell.call(["test", "-f", path], sudo=True)[0] == 0)
-
-# os.stat
-def stat(path):
-  class Stat:
-    RETRY_COUNT = 5
-    def __init__(self, path):
-      # Sometimes (on heavy load) stat call returns an empty output with zero return code
-      for i in range(0, self.RETRY_COUNT):
-        out = shell.checked_call(["stat", "-c", "%u %g %a", path], sudo=True)[1]
-        values = out.split(' ')
-        if len(values) == 3:
-          uid_str, gid_str, mode_str = values
-          self.st_uid, self.st_gid, self.st_mode = int(uid_str), int(gid_str), int(mode_str, 8)
-          break
-      else:
-        warning_message = "Can not parse a sudo stat call output: \"{0}\"".format(out)
-        Logger.warning(warning_message)
+  # os.makedir replacement
+  def makedir(path, mode):
+    shell.checked_call(["mkdir", path], sudo=True)
+    chmod(path, mode)
+    
+  # os.symlink replacement
+  def symlink(source, link_name):
+    shell.checked_call(["ln","-sf", source, link_name], sudo=True)
+    
+  # os.link replacement
+  def link(source, link_name):
+    shell.checked_call(["ln", "-f", source, link_name], sudo=True)
+    
+  # os unlink
+  def unlink(path):
+    shell.checked_call(["rm","-f", path], sudo=True)
+    
+  # shutil.rmtree
+  def rmtree(path):
+    shell.checked_call(["rm","-rf", path], sudo=True)
+    
+  def stat(path):
+    class Stat:
+      def __init__(self, path):
         stat_val = os.stat(path)
         self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777
-  return Stat(path)
+        
+    return Stat(path)
+    
+  # fp.write replacement
+  def create_file(filename, content, encoding=None):
+    """
+    if content is None, create empty file
+    """
+    tmpf = tempfile.NamedTemporaryFile()
+    
+    if content:
+      content = content.encode(encoding) if encoding else content
+      with open(tmpf.name, "wb") as fp:
+        fp.write(content)
+    
+    with tmpf:    
+      shell.checked_call(["cp", "-f", tmpf.name, filename], sudo=True)
+      
+  # fp.read replacement
+  def read_file(filename, encoding=None):
+    tmpf = tempfile.NamedTemporaryFile()
+    shell.checked_call(["cp", "-f", filename, tmpf.name], sudo=True)
+    
+    with tmpf:
+      with open(tmpf.name, "rb") as fp:
+        content = fp.read()
+        
+    content = content.decode(encoding) if encoding else content
+    return content
+      
+  # os.path.exists
+  def path_exists(path):
+    return (shell.call(["test", "-e", path], sudo=True)[0] == 0)
+  
+  # os.path.isdir
+  def path_isdir(path):
+    return (shell.call(["test", "-d", path], sudo=True)[0] == 0)
+  
+  # os.path.lexists
+  def path_lexists(path):
+    return (shell.call(["test", "-L", path], sudo=True)[0] == 0)
+  
+  # os.path.isfile
+  def path_isfile(path):
+    return (shell.call(["test", "-f", path], sudo=True)[0] == 0)
+  
+  # os.stat
+  def stat(path):
+    class Stat:
+      RETRY_COUNT = 5
+      def __init__(self, path):
+        # Sometimes (on heavy load) stat call returns an empty output with zero return code
+        for i in range(0, self.RETRY_COUNT):
+          out = shell.checked_call(["stat", "-c", "%u %g %a", path], sudo=True)[1]
+          values = out.split(' ')
+          if len(values) == 3:
+            uid_str, gid_str, mode_str = values
+            self.st_uid, self.st_gid, self.st_mode = int(uid_str), int(gid_str), int(mode_str, 8)
+            break
+        else:
+          warning_message = "Can not parse a sudo stat call output: \"{0}\"".format(out)
+          Logger.warning(warning_message)
+          stat_val = os.stat(path)
+          self.st_uid, self.st_gid, self.st_mode = stat_val.st_uid, stat_val.st_gid, stat_val.st_mode & 07777
+    return Stat(path)

ambari-common/src/main/python/resource_management/core/utils.py

@@ -20,6 +20,8 @@ Ambari Agent
 
 """
 
+import grp
+import pwd
 from resource_management.core.exceptions import Fail
 
 class AttributeDictionary(object):
@@ -106,3 +108,24 @@ def checked_unite(dict1, dict2):
   result.update(dict2)
   
   return result
+
+def _coerce_uid(user):
+  try:
+    uid = int(user)
+  except ValueError:
+    try:
+      uid = pwd.getpwnam(user).pw_uid
+    except KeyError:
+      raise Fail("User %s doesn't exist." % user)
+  return uid
+
+
+def _coerce_gid(group):
+  try:
+    gid = int(group)
+  except ValueError:
+    try:
+      gid = grp.getgrnam(group).gr_gid
+    except KeyError:
+      raise Fail("Group %s doesn't exist." % group)
+  return gid