AMBARI-4487. When logging certain operations, need to mask sensitive

properties (aonishuk)

ambari-agent/src/main/python/resource_management/core/__init__.py

@@ -28,5 +28,6 @@ from resource_management.core.resources import *
 from resource_management.core.source import *
 from resource_management.core.system import *
 from resource_management.core.shell import *
+from resource_management.core.logger import *
 
 __version__ = "0.4.1"

ambari-agent/src/main/python/resource_management/core/base.py

@@ -23,9 +23,9 @@ Ambari Agent
 __all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
            "BooleanArgument"]
 
-import logging
 from resource_management.core.exceptions import Fail, InvalidArgument
 from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
 
 class ResourceArgument(object):
   def __init__(self, default=None, required=False):
@@ -93,8 +93,6 @@ class ResourceMetaclass(type):
 class Resource(object):
   __metaclass__ = ResourceMetaclass
 
-  log = logging.getLogger("resource_management.resource")
-
   action = ForcedListArgument(default="nothing")
   ignore_failures = BooleanArgument(default=False)
   not_if = ResourceArgument() # pass command e.g. not_if = ('ls','/root/jdk')
@@ -146,7 +144,7 @@ class Resource(object):
         except InvalidArgument, exc:
           raise InvalidArgument("%s %s" % (self, exc))
 
-    Resource.log.debug("New resource %s: %s" % (self, self.arguments))
+    Logger.debug("New resource %s: %s" % (self, self.arguments))
     
     if not self.env.test_mode:
       self.env.run()
@@ -174,6 +172,4 @@ class Resource(object):
     self.arguments = state['arguments']
     self.env = state['env']
 
-    Resource.log = logging.getLogger("resource_management.resource")
-
     self.validate()

ambari-agent/src/main/python/resource_management/core/environment.py

@@ -22,7 +22,6 @@ Ambari Agent
 
 __all__ = ["Environment"]
 
-import logging
 import os
 import shutil
 import time
@@ -33,6 +32,7 @@ from resource_management.core.exceptions import Fail
 from resource_management.core.providers import find_provider
 from resource_management.core.utils import AttributeDictionary
 from resource_management.core.system import System
+from resource_management.core.logger import Logger
 
 
 class Environment(object):
@@ -44,7 +44,6 @@ class Environment(object):
     are looked up
     @param test_mode: if this is enabled, resources won't be executed until manualy running env.run().
     """
-    self.log = logging.getLogger("resource_management")
     self.reset(basedir, test_mode)
 
   def reset(self, basedir, test_mode):
@@ -73,7 +72,7 @@ class Environment(object):
         os.makedirs(self.config.backup.path, 0700)
       new_name = self.config.backup.prefix + path.replace('/', '-')
       backup_path = os.path.join(self.config.backup.path, new_name)
-      self.log.info("backing up %s to %s" % (path, backup_path))
+      Logger.info("backing up %s to %s" % (path, backup_path))
       shutil.copy(path, backup_path)
 
   def update_config(self, attributes, overwrite=True):
@@ -104,7 +103,7 @@ class Environment(object):
         self.config.params[variable] = value
         
   def run_action(self, resource, action):
-    self.log.debug("Performing action %s on %s" % (action, resource))
+    Logger.debug("Performing action %s on %s" % (action, resource))
 
     provider_class = find_provider(self, resource.__class__.__name__,
                                    resource.provider)
@@ -123,51 +122,26 @@ class Environment(object):
       ret, out = shell.call(cond)
       return ret == 0
 
-    raise Exception("Unknown condition type %r" % cond)
-
-  def _get_resource_repr(self, resource):
-    arguments_str = ""
-    for x,y in resource.arguments.iteritems():
-      
-      # strip unicode 'u' sign
-      if isinstance(y, unicode):
-        val = repr(y).lstrip('u')
-      # don't show dicts of configurations
-      # usually too long  
-      elif isinstance(y, dict):
-        val = "..."
-      # correctly output 'mode' (as they are octal values like 0755)
-      elif y and x == 'mode':
-        val = oct(y)
-      else:
-        val = repr(y)
-      
-      
-      arguments_str += "'{0}': {1}, ".format(x, val)
-      
-    if arguments_str:  
-      arguments_str = arguments_str[:-2]
-    
-    return "{0} {{{1}}}".format(resource, arguments_str)  
+    raise Exception("Unknown condition type %r" % cond) 
     
   def run(self):
     with self:
       # Run resource actions
       while self.resource_list:
         resource = self.resource_list.pop(0)
-        self.log.info(self._get_resource_repr(resource))
+        Logger.info_resource(resource)
         
         if resource.initial_wait:
           time.sleep(resource.initial_wait)
 
         if resource.not_if is not None and self._check_condition(
           resource.not_if):
-          self.log.info("Skipping %s due to not_if" % resource)
+          Logger.info("Skipping %s due to not_if" % resource)
           continue
 
         if resource.only_if is not None and not self._check_condition(
           resource.only_if):
-          self.log.info("Skipping %s due to only_if" % resource)
+          Logger.info("Skipping %s due to only_if" % resource)
           continue
 
         for action in resource.action:

ambari-agent/src/main/python/resource_management/core/logger.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+__all__ = ["Logger"]
+import logging
+
+class Logger:
+  logger = logging.getLogger("resource_management")
+  
+  # unprotected_strings : protected_strings map
+  sensitive_strings = {}
+  
+  @staticmethod
+  def info(text):
+    Logger.logger.info(Logger.get_protected_text(text))
+  
+  @staticmethod  
+  def debug(text):
+    Logger.logger.debug(Logger.get_protected_text(text))
+
+  @staticmethod
+  def info_resource(resource):
+    Logger.info(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+  
+  @staticmethod  
+  def debug_resource(resource):
+    Logger.debug(Logger.get_protected_text(Logger._get_resource_repr(resource)))
+    
+  @staticmethod
+  def get_protected_text(text):
+    """
+    Replace passwords with [PROTECTED]
+    """
+    for unprotected_string, protected_string in Logger.sensitive_strings.iteritems():
+      text = text.replace(unprotected_string, protected_string)
+      
+    return text
+    
+  @staticmethod  
+  def _get_resource_repr(resource):
+    arguments_str = ""
+    for x,y in resource.arguments.iteritems():
+      
+      # strip unicode 'u' sign
+      if isinstance(y, unicode):
+        val = repr(y).lstrip('u')
+      # don't show dicts of configurations
+      # usually too long  
+      elif isinstance(y, dict):
+        val = "..."
+      # correctly output 'mode' (as they are octal values like 0755)
+      elif y and x == 'mode':
+        val = oct(y)
+      else:
+        val = repr(y)
+      
+      
+      arguments_str += "'{0}': {1}, ".format(x, val)
+      
+    if arguments_str:  
+      arguments_str = arguments_str[:-2]
+    
+    return "{0} {{{1}}}".format(resource, arguments_str)

ambari-agent/src/main/python/resource_management/core/providers/__init__.py

@@ -22,14 +22,12 @@ Ambari Agent
 
 __all__ = ["Provider", "find_provider"]
 
-import logging
 from resource_management.core.exceptions import Fail
 from resource_management.libraries.providers import PROVIDERS as LIBRARY_PROVIDERS
 
 
 class Provider(object):
   def __init__(self, resource):
-    self.log = logging.getLogger("resource_management.provider")
     self.resource = resource
 
   def action_nothing(self):

ambari-agent/src/main/python/resource_management/core/providers/accounts.py

@@ -26,16 +26,17 @@ import grp
 import pwd
 from resource_management.core import shell
 from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
 
 
 class UserProvider(Provider):
   def action_create(self):
     if not self.user:
       command = ['useradd', "-m"]
-      self.log.info("Adding user %s" % self.resource)
+      Logger.info("Adding user %s" % self.resource)
     else:
       command = ['usermod']
-      self.log.info("Modifying user %s" % (self.resource.username))
+      Logger.info("Modifying user %s" % (self.resource.username))
 
     options = dict(
       comment="-c",
@@ -65,7 +66,7 @@ class UserProvider(Provider):
     if self.user:
       command = ['userdel', self.resource.username]
       shell.checked_call(command)
-      self.log.info("Removed user %s" % self.resource)
+      Logger.info("Removed user %s" % self.resource)
 
   @property
   def user(self):
@@ -80,10 +81,10 @@ class GroupProvider(Provider):
     group = self.group
     if not group:
       command = ['groupadd']
-      self.log.info("Adding group %s" % self.resource)
+      Logger.info("Adding group %s" % self.resource)
     else:
       command = ['groupmod']
-      self.log.info("Modifying group %s" % (self.resource.group_name))
+      Logger.info("Modifying group %s" % (self.resource.group_name))
       
     options = dict(
         gid="-g",
@@ -105,7 +106,7 @@ class GroupProvider(Provider):
     if self.group:
       command = ['groupdel', self.resource.group_name]
       shell.checked_call(command)
-      self.log.info("Removed group %s" % self.resource)
+      Logger.info("Removed group %s" % self.resource)
 
   @property
   def group(self):

ambari-agent/src/main/python/resource_management/core/providers/mount.py

@@ -26,6 +26,7 @@ import os
 import re
 from resource_management.core.base import Fail
 from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
 
 
 class MountProvider(Provider):
@@ -34,7 +35,7 @@ class MountProvider(Provider):
       os.makedirs(self.resource.mount_point)
 
     if self.is_mounted():
-      self.log.debug("%s already mounted" % self)
+      Logger.debug("%s already mounted" % self)
     else:
       args = ["mount"]
       if self.resource.fstype:
@@ -47,19 +48,19 @@ class MountProvider(Provider):
 
       check_call(args)
 
-      self.log.info("%s mounted" % self)
+      Logger.info("%s mounted" % self)
 
   def action_umount(self):
     if self.is_mounted():
       check_call(["umount", self.resource.mount_point])
 
-      self.log.info("%s unmounted" % self)
+      Logger.info("%s unmounted" % self)
     else:
-      self.log.debug("%s is not mounted" % self)
+      Logger.debug("%s is not mounted" % self)
 
   def action_enable(self):
     if self.is_enabled():
-      self.log.debug("%s already enabled" % self)
+      Logger.debug("%s already enabled" % self)
     else:
       if not self.resource.device:
         raise Fail("[%s] device not set but required for enable action" % self)
@@ -76,7 +77,7 @@ class MountProvider(Provider):
           self.resource.passno,
         ))
 
-      self.log.info("%s enabled" % self)
+      Logger.info("%s enabled" % self)
 
   def action_disable(self):
     pass # TODO

ambari-agent/src/main/python/resource_management/core/providers/package/yumrpm.py

@@ -22,6 +22,7 @@ Ambari Agent
 
 from resource_management.core.providers.package import PackageProvider
 from resource_management.core import shell
+from resource_management.core.logger import Logger
 
 INSTALL_CMD = "/usr/bin/yum -d 0 -e 0 -y install %s"
 REMOVE_CMD = "/usr/bin/yum -d 0 -e 0 -y erase %s"
@@ -29,7 +30,7 @@ REMOVE_CMD = "/usr/bin/yum -d 0 -e 0 -y erase %s"
 class YumProvider(PackageProvider):
   def install_package(self, name):
     cmd = INSTALL_CMD % (name)
-    self.log.info("Installing package %s ('%s')", name, cmd)
+    Logger.info("Installing package %s ('%s')" % (name, cmd))
     shell.checked_call(cmd)
 
   def upgrade_package(self, name):
@@ -37,5 +38,5 @@ class YumProvider(PackageProvider):
   
   def remove_package(self, name):
     cmd = REMOVE_CMD % (name)
-    self.log.info("Removing package %s ('%s')", name, cmd)
+    Logger.info("Removing package %s ('%s')" % (name, cmd))
     shell.checked_call(cmd)    

ambari-agent/src/main/python/resource_management/core/providers/package/zypper.py

@@ -22,6 +22,7 @@ Ambari Agent
 
 from resource_management.core.providers.package import PackageProvider
 from resource_management.core import shell
+from resource_management.core.logger import Logger
 
 INSTALL_CMD = "/usr/bin/zypper --quiet install --auto-agree-with-licenses --no-confirm %s"
 REMOVE_CMD = "/usr/bin/zypper --quiet remove --no-confirm %s"
@@ -29,7 +30,7 @@ REMOVE_CMD = "/usr/bin/zypper --quiet remove --no-confirm %s"
 class ZypperProvider(PackageProvider):
   def install_package(self, name):
     cmd = INSTALL_CMD % (name)
-    self.log.info("Installing package %s ('%s')", name, cmd)
+    Logger.info("Installing package %s ('%s')", name, cmd)
     shell.checked_call(cmd)
 
   def upgrade_package(self, name):
@@ -37,5 +38,5 @@ class ZypperProvider(PackageProvider):
   
   def remove_package(self, name):
     cmd = REMOVE_CMD % (name)
-    self.log.info("Removing package %s ('%s')", name, cmd)
+    Logger.info("Removing package %s ('%s')", name, cmd)
     shell.checked_call(cmd)

ambari-agent/src/main/python/resource_management/core/providers/service.py

@@ -25,6 +25,7 @@ import os
 from resource_management.core import shell
 from resource_management.core.base import Fail
 from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
 
 
 class ServiceProvider(Provider):
@@ -53,11 +54,11 @@ class ServiceProvider(Provider):
 
   def _exec_cmd(self, command, expect=None):
     if command != "status":
-      self.log.info("%s command '%s'" % (self.resource, command))
+      Logger.info("%s command '%s'" % (self.resource, command))
 
     custom_cmd = getattr(self.resource, "%s_command" % command, None)
     if custom_cmd:
-      self.log.debug("%s executing '%s'" % (self.resource, custom_cmd))
+      Logger.debug("%s executing '%s'" % (self.resource, custom_cmd))
       if hasattr(custom_cmd, "__call__"):
         if custom_cmd():
           ret = 0

ambari-agent/src/main/python/resource_management/core/providers/system.py

@@ -30,6 +30,7 @@ import shutil
 from resource_management.core import shell
 from resource_management.core.base import Fail
 from resource_management.core.providers import Provider
+from resource_management.core.logger import Logger
 
 
 def _coerce_uid(user):
@@ -54,27 +55,27 @@ def _coerce_gid(group):
   return gid
 
 
-def _ensure_metadata(path, user, group, mode=None, log=None):
+def _ensure_metadata(path, user, group, mode=None):
   stat = os.stat(path)
 
   if mode:
     existing_mode = stat.st_mode & 07777
     if existing_mode != mode:
-      log and log.info("Changing permission for %s from %o to %o" % (
+      Logger.info("Changing permission for %s from %o to %o" % (
       path, existing_mode, mode))
       os.chmod(path, mode)
 
   if user:
     uid = _coerce_uid(user)
     if stat.st_uid != uid:
-      log and log.info(
+      Logger.info(
         "Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
       os.chown(path, uid, -1)
 
   if group:
     gid = _coerce_gid(group)
     if stat.st_gid != gid:
-      log and log.info(
+      Logger.info(
         "Changing group for %s from %d to %s" % (path, stat.st_gid, group))
       os.chown(path, -1, gid)
 
@@ -106,14 +107,13 @@ class FileProvider(Provider):
             self.resource.env.backup_file(path)
 
     if write:
-      self.log.info("Writing %s because %s" % (self.resource, reason))
+      Logger.info("Writing %s because %s" % (self.resource, reason))
       with open(path, "wb") as fp:
         if content:
           fp.write(content)
 
     _ensure_metadata(self.resource.path, self.resource.owner,
-                        self.resource.group, mode=self.resource.mode,
-                        log=self.log)
+                        self.resource.group, mode=self.resource.mode)
 
   def action_delete(self):
     path = self.resource.path
@@ -122,7 +122,7 @@ class FileProvider(Provider):
       raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
     
     if os.path.exists(path):
-      self.log.info("Deleting %s" % self.resource)
+      Logger.info("Deleting %s" % self.resource)
       os.unlink(path)
 
   def _get_content(self):
@@ -140,7 +140,7 @@ class DirectoryProvider(Provider):
   def action_create(self):
     path = self.resource.path
     if not os.path.exists(path):
-      self.log.info("Creating directory %s" % self.resource)
+      Logger.info("Creating directory %s" % self.resource)
       if self.resource.recursive:
         os.makedirs(path, self.resource.mode or 0755)
       else:
@@ -154,7 +154,7 @@ class DirectoryProvider(Provider):
       raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
 
     _ensure_metadata(path, self.resource.owner, self.resource.group,
-                        mode=self.resource.mode, log=self.log)
+                        mode=self.resource.mode)
 
   def action_delete(self):
     path = self.resource.path
@@ -162,7 +162,7 @@ class DirectoryProvider(Provider):
       if not os.path.isdir(path):
         raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
       
-      self.log.info("Removing directory %s and all its content" % self.resource)
+      Logger.info("Removing directory %s and all its content" % self.resource)
       shutil.rmtree(path)
 
 
@@ -177,7 +177,7 @@ class LinkProvider(Provider):
       if not os.path.islink(path):
         raise Fail(
           "%s trying to create a symlink with the same name as an existing file or directory" % self)
-      self.log.info("%s replacing old symlink to %s" % (self.resource, oldpath))
+      Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
       os.unlink(path)
       
     if self.resource.hard:
@@ -186,19 +186,19 @@ class LinkProvider(Provider):
       if os.path.isdir(self.resource.to):
         raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
       
-      self.log.info("Creating hard %s" % self.resource)
+      Logger.info("Creating hard %s" % self.resource)
       os.link(self.resource.to, path)
     else:
       if not os.path.exists(self.resource.to):
-        self.log.info("Warning: linking to nonexistent location %s", self.resource.to)
+        Logger.info("Warning: linking to nonexistent location %s", self.resource.to)
         
-      self.log.info("Creating symbolic %s" % self.resource)
+      Logger.info("Creating symbolic %s" % self.resource)
       os.symlink(self.resource.to, path)
 
   def action_delete(self):
     path = self.resource.path
     if os.path.exists(path):
-      self.log.info("Deleting %s" % self.resource)
+      Logger.info("Deleting %s" % self.resource)
       os.unlink(path)
 
 
@@ -218,7 +218,7 @@ class ExecuteProvider(Provider):
       if os.path.exists(self.resource.creates):
         return
 
-    self.log.debug("Executing %s" % self.resource)
+    Logger.debug("Executing %s" % self.resource)
     
     if self.resource.path != []:
       if not self.resource.environment:
@@ -236,7 +236,7 @@ class ExecuteProvider(Provider):
         if i == self.resource.tries-1: # last try
           raise ex
         else:
-          self.log.info("Retrying after %d seconds. Reason: %s", self.resource.try_sleep, str(ex))
+          Logger.info("Retrying after %d seconds. Reason: %s", self.resource.try_sleep, str(ex))
           time.sleep(self.resource.try_sleep)
        
 
@@ -244,7 +244,7 @@ class ExecuteScriptProvider(Provider):
   def action_run(self):
     from tempfile import NamedTemporaryFile
 
-    self.log.info("Running script %s" % self.resource)
+    Logger.info("Running script %s" % self.resource)
     with NamedTemporaryFile(prefix="resource_management-script", bufsize=0) as tf:
       tf.write(self.resource.code)
       tf.flush()

ambari-agent/src/main/python/resource_management/core/shell.py

@@ -22,12 +22,10 @@ Ambari Agent
 
 __all__ = ["checked_call"]
 
-import logging
 import subprocess
 import pipes
 from exceptions import Fail
-
-log = logging.getLogger("resource_management.provider")
+from resource_management.core.logger import Logger
 
 def checked_call(command, logoutput=False, 
          cwd=None, env=None, preexec_fn=None, user=None):
@@ -67,7 +65,7 @@ def _call(command, logoutput=False, throw_on_failure=True,
   code = proc.returncode
   
   if logoutput and out and out!="":
-    log.info(out)
+    Logger.info(out)
   
   if throw_on_failure and code:
     err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out)

ambari-agent/src/main/python/resource_management/libraries/functions/check_process_status.py

@@ -21,11 +21,10 @@ Ambari Agent
 """
 
 from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
 __all__ = ["check_process_status"]
 
-import os, logging
-
-log = logging.getLogger('resource_management')
+import os
 
 def check_process_status(pid_file):
   """
@@ -42,7 +41,7 @@ def check_process_status(pid_file):
     try:
       pid = int(f.read())
     except:
-      log.debug("Pid file {0} does not exist".format(pid_file))
+      Logger.debug("Pid file {0} does not exist".format(pid_file))
       raise ComponentIsNotRunning()
     try:
       # Kill will not actually kill the process
@@ -52,7 +51,7 @@ def check_process_status(pid_file):
       # process ID or process group ID.
       os.kill(pid, 0)
     except OSError:
-      log.debug("Process with pid {0} is not running. Stale pid file"
+      Logger.debug("Process with pid {0} is not running. Stale pid file"
                 " at {1}".format(pid, pid_file))
       raise ComponentIsNotRunning()
   pass

ambari-agent/src/main/python/resource_management/libraries/functions/default.py

@@ -21,12 +21,11 @@ Ambari Agent
 """
 
 __all__ = ["default"]
-import logging
 from resource_management.libraries.script import Script
 from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+from resource_management.core.logger import Logger
 
 default_subdict='/configurations/global'
-log = logging.getLogger('resource_management')
 
 def default(name, default_value):
   subdicts = filter(None, name.split('/'))
@@ -40,7 +39,7 @@ def default(name, default_value):
       curr_dict = curr_dict[x]
     else:
       if not isinstance(default_value, UnknownConfiguration):
-        log.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value))
+        Logger.debug("Cannot find configuration: '%s'. Using '%s' value as default" % (name, default_value))
       return default_value
 
   return curr_dict

ambari-agent/src/main/python/resource_management/libraries/functions/format.py

@@ -26,6 +26,7 @@ from string import Formatter
 from resource_management.core.exceptions import Fail
 from resource_management.core.utils import checked_unite
 from resource_management.core.environment import Environment
+from resource_management.core.logger import Logger
 
 
 class ConfigurationFormatter(Formatter):
@@ -33,9 +34,34 @@ class ConfigurationFormatter(Formatter):
     env = Environment.get_instance()
     variables = kwargs
     params = env.config.params
+    all_params = checked_unite(variables, params)
     
-    result = checked_unite(variables, params)
-    return self.vformat(format_string, args, result)
+    self.convert_field = self.convert_field_protected
+    result_protected = self.vformat(format_string, args, all_params)
+    
+    self.convert_field = self.convert_field_unprotected
+    result_unprotected = self.vformat(format_string, args, all_params)
+    
+    if result_protected != result_unprotected:
+      Logger.sensitive_strings[result_unprotected] = result_protected
+      
+    return result_unprotected
+  
+  def convert_field_unprotected(self, value, conversion):
+    if conversion == 'p':
+      return value
+      
+    return super(ConfigurationFormatter, self).convert_field(value, conversion)
+  
+  def convert_field_protected(self, value, conversion):
+    """
+    Enable masking sensitive information like
+    passwords from logs via !p (password) format flag.
+    """
+    if conversion == 'p':
+      return "[PROTECTED]"
+      
+    return super(ConfigurationFormatter, self).convert_field(value, conversion)
       
   
 def format(format_string, *args, **kwargs):

ambari-agent/src/main/python/resource_management/libraries/providers/properties_file.py

@@ -38,7 +38,7 @@ class PropertiesFileProvider(Provider):
 {{key}}={{value}}{% endfor %}
     ''', extra_imports=[time], properties_dict=self.resource.properties)
 
-    self.log.debug(format("Generating properties file: {filepath}"))
+    Logger.info(format("Generating properties file: {filepath}"))
 
     with Environment.get_instance_copy() as env:
       File (format("{filepath}"),

ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py

@@ -40,7 +40,7 @@ class XmlConfigProvider(Provider):
   </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations)
    
   
-    self.log.info(format("Generating config: {conf_dir}/{filename}"))
+    Logger.info(format("Generating config: {conf_dir}/{filename}"))
     
     with Environment.get_instance_copy() as env:
       File (format("{conf_dir}/{filename}"),

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py

@@ -46,7 +46,7 @@ def hive_service(
 
     if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
       db_connection_check_command = format(
-        "{java64_home}/bin/java -cp {check_db_connection_jar}:/usr/share/java/{jdbc_jar_name} org.apache.ambari.server.DBConnectionVerification {hive_jdbc_connection_url} {hive_metastore_user_name} {hive_metastore_user_passwd} {hive_jdbc_driver}")
+        "{java64_home}/bin/java -cp {check_db_connection_jar}:/usr/share/java/{jdbc_jar_name} org.apache.ambari.server.DBConnectionVerification {hive_jdbc_connection_url} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
       Execute(db_connection_check_command,
               path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin')
 

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/mysql_server.py

@@ -25,11 +25,6 @@ from mysql_service import mysql_service
 
 class MysqlServer(Script):
 
-  if System.get_instance().os_family == "suse":
-    daemon_name = 'mysql'
-  else:
-    daemon_name = 'mysqld'
-
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -38,40 +33,38 @@ class MysqlServer(Script):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action='start')
+    mysql_service(daemon_name=params.daemon_name, action='start')
 
     File(params.mysql_adduser_path,
          mode=0755,
          content=StaticFile('addMysqlUser.sh')
     )
 
-    # Autoescaping
-    cmd = ("bash", "-x", params.mysql_adduser_path, self.daemon_name,
-           params.hive_metastore_user_name, str(params.hive_metastore_user_passwd) , params.mysql_host[0])
+    cmd = format("bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {mysql_host[0]}")
 
     Execute(cmd,
             tries=3,
             try_sleep=5,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-            logoutput=True
+            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
     )
 
-    mysql_service(daemon_name=self.daemon_name, action='stop')
+    mysql_service(daemon_name=params.daemon_name, action='stop')
 
   def start(self, env):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action = 'start')
+    mysql_service(daemon_name=params.daemon_name, action = 'start')
 
   def stop(self, env):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action = 'stop')
+    mysql_service(daemon_name=params.daemon_name, action = 'stop')
 
   def status(self, env):
-    mysql_service(daemon_name=self.daemon_name, action = 'status')
+    import status_params
+    mysql_service(daemon_name=status_params.daemon_name, action = 'status')
 
 if __name__ == "__main__":
   MysqlServer().execute()

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py

@@ -133,3 +133,5 @@ if ('hive-exec-log4j' in config['configurations']):
   log4j_exec_props = config['configurations']['hive-exec-log4j']
 else:
   log4j_exec_props = None
+  
+daemon_name = status_params.daemon_name

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/status_params.py

@@ -28,3 +28,8 @@ hive_pid = 'hive-server.pid'
 hive_metastore_pid = 'hive.pid'
 
 hcat_pid_dir = config['configurations']['global']['hcat_pid_dir'] #hcat_pid_dir
+
+if System.get_instance().os_family == "suse":
+  daemon_name = 'mysql'
+else:
+  daemon_name = 'mysqld'

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/scripts/nagios.py

@@ -77,7 +77,7 @@ def nagios():
 def set_web_permisssions():
   import params
 
-  cmd = format("{htpasswd_cmd} -c -b  /etc/nagios/htpasswd.users {nagios_web_login} {nagios_web_password}")
+  cmd = format("{htpasswd_cmd} -c -b  /etc/nagios/htpasswd.users {nagios_web_login} {nagios_web_password!p}")
   test = format("grep {nagios_web_login} /etc/nagios/htpasswd.users")
   Execute( cmd,
     not_if = test

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie_service.py

@@ -10,7 +10,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
     start_cmd = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-start.sh")
     
     if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
-      db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification {oozie_jdbc_connection_url} {oozie_metastore_user_name} {oozie_metastore_user_passwd} {jdbc_driver_name}")
+      db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification {oozie_jdbc_connection_url} {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}")
     else:
       db_connection_check_command = None
       

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py

@@ -46,7 +46,7 @@ def hive_service(
 
     if params.hive_jdbc_driver == "com.mysql.jdbc.Driver" or params.hive_jdbc_driver == "oracle.jdbc.driver.OracleDriver":
       db_connection_check_command = format(
-        "{java64_home}/bin/java -cp {check_db_connection_jar}:/usr/share/java/{jdbc_jar_name} org.apache.ambari.server.DBConnectionVerification {hive_jdbc_connection_url} {hive_metastore_user_name} {hive_metastore_user_passwd} {hive_jdbc_driver}")
+        "{java64_home}/bin/java -cp {check_db_connection_jar}:/usr/share/java/{jdbc_jar_name} org.apache.ambari.server.DBConnectionVerification {hive_jdbc_connection_url} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {hive_jdbc_driver}")
       Execute(db_connection_check_command,
               path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin')
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/mysql_server.py

@@ -25,11 +25,6 @@ from mysql_service import mysql_service
 
 class MysqlServer(Script):
 
-  if System.get_instance().os_family == "suse":
-    daemon_name = 'mysql'
-  else:
-    daemon_name = 'mysqld'
-
   def install(self, env):
     self.install_packages(env)
     self.configure(env)
@@ -38,40 +33,38 @@ class MysqlServer(Script):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action='start')
+    mysql_service(daemon_name=params.daemon_name, action='start')
 
     File(params.mysql_adduser_path,
          mode=0755,
          content=StaticFile('addMysqlUser.sh')
     )
 
-    # Autoescaping
-    cmd = ("bash", "-x", params.mysql_adduser_path, self.daemon_name,
-           params.hive_metastore_user_name, str(params.hive_metastore_user_passwd) , params.mysql_host[0])
+    cmd = format("bash -x {mysql_adduser_path} {daemon_name} {hive_metastore_user_name} {hive_metastore_user_passwd!p} {mysql_host[0]}")
 
     Execute(cmd,
             tries=3,
             try_sleep=5,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-            logoutput=True
+            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
     )
 
-    mysql_service(daemon_name=self.daemon_name, action='stop')
+    mysql_service(daemon_name=params.daemon_name, action='stop')
 
   def start(self, env):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action = 'start')
+    mysql_service(daemon_name=params.daemon_name, action = 'start')
 
   def stop(self, env):
     import params
     env.set_params(params)
 
-    mysql_service(daemon_name=self.daemon_name, action = 'stop')
+    mysql_service(daemon_name=params.daemon_name, action = 'stop')
 
   def status(self, env):
-    mysql_service(daemon_name=self.daemon_name, action = 'status')
+    import status_params
+    mysql_service(daemon_name=status_params.daemon_name, action = 'status')
 
 if __name__ == "__main__":
   MysqlServer().execute()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py

@@ -133,3 +133,5 @@ if ('hive-exec-log4j' in config['configurations']):
   log4j_exec_props = config['configurations']['hive-exec-log4j']
 else:
   log4j_exec_props = None
+
+daemon_name = status_params.daemon_name

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/status_params.py

@@ -28,3 +28,8 @@ hive_pid = 'hive-server.pid'
 hive_metastore_pid = 'hive.pid'
 
 hcat_pid_dir = config['configurations']['global']['hcat_pid_dir'] #hcat_pid_dir
+
+if System.get_instance().os_family == "suse":
+  daemon_name = 'mysql'
+else:
+  daemon_name = 'mysqld'

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/scripts/nagios.py

@@ -77,7 +77,7 @@ def nagios():
 def set_web_permisssions():
   import params
 
-  cmd = format("{htpasswd_cmd} -c -b  /etc/nagios/htpasswd.users {nagios_web_login} {nagios_web_password}")
+  cmd = format("{htpasswd_cmd} -c -b  /etc/nagios/htpasswd.users {nagios_web_login} {nagios_web_password!p}")
   test = format("grep {nagios_web_login} /etc/nagios/htpasswd.users")
   Execute( cmd,
     not_if = test

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie_service.py

@@ -30,7 +30,7 @@ def oozie_service(action = 'start'): # 'start' or 'stop'
     start_cmd = format("cd {oozie_tmp_dir} && /usr/lib/oozie/bin/oozie-start.sh")
     
     if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
-      db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification {oozie_jdbc_connection_url} {oozie_metastore_user_name} {oozie_metastore_user_passwd} {jdbc_driver_name}")
+      db_connection_check_command = format("{java_home}/bin/java -cp {check_db_connection_jar}:{jdbc_driver_jar} org.apache.ambari.server.DBConnectionVerification {oozie_jdbc_connection_url} {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}")
     else:
       db_connection_check_command = None
       

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_mysql_server.py

@@ -106,8 +106,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', 'hive', 'q', 'c6402.ambari.apache.org'),
-      logoutput = True,
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive q c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
@@ -128,8 +127,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', 'hive', 'q', 'c6402.ambari.apache.org'),
-      logoutput = True,
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive q c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_mysql_server.py

@@ -107,8 +107,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
-      logoutput = True,
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive asd c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
@@ -129,8 +128,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', ('bash', '-x', '/tmp/addMysqlUser.sh', 'mysql', u'hive', 'asd', u'c6402.ambari.apache.org'),
-      logoutput = True,
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive asd c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,