AMBARI-3656. Resource Manager. Params subtitution: allow dictionaries in dictionaries, allow default prefixes (Andrew Onischuk via dlysnichenko)

ambari-agent/src/main/python/resource_management/base.py

@@ -4,10 +4,8 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
            "BooleanArgument"]
 
 import logging
-from string import Template
-from resource_management.environment import Environment
 from resource_management.exceptions import Fail, InvalidArgument
-
+from resource_management.environment import Environment, Substitutor
 
 class ResourceArgument(object):
   def __init__(self, default=None, required=False, allow_override=False):
@@ -71,8 +69,8 @@ class ResourceMetaclass(type):
         value.name = key
         mcs._arguments[key] = value
         setattr(mcs, key, Accessor(key))
-
-
+  
+  
 class Resource(object):
   __metaclass__ = ResourceMetaclass
 
@@ -96,8 +94,8 @@ class Resource(object):
         
       name = name[0]
     
-    name = Resource.subsitute_params(name)
     env = env or Environment.get_instance()
+    name = Substitutor.substitute(name)
     provider = provider or getattr(cls, 'provider', None)
     
     r_type = cls.__name__
@@ -124,8 +122,9 @@ class Resource(object):
     if hasattr(self, 'name'):
       return
 
-    self.name = Resource.subsitute_params(name)
     self.env = env or Environment.get_instance()
+    self.name = Substitutor.substitute(name)
+     
     self.provider = provider or getattr(self, 'provider', None)
 
     self.arguments = {}
@@ -136,7 +135,7 @@ class Resource(object):
         raise Fail("%s received unsupported argument %s" % (self, key))
       else:
         try:
-          self.arguments[key] = Resource.subsitute_params(arg.validate(value))
+          self.arguments[key] = Substitutor.substitute(arg.validate(value))
         except InvalidArgument, exc:
           raise InvalidArgument("%s %s" % (self, exc))
 
@@ -156,25 +155,6 @@ class Resource(object):
       self.subscribe(*sub)
 
     self.validate()
-    
-  @staticmethod
-  def subsitute_params(val):
-    env = Environment.get_instance()
-    
-    if env.config.params and isinstance(val, str):
-      try:
-        # use 'safe_substitute' to ignore failures
-        result = Template(val).substitute(env.config.params)
-        if '$' in val:
-          Resource.log.debug("%s after substitution is %s", val, result)
-        return result
-      except KeyError as ex:
-        key_name = '$'+str(ex).strip("'")
-        raise Fail("Configuration %s not found" % key_name)
-        
-    return val
-      
-  
 
   def validate(self):
     pass

ambari-agent/src/main/python/resource_management/environment.py

@@ -11,8 +11,9 @@ from datetime import datetime
 from resource_management import shell
 from resource_management.exceptions import Fail
 from resource_management.providers import find_provider
-from resource_management.utils import AttributeDictionary
+from resource_management.utils import AttributeDictionary, ParamsAttributeDictionary
 from resource_management.system import System
+from string import Template
 
 
 class Environment(object):
@@ -32,6 +33,7 @@ class Environment(object):
     self.config = AttributeDictionary()
     self.resources = {}
     self.resource_list = []
+    Substitutor.default_prefixes = []
     self.delayed_actions = set()
     self.update_config({
       # current time
@@ -43,7 +45,7 @@ class Environment(object):
       # dir where templates,failes dirs are 
       'basedir': basedir, 
       # variables, which can be used in templates
-      'params': params, 
+      'params': ParamsAttributeDictionary(Substitutor, params), 
     })
 
   def backup_file(self, path):
@@ -87,6 +89,9 @@ class Environment(object):
         self.log.info(
           "%s sending %s action to %s (delayed)" % (resource, action, res))
       self.delayed_actions |= resource.subscriptions['delayed']
+      
+  def set_default_prefixes(self, dict):
+    Substitutor.default_prefixes = dict
 
   def _check_condition(self, cond):
     if hasattr(cond, '__call__'):
@@ -157,3 +162,78 @@ class Environment(object):
     self.resources = state['resources']
     self.resource_list = state['resource_list']
     self.delayed_actions = state['delayed_actions']
+    
+
+class Substitutor():
+  log = logging.getLogger("resource_management.resource")
+  default_prefixes = []
+  
+  class ExtendedTemplate(Template):
+    """
+    This is done to support substitution of dictionaries in dictionaries 
+    ( ':' sign)
+    
+    default is:
+    idpattern = r'[_a-z][_a-z0-9]*'
+    """
+    idpattern = r'[_a-z][_a-z0-9:]*'
+    
+  @staticmethod 
+  def _get_subdict(name, dic):
+    """
+    "a:b:c" => a[b][c]
+    
+    doesn't use prefixes
+    """
+    name_parts = name.split(':')
+    curr = dic
+    
+    for x in name_parts:
+      curr = curr[x]
+    return curr
+      
+  @staticmethod
+  def get_subdict(name, dic):
+    """
+    "a:b:c" => a[b][c]
+    
+    can use prefixes
+    """
+    prefixes = list(Substitutor.default_prefixes)
+    prefixes.insert(0, None) # for not prefixed case
+    name_parts = name.split(':')
+    is_found = False
+    result = None
+    
+    for prefix in prefixes:
+      curr = Substitutor._get_subdict(prefix,dic) if prefix else dic
+      
+      try:
+        for x in name_parts:
+          curr = curr[x]
+      except (KeyError, TypeError):
+        continue
+      
+      if is_found:
+        raise Fail("Variable ${%s} found more than one time, please check your default prefixes!" % name)
+      
+      is_found = True
+      result = curr
+      
+    if not result:
+      raise Fail("Configuration on ${%s} cannot be resolved" % name)
+    
+    return result
+
+  @staticmethod
+  def substitute(val):
+    env = Environment.get_instance()
+    dic = env.config.params
+    
+    if dic and isinstance(val, str):
+      result = Substitutor.ExtendedTemplate(val).substitute(dic)
+      if '$' in val:
+        Substitutor.log.debug("%s after substitution is %s", val, result)
+      return result
+        
+    return val

ambari-agent/src/main/python/resource_management/utils.py

@@ -14,8 +14,7 @@ class AttributeDictionary(object):
     try:
       return self[name]
     except KeyError:
-      raise AttributeError(
-        "'%s' object has no attribute '%s'" % (self.__class__.__name__, name))
+      raise AttributeError("'%s' object has no attribute '%s'" % (self.__class__.__name__, name))
 
   def __setitem__(self, name, value):
     self._dict[name] = self._convert_value(value)
@@ -66,3 +65,69 @@ class AttributeDictionary(object):
 
   def __setstate__(self, state):
     super(AttributeDictionary, self).__setattr__("_dict", state)
+    
+class ParamsAttributeDictionary(object):
+  """
+  This class can store user parameters
+  and support some features necessary for substitution to work.
+  """
+  def __init__(self, substitutor, *args, **kwargs):
+    d = kwargs
+    if len(args)==1:
+      d = args[0]
+    super(ParamsAttributeDictionary, self).__setattr__("_dict", d)
+    super(ParamsAttributeDictionary, self).__setattr__("substitutor", substitutor)
+
+  def __setattr__(self, name, value):
+    self[name] = value
+
+  def __setitem__(self, name, value):
+    self._dict[name] = self._convert_value(value)
+
+  def __getitem__(self, name):
+    val = self.substitutor.get_subdict(name, self._dict)
+    return self._convert_value(val)
+
+  def _convert_value(self, value):
+    if isinstance(value, dict) and not isinstance(value, ParamsAttributeDictionary):
+      return ParamsAttributeDictionary(self.substitutor, value)
+    return value
+
+  def copy(self):
+    return self.__class__(self._dict.copy())
+
+  def update(self, *args, **kwargs):
+    self._dict.update(*args, **kwargs)
+
+  def items(self):
+    return self._dict.items()
+
+  def values(self):
+    return self._dict.values()
+
+  def keys(self):
+    return self._dict.keys()
+
+  def pop(self, *args, **kwargs):
+    return self._dict.pop(*args, **kwargs)
+
+  def get(self, *args, **kwargs):
+    return self._dict.get(*args, **kwargs)
+
+  def __repr__(self):
+    return self._dict.__repr__()
+
+  def __unicode__(self):
+    return self._dict.__unicode__()
+
+  def __str__(self):
+    return self._dict.__str__()
+
+  def __iter__(self):
+    return self._dict.__iter__()
+
+  def __getstate__(self):
+    return self._dict
+
+  def __setstate__(self, state):
+    super(ParamsAttributeDictionary, self).__setattr__("_dict", state)