AMBARI-3663. Resource Management. Easy formatting of configurations (Andrew Onischuk via dlysnichenko)

ambari-agent/src/main/python/resource_management/core/base.py

@@ -5,7 +5,7 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument",
 
 import logging
 from resource_management.core.exceptions import Fail, InvalidArgument
-from resource_management.core.environment import Environment, Substitutor
+from resource_management.core.environment import Environment
 
 class ResourceArgument(object):
   def __init__(self, default=None, required=False, allow_override=False):
@@ -95,7 +95,6 @@ class Resource(object):
       name = name[0]
     
     env = env or Environment.get_instance()
-    name = Substitutor.substitute(name)
     provider = provider or getattr(cls, 'provider', None)
     
     r_type = cls.__name__
@@ -123,7 +122,7 @@ class Resource(object):
       return
 
     self.env = env or Environment.get_instance()
-    self.name = Substitutor.substitute(name)
+    self.name = name
      
     self.provider = provider or getattr(self, 'provider', None)
 
@@ -135,7 +134,7 @@ class Resource(object):
         raise Fail("%s received unsupported argument %s" % (self, key))
       else:
         try:
-          self.arguments[key] = Substitutor.substitute(arg.validate(value))
+          self.arguments[key] = arg.validate(value)
         except InvalidArgument, exc:
           raise InvalidArgument("%s %s" % (self, exc))
 

ambari-agent/src/main/python/resource_management/core/environment.py

@@ -1,19 +1,20 @@
 #!/usr/bin/env python
 
-__all__ = ["Environment"]
+__all__ = ["Environment","format"]
 
 import logging
+import sys
 import os
 import shutil
 import time
 from datetime import datetime
+from string import Formatter
 
 from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 from resource_management.core.providers import find_provider
-from resource_management.core.utils import AttributeDictionary, ParamsAttributeDictionary
+from resource_management.core.utils import AttributeDictionary, checked_unite
 from resource_management.core.system import System
-from string import Template
 
 
 class Environment(object):
@@ -26,6 +27,7 @@ class Environment(object):
     @param params: configurations dictionary (this will be accessible in the templates)
     """
     self.log = logging.getLogger("resource_management")
+    self.formatter = ConfigurationFormatter()
     self.reset(basedir, params)
 
   def reset(self, basedir, params):
@@ -44,7 +46,7 @@ class Environment(object):
       # dir where templates,failes dirs are 
       'basedir': basedir, 
       # variables, which can be used in templates
-      'params': ParamsAttributeDictionary(Substitutor, params.copy()), 
+      'params': params.copy(),
     })
 
   def backup_file(self, path):
@@ -169,18 +171,20 @@ class Environment(object):
     self.resource_list = state['resource_list']
     self.delayed_actions = state['delayed_actions']
     
-
-class Substitutor():
-  log = logging.getLogger("resource_management.resource")
-  @staticmethod
-  def substitute(val):
+class ConfigurationFormatter(Formatter):
+  def format(self, format_string, *args, **kwargs):
     env = Environment.get_instance()
-    dic = env.config.params
+    variables = kwargs
+    params = env.config.params
     
-    if dic and isinstance(val, str):
-      result = Template(val).substitute(dic)
-      if '$' in val:
-        Substitutor.log.debug("%s after substitution is %s", val, result)
-      return result
-        
-    return val
+    result = checked_unite(variables, params)
+    return self.vformat(format_string, args, result)
+  
+def format(format_string, *args, **kwargs):
+  env = Environment.get_instance()
+  variables = sys._getframe(1).f_locals
+  
+  result = checked_unite(kwargs, variables)
+  result.pop("self", None) # self kwarg would result in an error
+  return env.formatter.format(format_string, args, **result)
+  

ambari-agent/src/main/python/resource_management/core/source.py

@@ -1,5 +1,6 @@
 from __future__ import with_statement
-from resource_management.core.environment import Substitutor, Environment
+from resource_management.core.environment import Environment
+from resource_management.core.utils import checked_unite
 
 __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"]
 
@@ -10,9 +11,9 @@ import urlparse
 
 
 class Source(object):
-  def __init__(self, name, env=None):
-    self.name = Substitutor.substitute(name)
-    self.env = env or Environment.get_instance()
+  def __init__(self, name):
+    self.env = Environment.get_instance()
+    self.name = name
     
   def get_content(self):
     raise NotImplementedError()
@@ -25,8 +26,8 @@ class Source(object):
 
 
 class StaticFile(Source):
-  def __init__(self, name, env=None):
-    super(StaticFile, self).__init__(name, env)
+  def __init__(self, name):
+    super(StaticFile, self).__init__(name)
 
   def get_content(self):
     # absolute path
@@ -42,7 +43,7 @@ class StaticFile(Source):
 
 
 try:
-  from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader
+  from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined
 except ImportError:
   class Template(Source):
     def __init__(self, name, variables=None, env=None):
@@ -73,14 +74,17 @@ else:
       return source, path, lambda: mtime == os.path.getmtime(path)
 
   class Template(Source):
-    def __init__(self, name, variables=None, env=None):
-      super(Template, self).__init__(name, env)
+    def __init__(self, name, **kwargs):
+      """
+      @param kwargs: Additional variables passed to template
+      """
+      super(Template, self).__init__(name)
       params = self.env.config.params
-      variables = params if params else variables
+      variables = checked_unite(params, kwargs)
       self.context = variables.copy() if variables else {}
       if not hasattr(self, 'template_env'):
         self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env),
-                                        autoescape=False)
+                                        autoescape=False, undefined=StrictUndefined)
       self.template = self.template_env.get_template(self.name)     
 
     def get_content(self):
@@ -94,13 +98,13 @@ else:
       return rendered + "\n" if not rendered.endswith('\n') else rendered
     
   class InlineTemplate(Template):
-    def __init__(self, name, variables=None, env=None):
+    def __init__(self, name, **kwargs):
       self.template_env = JinjaEnvironment(loader=FunctionLoader(lambda text: text))
-      super(InlineTemplate, self).__init__(name, variables, env) 
+      super(InlineTemplate, self).__init__(name, **kwargs) 
 
 
 class DownloadSource(Source):
-  def __init__(self, name, cache=True, md5sum=None, env=None):
+  def __init__(self, name, cache=True, md5sum=None):
     super(DownloadSource, self).__init__(name)
     self.url = self.name
     self.md5sum = md5sum

ambari-agent/src/main/python/resource_management/core/utils.py

@@ -54,7 +54,10 @@ class AttributeDictionary(object):
     return self._dict.__repr__()
 
   def __unicode__(self):
-    return self._dict.__unicode__()
+    if isinstance(self._dict, str):
+      return self._dict.__unicode__()
+    else:
+      return str(self._dict)
 
   def __str__(self):
     return self._dict.__str__()
@@ -68,27 +71,12 @@ class AttributeDictionary(object):
   def __setstate__(self, state):
     super(AttributeDictionary, self).__setattr__("_dict", state)
     
-class ParamsAttributeDictionary(AttributeDictionary):
-  """
-  This class can store user parameters
-  and it supports some features necessary for substitution to work.
-  """
-  def __init__(self, substitutor, *args, **kwargs):
-    super(ParamsAttributeDictionary, self).__init__(*args, **kwargs)
-    super(AttributeDictionary, self).__setattr__("substitutor", substitutor)
-
-  def __getitem__(self, name):
-    try:
-      return self._convert_value(self._dict[name])
-    except KeyError as ex:
-      raise Fail("Configuration $%s not found!" % str(ex).strip("'"))
+def checked_unite(dict1, dict2):
+  for key in dict1:
+    if key in dict2:
+      raise Fail("Variable '%s' already exists more than once as a variable/configuration/kwarg parameter. Cannot evaluate it." % key)
   
-  def copy(self):
-    # don't allow real copying to be able to change params passed to jinja2
-    return self
-
-  def __unicode__(self):
-    if isinstance(self._dict, str):
-      return self._dict.__unicode__()
-    else:
-      return str(self._dict)
+  result = dict1.copy()
+  result.update(dict2)
+  
+  return result

ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py

@@ -2,21 +2,19 @@ from resource_management import *
 
 class ExecuteHadoopProvider(Provider):
   def action_run(self):
-    if self.resource.security_enabled and not self.resource.kinit_override:
-      kinit_if_needed = "su - {user} -c '{kinit_path_local} -kt {keytab} {principal}'".format(
-            user=self.resource.user, kinit_path_local=self.resource.kinit_path_local, 
-            keytab=self.resource.keytab, principal=self.resource.principal)
-    else:
-      kinit_if_needed = ""
-
-    cmd = "hadoop --config {conf_dir} {command}".format(conf_dir=self.resource.conf_dir, command=self.resource.command)
+    kinit_path_local = self.resource.kinit_path_local
+    keytab = self.resource.keytab
+    principal = self.resource.principal
+    conf_dir = self.resource.conf_dir
+    command = self.resource.command
     
-    if kinit_if_needed != "":
-      Execute ( kinit_if_needed,
-        path = ['/bin']
+    if self.resource.security_enabled and not self.resource.kinit_override:
+      Execute ((kinit_path_local, '-kt', keytab, principal),
+        path = ['/bin'],
+        user = self.resource.user
       )
-  
-    Execute ( cmd,
+
+    Execute (('hadoop', '--config', conf_dir, command),
       user        = self.resource.user,
       tries       = self.resource.tries,
       try_sleep   = self.resource.try_sleep,