AMBARI-4036. RMF: change the way we work with not existant configs

(aonishuk)

ambari-agent/src/main/python/resource_management/core/environment.py

@@ -100,7 +100,6 @@ class Environment(object):
       # don't include system variables, methods, classes, modules
       if not variable.startswith("__") and \
           not hasattr(value, '__call__')and \
-          not hasattr(value, '__module__') and \
           not hasattr(value, '__file__'):
         self.config.params[variable] = value
         

ambari-agent/src/main/python/resource_management/libraries/functions/default.py

@@ -27,7 +27,7 @@ from resource_management.libraries.script import Script
 default_subdict='/configurations/global'
 log = logging.getLogger('resource_management')
 
-def default(name, default_value=None):
+def default(name, default_value):
   subdicts = filter(None, name.split('/'))
   
   if not name.startswith('/'):

ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py

@@ -39,9 +39,14 @@ class ConfigDictionary(dict):
 
   def __getitem__(self, name):
     """
-    Use Python types
+    - use Python types
+    - enable lazy failure for unknown configs. 
     """
-    value = super(ConfigDictionary, self).__getitem__(name)
+    try:
+      value = super(ConfigDictionary, self).__getitem__(name)
+    except KeyError:
+      return UnknownConfiguration(name)
+      
     
     if value == "true":
       value = True
@@ -56,4 +61,15 @@ class ConfigDictionary(dict):
         except (ValueError, TypeError):
           pass
     
-    return value
+    return value
+  
+  
+class UnknownConfiguration():
+  """
+  Lazy failing for unknown configs.
+  """
+  def __init__(self, name):
+    self.name = name
+   
+  def __getattr__(self, name):
+    raise Fail("Configuration parameter '"+self.name+"' was not found in configurations dictionary!")

ambari-agent/src/main/python/resource_management/libraries/script/repo_installer.py

@@ -41,7 +41,6 @@ class RepoInstaller():
     @param action: "delete" or "create"
     @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
     """
-    print repo_string
     repo_dicts = json.loads(repo_string)
     
     if not isinstance(repo_dicts, list):

ambari-agent/src/test/python/resource_management/TestScript.py

@@ -47,40 +47,48 @@ class TestScript(TestCase):
 
   @patch("resource_management.core.providers.package.PackageProvider")
   def test_install_packages(self, package_provider_mock):
-    no_such_entry_config = {
+    no_packages_config = {
+      'hostLevelParams' : {
+        'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+      }
     }
     empty_config = {
       'hostLevelParams' : {
-        'package_list' : ''
+        'package_list' : '',
+        'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
       }
     }
     dummy_config = {
       'hostLevelParams' : {
         'package_list' : "[{\"type\":\"rpm\",\"name\":\"hbase\"},"
-                         "{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]"
+                         "{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]",
+        'repo_info' : "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
+        'service_repo_info' : "[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
       }
     }
 
     # Testing config without any keys
     with Environment(".", test_mode=True) as env:
       script = Script()
-      Script.config = no_such_entry_config
+      Script.config = no_packages_config
       script.install_packages(env)
-    self.assertEquals(len(env.resource_list), 0)
+    resource_dump = pprint.pformat(env.resource_list)
+    self.assertEquals(resource_dump, "[Repository['HDP-2.0._'], Repository['HDP-2.0._']]")
 
     # Testing empty package list
     with Environment(".", test_mode=True) as env:
       script = Script()
       Script.config = empty_config
       script.install_packages(env)
-    self.assertEquals(len(env.resource_list), 0)
+    resource_dump = pprint.pformat(env.resource_list)
+    self.assertEquals(resource_dump, "[Repository['HDP-2.0._'], Repository['HDP-2.0._']]")
 
     # Testing installing of a list of packages
     with Environment(".", test_mode=True) as env:
       Script.config = dummy_config
       script.install_packages("env")
     resource_dump = pprint.pformat(env.resource_list)
-    self.assertEqual(resource_dump, "[Package['hbase'], Package['yet-another-package']]")
+    self.assertEqual(resource_dump, "[Repository['HDP-2.0._'],\n Repository['HDP-2.0._'],\n Package['hbase'],\n Package['yet-another-package'],\n Repository['HDP-2.0._'],\n Repository['HDP-2.0._']]")
 
   @patch("__builtin__.open")
   def test_structured_out(self, open_mock):

ambari-server/src/main/resources/stacks/HDP/1.3._/services/SQOOP/package/scripts/params.py

@@ -31,7 +31,6 @@ zoo_conf_dir = "/etc/zookeeper"
 sqoop_lib = "/usr/lib/sqoop/lib"
 sqoop_user = "sqoop"
 
-keytab_path = default("keytab_path")
-smoke_user_keytab = default('smokeuser_keytab')
-kinit_path_local = functions.get_kinit_path(
-  [default('kinit_path_local'), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+keytab_path = config['configurations']['global']['keytab_path']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.0._/metainfo.xml

@@ -17,6 +17,6 @@
 -->
 <metainfo>
     <versions>
-	  <active>false</active>
+	  <active>true</active>
     </versions>
 </metainfo>

ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/hbase_client.py

@@ -41,10 +41,12 @@ class HbaseClient(Script):
 def main():
   command_type = 'install'
   command_data_file = '/root/workspace/HBase/input.json'
-  basedir = '/root/workspace/HBase/main'
-  sys.argv = ["", command_type, command_data_file, basedir]
+  basedir = '/root/workspace/HBase/'
+  stdoutfile = '/1.txt'
+  sys.argv = ["", command_type, command_data_file, basedir, stdoutfile]
   
   HbaseClient().execute()
   
 if __name__ == "__main__":
-  HbaseClient().execute()
+  #HbaseClient().execute()
+  main()

ambari-server/src/main/resources/stacks/HDP/2.0._/services/HBASE/package/scripts/params.py

@@ -26,6 +26,8 @@ import functions
 # server configurations
 config = Script.get_config()
 
+abc = config['configurations']['global']['wtf']
+
 conf_dir = "/etc/hbase/conf"
 daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
 
@@ -63,7 +65,7 @@ if security_enabled:
   
   _use_hostname_in_principal = default('instance_name', True)
   _master_primary_name = config['configurations']['global']['hbase_master_primary_name']
-  _hostname = default('/hostname')
+  _hostname = config['hostname']
   _kerberos_domain = config['configurations']['global']['kerberos_domain']
   _master_principal_name = config['configurations']['global']['hbase_master_principal_name']
   _regionserver_primary_name = config['configurations']['global']['hbase_regionserver_primary_name']
@@ -75,11 +77,8 @@ if security_enabled:
     master_jaas_princ = format("{_master_principal_name}@{_kerberos_domain}")
     regionserver_jaas_princ = format("{_regionserver_primary_name}@{_kerberos_domain}")
     
-master_keytab_path = default('/configurations/hbase-site/hbase.master.keytab.file')
-regionserver_keytab_path = default('/configurations/hbase-site/hbase.regionserver.keytab.file')
-smoke_user_keytab = default('smokeuser_keytab')
-hbase_user_keytab = default('hbase_user_keytab')
-kinit_path_local = get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-
-# fix exeuteHadoop calls for secured cluster
-# to string template...
+master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
+regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+hbase_user_keytab = config['configurations']['global']['hbase_user_keytab']
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.0._/services/HIVE/package/scripts/params.py

@@ -29,7 +29,7 @@ hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.opti
 hive_server_conf_dir = "/etc/hive/conf.server"
 hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
 
-hive_metastore_user_passwd = default("/configurations/hive-site/javax.jdo.option.ConnectionPassword")
+hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
 
 #users
 hive_user = config['configurations']['global']['hive_user']
@@ -53,11 +53,11 @@ hive_url = format("jdbc:hive2://{hive_server_host}:10000")
 smokeuser = config['configurations']['global']['smokeuser']
 smoke_test_sql = "/tmp/hiveserver2.sql"
 smoke_test_path = "/tmp/hiveserver2Smoke.sh"
-smoke_user_keytab = default("smokeuser_keytab")
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
 hive_metatore_keytab_path = default("hive-site/hive.metastore.kerberos.keytab.file","/etc/security/keytabs/hive.service.keytab")
 
 #hive_env
@@ -93,13 +93,13 @@ driver_curl_source = format("{jdk_location}/{jdbc_jar_name}")
 start_hiveserver2_path = "/tmp/start_hiveserver2_script"
 start_metastore_path = "/tmp/start_metastore_script"
 
-hive_aux_jars_path = config['configurations']['global']['hive_aux_jars_path'] #'/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar'
-hadoop_heapsize = config['configurations']['global']['hadoop_heapsize'] #default("hadoop_heapsize","1024")
-java64_home = config['configurations']['global']['java64_home'] #default("java64_home","/usr/jdk64/jdk1.6.0_31")
+hive_aux_jars_path = config['configurations']['global']['hive_aux_jars_path']
+hadoop_heapsize = config['configurations']['global']['hadoop_heapsize']
+java64_home = config['configurations']['global']['java64_home']
 
 ##### MYSQL
 
-db_name = config['configurations']['global']['hive_database_name'] #default("hive_database_name","hive")
+db_name = config['configurations']['global']['hive_database_name']
 mysql_user = "mysql"
 mysql_group = 'mysql'
 mysql_host = config['clusterHostInfo']['hive_mysql_host']

ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/params.py

@@ -23,11 +23,11 @@ oozie_libext_dir = "/usr/lib/oozie/libext"
 lzo_enabled = config['configurations']['global']['lzo_enabled']
 security_enabled = config['configurations']['global']['security_enabled']
 
-kinit_path_local = get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-oozie_service_keytab = default("/configurations/oozie-site/oozie.service.HadoopAccessorService.keytab.file")
-oozie_principal = default("/configurations/oozie-site/oozie.service.HadoopAccessorService.kerberos.principal")
-smokeuser_keytab = default("/configurations/global/smokeuser_keytab")
-oozie_keytab = default("keytab_path")
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+oozie_service_keytab = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.keytab.file']
+oozie_principal = config['configurations']['oozie-site']['oozie.service.HadoopAccessorService.kerberos.principal']
+smokeuser_keytab = config['configurations']['global']['smokeuser_keytab']
+oozie_keytab = config['configurations']['global']['keytab_path']
 
 oracle_driver_jar_name = "ojdbc6.jar"
 java_share_dir = "/usr/share/java"

ambari-server/src/main/resources/stacks/HDP/2.0._/services/OOZIE/package/scripts/service_check.py

@@ -19,9 +19,14 @@ def oozie_smoke_shell_file(
     content = StaticFile(file_name),
     mode = 0755
   )
+  
+  if params.security_enabled:
+    sh_cmd = format("sh /tmp/{file_name} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local}")
+  else:
+    sh_cmd = format("sh /tmp/{file_name} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled}")
 
   Execute( format("/tmp/{file_name}"),
-    command   = format("sh /tmp/{file_name} {conf_dir} {hadoop_conf_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local}"),
+    command   = sh_cmd,
     tries     = 3,
     try_sleep = 5,
     logoutput = True

ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py

@@ -34,7 +34,6 @@ zoo_conf_dir = "/etc/zookeeper"
 sqoop_lib = "/usr/lib/sqoop/lib"
 sqoop_user = "sqoop"
 
-keytab_path = default("keytab_path")
-smoke_user_keytab = default('smokeuser_keytab')
-kinit_path_local = functions.get_kinit_path(
-  [default('kinit_path_local'), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+keytab_path = config['configurations']['global']['keytab_path']
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])

ambari-server/src/main/resources/stacks/HDP/2.0._/services/WEBHCAT/package/scripts/params.py

@@ -44,7 +44,7 @@ user_group = config['configurations']['global']['user_group']
 webhcat_server_host = config['clusterHostInfo']['webhcat_server_host']
 
 webhcat_apps_dir = "/apps/webhcat"
-smoke_user_keytab = default("smokeuser_keytab")
+smoke_user_keytab = config['configurations']['global']['smokeuser_keytab']
 smokeuser = config['configurations']['global']['smokeuser']
 security_enabled = config['configurations']['global']['security_enabled']
-kinit_path_local = get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+kinit_path_local = get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])