AMBARI-4684. Password with special symbols cause deploy fail (aonishuk)

ambari-agent/src/main/python/resource_management/libraries/functions/format.py

@@ -22,6 +22,7 @@ Ambari Agent
 
 __all__ = ["format"]
 import sys
+import pipes
 from string import Formatter
 from resource_management.core.exceptions import Fail
 from resource_management.core.utils import checked_unite
@@ -30,6 +31,12 @@ from resource_management.core.logger import Logger
 
 
 class ConfigurationFormatter(Formatter):
+  """
+  Flags:
+  !e - escape bash properties flag
+  !h - hide sensitive information from the logs
+  !p - password flag, !p=!s+!e. Has both !e, !h effect
+  """
   def format(self, format_string, *args, **kwargs):
     env = Environment.get_instance()
     variables = kwargs
@@ -48,18 +55,22 @@ class ConfigurationFormatter(Formatter):
     return result_unprotected
   
   def convert_field_unprotected(self, value, conversion):
-    if conversion == 'p':
-      return value
-      
-    return super(ConfigurationFormatter, self).convert_field(value, conversion)
+    return self._convert_field(value, conversion, False)
   
   def convert_field_protected(self, value, conversion):
     """
     Enable masking sensitive information like
     passwords from logs via !p (password) format flag.
     """
-    if conversion == 'p':
-      return "[PROTECTED]"
+    return self._convert_field(value, conversion, True)
+  
+  def _convert_field(self, value, conversion, is_protected):
+    if conversion == 'e':
+      return pipes.quote(value)
+    elif conversion == 'h':
+      return "[PROTECTED]" if is_protected else value
+    elif conversion == 'p':
+      return "[PROTECTED]" if is_protected else self._convert_field(value, 'e', is_protected)
       
     return super(ConfigurationFormatter, self).convert_field(value, conversion)
       

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_metastore.py

@@ -76,7 +76,7 @@ class TestHiveMetastore(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive q com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 
@@ -146,7 +146,7 @@ class TestHiveMetastore(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive q com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py

@@ -76,7 +76,7 @@ class TestHiveServer(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive q com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 
@@ -147,7 +147,7 @@ class TestHiveServer(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive q com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_mysql_server.py

@@ -106,7 +106,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive q c6402.ambari.apache.org',
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive \'!`"\'"\'"\' 1\' c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
@@ -127,7 +127,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive q c6402.ambari.apache.org',
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive \'!`"\'"\'"\' 1\' c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,

ambari-server/src/test/python/stacks/1.3.2/NAGIOS/test_nagios_server.py

@@ -248,7 +248,7 @@ class TestNagiosServer(RMFTestCase):
                               mode=0755
     )
     self.assertResourceCalled('Execute',
-                              'htpasswd2 -c -b  /etc/nagios/htpasswd.users nagiosadmin q',
+                              'htpasswd2 -c -b  /etc/nagios/htpasswd.users nagiosadmin \'!`"\'"\'"\' 1\'',
                               not_if="grep nagiosadmin /etc/nagios/htpasswd.users"
     )
 

ambari-server/src/test/python/stacks/1.3.2/configs/default.json

@@ -196,7 +196,7 @@
             "nagios_web_login": "nagiosadmin", 
             "nagios_contact": "q@q.q", 
             "hive_database": "New MySQL Database", 
-            "nagios_web_password": "q", 
+            "nagios_web_password": "!`\"' 1",
             "clientPort": "2181", 
             "oozie_derby_database": "Derby", 
             "snappy_enabled": "true", 
@@ -320,7 +320,7 @@
         }, 
         "hive-site": {
             "hive.enforce.sorting": "true", 
-            "javax.jdo.option.ConnectionPassword": "q", 
+            "javax.jdo.option.ConnectionPassword": "!`\"' 1", 
             "javax.jdo.option.ConnectionDriverName": "com.mysql.jdbc.Driver", 
             "hive.optimize.bucketmapjoin.sortedmerge": "true", 
             "fs.file.impl.disable.cache": "true", 

ambari-server/src/test/python/stacks/1.3.2/configs/secured.json

@@ -287,7 +287,7 @@
             "user_group": "hadoop", 
             "hive_user": "hive", 
             "webHCat_http_primary_name": "HTTP", 
-            "nagios_web_password": "q", 
+            "nagios_web_password": "!`\"' 1", 
             "smokeuser": "ambari-qa", 
             "ganglia_conf_dir": "/etc/ganglia/hdp", 
             "hbase_master_heapsize": "1024m", 
@@ -419,7 +419,7 @@
         }, 
         "hive-site": {
             "hive.enforce.sorting": "true", 
-            "javax.jdo.option.ConnectionPassword": "q", 
+            "javax.jdo.option.ConnectionPassword": "!`\"' 1", 
             "javax.jdo.option.ConnectionDriverName": "com.mysql.jdbc.Driver", 
             "hive.optimize.bucketmapjoin.sortedmerge": "true", 
             "fs.file.impl.disable.cache": "true", 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_metastore.py

@@ -59,7 +59,7 @@ class TestHiveMetastore(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 
@@ -129,7 +129,7 @@ class TestHiveMetastore(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -76,7 +76,7 @@ class TestHiveServer(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 
@@ -147,7 +147,7 @@ class TestHiveServer(RMFTestCase):
                               user = 'hive'
     )
 
-    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive asd com.mysql.jdbc.Driver',
+    self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin']
     )
 

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_mysql_server.py

@@ -107,7 +107,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive asd c6402.ambari.apache.org',
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive \'!`"\'"\'"\' 1\' c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,
@@ -128,7 +128,7 @@ class TestMySqlServer(RMFTestCase):
       content = StaticFile('addMysqlUser.sh'),
       mode = 493,
     )
-    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive asd c6402.ambari.apache.org',
+    self.assertResourceCalled('Execute', 'bash -x /tmp/addMysqlUser.sh mysql hive \'!`"\'"\'"\' 1\' c6402.ambari.apache.org',
       path = ['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
       tries = 3,
       try_sleep = 5,

ambari-server/src/test/python/stacks/2.0.6/NAGIOS/test_nagios_server.py

@@ -247,7 +247,7 @@ class TestNagiosServer(RMFTestCase):
                               mode=0755
     )
     self.assertResourceCalled('Execute',
-                              'htpasswd2 -c -b  /etc/nagios/htpasswd.users nagiosadmin asd',
+                              'htpasswd2 -c -b  /etc/nagios/htpasswd.users nagiosadmin \'!`"\'"\'"\' 1\'',
                               not_if="grep nagiosadmin /etc/nagios/htpasswd.users"
     )
     self.assertResourceCalled('File', '/etc/nagios/htpasswd.users',

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -231,7 +231,7 @@
             "yarn_log_dir_prefix": "/var/log/hadoop-yarn", 
             "zk_log_dir": "/var/log/zookeeper", 
             "hive_aux_jars_path": "/usr/lib/hcatalog/share/hcatalog/hcatalog-core.jar", 
-            "nagios_web_password": "asd", 
+            "nagios_web_password": "!`\"' 1", 
             "tickTime": "2000", 
             "hive_ambari_database": "MySQL", 
             "falcon_user": "falcon", 
@@ -410,7 +410,7 @@
         }, 
         "hive-site": {
             "hive.enforce.sorting": "true", 
-            "javax.jdo.option.ConnectionPassword": "asd", 
+            "javax.jdo.option.ConnectionPassword": "!`\"' 1", 
             "javax.jdo.option.ConnectionDriverName": "com.mysql.jdbc.Driver", 
             "hive.optimize.bucketmapjoin.sortedmerge": "true", 
             "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider", 

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -376,7 +376,7 @@
             "hive_user": "hive", 
             "resourcemanager_http_principal_name": "HTTP/_HOST", 
             "webHCat_http_primary_name": "HTTP", 
-            "nagios_web_password": "asd", 
+            "nagios_web_password": "!`\"' 1", 
             "smokeuser": "ambari-qa", 
             "hbase_master_heapsize": "1024m", 
             "kerberos_install_type": "MANUALLY_SET_KERBEROS", 
@@ -529,7 +529,7 @@
         }, 
         "hive-site": {
             "hive.enforce.sorting": "true", 
-            "javax.jdo.option.ConnectionPassword": "asd", 
+            "javax.jdo.option.ConnectionPassword": "!`\"' 1", 
             "javax.jdo.option.ConnectionDriverName": "com.mysql.jdbc.Driver", 
             "hive.optimize.bucketmapjoin.sortedmerge": "true", 
             "hive.security.metastore.authorization.manager": "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider",