AMBARI-9576 - LDAP Sync: Port and Protocol hard coded /usr/sbin/ambari-server.py (tbeerbower)

ambari-server/src/main/python/ambari-server.py

@@ -336,13 +336,13 @@ def init_parser_options(parser):
                     help="Start ambari-server in debug mode")
   parser.add_option('-y', '--suspend-start', action="store_true", dest='suspend_start', default=False,
                     help="Freeze ambari-server Java process at startup in debug mode")
-  parser.add_option('--all', action="store_true", default=False, help="LDAP sync all Ambari users and groups",
+  parser.add_option('--all', action="store_true", default=False, help="LDAP sync all option.  Synchronize all LDAP users and groups.",
                     dest="ldap_sync_all")
   parser.add_option('--existing', action="store_true", default=False,
-                    help="LDAP sync existing Ambari users and groups only", dest="ldap_sync_existing")
-  parser.add_option('--users', default=None, help="Specifies the path to the LDAP sync users CSV file.",
+                    help="LDAP sync existing option.  Synchronize existing Ambari users and groups only.", dest="ldap_sync_existing")
+  parser.add_option('--users', default=None, help="LDAP sync users option. Specifies the path to a CSV file of user names to be synchronized.",
                     dest="ldap_sync_users")
-  parser.add_option('--groups', default=None, help="Specifies the path to the LDAP sync groups CSV file.",
+  parser.add_option('--groups', default=None, help="LDAP sync groups option.  Specifies the path to a CSV file of group names to be synchronized.",
                     dest="ldap_sync_groups")
   parser.add_option('--database', default=None, help="Database to use embedded|oracle|mysql|postgres", dest="dbms")
   parser.add_option('--databasehost', default=None, help="Hostname of database server", dest="database_host")

ambari-server/src/main/python/ambari_server/serverConfiguration.py

@@ -156,6 +156,11 @@ SSL_TRUSTSTORE_PATH_PROPERTY = "ssl.trustStore.path"
 SSL_TRUSTSTORE_PASSWORD_PROPERTY = "ssl.trustStore.password"
 SSL_TRUSTSTORE_TYPE_PROPERTY = "ssl.trustStore.type"
 
+# SSL common
+SSL_API = 'api.ssl'
+SSL_API_PORT = 'client.api.ssl.port'
+DEFAULT_SSL_API_PORT = 8443
+
 # JDK
 JDK_RELEASES="java.releases"
 

ambari-server/src/main/python/ambari_server/setupHttps.py

@@ -33,6 +33,7 @@ from ambari_commons.os_utils import is_root, run_os_command, copy_file, set_file
 from ambari_server.serverConfiguration import get_ambari_properties, find_properties_file, read_ambari_user, \
     SSL_TRUSTSTORE_PASSWORD_PROPERTY, get_is_secure, decrypt_password_for_alias, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
     SSL_TRUSTSTORE_PATH_PROPERTY, get_value_from_properties, SSL_TRUSTSTORE_TYPE_PROPERTY, find_jdk, configDefaults, \
+    SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, \
     get_encrypted_password, GET_FQDN_SERVICE_URL
 from ambari_server.setupSecurity import adjust_directory_permissions
 from ambari_server.userInput import get_YN_input, get_validated_string_input, read_password, get_prompt_default, \
@@ -58,8 +59,6 @@ KEYTOOL_DELETE_CERT_CMD = "{0} -delete -alias '{1}' -storepass '{2}' -noprompt"
 KEYTOOL_KEYSTORE = " -keystore '{0}'"
 
 SSL_KEY_DIR = 'security.server.keys_dir'
-SSL_API_PORT = 'client.api.ssl.port'
-SSL_API = 'api.ssl'
 SSL_SERVER_CERT_NAME = 'client.api.ssl.cert_name'
 SSL_SERVER_KEY_NAME = 'client.api.ssl.key_name'
 SSL_CERT_FILE_NAME = "https.crt"
@@ -67,7 +66,6 @@ SSL_KEY_FILE_NAME = "https.key"
 SSL_KEYSTORE_FILE_NAME = "https.keystore.p12"
 SSL_KEY_PASSWORD_FILE_NAME = "https.pass.txt"
 SSL_KEY_PASSWORD_LENGTH = 50
-DEFAULT_SSL_API_PORT = 8443
 SSL_DATE_FORMAT = '%b  %d %H:%M:%S %Y GMT'
 
 #SSL certificate metainfo

ambari-server/src/main/python/ambari_server/setupSecurity.py

@@ -43,7 +43,8 @@ from ambari_server.serverConfiguration import configDefaults, \
   LDAP_MGR_PASSWORD_ALIAS, LDAP_MGR_PASSWORD_FILENAME, LDAP_MGR_PASSWORD_PROPERTY, LDAP_MGR_USERNAME_PROPERTY, \
   LDAP_PRIMARY_URL_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED, SECURITY_KEY_ENV_VAR_NAME, SECURITY_KERBEROS_JASS_FILENAME, \
   SECURITY_PROVIDER_KEY_CMD, SECURITY_MASTER_KEY_FILENAME, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
-  SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY
+  SSL_TRUSTSTORE_PASSWORD_PROPERTY, SSL_TRUSTSTORE_PATH_PROPERTY, SSL_TRUSTSTORE_TYPE_PROPERTY, \
+  SSL_API, SSL_API_PORT, DEFAULT_SSL_API_PORT, CLIENT_API_PORT
 from ambari_server.serverUtils import is_server_runing
 from ambari_server.setupActions import SETUP_ACTION, LDAP_SETUP_ACTION
 from ambari_server.userInput import get_validated_string_input, get_prompt_default, read_password, get_YN_input
@@ -57,10 +58,10 @@ REGEX_ANYTHING = ".*"
 
 CLIENT_SECURITY_KEY = "client.security"
 
-# api properties
+# Ambari server API properties
 SERVER_API_HOST = '127.0.0.1'
 SERVER_API_PROTOCOL = 'http'
-SERVER_API_PORT = '8080'
+SERVER_API_SSL_PROTOCOL = 'https'
 SERVER_API_LDAP_URL = '/api/v1/ldap_sync_events'
 
 
@@ -232,7 +233,7 @@ def sync_ldap(options):
   ldap_sync_options = LdapSyncOptions(options)
 
   if ldap_sync_options.no_ldap_sync_options_set():
-    err = 'Must specify a sync option.  Please see help for more information.'
+    err = 'Must specify a sync option (all, existing, users or groups).  Please invoke ambari-server.py --help to print the options.'
     raise FatalException(1, err)
 
   admin_login = get_validated_string_input(prompt="Enter Ambari Admin login: ", default=None,
@@ -242,7 +243,26 @@ def sync_ldap(options):
                                               pattern=None, description=None,
                                               is_pass=True, allowEmpty=False)
 
-  url = '{0}://{1}:{2!s}{3}'.format(SERVER_API_PROTOCOL, SERVER_API_HOST, SERVER_API_PORT, SERVER_API_LDAP_URL)
+  properties = get_ambari_properties()
+  if properties == -1:
+    raise FatalException(1, "Failed to read properties file.")
+
+  api_protocol = SERVER_API_PROTOCOL
+  api_port = CLIENT_API_PORT
+
+  api_ssl = False
+  api_ssl_prop = properties.get_property(SSL_API)
+  if api_ssl_prop is not None:
+    api_ssl = api_ssl_prop.lower() == "true"
+
+  if api_ssl:
+    api_protocol = SERVER_API_SSL_PROTOCOL
+    api_port = DEFAULT_SSL_API_PORT
+    api_port_prop = properties.get_property(SSL_API_PORT)
+    if api_port_prop is not None:
+      api_port = api_port_prop
+
+  url = '{0}://{1}:{2!s}{3}'.format(api_protocol, SERVER_API_HOST, api_port, SERVER_API_LDAP_URL)
   admin_auth = base64.encodestring('%s:%s' % (admin_login, admin_password)).replace('\n', '')
   request = urllib2.Request(url)
   request.add_header('Authorization', 'Basic %s' % admin_auth)
@@ -263,12 +283,13 @@ def sync_ldap(options):
 
     if ldap_sync_options.ldap_sync_users is not None:
       new_specs = [{"principal_type":"users","sync_type":"specific","names":""}]
+      get_ldap_event_spec_names(ldap_sync_options.ldap_sync_users, specs, new_specs)
     if ldap_sync_options.ldap_sync_groups is not None:
       new_specs = [{"principal_type":"groups","sync_type":"specific","names":""}]
       get_ldap_event_spec_names(ldap_sync_options.ldap_sync_groups, specs, new_specs)
 
   if get_verbose():
-    sys.stdout.write('\nCalling API ' + SERVER_API_LDAP_URL + ' : ' + str(bodies) + '\n')
+    sys.stdout.write('\nCalling API ' + url + ' : ' + str(bodies) + '\n')
 
   request.add_data(json.dumps(bodies))
   request.get_method = lambda: 'POST'

ambari-server/src/test/python/TestAmbariServer.py

@@ -69,6 +69,7 @@ with patch("platform.linux_distribution", return_value = os_distro_value):
           PERSISTENCE_TYPE_PROPERTY, JDBC_URL_PROPERTY, get_conf_dir, JDBC_USER_NAME_PROPERTY, JDBC_PASSWORD_PROPERTY, \
           JDBC_DATABASE_NAME_PROPERTY, OS_TYPE_PROPERTY, validate_jdk, JDBC_POSTGRES_SCHEMA_PROPERTY, \
           RESOURCES_DIR_PROPERTY, JDBC_RCA_PASSWORD_ALIAS, JDBC_RCA_SCHEMA_PROPERTY, IS_LDAP_CONFIGURED, \
+          SSL_API, SSL_API_PORT, \
           LDAP_MGR_PASSWORD_PROPERTY, LDAP_MGR_PASSWORD_ALIAS, JDBC_PASSWORD_FILENAME, NR_USER_PROPERTY, SECURITY_KEY_IS_PERSISTED, \
           SSL_TRUSTSTORE_PASSWORD_PROPERTY, SECURITY_IS_ENCRYPTION_ENABLED, SSL_TRUSTSTORE_PASSWORD_ALIAS, \
           SECURITY_MASTER_KEY_LOCATION, SECURITY_KEYS_DIR, LDAP_PRIMARY_URL_PROPERTY, store_password_file, \
@@ -5687,6 +5688,147 @@ MIIFHjCCAwYCCQDpHKOBI+Lt0zANBgkqhkiG9w0BAQUFADBRMQswCQYDVQQGEwJV
 
     sync_ldap(options)
 
+    url = '{0}://{1}:{2!s}{3}'.format('http', '127.0.0.1', '8080', '/api/v1/ldap_sync_events')
+    request = urlopen_mock.call_args_list[0][0][0]
+
+    self.assertEquals(url, str(request.get_full_url()))
+    self.assertEquals('[{"Event": {"specs": [{"principal_type": "users", "sync_type": "all"}, {"principal_type": "groups", "sync_type": "all"}]}}]', request.data)
+
+    self.assertTrue(response.getcode.called)
+    self.assertTrue(response.read.called)
+    pass
+
+  @patch("__builtin__.open")
+  @patch("os.path.exists")
+  @patch("urllib2.urlopen")
+  @patch("ambari_server.setupSecurity.get_validated_string_input")
+  @patch("ambari_server.setupSecurity.get_ambari_properties")
+  @patch("ambari_server.setupSecurity.is_server_runing")
+  @patch("ambari_server.setupSecurity.is_root")
+  def test_ldap_sync_users(self, is_root_method, is_server_runing_mock, get_ambari_properties_mock,
+                         get_validated_string_input_mock, urlopen_mock, os_path_exists_mock, open_mock):
+
+    os_path_exists_mock.return_value = 1
+    f = MagicMock()
+    f.__enter__().read.return_value = "bob, tom"
+
+    open_mock.return_value = f
+    is_root_method.return_value = True
+    is_server_runing_mock.return_value = (True, 0)
+    properties = Properties()
+    properties.process_pair(IS_LDAP_CONFIGURED, 'true')
+    get_ambari_properties_mock.return_value = properties
+    get_validated_string_input_mock.side_effect = ['admin', 'admin']
+
+    response = MagicMock()
+    response.getcode.side_effect = [201, 200, 200]
+    response.read.side_effect = ['{"resources" : [{"href" : "http://c6401.ambari.apache.org:8080/api/v1/ldap_sync_events/16","Event" : {"id" : 16}}]}',
+                                 '{"Event":{"status" : "RUNNING","summary" : {"groups" : {"created" : 0,"removed" : 0,"updated" : 0},"memberships" : {"created" : 0,"removed" : 0},"users" : {"created" : 0,"removed" : 0,"updated" : 0}}}}',
+                                 '{"Event":{"status" : "COMPLETE","summary" : {"groups" : {"created" : 1,"removed" : 0,"updated" : 0},"memberships" : {"created" : 5,"removed" : 0},"users" : {"created" : 5,"removed" : 0,"updated" : 0}}}}']
+
+    urlopen_mock.return_value = response
+
+    options = MagicMock()
+    options.ldap_sync_all = False
+    options.ldap_sync_existing = False
+    options.ldap_sync_users = 'users.txt'
+    options.ldap_sync_groups = None
+
+    sync_ldap(options)
+
+    request = urlopen_mock.call_args_list[0][0][0]
+
+    self.assertEquals('[{"Event": {"specs": [{"principal_type": "users", "sync_type": "specific", "names": "bob, tom"}]}}]', request.data)
+
+    self.assertTrue(response.getcode.called)
+    self.assertTrue(response.read.called)
+    pass
+
+  @patch("__builtin__.open")
+  @patch("os.path.exists")
+  @patch("urllib2.urlopen")
+  @patch("ambari_server.setupSecurity.get_validated_string_input")
+  @patch("ambari_server.setupSecurity.get_ambari_properties")
+  @patch("ambari_server.setupSecurity.is_server_runing")
+  @patch("ambari_server.setupSecurity.is_root")
+  def test_ldap_sync_groups(self, is_root_method, is_server_runing_mock, get_ambari_properties_mock,
+                           get_validated_string_input_mock, urlopen_mock, os_path_exists_mock, open_mock):
+
+    os_path_exists_mock.return_value = 1
+    f = MagicMock()
+    f.__enter__().read.return_value = "group1, group2"
+
+    open_mock.return_value = f
+    is_root_method.return_value = True
+    is_server_runing_mock.return_value = (True, 0)
+    properties = Properties()
+    properties.process_pair(IS_LDAP_CONFIGURED, 'true')
+    get_ambari_properties_mock.return_value = properties
+    get_validated_string_input_mock.side_effect = ['admin', 'admin']
+
+    response = MagicMock()
+    response.getcode.side_effect = [201, 200, 200]
+    response.read.side_effect = ['{"resources" : [{"href" : "http://c6401.ambari.apache.org:8080/api/v1/ldap_sync_events/16","Event" : {"id" : 16}}]}',
+                                 '{"Event":{"status" : "RUNNING","summary" : {"groups" : {"created" : 0,"removed" : 0,"updated" : 0},"memberships" : {"created" : 0,"removed" : 0},"users" : {"created" : 0,"removed" : 0,"updated" : 0}}}}',
+                                 '{"Event":{"status" : "COMPLETE","summary" : {"groups" : {"created" : 1,"removed" : 0,"updated" : 0},"memberships" : {"created" : 5,"removed" : 0},"users" : {"created" : 5,"removed" : 0,"updated" : 0}}}}']
+
+    urlopen_mock.return_value = response
+
+    options = MagicMock()
+    options.ldap_sync_all = False
+    options.ldap_sync_existing = False
+    options.ldap_sync_users = None
+    options.ldap_sync_groups = 'groups.txt'
+
+    sync_ldap(options)
+
+    request = urlopen_mock.call_args_list[0][0][0]
+
+    self.assertEquals('[{"Event": {"specs": [{"principal_type": "groups", "sync_type": "specific", "names": "group1, group2"}]}}]', request.data)
+
+    self.assertTrue(response.getcode.called)
+    self.assertTrue(response.read.called)
+    pass
+
+  @patch("urllib2.urlopen")
+  @patch("ambari_server.setupSecurity.get_validated_string_input")
+  @patch("ambari_server.setupSecurity.get_ambari_properties")
+  @patch("ambari_server.setupSecurity.is_server_runing")
+  @patch("ambari_server.setupSecurity.is_root")
+  def test_ldap_sync_ssl(self, is_root_method, is_server_runing_mock, get_ambari_properties_mock,
+                         get_validated_string_input_mock, urlopen_mock):
+
+    is_root_method.return_value = True
+    is_server_runing_mock.return_value = (True, 0)
+    properties = Properties()
+    properties.process_pair(IS_LDAP_CONFIGURED, 'true')
+    properties.process_pair(SSL_API, 'true')
+    properties.process_pair(SSL_API_PORT, '8443')
+    get_ambari_properties_mock.return_value = properties
+    get_validated_string_input_mock.side_effect = ['admin', 'admin']
+
+
+    response = MagicMock()
+    response.getcode.side_effect = [201, 200, 200]
+    response.read.side_effect = ['{"resources" : [{"href" : "https://c6401.ambari.apache.org:8443/api/v1/ldap_sync_events/16","Event" : {"id" : 16}}]}',
+                                 '{"Event":{"status" : "RUNNING","summary" : {"groups" : {"created" : 0,"removed" : 0,"updated" : 0},"memberships" : {"created" : 0,"removed" : 0},"users" : {"created" : 0,"removed" : 0,"updated" : 0}}}}',
+                                 '{"Event":{"status" : "COMPLETE","summary" : {"groups" : {"created" : 1,"removed" : 0,"updated" : 0},"memberships" : {"created" : 5,"removed" : 0},"users" : {"created" : 5,"removed" : 0,"updated" : 0}}}}']
+
+    urlopen_mock.return_value = response
+
+    options = MagicMock()
+    options.ldap_sync_all = True
+    options.ldap_sync_existing = False
+    options.ldap_sync_users = None
+    options.ldap_sync_groups = None
+
+    sync_ldap(options)
+
+    url = '{0}://{1}:{2!s}{3}'.format('https', '127.0.0.1', '8443', '/api/v1/ldap_sync_events')
+    request = urlopen_mock.call_args_list[0][0][0]
+
+    self.assertEquals(url, str(request.get_full_url()))
+
     self.assertTrue(response.getcode.called)
     self.assertTrue(response.read.called)
     pass