AMBARI-10562. Summary:Issues when log/pid dirs are customized - Kafka and Knox (Emil Anca via rlevas)

ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java

@@ -28,6 +28,7 @@ import java.util.Set;
 
 import javax.naming.NamingException;
 import javax.naming.directory.Attributes;
+import javax.naming.directory.SearchControls;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.configuration.Configuration;
@@ -37,6 +38,7 @@ import org.apache.ambari.server.security.authorization.User;
 import org.apache.ambari.server.security.authorization.Users;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.springframework.ldap.control.PagedResultsDirContextProcessor;
 import org.springframework.ldap.core.AttributesMapper;
 import org.springframework.ldap.core.ContextMapper;
 import org.springframework.ldap.core.DirContextAdapter;
@@ -83,6 +85,7 @@ public class AmbariLdapDataPopulator {
   // Constants
   private static final String UID_ATTRIBUTE          = "uid";
   private static final String OBJECT_CLASS_ATTRIBUTE = "objectClass";
+  private static final int USERS_PAGE_SIZE = 500;
 
   /**
    * Construct an AmbariLdapDataPopulator.
@@ -515,12 +518,19 @@ public class AmbariLdapDataPopulator {
     final Set<LdapUserDto> users = new HashSet<LdapUserDto>();
     final LdapTemplate ldapTemplate = loadLdapTemplate();
     String baseDn = ldapServerProperties.getBaseDN();
-
-    for (Object dto: ldapTemplate.search(baseDn, filter.encode(), new LdapUserContextMapper(ldapServerProperties))) {
-      if (dto != null ) {
-        users.add((LdapUserDto) dto);
+    PagedResultsDirContextProcessor processor = createPagingProcessor();
+    SearchControls searchControls = new SearchControls();
+    searchControls.setSearchScope(SearchControls.SUBTREE_SCOPE);
+    LdapUserContextMapper ldapUserContextMapper = new LdapUserContextMapper(ldapServerProperties);
+    String encodedFilter = filter.encode();
+    
+    do {
+      for (Object dto : ldapTemplate.search(baseDn, encodedFilter, searchControls, ldapUserContextMapper, processor)) {
+        if (dto != null) {
+          users.add((LdapUserDto)dto);
+        }
       }
-    }
+    } while (processor.getCookie().getCookie() != null);
     return users;
   }
 
@@ -583,6 +593,11 @@ public class AmbariLdapDataPopulator {
       ldapServerProperties = properties;
 
       final LdapContextSource ldapContextSource = createLdapContextSource();
+	  
+      // The LdapTemplate by design will close the connection after each call to the LDAP Server
+      // In order to have the interaction work with large/paged results, said connection must be pooled and reused
+      ldapContextSource.setPooled(true);
+
       final List<String> ldapUrls = ldapServerProperties.getLdapUrls();
       ldapContextSource.setUrls(ldapUrls.toArray(new String[ldapUrls.size()]));
 
@@ -616,6 +631,14 @@ public class AmbariLdapDataPopulator {
     return new LdapContextSource();
   }
 
+  /**
+   * PagedResultsDirContextProcessor factory method.
+   * @return new processor;
+   */
+  protected PagedResultsDirContextProcessor createPagingProcessor() {
+    return new PagedResultsDirContextProcessor(USERS_PAGE_SIZE, null);
+  }
+
   /**
    * LdapTemplate factory method.
    *

ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java

@@ -43,12 +43,16 @@ import org.easymock.Capture;
 import org.easymock.EasyMock;
 import org.easymock.IAnswer;
 import org.junit.Test;
+import org.springframework.ldap.control.PagedResultsCookie;
+import org.springframework.ldap.control.PagedResultsDirContextProcessor;
 import org.springframework.ldap.core.AttributesMapper;
 import org.springframework.ldap.core.ContextMapper;
 import org.springframework.ldap.core.DirContextAdapter;
 import org.springframework.ldap.core.LdapTemplate;
 import org.springframework.ldap.core.support.LdapContextSource;
 
+import javax.naming.directory.SearchControls;
+
 import static junit.framework.Assert.*;
 import static org.easymock.EasyMock.*;
 import static org.easymock.EasyMock.createNiceMock;
@@ -69,6 +73,7 @@ public class AmbariLdapDataPopulatorTest {
 
     protected LdapTemplate ldapTemplate;
     private LdapContextSource ldapContextSource;
+    private PagedResultsDirContextProcessor processor;
 
     public TestAmbariLdapDataPopulator(Configuration configuration, Users users) {
       super(configuration, users);
@@ -85,10 +90,19 @@ public class AmbariLdapDataPopulatorTest {
       return ldapTemplate;
     }
 
+    @Override
+    protected PagedResultsDirContextProcessor createPagingProcessor() {
+      return processor;
+    }
+
     public void setLdapContextSource(LdapContextSource ldapContextSource) {
       this.ldapContextSource = ldapContextSource;
     }
 
+    public void setProcessor(PagedResultsDirContextProcessor processor) {
+      this.processor = processor;
+    }
+
     public void setLdapTemplate(LdapTemplate ldapTemplate) {
       this.ldapTemplate = ldapTemplate;
     }
@@ -1469,6 +1483,9 @@ public class AmbariLdapDataPopulatorTest {
     LdapTemplate ldapTemplate = createNiceMock(LdapTemplate.class);
     LdapServerProperties ldapServerProperties = createNiceMock(LdapServerProperties.class);
     Capture<ContextMapper> contextMapperCapture = new Capture<ContextMapper>();
+    Capture<SearchControls> searchControlsCapture = new Capture<SearchControls>();
+    PagedResultsDirContextProcessor processor = createNiceMock(PagedResultsDirContextProcessor.class);
+    PagedResultsCookie cookie = createNiceMock(PagedResultsCookie.class);
     LdapUserDto dto = new LdapUserDto();
 
     List<LdapUserDto> list = new LinkedList<LdapUserDto>();
@@ -1478,22 +1495,25 @@ public class AmbariLdapDataPopulatorTest {
     expect(ldapServerProperties.getUserObjectClass()).andReturn("objectClass").anyTimes();
     expect(ldapServerProperties.getDnAttribute()).andReturn("dn").anyTimes();
     expect(ldapServerProperties.getBaseDN()).andReturn("baseDN").anyTimes();
+    expect(processor.getCookie()).andReturn(cookie).anyTimes();
+    expect(cookie.getCookie()).andReturn(null).anyTimes();
 
     expect(ldapTemplate.lookup(eq("uid=foo,dc=example,dc=com"), capture(contextMapperCapture))).andReturn(dto);
 
     expect(ldapTemplate.lookup(eq("foo"), capture(contextMapperCapture))).andReturn(null);
-    expect(ldapTemplate.search(eq("baseDN"), eq("(&(objectClass=objectClass)(|(dn=foo)(uid=foo)))"), capture(contextMapperCapture))).andReturn(list);
+    expect(ldapTemplate.search(eq("baseDN"), eq("(&(objectClass=objectClass)(|(dn=foo)(uid=foo)))"), anyObject(SearchControls.class), capture(contextMapperCapture), eq(processor))).andReturn(list);
 
-    replay(ldapTemplate, ldapServerProperties, users, configuration);
+    replay(ldapTemplate, ldapServerProperties, users, configuration, processor, cookie);
 
     AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
 
     populator.setLdapTemplate(ldapTemplate);
+    populator.setProcessor(processor);
 
     assertEquals(dto, populator.getLdapUserByMemberAttr("uid=foo,dc=example,dc=com"));
     assertEquals(dto, populator.getLdapUserByMemberAttr("foo"));
 
-    verify(ldapTemplate, ldapServerProperties, users, configuration);
+    verify(ldapTemplate, ldapServerProperties, users, configuration, processor, cookie);
   }
 
   @Test