AMBARI-16875. LDAP sync cannot handle if the member attribute value is not DN or id (oleewere)

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -228,6 +228,10 @@ public class Configuration {
   public static final String LDAP_GROUP_SEARCH_FILTER_KEY = "authorization.ldap.groupSearchFilter";
   public static final String LDAP_REFERRAL_KEY = "authentication.ldap.referral";
   public static final String LDAP_PAGINATION_ENABLED_KEY = "authentication.ldap.pagination.enabled";
+  public static final String LDAP_SYCN_USER_MEMBER_REPLACE_PATTERN = "authentication.ldap.sync.userMemberReplacePattern";
+  public static final String LDAP_SYCN_GROUP_MEMBER_REPLACE_PATTERN = "authentication.ldap.sync.groupMemberReplacePattern";
+  public static final String LDAP_SYCN_USER_MEMBER_FILTER = "authentication.ldap.sync.userMemberFilter";
+  public static final String LDAP_SYCN_GROUP_MEMBER_FILTER = "authentication.ldap.sync.groupMemberFilter";
   public static final String SERVER_EC_CACHE_SIZE = "server.ecCacheSize";
   public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_ENABLED = "server.hrcStatusSummary.cache.enabled";
   public static final String SERVER_HRC_STATUS_SUMMARY_CACHE_SIZE = "server.hrcStatusSummary.cache.size";
@@ -536,6 +540,9 @@ public class Configuration {
   private static final String LDAP_GROUP_SEARCH_FILTER_DEFAULT = "";
   private static final String LDAP_REFERRAL_DEFAULT = "follow";
 
+  private static final String LDAP_SYNC_MEMBER_REPLACE_PATTERN_DEFAULT = "";
+  private static final String LDAP_SYNC_MEMBER_FILTER_DEFAULT = "";
+
   /**
    * !!! TODO: for development purposes only, should be changed to 'false'
    */
@@ -1854,6 +1861,14 @@ public class Configuration {
       LDAP_GROUP_SEARCH_FILTER_KEY, LDAP_GROUP_SEARCH_FILTER_DEFAULT));
     ldapServerProperties.setReferralMethod(properties.getProperty(
       LDAP_REFERRAL_KEY, LDAP_REFERRAL_DEFAULT));
+    ldapServerProperties.setSyncUserMemberReplacePattern(properties.getProperty(
+      LDAP_SYCN_USER_MEMBER_REPLACE_PATTERN, LDAP_SYNC_MEMBER_REPLACE_PATTERN_DEFAULT));
+    ldapServerProperties.setSyncGroupMemberReplacePattern(properties.getProperty(
+      LDAP_SYCN_GROUP_MEMBER_REPLACE_PATTERN, LDAP_SYNC_MEMBER_REPLACE_PATTERN_DEFAULT));
+    ldapServerProperties.setSyncUserMemberFilter(properties.getProperty(
+      LDAP_SYCN_USER_MEMBER_FILTER, LDAP_SYNC_MEMBER_FILTER_DEFAULT));
+    ldapServerProperties.setSyncGroupMemberFilter(properties.getProperty(
+      LDAP_SYCN_GROUP_MEMBER_FILTER, LDAP_SYNC_MEMBER_FILTER_DEFAULT));
     ldapServerProperties.setPaginationEnabled("true".equalsIgnoreCase(
       properties.getProperty(LDAP_PAGINATION_ENABLED_KEY, LDAP_PAGINATION_ENABLED_DEFAULT)));
 

ambari-server/src/main/java/org/apache/ambari/server/security/authorization/LdapServerProperties.java

@@ -52,10 +52,15 @@ public class LdapServerProperties {
   private String usernameAttribute;
   private String userSearchBase = "";
 
+  private String syncGroupMemberReplacePattern = "";
+  private String syncUserMemberReplacePattern = "";
+
   private String groupSearchFilter;
   private String userSearchFilter;
   private String alternateUserSearchFilter; // alternate user search filter to be used when users use their alternate login id (e.g. User Principal Name)
 
+  private String syncUserMemberFilter = "";
+  private String syncGroupMemberFilter = "";
   //LDAP pagination properties
   private boolean paginationEnabled = true;
 
@@ -264,6 +269,38 @@ public class LdapServerProperties {
     this.paginationEnabled = paginationEnabled;
   }
 
+  public String getSyncGroupMemberReplacePattern() {
+    return syncGroupMemberReplacePattern;
+  }
+
+  public void setSyncGroupMemberReplacePattern(String syncGroupMemberReplacePattern) {
+    this.syncGroupMemberReplacePattern = syncGroupMemberReplacePattern;
+  }
+
+  public String getSyncUserMemberReplacePattern() {
+    return syncUserMemberReplacePattern;
+  }
+
+  public void setSyncUserMemberReplacePattern(String syncUserMemberReplacePattern) {
+    this.syncUserMemberReplacePattern = syncUserMemberReplacePattern;
+  }
+
+  public String getSyncUserMemberFilter() {
+    return syncUserMemberFilter;
+  }
+
+  public void setSyncUserMemberFilter(String syncUserMemberFilter) {
+    this.syncUserMemberFilter = syncUserMemberFilter;
+  }
+
+  public String getSyncGroupMemberFilter() {
+    return syncGroupMemberFilter;
+  }
+
+  public void setSyncGroupMemberFilter(String syncGroupMemberFilter) {
+    this.syncGroupMemberFilter = syncGroupMemberFilter;
+  }
+
   @Override
   public boolean equals(Object obj) {
     if (this == obj) return true;
@@ -299,6 +336,14 @@ public class LdapServerProperties {
         that.groupSearchFilter) : that.groupSearchFilter != null) return false;
     if (dnAttribute != null ? !dnAttribute.equals(
         that.dnAttribute) : that.dnAttribute != null) return false;
+    if (syncGroupMemberReplacePattern != null ? !syncGroupMemberReplacePattern.equals(
+      that.syncGroupMemberReplacePattern) : that.syncGroupMemberReplacePattern != null) return false;
+    if (syncUserMemberReplacePattern != null ? !syncUserMemberReplacePattern.equals(
+      that.syncUserMemberReplacePattern) : that.syncUserMemberReplacePattern != null) return false;
+    if (syncUserMemberFilter != null ? !syncUserMemberFilter.equals(
+      that.syncUserMemberFilter) : that.syncUserMemberFilter != null) return false;
+    if (syncGroupMemberFilter != null ? !syncGroupMemberFilter.equals(
+      that.syncGroupMemberFilter) : that.syncGroupMemberFilter != null) return false;
     if (referralMethod != null ? !referralMethod.equals(that.referralMethod) : that.referralMethod != null) return false;
 
     if (groupMappingEnabled != that.isGroupMappingEnabled()) return false;
@@ -331,6 +376,10 @@ public class LdapServerProperties {
     result = 31 * result + (adminGroupMappingRules != null ? adminGroupMappingRules.hashCode() : 0);
     result = 31 * result + (groupSearchFilter != null ? groupSearchFilter.hashCode() : 0);
     result = 31 * result + (dnAttribute != null ? dnAttribute.hashCode() : 0);
+    result = 31 * result + (syncUserMemberReplacePattern != null ? syncUserMemberReplacePattern.hashCode() : 0);
+    result = 31 * result + (syncGroupMemberReplacePattern != null ? syncGroupMemberReplacePattern.hashCode() : 0);
+    result = 31 * result + (syncUserMemberFilter != null ? syncUserMemberFilter.hashCode() : 0);
+    result = 31 * result + (syncGroupMemberFilter != null ? syncGroupMemberFilter.hashCode() : 0);
     result = 31 * result + (referralMethod != null ? referralMethod.hashCode() : 0);
     result = 31 * result + (userSearchFilter != null ? userSearchFilter.hashCode() : 0);
     result = 31 * result + (alternateUserSearchFilter != null ? alternateUserSearchFilter.hashCode() : 0);

ambari-server/src/main/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulator.java

@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import javax.naming.NamingException;
@@ -39,6 +40,7 @@ import org.apache.ambari.server.security.authorization.Group;
 import org.apache.ambari.server.security.authorization.LdapServerProperties;
 import org.apache.ambari.server.security.authorization.User;
 import org.apache.ambari.server.security.authorization.Users;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -52,6 +54,7 @@ import org.springframework.ldap.core.support.LdapContextSource;
 import org.springframework.ldap.filter.AndFilter;
 import org.springframework.ldap.filter.EqualsFilter;
 import org.springframework.ldap.filter.Filter;
+import org.springframework.ldap.filter.HardcodedFilter;
 import org.springframework.ldap.filter.LikeFilter;
 import org.springframework.ldap.filter.OrFilter;
 import org.springframework.security.core.userdetails.UsernameNotFoundException;
@@ -100,6 +103,9 @@ public class AmbariLdapDataPopulator {
   // REGEXP to check member attribute starts with "cn=" or "uid=" - case insensitive
   private static final String IS_MEMBER_DN_REGEXP = "^(?i)(%s|%s)=.*$";
 
+  private static final String MEMBER_ATTRIBUTE_REPLACE_STRING = "${member}";
+  private static final String MEMBER_ATTRIBUTE_VALUE_PLACEHOLDER = "{member}";
+
   /**
    * Construct an AmbariLdapDataPopulator.
    *
@@ -448,7 +454,17 @@ public class AmbariLdapDataPopulator {
    */
   protected LdapUserDto getLdapUserByMemberAttr(String memberAttributeValue) {
     Set<LdapUserDto> filteredLdapUsers = new HashSet<LdapUserDto>();
-    if (memberAttributeValue!= null && isMemberAttributeBaseDn(memberAttributeValue)) {
+
+    memberAttributeValue = getUniqueIdByMemberPattern(memberAttributeValue,
+      ldapServerProperties.getSyncUserMemberReplacePattern());
+    Filter syncMemberFilter = createCustomMemberFilter(memberAttributeValue,
+      ldapServerProperties.getSyncUserMemberFilter());
+
+    if (memberAttributeValue != null && syncMemberFilter != null) {
+      LOG.trace("Use custom filter '{}' for getting member user with default baseDN ('{}')",
+        syncMemberFilter.encode(), ldapServerProperties.getBaseDN());
+      filteredLdapUsers = getFilteredLdapUsers(ldapServerProperties.getBaseDN(), syncMemberFilter);
+    } else if (memberAttributeValue!= null && isMemberAttributeBaseDn(memberAttributeValue)) {
       LOG.trace("Member can be used as baseDn: {}", memberAttributeValue);
       Filter filter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getUserObjectClass());
       filteredLdapUsers = getFilteredLdapUsers(memberAttributeValue, filter);
@@ -471,7 +487,17 @@ public class AmbariLdapDataPopulator {
    */
   protected LdapGroupDto getLdapGroupByMemberAttr(String memberAttributeValue) {
     Set<LdapGroupDto> filteredLdapGroups = new HashSet<LdapGroupDto>();
-    if (memberAttributeValue != null && isMemberAttributeBaseDn(memberAttributeValue)) {
+
+    memberAttributeValue = getUniqueIdByMemberPattern(memberAttributeValue,
+      ldapServerProperties.getSyncGroupMemberReplacePattern());
+    Filter syncMemberFilter = createCustomMemberFilter(memberAttributeValue,
+      ldapServerProperties.getSyncGroupMemberFilter());
+
+    if (memberAttributeValue != null && syncMemberFilter != null) {
+      LOG.trace("Use custom filter '{}' for getting member group with default baseDN ('{}')",
+        syncMemberFilter.encode(), ldapServerProperties.getBaseDN());
+      filteredLdapGroups = getFilteredLdapGroups(ldapServerProperties.getBaseDN(), syncMemberFilter);
+    } else if (memberAttributeValue != null && isMemberAttributeBaseDn(memberAttributeValue)) {
       LOG.trace("Member can be used as baseDn: {}", memberAttributeValue);
       Filter filter = new EqualsFilter(OBJECT_CLASS_ATTRIBUTE, ldapServerProperties.getGroupObjectClass());
       filteredLdapGroups = getFilteredLdapGroups(memberAttributeValue, filter);
@@ -485,6 +511,42 @@ public class AmbariLdapDataPopulator {
     return (filteredLdapGroups.isEmpty()) ? null : filteredLdapGroups.iterator().next();
   }
 
+  /**
+   * Use custom member filter. Replace {member} with the member attribute.
+   * E.g.: (&(objectclass=posixaccount)(dn={member})) -> (&(objectclass=posixaccount)(dn=cn=mycn,dc=apache,dc=org))
+   */
+  protected Filter createCustomMemberFilter(String memberAttributeValue, String syncMemberFilter) {
+    Filter filter = null;
+    if (StringUtils.isNotEmpty(syncMemberFilter)) {
+      filter = new HardcodedFilter(syncMemberFilter.replace(MEMBER_ATTRIBUTE_VALUE_PLACEHOLDER, memberAttributeValue));
+    }
+    return filter;
+  }
+
+  /**
+   * Replace memberAttribute value by a custom pattern to get the DN or id (like memberUid) of a user/group.
+   * E.g.: memberAttribute="<sid=...><guid=...>,cn=mycn,dc=org,dc=apache"
+   * Apply on (?<sid>.*);(?<guid>.*);(?<member>.*) pattern, then the result will be: "${member}"
+   */
+  protected String getUniqueIdByMemberPattern(String memberAttributeValue, String pattern) {
+    if (StringUtils.isNotEmpty(memberAttributeValue) && StringUtils.isNotEmpty(pattern)) {
+      try {
+        Pattern p = Pattern.compile(pattern);
+        Matcher m = p.matcher(memberAttributeValue);
+        LOG.debug("Apply replace pattern '{}' on '{}' membership attribbute value.", memberAttributeValue, pattern);
+        if (m.matches()) {
+          memberAttributeValue = m.replaceAll(MEMBER_ATTRIBUTE_REPLACE_STRING);
+          LOG.debug("Membership attribute value after replace pattern applied: '{}'", memberAttributeValue);
+        } else {
+          LOG.warn("Membership attribute value pattern is not matched ({}) on '{}'", pattern, memberAttributeValue);
+        }
+      } catch (Exception e) {
+        LOG.error("Error during replace memberAttribute '{}' with pattern '{}'", memberAttributeValue, pattern);
+      }
+    }
+    return memberAttributeValue;
+  }
+
   /**
    * Removes synced users which are not present in any of group.
    *

ambari-server/src/test/java/org/apache/ambari/server/security/ldap/AmbariLdapDataPopulatorTest.java

@@ -56,6 +56,7 @@ import org.springframework.ldap.core.ContextMapper;
 import org.springframework.ldap.core.DirContextAdapter;
 import org.springframework.ldap.core.LdapTemplate;
 import org.springframework.ldap.core.support.LdapContextSource;
+import org.springframework.ldap.filter.Filter;
 
 import javax.naming.directory.SearchControls;
 
@@ -63,6 +64,7 @@ import static junit.framework.Assert.*;
 import static org.easymock.EasyMock.*;
 import static org.easymock.EasyMock.anyBoolean;
 import static org.easymock.EasyMock.createNiceMock;
+import static org.junit.Assert.assertEquals;
 
 @RunWith(PowerMockRunner.class)
 @PrepareForTest(AmbariLdapUtils.class)
@@ -1805,6 +1807,72 @@ public class AmbariLdapDataPopulatorTest {
     assertFalse(result);
   }
 
+  @Test
+  public void testGetUniqueIdMemberPattern() {
+    // GIVEN
+    Configuration configuration = createNiceMock(Configuration.class);
+    Users users = createNiceMock(Users.class);
+    String  syncUserMemberPattern = "(?<sid>.*);(?<guid>.*);(?<member>.*)";
+    String memberAttribute = "<SID=...>;<GUID=...>;cn=member,dc=apache,dc=org";
+    AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
+    // WHEN
+    String result = populator.getUniqueIdByMemberPattern(memberAttribute, syncUserMemberPattern);
+    // THEN
+    assertEquals("cn=member,dc=apache,dc=org", result);
+  }
+
+  @Test
+  public void testGetUniqueIdByMemberPatternWhenPatternIsWrong() {
+    // GIVEN
+    Configuration configuration = createNiceMock(Configuration.class);
+    Users users = createNiceMock(Users.class);
+    String  syncUserMemberPattern = "(?<sid>.*);(?<guid>.*);(?<mem>.*)";
+    String memberAttribute = "<SID=...>;<GUID=...>;cn=member,dc=apache,dc=org";
+    AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
+    // WHEN
+    String result = populator.getUniqueIdByMemberPattern(memberAttribute, syncUserMemberPattern);
+    // THEN
+    assertEquals(memberAttribute, result);
+  }
+
+  @Test
+  public void testGetUniqueIdByMemberPatternWhenPatternIsEmpty() {
+    // GIVEN
+    Configuration configuration = createNiceMock(Configuration.class);
+    Users users = createNiceMock(Users.class);
+    String memberAttribute = "<SID=...>;<GUID=...>;cn=member,dc=apache,dc=org";
+    AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
+    // WHEN
+    String result = populator.getUniqueIdByMemberPattern(memberAttribute, "");
+    // THEN
+    assertEquals(memberAttribute, result);
+  }
+
+  @Test
+  public void testGetUniqueIdByMemberPatternWhenMembershipAttributeIsNull() {
+    // GIVEN
+    Configuration configuration = createNiceMock(Configuration.class);
+    Users users = createNiceMock(Users.class);
+    String  syncUserMemberPattern = "(?<sid>.*);(?<guid>.*);(?<member>.*)";
+    AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
+    // WHEN
+    String result = populator.getUniqueIdByMemberPattern(null, syncUserMemberPattern);
+    // THEN
+    assertNull(result);
+  }
+
+  @Test
+  public void testCreateCustomMemberFilter() {
+    // GIVEN
+    Configuration configuration = createNiceMock(Configuration.class);
+    Users users = createNiceMock(Users.class);
+    AmbariLdapDataPopulatorTestInstance populator = new AmbariLdapDataPopulatorTestInstance(configuration, users);
+    // WHEN
+    Filter result = populator.createCustomMemberFilter("myUid", "(&(objectclass=posixaccount)(uid={member}))");
+    // THEN
+    assertEquals("(&(objectclass=posixaccount)(uid=myUid))", result.encode());
+  }
+
   private static int userIdCounter = 1;
 
   private User createUser(String name, boolean ldapUser, GroupEntity group) {