AMBARI-19077: Ambari-server: Gather dependent configuration types and password properties for a service component

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -72,6 +72,9 @@ class CustomServiceOrchestrator():
   DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
   DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
 
+  # The property name used by the hadoop credential provider
+  CREDENTIAL_PROVIDER_PROPERTY_NAME = 'hadoop.security.credential.provider.path'
+
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
@@ -147,21 +150,97 @@ class CustomServiceOrchestrator():
 
     return conf_dir
 
-  def getAffectedConfigTypes(self, commandJson):
+  def getConfigTypeCredentials(self, commandJson):
     """
     Gets the affected config types for the service in this command
+    with the password aliases and values.
+
+    Input:
+    {
+        "config-type1" : {
+          "password_key_name1":"password_value_name1",
+          "password_key_name2":"password_value_name2",
+            :
+        },
+        "config-type2" : {
+          "password_key_name1":"password_value_name1",
+          "password_key_name2":"password_value_name2",
+            :
+        },
+           :
+    }
+
+    Output:
+    {
+        "config-type1" : {
+          "alias1":"password1",
+          "alias2":"password2",
+            :
+        },
+        "config-type2" : {
+          "alias1":"password1",
+          "alias2":"password2",
+            :
+        },
+           :
+    }
+
+    If password_key_name is the same as password_value_name, then password_key_name is the password alias itself.
+    The value it points to is the password value.
+
+    If password_key_name is not the same as the password_value_name, then password_key_name points to the alias.
+    The value is pointed to by password_value_name.
+
+    For example:
+    Input:
+    {
+      "oozie-site" : {"oozie.service.JPAService.jdbc.password" : "oozie.service.JPAService.jdbc.password"},
+      "admin-properties" {"db_user":"db_password", "ranger.jpa.jdbc.credential.alias:ranger-admin-site" : "db_password"}
+    }
+
+    Output:
+    {
+      "oozie-site" : {"oozie.service.JPAService.jdbc.password" : "MyOozieJdbcPassword"},
+      "admin-properties" {"rangerdba" : "MyRangerDbaPassword", "rangeradmin":"MyRangerDbaPassword"},
+    }
 
     :param commandJson:
     :return:
     """
-    return commandJson.get('configuration_attributes')
-
-  def getCredentialProviderPropertyName(self):
-    """
-    Gets the property name used by the hadoop credential provider
-    :return:
-    """
-    return 'hadoop.security.credential.provider.path'
+    configtype_credentials = {}
+    if 'configuration_credentials' in commandJson:
+      for config_type, password_properties in commandJson['configuration_credentials'].items():
+        if config_type in commandJson['configurations']:
+          value_names = []
+          config = commandJson['configurations'][config_type]
+          credentials = {}
+          for key_name, value_name in password_properties.items():
+            if key_name == value_name:
+              if value_name in config:
+                # password name is the alias
+                credentials[key_name] = config[value_name]
+                value_names.append(value_name) # Gather the value_name for deletion
+            else:
+              keyname_keyconfig = key_name.split(':')
+              key_name = keyname_keyconfig[0]
+              # if the key is in another configuration (cross reference),
+              # get the value of the key from that configuration
+              if (len(keyname_keyconfig) > 1):
+                if keyname_keyconfig[1] not in commandJson['configurations']:
+                  continue
+                key_config = commandJson['configurations'][keyname_keyconfig[1]]
+              else:
+                key_config = config
+              if key_name in key_config and value_name in config:
+                # password name points to the alias
+                credentials[key_config[key_name]] = config[value_name]
+                value_names.append(value_name) # Gather the value_name for deletion
+          if len(credentials) > 0:
+            configtype_credentials[config_type] = credentials
+          for value_name in value_names:
+            # Remove the clear text password
+            config.pop(value_name, None)
+    return configtype_credentials
 
   def generateJceks(self, commandJson):
     """
@@ -178,16 +257,6 @@ class CustomServiceOrchestrator():
 
     logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
 
-    # Password properties for a config type, if present,
-    # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
-    passwordProperties = {}
-    config_types = self.getAffectedConfigTypes(commandJson)
-    for config_type in config_types:
-      elem = config_types.get(config_type)
-      hidden = elem.get('hidden')
-      if hidden is not None:
-        passwordProperties[config_type] = hidden
-
     # Set up the variables for the external command to generate a JCEKS file
     java_home = commandJson['hostLevelParams']['java_home']
     java_bin = '{java_home}/bin/java'.format(java_home=java_home)
@@ -196,31 +265,30 @@ class CustomServiceOrchestrator():
     serviceName = commandJson['serviceName']
 
     # Gather the password values and remove them from the configuration
-    configs = commandJson.get('configurations')
-    for key, value in passwordProperties.items():
-      config = configs.get(key)
-      if config is not None:
-        file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
-        if os.path.exists(file_path):
-          os.remove(file_path)
-        provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
-        logger.info('provider_path={0}'.format(provider_path))
-        for alias in value:
-          pwd = config.get(alias)
-          if pwd is not None:
-            # Remove the clear text password
-            config.pop(alias, None)
-            # Add JCEKS provider path instead
-            config[self.getCredentialProviderPropertyName()] = provider_path
-            logger.debug("config={0}".format(config))
-            protected_pwd = PasswordString(pwd)
-            # Generate the JCEKS file
-            cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
-                   alias, '-value', protected_pwd, '-provider', provider_path)
-            logger.info(cmd)
-            cmd_result = subprocess.call(cmd)
-            logger.info('cmd_result = {0}'.format(cmd_result))
-            os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+    provider_paths = [] # A service may depend on multiple configs
+    configtype_credentials = self.getConfigTypeCredentials(commandJson)
+    for config_type, credentials in configtype_credentials.items():
+      config = commandJson['configurations'][config_type]
+      file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(config_type))
+      if os.path.exists(file_path):
+        os.remove(file_path)
+      provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
+      provider_paths.append(provider_path)
+      logger.info('provider_path={0}'.format(provider_path))
+      for alias, pwd in credentials.items():
+        logger.debug("config={0}".format(config))
+        protected_pwd = PasswordString(pwd)
+        # Generate the JCEKS file
+        cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
+               alias, '-value', protected_pwd, '-provider', provider_path)
+        logger.info(cmd)
+        cmd_result = subprocess.call(cmd)
+        logger.info('cmd_result = {0}'.format(cmd_result))
+        os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+
+    if provider_paths:
+      # Add JCEKS provider paths instead
+      config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = ','.join(provider_paths)
 
     return cmd_result
 

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -119,6 +119,39 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("credentialStoreEnabled")
   private String credentialStoreEnabled;
 
+  /**
+   * Map of config type to list of password properties
+   *   <pre>
+   *     {@code
+   *       {
+   *         "config_type1" :
+   *           {
+   *             "password_alias_name1:type1":"password_value_name1",
+   *             "password_alias_name2:type2":"password_value_name2",
+   *                 :
+   *           },
+   *         "config_type2" :
+   *           {
+   *             "password_alias_name1:type1":"password_value_name1",
+   *             "password_alias_name2:type2":"password_value_name2",
+   *                 :
+   *           },
+   *                 :
+   *       }
+   *     }
+   *   </pre>
+   */
+  @SerializedName("configuration_credentials")
+  private Map<String, Map<String, String>> configurationCredentials;
+
+  public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials) {
+    this.configurationCredentials = configurationCredentials;
+  }
+
+  public Map<String, Map<String, String>> getConfigurationCredentials() {
+    return this.configurationCredentials;
+  }
+
   public String getCommandId() {
     return commandId;
   }

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java

@@ -341,6 +341,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
   @Inject
   private AmbariActionExecutionHelper actionExecutionHelper;
 
+  private Map<String, Map<String, Map<String, String>>> configCredentialsForService = new HashMap<>();
+
   @Inject
   public AmbariManagementControllerImpl(ActionManager actionManager,
       Clusters clusters, Injector injector) throws Exception {
@@ -2143,6 +2145,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     Service clusterService = cluster.getService(serviceName);
     execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));
 
+    // Get the map of service config type to password properties for the service
+    Map<String, Map<String, String>> configCredentials;
+    configCredentials = configCredentialsForService.get(clusterService.getName());
+    if (configCredentials == null) {
+      configCredentials = configHelper.getPropertiesWithPropertyType(stackId, clusterService,
+              PropertyType.PASSWORD);
+      configCredentialsForService.put(clusterService.getName(), configCredentials);
+    }
+
+    execCmd.setConfigurationCredentials(configCredentials);
+
     // Create a local copy for each command
     Map<String, String> commandParams = new TreeMap<String, String>();
     if (commandParamsInp != null) { // if not defined

ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java

@@ -41,6 +41,7 @@ import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
 import org.apache.ambari.server.state.PropertyInfo.PropertyType;
 import org.apache.ambari.server.state.configgroup.ConfigGroup;
 import org.apache.ambari.server.utils.SecretReference;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -488,6 +489,50 @@ public class ConfigHelper {
     return result;
   }
 
+  /**
+   * Gets a map of config types to password property names to password property value names.
+   *
+   * @param stackId
+   * @param service
+   * @param propertyType
+   * @return
+   * @throws AmbariException
+     */
+  public Map<String, Map<String, String>> getPropertiesWithPropertyType(StackId stackId, Service service, PropertyType propertyType)
+          throws AmbariException {
+    StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
+    Map<String, Map<String, String>> result = new HashMap<>();
+    Map<String, String> passwordProperties;
+    Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), service.getName());
+    for (PropertyInfo serviceProperty : serviceProperties) {
+      if (serviceProperty.getPropertyTypes().contains(propertyType)) {
+        String stackPropertyConfigType = fileNameToConfigType(serviceProperty.getFilename());
+        passwordProperties = result.get(stackPropertyConfigType);
+        if (passwordProperties == null) {
+          passwordProperties = new HashMap<>();
+          result.put(stackPropertyConfigType, passwordProperties);
+        }
+        // If the password property is used by another property, it means the password property
+        // is a password value name while the use is the password alias name. If the user property
+        // is from another config type, include that in the password alias name as name:type.
+        if (serviceProperty.getUsedByProperties().size() > 0) {
+          for (PropertyDependencyInfo usedByProperty : serviceProperty.getUsedByProperties()) {
+            String propertyName = usedByProperty.getName();
+            if (!StringUtils.isEmpty(usedByProperty.getType())) {
+              propertyName += ':' + usedByProperty.getType();
+            }
+            passwordProperties.put(propertyName, serviceProperty.getName());
+          }
+        }
+        else {
+          passwordProperties.put(serviceProperty.getName(), serviceProperty.getName());
+        }
+      }
+    }
+
+    return result;
+  }
+
   public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType,
       Cluster cluster, Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
     StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());

ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java

@@ -76,6 +76,17 @@ public class PropertyInfo {
   private Set<PropertyDependencyInfo> dependedByProperties =
     new HashSet<PropertyDependencyInfo>();
 
+  /**
+   * The list of properties that use this property.
+   * Password properties may be used by other properties in
+   * the same config type or different config type, typically
+   * when asking for user name and password pairs.
+   */
+  @XmlElementWrapper(name="used-by")
+  @XmlElement(name = "property")
+  private Set<PropertyDependencyInfo> usedByProperties =
+          new HashSet<>();
+
   //This method is called after all the properties (except IDREF) are unmarshalled for this object,
   //but before this object is set to the parent object.
   void afterUnmarshal(Unmarshaller unmarshaller, Object parent) {
@@ -96,6 +107,10 @@ public class PropertyInfo {
     this.name = name;
   }
 
+  public Set<PropertyDependencyInfo> getUsedByProperties() {
+    return usedByProperties;
+  }
+
   public String getValue() {
     return value;
   }