Revert "AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information"

This reverts commit 033989e7fd5a28a431eb9bdf64bd68d732e5ee69.

ambari-agent/conf/unix/ambari-agent.ini

@@ -45,9 +45,6 @@ keysdir=/var/lib/ambari-agent/keys
 server_crt=ca.crt
 passphrase_env_var_name=AMBARI_PASSPHRASE
 ssl_verify_cert=0
-credential_lib_dir=/var/lib/ambari-agent/cred/lib
-credential_conf_dir=/var/lib/ambari-agent/cred/conf
-credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell
 
 [services]
 pidLookupPath=/var/run/

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -29,8 +29,6 @@ from FileCache import FileCache
 from AgentException import AgentException
 from PythonExecutor import PythonExecutor
 from PythonReflectiveExecutor import PythonReflectiveExecutor
-from resource_management.core.utils import PasswordString
-import subprocess
 import Constants
 import hostname
 
@@ -67,11 +65,6 @@ class CustomServiceOrchestrator():
   REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot and often (this increases their speed)
   DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS
 
-  # Path where hadoop credential JARS will be available
-  DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib'
-  DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
-  DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
-
   def __init__(self, config, controller):
     self.config = config
     self.tmp_dir = config.get('agent', 'prefix')
@@ -85,14 +78,6 @@ class CustomServiceOrchestrator():
     # cache reset will be called on every agent registration
     controller.registration_listeners.append(self.file_cache.reset)
 
-    # Construct the hadoop credential lib JARs path
-    self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir',
-                                                             self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
-
-    self.credential_conf_dir = config.get('security', 'credential_conf_dir', self.DEFAULT_CREDENTIAL_CONF_DIR)
-
-    self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', self.DEFAULT_CREDENTIAL_SHELL_CMD)
-
     # Clean up old status command files if any
     try:
       os.unlink(self.status_commands_stdout)
@@ -129,102 +114,6 @@ class CustomServiceOrchestrator():
     else:
       return PythonExecutor(self.tmp_dir, self.config)
 
-  def getProviderDirectory(self, service_name):
-    """
-    Gets the path to the service conf folder where the JCEKS file will be created.
-
-    :param service_name: Name of the service, for example, HIVE
-    :return: lower case path to the service conf folder
-    """
-
-    # The stack definition scripts of the service can move the
-    # JCEKS file around to where it wants, which is usually
-    # /etc/<service_name>/conf
-
-    conf_dir = os.path.join(self.credential_conf_dir, service_name.lower())
-    if not os.path.exists(conf_dir):
-      os.makedirs(conf_dir, 0644)
-
-    return conf_dir
-
-  def getAffectedConfigTypes(self, commandJson):
-    """
-    Gets the affected config types for the service in this command
-
-    :param commandJson:
-    :return:
-    """
-    return commandJson.get('configuration_attributes')
-
-  def getCredentialProviderPropertyName(self):
-    """
-    Gets the property name used by the hadoop credential provider
-    :return:
-    """
-    return 'hadoop.security.credential.provider.path'
-
-  def generateJceks(self, commandJson):
-    """
-    Generates the JCEKS file with passwords for the service specified in commandJson
-
-    :param commandJson: command JSON
-    :return: An exit value from the external process that generated the JCEKS file. None if
-    there are no passwords in the JSON.
-    """
-    cmd_result = None
-    roleCommand = None
-    if 'roleCommand' in commandJson:
-      roleCommand = commandJson['roleCommand']
-
-    logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
-
-    # Password properties for a config type, if present,
-    # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
-    passwordProperties = {}
-    config_types = self.getAffectedConfigTypes(commandJson)
-    for config_type in config_types:
-      elem = config_types.get(config_type)
-      hidden = elem.get('hidden')
-      if hidden is not None:
-        passwordProperties[config_type] = hidden
-
-    # Set up the variables for the external command to generate a JCEKS file
-    java_home = commandJson['hostLevelParams']['java_home']
-    java_bin = '{java_home}/bin/java'.format(java_home=java_home)
-
-    cs_lib_path = self.credential_shell_lib_path
-    serviceName = commandJson['serviceName']
-
-    # Gather the password values and remove them from the configuration
-    configs = commandJson.get('configurations')
-    for key, value in passwordProperties.items():
-      config = configs.get(key)
-      if config is not None:
-        file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
-        if os.path.exists(file_path):
-          os.remove(file_path)
-        provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
-        logger.info('provider_path={0}'.format(provider_path))
-        for alias in value:
-          pwd = config.get(alias)
-          if pwd is not None:
-            # Remove the clear text password
-            config.pop(alias, None)
-            # Add JCEKS provider path instead
-            config[self.getCredentialProviderPropertyName()] = provider_path
-            logger.debug("config={0}".format(config))
-            protected_pwd = PasswordString(pwd)
-            # Generate the JCEKS file
-            cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
-                   alias, '-value', protected_pwd, '-provider', provider_path)
-            logger.info(cmd)
-            cmd_result = subprocess.call(cmd)
-            logger.info('cmd_result = {0}'.format(cmd_result))
-            os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
-
-    return cmd_result
-
-
   def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None,
                  override_output_files=True, retry=False):
     """
@@ -290,15 +179,6 @@ class CustomServiceOrchestrator():
         handle.on_background_command_started = self.map_task_to_process
         del command['__handle']
 
-      # If command contains credentialStoreEnabled, then
-      # generate the JCEKS file for the configurations.
-      credentialStoreEnabled = False
-      if 'credentialStoreEnabled' in command:
-        credentialStoreEnabled = (command['credentialStoreEnabled'] == "true")
-
-      if credentialStoreEnabled == True:
-        self.generateJceks(command)
-
       json_path = self.dump_command_to_json(command, retry)
       pre_hook_tuple = self.resolve_hook_script_path(hook_dir,
           self.PRE_HOOK_PREFIX, command_name, script_type)

ambari-agent/src/packages/tarball/all.xml

@@ -190,34 +190,4 @@
       <outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory>
     </file>
   </files>
-  <moduleSets>
-    <moduleSet>
-      <binaries>
-        <includeDependencies>false</includeDependencies>
-        <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
-        <unpack>false</unpack>
-        <directoryMode>755</directoryMode>
-        <fileMode>644</fileMode>
-        <dependencySets>
-          <dependencySet>
-            <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
-            <unpack>false</unpack>
-            <includes>
-              <include>commons-cli:commons-cli</include>
-              <include>commons-collections:commons-collections</include>
-              <include>commons-configuration:commons-configuration</include>
-              <include>commons-io:commons-io:jar:${commons.io.version}</include>
-              <include>commons-lang:commons-lang</include>
-              <include>commons-logging:commons-logging</include>
-              <include>com.google.guava:guava</include>
-              <include>org.slf4j:slf4j-api</include>
-              <include>org.apache.hadoop:hadoop-common</include>
-              <include>org.apache.hadoop:hadoop-auth</include>
-              <include>org.apache.htrace:htrace-core</include>
-            </includes>
-          </dependencySet>
-        </dependencySets>
-      </binaries>
-    </moduleSet>
-  </moduleSets>
 </assembly>

ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java

@@ -112,13 +112,6 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("availableServices")
   private Map<String, String> availableServices = new HashMap<>();
 
-  /**
-   * "true" or "false" indicating whether this
-   * service is enabled for credential store use.
-   */
-  @SerializedName("credentialStoreEnabled")
-  private String credentialStoreEnabled;
-
   public String getCommandId() {
     return commandId;
   }
@@ -302,27 +295,6 @@ public class ExecutionCommand extends AgentCommand {
 	this.serviceType = serviceType;
   }
 
-  /**
-   * Get a value indicating whether this service is enabled
-   * for credential store use.
-   *
-   * @return "true" or "false", any other value is
-   * considered as "false"
-   */
-  public String getCredentialStoreEnabled() {
-    return credentialStoreEnabled;
-  }
-
-  /**
-   * Set a value indicating whether this service is enabled
-   * for credential store use.
-   *
-   * @param credentialStoreEnabled
-   */
-  public void setCredentialStoreEnabled(String credentialStoreEnabled) {
-    this.credentialStoreEnabled = credentialStoreEnabled;
-  }
-
   public String getComponentName() {
     return componentName;
   }

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -329,15 +329,7 @@ public class HeartBeatHandler {
           case BACKGROUND_EXECUTION_COMMAND:
           case EXECUTION_COMMAND: {
             ExecutionCommand ec = (ExecutionCommand)ac;
-            /*
-             * Set the value of credentialStore enabled before sending the command to agent.
-             */
-            Cluster cluster = clusterFsm.getCluster(ec.getClusterName());
-            Service service = cluster.getService(ec.getServiceName());
-            if (service != null) {
-              ec.setCredentialStoreEnabled(String.valueOf(service.isCredentialStoreEnabled()));
-            }
-            LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, task ID {}",
+            LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, taskId {}",
                      ec.getHostname(), ec.getRole(), ec.getRoleCommand(), ec.getCommandId(), ec.getTaskId());
             Map<String, String> hlp = ec.getHostLevelParams();
             if (hlp != null) {

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java

@@ -48,7 +48,6 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -172,31 +171,22 @@ public class TestHeartbeatHandler {
     ActionManager am = actionManagerTestHelper.getMockActionManager();
     expect(am.getTasks(anyObject(List.class))).andReturn(new ArrayList<HostRoleCommand>());
     replay(am);
-
-    Cluster cluster = heartbeatTestHelper.getDummyCluster();
-    Service hdfs = cluster.addService(HDFS);
-    hdfs.addServiceComponent(DATANODE);
-    hdfs.addServiceComponent(NAMENODE);
-    hdfs.addServiceComponent(SECONDARY_NAMENODE);
-    Collection<Host> hosts = cluster.getHosts();
-    assertEquals(hosts.size(), 1);
-
     Clusters fsm = clusters;
-    Host hostObject = hosts.iterator().next();
+    fsm.addHost(DummyHostname1);
+    Host hostObject = clusters.getHost(DummyHostname1);
     hostObject.setIPv4("ipv4");
     hostObject.setIPv6("ipv6");
     hostObject.setOsType(DummyOsType);
 
-    String hostname = hostObject.getHostName();
     ActionQueue aq = new ActionQueue();
 
     HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
     Register reg = new Register();
     HostInfo hi = new HostInfo();
-    hi.setHostName(hostname);
+    hi.setHostName(DummyHostname1);
     hi.setOS(DummyOs);
     hi.setOSRelease(DummyOSRelease);
-    reg.setHostname(hostname);
+    reg.setHostname(DummyHostname1);
     reg.setHardwareProfile(hi);
     reg.setAgentVersion(metaInfo.getServerVersion());
     handler.handleRegistration(reg);
@@ -205,21 +195,19 @@ public class TestHeartbeatHandler {
 
     ExecutionCommand execCmd = new ExecutionCommand();
     execCmd.setRequestAndStage(2, 34);
-    execCmd.setHostname(hostname);
-    execCmd.setClusterName(cluster.getClusterName());
-    execCmd.setServiceName(HDFS);
-    aq.enqueue(hostname, execCmd);
+    execCmd.setHostname(DummyHostname1);
+    aq.enqueue(DummyHostname1, new ExecutionCommand());
     HeartBeat hb = new HeartBeat();
     hb.setResponseId(0);
     HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
     List<Alert> al = new ArrayList<Alert>();
     al.add(new Alert());
     hb.setNodeStatus(hs);
-    hb.setHostname(hostname);
+    hb.setHostname(DummyHostname1);
 
     handler.handleHeartBeat(hb);
     assertEquals(HostState.HEALTHY, hostObject.getState());
-    assertEquals(0, aq.dequeueAll(hostname).size());
+    assertEquals(0, aq.dequeueAll(DummyHostname1).size());
   }