AMBARI-6533. Add API to get file log location in the task execution output, currently hardcoded in UI. (Alejandro Fernandez via swagle)

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -122,9 +122,6 @@ class ActionQueue(threading.Thread):
       except (Queue.Empty):
         pass
 
-
-
-
   def process_command(self, command):
     logger.debug("Took an element of Queue: " + pprint.pformat(command))
     # make sure we log failures
@@ -157,6 +154,8 @@ class ActionQueue(threading.Thread):
     taskId = command['taskId']
     # Preparing 'IN_PROGRESS' report
     in_progress_status = self.commandStatuses.generate_report_template(command)
+    # The path of the files that contain the output log and error log use a prefix that the agent advertises to the
+    # server. The prefix is defined in agent-config.ini
     in_progress_status.update({
       'tmpout': self.tmpdir + os.sep + 'output-' + str(taskId) + '.txt',
       'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',

ambari-agent/src/main/python/ambari_agent/Register.py

@@ -52,7 +52,8 @@ class Register:
                  'publicHostname'    : hostname.public_hostname(),
                  'hardwareProfile'   : self.hardware.get(),
                  'agentEnv'          : agentEnv,
-                 'agentVersion'      : version
+                 'agentVersion'      : version,
+                 'prefix'            : self.config.get('agent', 'prefix')
                }
     return register
 

ambari-agent/src/test/python/ambari_agent/TestRegistration.py

@@ -60,6 +60,7 @@ class TestRegistration(TestCase):
     print data['agentEnv']['umask']
     self.assertEquals(not data['agentEnv']['umask']== "", True, "agents umask should not be empty")
     self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
-    self.assertEquals(len(data), 8)
+    self.assertEquals(data['prefix'], config.get('agent', 'prefix'), 'The prefix path does not match')
+    self.assertEquals(len(data), 9)
 
     os.remove(ver_file)

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java

@@ -23,6 +23,7 @@ import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import com.google.inject.name.Named;
 import com.google.inject.persist.Transactional;
+import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.agent.CommandReport;
 import org.apache.ambari.server.agent.ExecutionCommand;
 import org.apache.ambari.server.orm.dao.ClusterDAO;
@@ -42,6 +43,7 @@ import org.apache.ambari.server.orm.entities.RequestScheduleEntity;
 import org.apache.ambari.server.orm.entities.RoleSuccessCriteriaEntity;
 import org.apache.ambari.server.orm.entities.StageEntity;
 import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Host;
 import org.apache.ambari.server.utils.StageUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -243,8 +245,25 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
         hostRoleCommandDAO.create(hostRoleCommandEntity);
 
         assert hostRoleCommandEntity.getTaskId() != null;
-
         hostRoleCommand.setTaskId(hostRoleCommandEntity.getTaskId());
+
+        try {
+          // Get the in-memory host object and its prefix to construct the output and error log paths.
+          Host hostObject = clusters.getHost(hostRoleCommandEntity.getHostName());
+          String prefix = hostObject.getPrefix();
+          if (null != prefix && !prefix.isEmpty()) {
+            if (!prefix.endsWith("/")) {
+              prefix = prefix + "/";
+            }
+            hostRoleCommand.setOutputLog(prefix + "output-" + hostRoleCommandEntity.getTaskId() + ".txt");
+            hostRoleCommand.setErrorLog(prefix + "errors-" + hostRoleCommandEntity.getTaskId() + ".txt");
+            hostRoleCommandEntity.setOutputLog(hostRoleCommand.getOutputLog());
+            hostRoleCommandEntity.setErrorLog(hostRoleCommand.getErrorLog());
+          }
+        } catch (AmbariException e) {
+          LOG.warn("Exception in getting prefix for host and setting output and error log files.");
+        }
+
         ExecutionCommandEntity executionCommandEntity = hostRoleCommand.constructExecutionCommandEntity();
         executionCommandEntity.setHostRoleCommand(hostRoleCommandEntity);
 

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java

@@ -46,6 +46,8 @@ public class HostRoleCommand {
   private HostRoleStatus status = HostRoleStatus.PENDING;
   private String stdout = "";
   private String stderr = "";
+  public String outputLog = null;
+  public String errorLog = null;
   private String structuredOut = "";
   private int exitCode = 999; //Default is unknown
   private long startTime = -1;
@@ -76,6 +78,8 @@ public class HostRoleCommand {
     status = hostRoleCommandEntity.getStatus();
     stdout = hostRoleCommandEntity.getStdOut() != null ? new String(hostRoleCommandEntity.getStdOut()) : "";
     stderr = hostRoleCommandEntity.getStdError() != null ? new String(hostRoleCommandEntity.getStdError()) : "";
+    outputLog = hostRoleCommandEntity.getOutputLog();
+    errorLog = hostRoleCommandEntity.getErrorLog();
     structuredOut = hostRoleCommandEntity.getStructuredOut() != null ? new String(hostRoleCommandEntity.getStructuredOut()) : "";
     exitCode = hostRoleCommandEntity.getExitcode();
     startTime = hostRoleCommandEntity.getStartTime();
@@ -185,6 +189,18 @@ public class HostRoleCommand {
     this.stderr = stderr;
   }
 
+  public String getOutputLog() { return outputLog; }
+
+  public void setOutputLog(String outputLog)  {
+    this.outputLog = outputLog;
+  }
+
+  public String getErrorLog() { return errorLog; }
+
+  public void setErrorLog(String errorLog) {
+      this.errorLog = errorLog;
+  }
+
   public int getExitCode() {
     return exitCode;
   }
@@ -291,6 +307,8 @@ public class HostRoleCommand {
     builder.append("  Role: ").append(role).append("\n");
     builder.append("  Status: ").append(status).append("\n");
     builder.append("  Event: ").append(event).append("\n");
+    builder.append("  Output log: ").append(outputLog).append("\n");
+    builder.append("  Error log: ").append(errorLog).append("\n");
     builder.append("  stdout: ").append(stdout).append("\n");
     builder.append("  stderr: ").append(stderr).append("\n");
     builder.append("  exitcode: ").append(exitCode).append("\n");

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java

@@ -641,6 +641,9 @@ public class HeartBeatHandler {
     // Get status of service components
     List<StatusCommand> cmds = heartbeatMonitor.generateStatusCommands(hostname);
 
+    // Save the prefix of the log file paths
+    hostObject.setPrefix(register.getPrefix());
+
     hostObject.handleEvent(new HostRegistrationRequestEvent(hostname,
         null != register.getPublicHostname() ? register.getPublicHostname() : hostname,
         new AgentVersion(register.getAgentVersion()), now, register.getHardwareProfile(),

ambari-server/src/main/java/org/apache/ambari/server/agent/Register.java

@@ -34,6 +34,7 @@ public class Register {
   private String publicHostname;
   private AgentEnv agentEnv;
   private String agentVersion;
+  private String prefix;
 
   @JsonProperty("responseId")
   public int getResponseId() {
@@ -89,6 +90,10 @@ public class Register {
     return agentVersion;
   }
 
+  public String getPrefix() { return prefix; }
+
+  public void setPrefix(String prefix) { this.prefix = prefix; }
+
   public void setAgentVersion(String agentVersion) {
     this.agentVersion = agentVersion;
   }
@@ -106,7 +111,8 @@ public class Register {
     String ret = "responseId=" + responseId + "\n" +
              "timestamp=" + timestamp + "\n" +
              "hostname="  + hostname + "\n" +
-             "currentPingPort=" + currentPingPort + "\n";
+             "currentPingPort=" + currentPingPort + "\n" +
+             "prefix=" + prefix + "\n";
 
     if (hardwareProfile != null)
       ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();

ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java

@@ -51,6 +51,13 @@ public class Configuration {
 
   public static final String CONFIG_FILE = "ambari.properties";
   public static final String BOOTSTRAP_DIR = "bootstrap.dir";
+
+  /**
+   *  PREFIX_DIR is shared in ambari-agent.ini and should only be called by unit tests.
+   *  For all server-side processing, it should be retrieved from <code>HostImpl.getPrefix()</code>
+   */
+  public static final String PREFIX_DIR = "/var/lib/ambari-agent/data";
+
   public static final String BOOTSTRAP_DIR_DEFAULT = "/var/run/ambari-server/bootstrap";
   public static final String VIEWS_DIR = "views.dir";
   public static final String VIEWS_DIR_DEFAULT = "/var/lib/ambari-server/resources/views";

ambari-server/src/main/java/org/apache/ambari/server/controller/ShortTaskStatus.java

@@ -28,12 +28,14 @@ public class ShortTaskStatus {
   protected String command;
   protected String status;
   protected String customCommandName;
+  protected String outputLog;
+  protected String errorLog;
 
   public ShortTaskStatus() {
   }
 
   public ShortTaskStatus(int taskId, long stageId, String hostName, String role, String command, String status,
-                         String customCommandName) {
+                         String customCommandName, String outputLog, String errorLog) {
     this.taskId = taskId;
     this.stageId = stageId;
     this.hostName = hostName;
@@ -41,6 +43,8 @@ public class ShortTaskStatus {
     this.command = command;
     this.status = status;
     this.customCommandName = customCommandName;
+    this.outputLog = outputLog;
+    this.errorLog = errorLog;
   }
 
   public ShortTaskStatus(HostRoleCommand hostRoleCommand) {
@@ -51,6 +55,8 @@ public class ShortTaskStatus {
     this.role = hostRoleCommand.getRole().toString();
     this.status = hostRoleCommand.getStatus().toString();
     this.customCommandName = hostRoleCommand.getCustomCommandName();
+    this.outputLog = hostRoleCommand.getOutputLog();
+    this.errorLog = hostRoleCommand.getErrorLog();
   }
 
   public String getCustomCommandName() {
@@ -101,6 +107,22 @@ public class ShortTaskStatus {
     this.command = command;
   }
 
+  public String getOutputLog() {
+    return outputLog;
+  }
+
+  public void setOutputLog(String outputLog) {
+    this.outputLog = outputLog;
+  }
+
+  public String getErrorLog() {
+    return errorLog;
+  }
+
+  public void setErrorLog(String errorLog) {
+    this.errorLog = errorLog;
+  }
+
   public String getStatus() {
     return status;
   }
@@ -117,7 +139,10 @@ public class ShortTaskStatus {
         + ", hostname=" + hostName
         + ", role=" + role
         + ", command=" + command
-        + ", status=" + status);
+        + ", status=" + status
+        + ", outputLog=" + outputLog
+        + ", errorLog=" + errorLog
+    );
     return sb.toString();
   }
 

ambari-server/src/main/java/org/apache/ambari/server/controller/TaskStatusResponse.java

@@ -37,8 +37,8 @@ public class TaskStatusResponse extends ShortTaskStatus {
   public TaskStatusResponse(long requestId,
                             int taskId, long stageId, String hostName, String role, String command, String status,
                             int exitCode, String stderr, String stdout, long startTime, short attemptCount,
-                            String commandDetail, String customCommandName) {
-    super(taskId, stageId, hostName, role, command, status, customCommandName);
+                            String commandDetail, String customCommandName, String outputLog, String errorLog) {
+    super(taskId, stageId, hostName, role, command, status, customCommandName, outputLog, errorLog);
     this.requestId = requestId;
     this.exitCode = exitCode;
     this.stderr = stderr;

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TaskResourceProvider.java

@@ -57,6 +57,8 @@ class TaskResourceProvider extends AbstractControllerResourceProvider {
   protected static final String TASK_EXIT_CODE_PROPERTY_ID    = PropertyHelper.getPropertyId("Tasks", "exit_code");
   protected static final String TASK_STDERR_PROPERTY_ID       = PropertyHelper.getPropertyId("Tasks", "stderr");
   protected static final String TASK_STOUT_PROPERTY_ID        = PropertyHelper.getPropertyId("Tasks", "stdout");
+  protected static final String TASK_OUTPUTLOG_PROPERTY_ID    = PropertyHelper.getPropertyId("Tasks", "output_log");
+  protected static final String TASK_ERRORLOG_PROPERTY_ID     = PropertyHelper.getPropertyId("Tasks", "error_log");
   protected static final String TASK_STRUCT_OUT_PROPERTY_ID   = PropertyHelper.getPropertyId("Tasks", "structured_out");
   protected static final String TASK_START_TIME_PROPERTY_ID   = PropertyHelper.getPropertyId("Tasks", "start_time");
   protected static final String TASK_END_TIME_PROPERTY_ID     = PropertyHelper.getPropertyId("Tasks", "end_time");
@@ -153,6 +155,8 @@ class TaskResourceProvider extends AbstractControllerResourceProvider {
         setResourceProperty(resource, TASK_EXIT_CODE_PROPERTY_ID, response.getExitCode(), requestedIds);
         setResourceProperty(resource, TASK_STDERR_PROPERTY_ID, response.getStderr(), requestedIds);
         setResourceProperty(resource, TASK_STOUT_PROPERTY_ID, response.getStdout(), requestedIds);
+        setResourceProperty(resource, TASK_OUTPUTLOG_PROPERTY_ID, response.getOutputLog(), requestedIds);
+        setResourceProperty(resource, TASK_ERRORLOG_PROPERTY_ID, response.getErrorLog(), requestedIds);
         setResourceProperty(resource, TASK_STRUCT_OUT_PROPERTY_ID, parseStructuredOutput(response.getStructuredOut()), requestedIds);
         setResourceProperty(resource, TASK_START_TIME_PROPERTY_ID, response.getStartTime(), requestedIds);
         setResourceProperty(resource, TASK_END_TIME_PROPERTY_ID, response.getEndTime(), requestedIds);

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java

@@ -101,6 +101,15 @@ public class HostRoleCommandEntity {
   @Basic
   private byte[] stdOut = new byte[0];
 
+  @Column(name = "output_log")
+  @Basic
+  private String outputLog = null;
+
+  @Column(name = "error_log")
+  @Basic
+  private String errorLog = null;
+
+
   @Column(name = "structured_out")
   @Lob
   @Basic
@@ -228,6 +237,14 @@ public class HostRoleCommandEntity {
     this.stdOut = stdOut;
   }
 
+  public String getOutputLog() { return outputLog; }
+
+  public void setOutputLog(String outputLog) { this.outputLog = outputLog; }
+
+  public String getErrorLog() { return errorLog; }
+
+  public void setErrorLog(String errorLog) { this.errorLog = errorLog; }
+
   public Long getStartTime() {
     return startTime;
   }
@@ -318,6 +335,8 @@ public class HostRoleCommandEntity {
     if (status != null ? !status.equals(that.status) : that.status != null) return false;
     if (stdError != null ? !Arrays.equals(stdError, that.stdError) : that.stdError != null) return false;
     if (stdOut != null ? !Arrays.equals(stdOut, that.stdOut) : that.stdOut != null) return false;
+    if (outputLog != null ? !outputLog.equals(that.outputLog) : that.outputLog != null) return false;
+    if (errorLog != null ? !errorLog.equals(that.errorLog) : that.errorLog != null) return false;
     if (taskId != null ? !taskId.equals(that.taskId) : that.taskId != null) return false;
     if (structuredOut != null ? !Arrays.equals(structuredOut, that.structuredOut) : that.structuredOut != null) return false;
     if (endTime != null ? !endTime.equals(that.endTime) : that.endTime != null) return false;
@@ -337,6 +356,8 @@ public class HostRoleCommandEntity {
     result = 31 * result + (status != null ? status.hashCode() : 0);
     result = 31 * result + (stdError != null ? Arrays.hashCode(stdError) : 0);
     result = 31 * result + (stdOut != null ? Arrays.hashCode(stdOut) : 0);
+    result = 31 * result + (outputLog != null ? outputLog.hashCode() : 0);
+    result = 31 * result + (errorLog != null ? errorLog.hashCode() : 0);
     result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
     result = 31 * result + (lastAttemptTime != null ? lastAttemptTime.hashCode() : 0);
     result = 31 * result + (attemptCount != null ? attemptCount.hashCode() : 0);

ambari-server/src/main/java/org/apache/ambari/server/state/Host.java

@@ -275,6 +275,18 @@ public interface Host {
    */
   public void setState(HostState state);
 
+  /**
+   * Get the prefix path of all logs
+   * @return prefix
+   */
+  public String getPrefix();
+
+  /**
+   * Set the prefix path of all logs of the host
+   * @param prefix the prefix path to set
+   */
+  public void setPrefix(String prefix);
+
   /**
    * Send an event to the Host's StateMachine
    * @param event HostEvent

ambari-server/src/main/java/org/apache/ambari/server/state/host/HostImpl.java

@@ -120,9 +120,12 @@ public class HostImpl implements Host {
 
   
   
-  //In-memory status, based on host components states
+  // In-memory status, based on host components states
   private String status;
 
+  // In-memory prefix of log file paths that is retrieved when the agent registers with the server
+  private String prefix;
+
   private static final StateMachineFactory
     <HostImpl, HostState, HostEventType, HostEvent>
       stateMachineFactory
@@ -861,6 +864,21 @@ public class HostImpl implements Host {
     }
   }
 
+  @Override
+  public String getPrefix() { return prefix; }
+
+  @Override
+  public void setPrefix(String prefix) {
+    if (prefix != null && !prefix.equals(this.prefix)) {
+      try {
+        writeLock.lock();
+        this.prefix = prefix;
+      } finally {
+        writeLock.unlock();
+      }
+    }
+  }
+
   @Override
   public Map<String, String> getHostAttributes() {
     try {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java

@@ -27,6 +27,7 @@ import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.orm.DBAccessor;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.ambari.server.configuration.Configuration;
 
 import com.google.inject.Inject;
 import com.google.inject.Injector;
@@ -68,6 +69,7 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
     // !!! TODO: alerting DDL upgrade
 
     List<DBAccessor.DBColumnInfo> columns;
+    String dbType = getDbType();
 
     // add admin tables and initial values prior to adding referencing columns on existing tables
     columns = new ArrayList<DBAccessor.DBColumnInfo>();
@@ -157,6 +159,20 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
         Integer.class, 1, 1, false));
     dbAccessor.addColumn("viewinstance", new DBAccessor.DBColumnInfo("resource_id",
         Long.class, 1, 1, false));
+
+    dbAccessor.addColumn("host_role_command", new DBAccessor.DBColumnInfo("output_log", String.class, 255, null, true));
+    dbAccessor.addColumn("host_role_command", new DBAccessor.DBColumnInfo("error_log", String.class, 255, null, true));
+
+    // Update historic records with the log paths, but only enough so as to not prolong the upgrade process
+    if (dbType.equals(Configuration.POSTGRES_DB_NAME) || dbType.equals(Configuration.ORACLE_DB_NAME)) {
+      // Postgres and Oracle use a different concatenation operator.
+      dbAccessor.executeQuery("UPDATE host_role_command SET output_log = ('/var/lib/ambari-agent/data/output-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);");
+      dbAccessor.executeQuery("UPDATE host_role_command SET error_log = ('/var/lib/ambari-agent/data/errors-' || CAST(task_id AS VARCHAR(20)) || '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);");
+    } else if (dbType.equals(Configuration.MYSQL_DB_NAME)) {
+      // MySQL uses a different concatenation operator.
+      dbAccessor.executeQuery("UPDATE host_role_command SET output_log = CONCAT('/var/lib/ambari-agent/data/output-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE output_log IS NULL OR output_log = '' ORDER BY task_id DESC LIMIT 1000);");
+      dbAccessor.executeQuery("UPDATE host_role_command SET error_log = CONCAT('/var/lib/ambari-agent/data/errors-', task_id, '.txt') WHERE task_id IN (SELECT task_id FROM host_role_command WHERE error_log IS NULL OR error_log = '' ORDER BY task_id DESC LIMIT 1000);");
+    }
   }
 
 

ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql

@@ -41,7 +41,7 @@ CREATE TABLE users (user_id INTEGER, principal_id BIGINT NOT NULL, create_time T
 CREATE TABLE groups (group_id INTEGER, principal_id BIGINT NOT NULL, group_name VARCHAR(255) NOT NULL, ldap_group INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (group_id));
 CREATE TABLE members (member_id INTEGER, group_id INTEGER NOT NULL, user_id INTEGER NOT NULL, PRIMARY KEY (member_id));
 CREATE TABLE execution_command (task_id BIGINT NOT NULL, command LONGBLOB, PRIMARY KEY (task_id));
-CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event LONGTEXT NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), role_command VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error LONGBLOB, std_out LONGBLOB, structured_out LONGBLOB, command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
+CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event LONGTEXT NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), role_command VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error LONGBLOB, std_out LONGBLOB, output_log VARCHAR(255) NULL, error_log VARCHAR(255) NULL, structured_out LONGBLOB, command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
 CREATE TABLE role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor DOUBLE NOT NULL, PRIMARY KEY (role, request_id, stage_id));
 CREATE TABLE stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), cluster_host_info LONGBLOB, PRIMARY KEY (stage_id, request_id));
 CREATE TABLE request (request_id BIGINT NOT NULL, cluster_id BIGINT, request_schedule_id BIGINT, command_name VARCHAR(255), create_time BIGINT NOT NULL, end_time BIGINT NOT NULL, inputs LONGBLOB, request_context VARCHAR(255), request_type VARCHAR(255), start_time BIGINT NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id));

ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql

@@ -31,7 +31,7 @@ CREATE TABLE users (user_id NUMBER(10) NOT NULL, principal_id NUMBER(19) NOT NUL
 CREATE TABLE groups (group_id NUMBER(10) NOT NULL, principal_id NUMBER(19) NOT NULL, group_name VARCHAR2(255) NOT NULL, ldap_group NUMBER(10) DEFAULT 0, PRIMARY KEY (group_id));
 CREATE TABLE members (member_id NUMBER(10), group_id NUMBER(10) NOT NULL, user_id NUMBER(10) NOT NULL, PRIMARY KEY (member_id));
 CREATE TABLE execution_command (task_id NUMBER(19) NOT NULL, command BLOB NULL, PRIMARY KEY (task_id));
-CREATE TABLE host_role_command (task_id NUMBER(19) NOT NULL, attempt_count NUMBER(5) NOT NULL, event CLOB NULL, exitcode NUMBER(10) NOT NULL, host_name VARCHAR2(255) NOT NULL, last_attempt_time NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, role VARCHAR2(255) NULL, role_command VARCHAR2(255) NULL, stage_id NUMBER(19) NOT NULL, start_time NUMBER(19) NOT NULL, end_time NUMBER(19), status VARCHAR2(255) NULL, std_error BLOB NULL, std_out BLOB NULL, structured_out BLOB NULL,  command_detail VARCHAR2(255) NULL, custom_command_name VARCHAR2(255) NULL, PRIMARY KEY (task_id));
+CREATE TABLE host_role_command (task_id NUMBER(19) NOT NULL, attempt_count NUMBER(5) NOT NULL, event CLOB NULL, exitcode NUMBER(10) NOT NULL, host_name VARCHAR2(255) NOT NULL, last_attempt_time NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, role VARCHAR2(255) NULL, role_command VARCHAR2(255) NULL, stage_id NUMBER(19) NOT NULL, start_time NUMBER(19) NOT NULL, end_time NUMBER(19), status VARCHAR2(255) NULL, std_error BLOB NULL, std_out BLOB NULL, output_log VARCHAR2(255) NULL, error_log VARCHAR2(255) NULL, structured_out BLOB NULL,  command_detail VARCHAR2(255) NULL, custom_command_name VARCHAR2(255) NULL, PRIMARY KEY (task_id));
 CREATE TABLE role_success_criteria (role VARCHAR2(255) NOT NULL, request_id NUMBER(19) NOT NULL, stage_id NUMBER(19) NOT NULL, success_factor NUMBER(19,4) NOT NULL, PRIMARY KEY (role, request_id, stage_id));
 CREATE TABLE stage (stage_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19) NULL, log_info VARCHAR2(255) NULL, request_context VARCHAR2(255) NULL, cluster_host_info BLOB NOT NULL, PRIMARY KEY (stage_id, request_id));
 CREATE TABLE request (request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19), request_schedule_id NUMBER(19), command_name VARCHAR(255), create_time NUMBER(19) NOT NULL, end_time NUMBER(19) NOT NULL, inputs BLOB, request_context VARCHAR(255), request_type VARCHAR(255), start_time NUMBER(19) NOT NULL, status VARCHAR(255), PRIMARY KEY (request_id));

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -49,7 +49,7 @@ CREATE TABLE members (member_id INTEGER, group_id INTEGER NOT NULL, user_id INTE
 
 CREATE TABLE execution_command (command BYTEA, task_id BIGINT NOT NULL, PRIMARY KEY (task_id));
 
-CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, structured_out BYTEA, role_command VARCHAR(255), command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
+CREATE TABLE host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, output_log VARCHAR(255) NULL, error_log VARCHAR(255) NULL, structured_out BYTEA, role_command VARCHAR(255), command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
 
 CREATE TABLE role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id));
 

ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql

@@ -76,7 +76,7 @@ GRANT ALL PRIVILEGES ON TABLE ambari.members TO :username;
 CREATE TABLE ambari.execution_command (command BYTEA, task_id BIGINT NOT NULL, PRIMARY KEY (task_id));
 GRANT ALL PRIVILEGES ON TABLE ambari.execution_command TO :username;
 
-CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, structured_out BYTEA, role_command VARCHAR(255), command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
+CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, output_log VARCHAR(255) NULL, error_log VARCHAR(255) NULL, structured_out BYTEA, role_command VARCHAR(255), command_detail VARCHAR(255), custom_command_name VARCHAR(255), PRIMARY KEY (task_id));
 GRANT ALL PRIVILEGES ON TABLE ambari.host_role_command TO :username;
 
 CREATE TABLE ambari.role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id));

ambari-server/src/main/resources/properties.json

@@ -147,6 +147,8 @@
         "Tasks/exit_code",
         "Tasks/stderr",
         "Tasks/stdout",
+        "Tasks/output_log",
+        "Tasks/error_log",
         "Tasks/start_time",
         "Tasks/end_time",
         "Tasks/structured_out",

ambari-server/src/main/resources/stacks/HDP/2.1/services/HIVE/metainfo.xml.orig

@@ -1,51 +0,0 @@
-<?xml version="1.0"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<metainfo>
-  <schemaVersion>2.0</schemaVersion>
-  <services>
-    <service>
-      <name>HIVE</name>
-      <comment>Data warehouse system for ad-hoc queries &amp; analysis of large datasets and table &amp; storage management service</comment>
-      <version>0.13.0.2.1</version>
-
-      <configuration-dependencies>
-        <config-type>hive-site</config-type>
-        <config-type>hive-log4j</config-type>
-        <config-type>hive-exec-log4j</config-type>
-        <config-type>global</config-type>
-      </configuration-dependencies>
-    </service>
-
-    <service>
-      <name>HCATALOG</name>
-      <comment>This is comment for HCATALOG service</comment>
-      <version>0.12.0.2.1</version>
-      <osSpecifics>
-        <osSpecific>
-          <osFamily>any</osFamily>
-          <packages>
-            <package>
-              <name>hive-hcatalog</name>
-            </package>
-          </packages>
-        </osSpecific>
-      </osSpecifics>
-    </service>
-
-  </services>
-</metainfo>

ambari-server/src/test/java/org/apache/ambari/server/agent/LocalAgentSimulator.java

@@ -18,6 +18,7 @@
 package org.apache.ambari.server.agent;
 
 import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.configuration.Configuration;
 import org.apache.ambari.server.state.fsm.InvalidStateTransitionException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -97,6 +98,7 @@ public class LocalAgentSimulator implements Runnable {
       reg.setTimestamp(System.currentTimeMillis());
       reg.setHostname(this.hostname);
       reg.setAgentVersion(this.agentVersion);
+      reg.setPrefix(Configuration.PREFIX_DIR);
     }
     RegistrationResponse response;
     try {

ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java

@@ -668,6 +668,7 @@ public class TestHeartbeatHandler {
     reg.setCurrentPingPort(DummyCurrentPingPort);
     reg.setHardwareProfile(hi);
     reg.setAgentVersion(metaInfo.getServerVersion());
+    reg.setPrefix(Configuration.PREFIX_DIR);
     handler.handleRegistration(reg);
     assertEquals(hostObject.getState(), HostState.HEALTHY);
     assertEquals(DummyOsType, hostObject.getOsType());
@@ -698,6 +699,7 @@ public class TestHeartbeatHandler {
     reg.setHostname(DummyHostname1);
     reg.setHardwareProfile(hi);
     reg.setAgentVersion(""); // Invalid agent version
+    reg.setPrefix(Configuration.PREFIX_DIR);
     try {
       handler.handleRegistration(reg);
       fail ("Expected failure for non compatible agent version");
@@ -823,12 +825,14 @@ public class TestHeartbeatHandler {
     reg.setHostname(DummyHostname1);
     reg.setHardwareProfile(hi);
     reg.setAgentVersion(metaInfo.getServerVersion());
+    reg.setPrefix(Configuration.PREFIX_DIR);
     RegistrationResponse response = handler.handleRegistration(reg);
 
     assertEquals(hostObject.getState(), HostState.HEALTHY);
     assertEquals("redhat5", hostObject.getOsType());
     assertEquals(RegistrationStatus.OK, response.getResponseStatus());
     assertEquals(0, response.getResponseId());
+    assertEquals(reg.getPrefix(), hostObject.getPrefix());
     assertTrue(response.getStatusCommands().isEmpty());
   }
 

ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java

@@ -1562,6 +1562,50 @@ public class AmbariManagementControllerTest {
 
   }
 
+  @Test
+  /**
+   * Create a cluster with a service, and verify that the request tasks have the correct output log and error log paths.
+   */
+  public void testRequestStatusLogs() throws Exception {
+    testCreateServiceComponentHostSimple();
+
+    String clusterName = "foo1";
+    String serviceName = "HDFS";
+
+    Cluster cluster = clusters.getCluster(clusterName);
+    for (Host h : clusters.getHosts()) {
+      // Simulate each agent registering and setting the prefix path on its host
+      h.setPrefix(Configuration.PREFIX_DIR);
+    }
+
+    Map<String, Config> configs = new HashMap<String, Config>();
+    Map<String, String> properties = new HashMap<String, String>();
+    Map<String, Map<String, String>> propertiesAttributes = new HashMap<String, Map<String,String>>();
+
+    Config c1 = new ConfigImpl(cluster, "hdfs-site", properties, propertiesAttributes, injector);
+    c1.setVersionTag("v1");
+    cluster.addConfig(c1);
+    c1.persist();
+    configs.put(c1.getType(), c1);
+
+    ServiceRequest r = new ServiceRequest(clusterName, serviceName, State.INSTALLED.toString());
+    Set<ServiceRequest> requests = new HashSet<ServiceRequest>();
+    requests.add(r);
+
+    Map<String, String> mapRequestProps = new HashMap<String, String>();
+    mapRequestProps.put("context", "Called from a test");
+
+    RequestStatusResponse trackAction =
+        ServiceResourceProviderTest.updateServices(controller, requests, mapRequestProps, true, false);
+
+    List<ShortTaskStatus> taskStatuses = trackAction.getTasks();
+    Assert.assertFalse(taskStatuses.isEmpty());
+    for (ShortTaskStatus task : taskStatuses) {
+      Assert.assertEquals("Task output logs don't match", Configuration.PREFIX_DIR + "/output-" + task.getTaskId() + ".txt", task.getOutputLog());
+      Assert.assertEquals("Task error logs don't match", Configuration.PREFIX_DIR + "/errors-" + task.getTaskId() + ".txt", task.getErrorLog());
+    }
+  }
+
   @Test
   public void testInstallAndStartService() throws Exception {
     testCreateServiceComponentHostSimple();

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/TaskResourceProviderTest.java

@@ -98,7 +98,7 @@ public class TaskResourceProviderTest {
 
     Set<TaskStatusResponse> allResponse = new HashSet<TaskStatusResponse>();
     allResponse.add(new TaskStatusResponse(100L, 100, 100L, "HostName100", "", "", "", 0, "", "", 0L, (short) 0,
-        "commandDetail", "customCommandName"));
+        "commandDetail", "customCommandName", null, null));
 
     // set expectations
     expect(managementController.getTaskStatus(AbstractResourceProviderTest.Matcher.getTaskRequestSet(100L, 100L))).

ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java

@@ -33,9 +33,9 @@ public class InMemoryDefaultTestModule extends AbstractModule {
   protected void configure() {
     properties.setProperty(Configuration.SERVER_PERSISTENCE_TYPE_KEY, "in-memory");
     properties.setProperty(Configuration.METADETA_DIR_PATH,
-        "src/test/resources/stacks");
+        "ambari-server/src/test/resources/stacks");
     properties.setProperty(Configuration.SERVER_VERSION_FILE,
-            "target/version");
+            "ambari-server/target/version");
     properties.setProperty(Configuration.OS_VERSION_KEY,
         "centos5");
     try {

ambari-web/app/controllers/global/background_operations_controller.js

@@ -170,6 +170,8 @@ App.BackgroundOperationsController = Em.Controller.extend({
     task.Tasks.status = data.Tasks.status;
     task.Tasks.stdout = data.Tasks.stdout;
     task.Tasks.stderr = data.Tasks.stderr;
+    task.Tasks.output_log = data.Tasks.output_log;
+    task.Tasks.error_log = data.Tasks.error_log;
     this.set('serviceTimestamp', App.dateTime());
   },
 

ambari-web/app/messages.js

@@ -203,7 +203,7 @@ Em.I18n.translations = {
   'common.additional': 'Additional',
   'common.time.start': 'Start Time',
   'common.time.end': 'End Time',
-  'common.hostLog.popup.logDir.path':'/var/lib/ambari-agent/data/',
+  'common.hostLog.popup.logDir.path':'/var/lib/ambari-agent/data/',   // TODO, this hardcoded path needs to be removed.
   'common.hostLog.popup.outputLog.value': 'output-{0}.txt',
   'common.hostLog.popup.errorLog.value': 'errors-{0}.txt',
   'common.maintenance.task': ' Toggle Maintenance Mode',

ambari-web/app/utils/host_progress_popup.js

@@ -301,8 +301,6 @@ App.HostPopup = Em.Object.create({
       commandDetail: App.format.commandDetail(_task.Tasks.command_detail),
       status: App.format.taskStatus(_task.Tasks.status),
       role: App.format.role(_task.Tasks.role),
-      outputLog: Em.I18n.t('common.hostLog.popup.logDir.path') + Em.I18n.t('common.hostLog.popup.outputLog.value').format(_task.Tasks.id),
-      errorLog: Em.I18n.t('common.hostLog.popup.logDir.path') + Em.I18n.t('common.hostLog.popup.errorLog.value').format(_task.Tasks.id),
       stderr: _task.Tasks.stderr,
       stdout: _task.Tasks.stdout,
       isVisible: true,
@@ -377,6 +375,9 @@ App.HostPopup = Em.Object.create({
                   existTask.set('status', App.format.taskStatus(_task.Tasks.status));
                   existTask.set('stdout', _task.Tasks.stdout);
                   existTask.set('stderr', _task.Tasks.stderr);
+                  // Verified that this is needed.
+                  existTask.set('outputLog', _task.Tasks.output_log);
+                  existTask.set('errorLog', _task.Tasks.error_log);
                   existTask.set('startTime', date.startTime(_task.Tasks.start_time));
                   existTask.set('duration', date.durationSummary(_task.Tasks.start_time, _task.Tasks.end_time));
                 } else {

ambari-web/app/views/wizard/step9/hostLogPopupBody_view.js

@@ -147,8 +147,6 @@ App.WizardStep9HostLogPopupBodyView = Em.View.extend({
         taskInfo.set('isVisible', true);
         taskInfo.set('icon', '');
         taskInfo.set('hostName', _task.Tasks.host_name);
-        taskInfo.set('outputLog', Em.I18n.t('common.hostLog.popup.logDir.path') + Em.I18n.t('common.hostLog.popup.outputLog.value').format(_task.Tasks.id));
-        taskInfo.set('errorLog', Em.I18n.t('common.hostLog.popup.logDir.path') + Em.I18n.t('common.hostLog.popup.errorLog.value').format(_task.Tasks.id));
         if (taskInfo.get('status') == 'pending' || taskInfo.get('status') == 'queued') {
           taskInfo.set('icon', 'icon-cog');
         } else if (taskInfo.get('status') == 'in_progress') {