AMBARI-3934 Custom Action: Enhance host_role_command table to record end time and structured output (dsen)

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ActionDBAccessorImpl.java

@@ -117,6 +117,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
    */
   @Override
   public void abortOperation(long requestId) {
+    long now = System.currentTimeMillis();
     List<HostRoleCommandEntity> commands =
         hostRoleCommandDAO.findByRequest(requestId);
     for (HostRoleCommandEntity command : commands) {
@@ -124,6 +125,7 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
           command.getStatus() == HostRoleStatus.IN_PROGRESS ||
           command.getStatus() == HostRoleStatus.PENDING) {
         command.setStatus(HostRoleStatus.ABORTED);
+        command.setEndTime(now);
         hostRoleCommandDAO.merge(command);
         LOG.info("Aborting command. Hostname " + command.getHostName()
             + " role " + command.getRole()
@@ -141,10 +143,12 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
   @Transactional
   public void timeoutHostRole(String host, long requestId, long stageId,
                               String role) {
+    long now = System.currentTimeMillis();
     List<HostRoleCommandEntity> commands =
         hostRoleCommandDAO.findByHostRole(host, requestId, stageId, role);
     for (HostRoleCommandEntity command : commands) {
       command.setStatus(HostRoleStatus.TIMEDOUT);
+      command.setEndTime(now);
       hostRoleCommandDAO.merge(command);
     }
   }
@@ -229,12 +233,20 @@ public class ActionDBAccessorImpl implements ActionDBAccessor {
           + "HostName " + hostname + " requestId " + requestId + " stageId "
           + stageId + " role " + role + " report " + report);
     }
+    long now = System.currentTimeMillis();
     List<HostRoleCommandEntity> commands = hostRoleCommandDAO.findByHostRole(
         hostname, requestId, stageId, role);
     for (HostRoleCommandEntity command : commands) {
       command.setStatus(HostRoleStatus.valueOf(report.getStatus()));
       command.setStdOut(report.getStdOut().getBytes());
       command.setStdError(report.getStdErr().getBytes());
+      command.setStructuredOut(report.getStructuredOut() == null ? null :
+        report.getStructuredOut().getBytes());           // ===================================
+      if (command.getStatus() == HostRoleStatus.COMPLETED ||
+          command.getStatus() == HostRoleStatus.ABORTED ||
+          command.getStatus() == HostRoleStatus.FAILED) {
+        command.setEndTime(now);
+      }
       command.setExitcode(report.getExitCode());
       hostRoleCommandDAO.merge(command);
     }

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleCommand.java

@@ -47,9 +47,11 @@ public class HostRoleCommand {
   private HostRoleStatus status = HostRoleStatus.PENDING;
   private String stdout = "";
   private String stderr = "";
+  private String structuredOut = "";
   private int exitCode = 999; //Default is unknown
   private final ServiceComponentHostEventWrapper event;
   private long startTime = -1;
+  private long endTime = -1;
   private long lastAttemptTime = -1;
   private short attemptCount = 0;
   private RoleCommand roleCommand;
@@ -76,8 +78,10 @@ public class HostRoleCommand {
     status = hostRoleCommandEntity.getStatus();
     stdout = hostRoleCommandEntity.getStdOut() != null ? new String(hostRoleCommandEntity.getStdOut()) : "";
     stderr = hostRoleCommandEntity.getStdError() != null ? new String(hostRoleCommandEntity.getStdError()) : "";
+    structuredOut = hostRoleCommandEntity.getStructuredOut() != null ? new String(hostRoleCommandEntity.getStructuredOut()) : "";
     exitCode = hostRoleCommandEntity.getExitcode();
     startTime = hostRoleCommandEntity.getStartTime();
+    endTime = hostRoleCommandEntity.getEndTime() != null ? hostRoleCommandEntity.getEndTime() : -1L;
     lastAttemptTime = hostRoleCommandEntity.getLastAttemptTime();
     attemptCount = hostRoleCommandEntity.getAttemptCount();
     roleCommand = hostRoleCommandEntity.getRoleCommand();
@@ -95,7 +99,9 @@ public class HostRoleCommand {
     hostRoleCommandEntity.setStdError(stderr.getBytes());
     hostRoleCommandEntity.setExitcode(exitCode);
     hostRoleCommandEntity.setStdOut(stdout.getBytes());
+    hostRoleCommandEntity.setStructuredOut(structuredOut.getBytes());
     hostRoleCommandEntity.setStartTime(startTime);
+    hostRoleCommandEntity.setEndTime(endTime);
     hostRoleCommandEntity.setLastAttemptTime(lastAttemptTime);
     hostRoleCommandEntity.setAttemptCount(attemptCount);
     hostRoleCommandEntity.setRoleCommand(roleCommand);
@@ -193,6 +199,22 @@ public class HostRoleCommand {
     this.attemptCount++;
   }
 
+  public String getStructuredOut() {
+    return structuredOut;
+  }
+
+  public void setStructuredOut(String structuredOut) {
+    this.structuredOut = structuredOut;
+  }
+
+  public long getEndTime() {
+    return endTime;
+  }
+
+  public void setEndTime(long endTime) {
+    this.endTime = endTime;
+  }
+
   public ExecutionCommandWrapper getExecutionCommandWrapper() {
     if (taskId != -1 && executionCommandWrapper == null) {
       ExecutionCommandEntity commandEntity = executionCommandDAO.findByPK(taskId);

ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java

@@ -28,6 +28,7 @@ public class CommandReport {
   private String actionId;
   private String stdout;
   private String stderr;
+  private String structuredOut;
   private String status;
   int exitCode;
   private String clusterName;
@@ -97,6 +98,17 @@ public class CommandReport {
     this.stdout = stdout;
   }
 
+  @JsonProperty("structuredOut")
+  public String getStructuredOut() {
+    return this.structuredOut;
+  }
+
+
+  @JsonProperty("structuredOut")
+  public void setStructuredOut(String structuredOut) {
+    this.structuredOut = structuredOut;
+  }
+
   @JsonProperty("roleCommand")
   public String getRoleCommand() {
     return this.roleCommand;

ambari-server/src/main/java/org/apache/ambari/server/controller/TaskStatusResponse.java

@@ -25,7 +25,9 @@ public class TaskStatusResponse extends ShortTaskStatus {
   private int exitCode;
   private String stderr;
   private String stdout;
+  private String structuredOut;
   private long startTime;
+  private Long endTime;
   private short attemptCount;
 
   public TaskStatusResponse() {
@@ -51,6 +53,8 @@ public class TaskStatusResponse extends ShortTaskStatus {
     this.stdout = hostRoleCommand.getStdout();
     this.startTime = hostRoleCommand.getStartTime();
     this.attemptCount = hostRoleCommand.getAttemptCount();
+    this.structuredOut = hostRoleCommand.getStructuredOut();
+    this.endTime = hostRoleCommand.getEndTime();
   }
 
   public long getRequestId() {
@@ -100,7 +104,23 @@ public class TaskStatusResponse extends ShortTaskStatus {
   public void setAttemptCount(short attemptCount) {
     this.attemptCount = attemptCount;
   }
-  
+
+  public String getStructuredOut() {
+    return structuredOut;
+  }
+
+  public void setStructuredOut(String structuredOut) {
+    this.structuredOut = structuredOut;
+  }
+
+  public Long getEndTime() {
+    return endTime;
+  }
+
+  public void setEndTime(Long endTime) {
+    this.endTime = endTime;
+  }
+
   @Override
   public String toString() {
       return super.toString();

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/TaskResourceProvider.java

@@ -56,7 +56,9 @@ class TaskResourceProvider extends AbstractControllerResourceProvider {
   protected static final String TASK_EXIT_CODE_PROPERTY_ID    = PropertyHelper.getPropertyId("Tasks", "exit_code");
   protected static final String TASK_STDERR_PROPERTY_ID       = PropertyHelper.getPropertyId("Tasks", "stderr");
   protected static final String TASK_STOUT_PROPERTY_ID        = PropertyHelper.getPropertyId("Tasks", "stdout");
+  protected static final String TASK_STRUCT_OUT_PROPERTY_ID   = PropertyHelper.getPropertyId("Tasks", "structured_out");
   protected static final String TASK_START_TIME_PROPERTY_ID   = PropertyHelper.getPropertyId("Tasks", "start_time");
+  protected static final String TASK_END_TIME_PROPERTY_ID     = PropertyHelper.getPropertyId("Tasks", "end_time");
   protected static final String TASK_ATTEMPT_CNT_PROPERTY_ID  = PropertyHelper.getPropertyId("Tasks", "attempt_cnt");
 
 
@@ -141,7 +143,9 @@ class TaskResourceProvider extends AbstractControllerResourceProvider {
         setResourceProperty(resource, TASK_EXIT_CODE_PROPERTY_ID, response.getExitCode(), requestedIds);
         setResourceProperty(resource, TASK_STDERR_PROPERTY_ID, response.getStderr(), requestedIds);
         setResourceProperty(resource, TASK_STOUT_PROPERTY_ID, response.getStdout(), requestedIds);
+        setResourceProperty(resource, TASK_STRUCT_OUT_PROPERTY_ID, response.getStructuredOut(), requestedIds);
         setResourceProperty(resource, TASK_START_TIME_PROPERTY_ID, response.getStartTime(), requestedIds);
+        setResourceProperty(resource, TASK_END_TIME_PROPERTY_ID, response.getEndTime(), requestedIds);
         setResourceProperty(resource, TASK_ATTEMPT_CNT_PROPERTY_ID, response.getAttemptCount(), requestedIds);
         resources.add(resource);
       }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostRoleCommandEntity.java

@@ -99,10 +99,19 @@ public class HostRoleCommandEntity {
   @Basic
   private byte[] stdOut = new byte[0];
 
+  @Column(name = "structured_out")
+  @Lob
+  @Basic
+  private byte[] structuredOut = new byte[0];
+
   @Basic
   @Column(name = "start_time", nullable = false)
   private Long startTime = -1L;
 
+  @Basic
+  @Column(name = "end_time", nullable = false)
+  private Long endTime = -1L;
+
   @Basic
   @Column(name = "last_attempt_time", nullable = false)
   private Long lastAttemptTime = -1L;
@@ -238,6 +247,22 @@ public class HostRoleCommandEntity {
     this.roleCommand = roleCommand;
   }
 
+  public byte[] getStructuredOut() {
+    return structuredOut;
+  }
+
+  public void setStructuredOut(byte[] structuredOut) {
+    this.structuredOut = structuredOut;
+  }
+
+  public Long getEndTime() {
+    return endTime;
+  }
+
+  public void setEndTime(Long endTime) {
+    this.endTime = endTime;
+  }
+
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;
@@ -259,6 +284,8 @@ public class HostRoleCommandEntity {
     if (stdError != null ? !Arrays.equals(stdError, that.stdError) : that.stdError != null) return false;
     if (stdOut != null ? !Arrays.equals(stdOut, that.stdOut) : that.stdOut != null) return false;
     if (taskId != null ? !taskId.equals(that.taskId) : that.taskId != null) return false;
+    if (structuredOut != null ? !Arrays.equals(structuredOut, that.structuredOut) : that.structuredOut != null) return false;
+    if (endTime != null ? !endTime.equals(that.endTime) : that.endTime != null) return false;
 
     return true;
   }
@@ -278,6 +305,8 @@ public class HostRoleCommandEntity {
     result = 31 * result + (startTime != null ? startTime.hashCode() : 0);
     result = 31 * result + (lastAttemptTime != null ? lastAttemptTime.hashCode() : 0);
     result = 31 * result + (attemptCount != null ? attemptCount.hashCode() : 0);
+    result = 31 * result + (endTime != null ? endTime.hashCode() : 0);
+    result = 31 * result + (structuredOut != null ? Arrays.hashCode(structuredOut) : 0);
     return result;
   }
 

ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql

@@ -29,7 +29,7 @@ CREATE TABLE servicedesiredstate (cluster_id NUMBER(19) NOT NULL, desired_host_r
 CREATE TABLE roles (role_name VARCHAR2(255) NOT NULL, PRIMARY KEY (role_name));
 CREATE TABLE users (user_id NUMBER(10) NOT NULL, create_time TIMESTAMP NULL, ldap_user NUMBER(10) DEFAULT 0, user_name VARCHAR2(255) NULL, user_password VARCHAR2(255) NULL, PRIMARY KEY (user_id));
 CREATE TABLE execution_command (task_id NUMBER(19) NOT NULL, command BLOB NULL, PRIMARY KEY (task_id));
-CREATE TABLE host_role_command (task_id NUMBER(19) NOT NULL, attempt_count NUMBER(5) NOT NULL, event CLOB NULL, exitcode NUMBER(10) NOT NULL, host_name VARCHAR2(255) NOT NULL, last_attempt_time NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, role VARCHAR2(255) NULL, role_command VARCHAR2(255) NULL, stage_id NUMBER(19) NOT NULL, start_time NUMBER(19) NOT NULL, status VARCHAR2(255) NULL, std_error BLOB NULL, std_out BLOB NULL, PRIMARY KEY (task_id));
+CREATE TABLE host_role_command (task_id NUMBER(19) NOT NULL, attempt_count NUMBER(5) NOT NULL, event CLOB NULL, exitcode NUMBER(10) NOT NULL, host_name VARCHAR2(255) NOT NULL, last_attempt_time NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, role VARCHAR2(255) NULL, role_command VARCHAR2(255) NULL, stage_id NUMBER(19) NOT NULL, start_time NUMBER(19) NOT NULL, end_time NUMBER(19), status VARCHAR2(255) NULL, std_error BLOB NULL, std_out BLOB NULL, structured_out BLOB NULL, PRIMARY KEY (task_id));
 CREATE TABLE role_success_criteria (role VARCHAR2(255) NOT NULL, request_id NUMBER(19) NOT NULL, stage_id NUMBER(19) NOT NULL, success_factor NUMBER(19,4) NOT NULL, PRIMARY KEY (role, request_id, stage_id));
 CREATE TABLE stage (stage_id NUMBER(19) NOT NULL, request_id NUMBER(19) NOT NULL, cluster_id NUMBER(19) NULL, log_info VARCHAR2(255) NULL, request_context VARCHAR2(255) NULL, cluster_host_info BLOB NOT NULL, PRIMARY KEY (stage_id, request_id));
 CREATE TABLE key_value_store ("key" VARCHAR2(255) NOT NULL, "value" CLOB NULL, PRIMARY KEY ("key"));

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -70,7 +70,7 @@ GRANT ALL PRIVILEGES ON TABLE ambari.users TO :username;
 CREATE TABLE ambari.execution_command (command BYTEA, task_id BIGINT NOT NULL, PRIMARY KEY (task_id));
 GRANT ALL PRIVILEGES ON TABLE ambari.execution_command TO :username;
 
-CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, status VARCHAR(255), std_error BYTEA, std_out BYTEA, role_command VARCHAR(255), PRIMARY KEY (task_id));
+CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, structured_out BYTEA, role_command VARCHAR(255), PRIMARY KEY (task_id));
 GRANT ALL PRIVILEGES ON TABLE ambari.host_role_command TO :username;
 
 CREATE TABLE ambari.role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id));

ambari-server/src/main/resources/Ambari-DDL-Postgres-REMOTE-CREATE.sql

@@ -33,7 +33,7 @@ CREATE TABLE ambari.servicedesiredstate (cluster_id BIGINT NOT NULL, desired_hos
 CREATE TABLE ambari.roles (role_name VARCHAR(255) NOT NULL, PRIMARY KEY (role_name));
 CREATE TABLE ambari.users (user_id INTEGER, ldap_user INTEGER NOT NULL DEFAULT 0, user_name VARCHAR(255) NOT NULL, create_time TIMESTAMP DEFAULT NOW(), user_password VARCHAR(255), PRIMARY KEY (user_id), UNIQUE (ldap_user, user_name));
 CREATE TABLE ambari.execution_command (command bytea, task_id BIGINT NOT NULL, PRIMARY KEY (task_id));
-CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, status VARCHAR(255), std_error bytea, std_out bytea, role_command VARCHAR(255), PRIMARY KEY (task_id));
+CREATE TABLE ambari.host_role_command (task_id BIGINT NOT NULL, attempt_count SMALLINT NOT NULL, event VARCHAR(32000) NOT NULL, exitcode INTEGER NOT NULL, host_name VARCHAR(255) NOT NULL, last_attempt_time BIGINT NOT NULL, request_id BIGINT NOT NULL, role VARCHAR(255), stage_id BIGINT NOT NULL, start_time BIGINT NOT NULL, end_time BIGINT, status VARCHAR(255), std_error BYTEA, std_out BYTEA, structured_out BYTEA, role_command VARCHAR(255), PRIMARY KEY (task_id));
 CREATE TABLE ambari.role_success_criteria (role VARCHAR(255) NOT NULL, request_id BIGINT NOT NULL, stage_id BIGINT NOT NULL, success_factor FLOAT NOT NULL, PRIMARY KEY (role, request_id, stage_id));
 CREATE TABLE ambari.stage (stage_id BIGINT NOT NULL, request_id BIGINT NOT NULL, cluster_id BIGINT NOT NULL, log_info VARCHAR(255) NOT NULL, request_context VARCHAR(255), PRIMARY KEY (stage_id, request_id));
 CREATE TABLE ambari.ClusterHostMapping (cluster_id BIGINT NOT NULL, host_name VARCHAR(255) NOT NULL, PRIMARY KEY (cluster_id, host_name));

ambari-server/src/main/resources/properties.json

@@ -118,6 +118,8 @@
         "Tasks/stderr",
         "Tasks/stdout",
         "Tasks/start_time",
+        "Tasks/end_time",
+        "Tasks/structured_out",
         "Tasks/attempt_cnt",
         "_"
     ],

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Oracle-UPGRADE.sql

@@ -76,4 +76,8 @@ ALTER TABLE stage MODIFY (cluster_host_info NOT NULL);
 
 ALTER TABLE ambari.hosts DROP COLUMN disks_info;
 
+--Added end_time and structured output support to command execution result
+ALTER TABLE host_role_command ADD (end_time NUMBER(19) DEFAULT NULL);
+ALTER TABLE host_role_command ADD (structured_out BLOB DEFAULT NULL);
+
 commit;

ambari-server/src/main/resources/upgrade/ddl/Ambari-DDL-Postgres-UPGRADE-1.3.0.sql

@@ -154,3 +154,7 @@ UPDATE ambari.stage sd
 ALTER TABLE ambari.stage ALTER COLUMN cluster_host_info SET NOT NULL;
 
 ALTER TABLE ambari.hosts DROP COLUMN disks_info;
+
+--Added end_time and structured output support to command execution result
+ALTER TABLE ambari.host_role_command ADD COLUMN end_time BIGINT;
+ALTER TABLE ambari.host_role_command ADD COLUMN structured_out BYTEA;

ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionDBAccessorImpl.java

@@ -238,6 +238,7 @@ public class TestActionDBAccessorImpl {
     commandReport.setStatus(HostRoleStatus.COMPLETED.toString());
     commandReport.setStdOut(largeString);
     commandReport.setStdErr(largeString);
+    commandReport.setStructuredOut(largeString);
     commandReport.setExitCode(123);
     db.updateHostRoleState(hostName, requestId, stageId, Role.HBASE_MASTER.toString(), commandReport);
 
@@ -249,6 +250,10 @@ public class TestActionDBAccessorImpl {
     assertNotNull(command);
 
     assertEquals(largeString, command.getStdout());
+    assertEquals(largeString, command.getStructuredOut());
+
+    //endTime for completed commands should be set
+    assertTrue(command.getEndTime() != -1);
 
   }
 

ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionManager.java

@@ -98,6 +98,7 @@ public class TestActionManager {
     cr.setStatus("COMPLETED");
     cr.setStdErr("ERROR");
     cr.setStdOut("OUTPUT");
+    cr.setStructuredOut("STRUCTURED_OUTPUT");
     cr.setExitCode(215);
     reports.add(cr);
     am.processTaskResponse(hostname, reports);
@@ -113,7 +114,10 @@ public class TestActionManager {
         "OUTPUT",
         am.getAction(requestId, stageId)
             .getHostRoleCommand(hostname, "HBASE_MASTER").getStdout());
-    
+    assertEquals(
+      "STRUCTURED_OUTPUT",
+      am.getAction(requestId, stageId)
+        .getHostRoleCommand(hostname, "HBASE_MASTER").getStructuredOut());
   }
   
   @Test
@@ -136,6 +140,7 @@ public class TestActionManager {
     String outLog = Arrays.toString(new byte[110000]);
     cr.setStdErr(errLog);
     cr.setStdOut(outLog);
+    cr.setStructuredOut(outLog);
     cr.setExitCode(215);
     reports.add(cr);
     am.processTaskResponse(hostname, reports);
@@ -151,7 +156,10 @@ public class TestActionManager {
         outLog.length(),
         am.getAction(requestId, stageId)
             .getHostRoleCommand(hostname, "HBASE_MASTER").getStdout().length());
-    
+    assertEquals(
+        outLog.length(),
+        am.getAction(requestId, stageId)
+            .getHostRoleCommand(hostname, "HBASE_MASTER").getStructuredOut().length());
   }
 
   private void populateActionDB(ActionDBAccessor db, String hostname) {