AMBARI-8422. Upgrade Execute: create database mappings for Upgrades to Requests (ncole)

ambari-server/src/main/java/org/apache/ambari/server/actionmanager/HostRoleStatus.java

@@ -22,23 +22,39 @@ import java.util.Collections;
 import java.util.List;
 
 public enum HostRoleStatus {
-  PENDING(0), //Not queued for a host
-  QUEUED(1), //Queued for a host (or has already been sent to host, but host did not answer yet)
-  IN_PROGRESS(2), //Host reported it is working (we received an IN_PROGRESS command status from host)
-  COMPLETED(3), //Host reported success
-  FAILED(4), //Failed
-  TIMEDOUT(5), //Host did not respond in time
-  ABORTED(6); //Operation was abandoned
-  private final int status;
+  /**
+   * Not queued for a host.
+   */
+  PENDING,
+  /**
+   * Queued for a host, or has already been sent to host, but host did not answer yet.
+   */
+  QUEUED,
+  /**
+   * Host reported it is working, received an IN_PROGRESS command status from host.
+   */
+  IN_PROGRESS,
+  /**
+   * Host reported success
+   */
+  COMPLETED,
+  /**
+   * Failed
+   */
+  FAILED,
+  /**
+   * Host did not respond in time
+   */
+  TIMEDOUT,
+  /**
+   * Operation was abandoned
+   */
+  ABORTED;
 
   private static List<HostRoleStatus> COMPLETED_STATES = Arrays.asList(FAILED, TIMEDOUT, ABORTED, COMPLETED);
   private static List<HostRoleStatus> FAILED_STATES = Arrays.asList(FAILED, TIMEDOUT, ABORTED);
 
 
-  private HostRoleStatus(int status) {
-    this.status = status;
-  }
-
   /**
    * Indicates whether or not it is a valid failure state.
    *

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeItemResourceProvider.java

@@ -21,10 +21,13 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.ambari.server.StaticallyInject;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
 import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.controller.spi.NoSuchParentResourceException;
@@ -39,6 +42,7 @@ import org.apache.ambari.server.controller.spi.UnsupportedPropertyException;
 import org.apache.ambari.server.orm.dao.UpgradeDAO;
 import org.apache.ambari.server.orm.entities.UpgradeEntity;
 import org.apache.ambari.server.orm.entities.UpgradeItemEntity;
+import org.apache.ambari.server.state.UpgradeState;
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
@@ -53,6 +57,7 @@ public class UpgradeItemResourceProvider extends AbstractControllerResourceProvi
   protected static final String UPGRADE_ITEM_ID = "UpgradeItem/id";
   protected static final String UPGRADE_ITEM_STATE = "UpgradeItem/state";
   protected static final String UPGRADE_ITEM_TEXT = "UpgradeItem/text";
+  protected static final String UPGRADE_ITEM_STAGE_ID= "UpgradeItem/stage_id";
 
   private static final Set<String> PK_PROPERTY_IDS = new HashSet<String>(
       Arrays.asList(UPGRADE_ID, UPGRADE_ITEM_ID));
@@ -119,7 +124,7 @@ public class UpgradeItemResourceProvider extends AbstractControllerResourceProvi
       Collection<UpgradeItemEntity> items = upgrade.getUpgradeItems();
       if (null != items) {
         for (UpgradeItemEntity entity : items) {
-          results.add(toResource(entity, upgradeId, requestPropertyIds));
+          results.add(toResource(upgrade, entity, requestPropertyIds));
         }
       }
     }
@@ -148,15 +153,77 @@ public class UpgradeItemResourceProvider extends AbstractControllerResourceProvi
     return PK_PROPERTY_IDS;
   }
 
-  private Resource toResource(UpgradeItemEntity entity, long upgradeId, Set<String> requestedIds) {
+  private Resource toResource(UpgradeEntity upgrade, UpgradeItemEntity item, Set<String> requestedIds) {
     ResourceImpl resource = new ResourceImpl(Resource.Type.UpgradeItem);
 
-    setResourceProperty(resource, UPGRADE_ID, Long.valueOf(upgradeId), requestedIds);
-    setResourceProperty(resource, UPGRADE_ITEM_ID, entity.getId(), requestedIds);
-    setResourceProperty(resource, UPGRADE_ITEM_STATE, entity.getState(), requestedIds);
-    setResourceProperty(resource, UPGRADE_ITEM_TEXT, entity.getText(), requestedIds);
+    setResourceProperty(resource, UPGRADE_ID, upgrade.getId(), requestedIds);
+    setResourceProperty(resource, UPGRADE_ITEM_ID, item.getId(), requestedIds);
+    if (isPropertyRequested(UPGRADE_ITEM_STATE, requestedIds)) {
+      UpgradeState state = calculateState(upgrade, item);
+      setResourceProperty(resource, UPGRADE_ITEM_STATE, state, requestedIds);
+    }
+    setResourceProperty(resource, UPGRADE_ITEM_TEXT, item.getText(), requestedIds);
 
     return resource;
   }
 
+  private UpgradeState calculateState(UpgradeEntity upgrade, UpgradeItemEntity item) {
+    long requestId = upgrade.getRequestId().longValue();
+    long stageId = item.getStageId().longValue();
+
+    List<HostRoleCommand> commands = getManagementController().getActionManager().getRequestTasks(requestId);
+
+    int pending = 0;
+    int complete = 0;
+    int in_progress = 0;
+    int failed = 0;
+
+    for (HostRoleCommand command : commands) {
+      if (stageId != command.getStageId()) {
+        continue;
+      }
+
+      HostRoleStatus status = command.getStatus();
+      if (status.isFailedState()) {
+        failed++;
+      } else {
+        switch (status) {
+          case COMPLETED:
+            complete++;
+            break;
+          case IN_PROGRESS:
+            in_progress++;
+            break;
+          case PENDING:
+            pending++;
+            break;
+          case QUEUED:
+            in_progress++;
+            break;
+          case FAILED:
+          case ABORTED:
+          case TIMEDOUT:
+            failed++;
+            break;
+          default:
+            break;
+        }
+      }
+    }
+
+    if (failed > 0) {
+      return UpgradeState.FAILED;
+    } else if (in_progress > 0) {
+      return UpgradeState.IN_PROGRESS;
+    } else if (pending > 0) {
+      return UpgradeState.PENDING;
+    } else if (complete > 0) {
+      return UpgradeState.COMPLETE;
+    }
+
+    return UpgradeState.NONE;
+  }
+
+
+
 }

ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java

@@ -77,7 +77,8 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
   protected static final String UPGRADE_ID = "Upgrade/id";
   protected static final String UPGRADE_CLUSTER_NAME = "Upgrade/cluster_name";
-  protected static final String UPGRADE_VERSION = "Upgrade/version";
+  protected static final String UPGRADE_VERSION = "Upgrade/repository_version";
+  protected static final String UPGRADE_REQUEST_ID = "Upgrade/request_id";
 
   private static final Set<String> PK_PROPERTY_IDS = new HashSet<String>(
       Arrays.asList(UPGRADE_ID, UPGRADE_CLUSTER_NAME));
@@ -100,13 +101,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
   @Inject
   private static Provider<AmbariCustomCommandExecutionHelper> commandExecutionHelper;
 
-
-
   static {
     // properties
     PROPERTY_IDS.add(UPGRADE_ID);
     PROPERTY_IDS.add(UPGRADE_CLUSTER_NAME);
     PROPERTY_IDS.add(UPGRADE_VERSION);
+    PROPERTY_IDS.add(UPGRADE_REQUEST_ID);
 
     // keys
     KEY_PROPERTY_IDS.put(Resource.Type.Upgrade, UPGRADE_ID);
@@ -219,6 +219,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
     setResourceProperty(resource, UPGRADE_ID, entity.getId(), requestedIds);
     setResourceProperty(resource, UPGRADE_CLUSTER_NAME, clusterName, requestedIds);
+    setResourceProperty(resource, UPGRADE_REQUEST_ID, entity.getRequestId(), requestedIds);
 
     return resource;
   }
@@ -349,8 +350,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     entity.setClusterId(Long.valueOf(cluster.getClusterId()));
     entity.setUpgradeItems(items);
 
-    m_upgradeDAO.create(entity);
-
     RequestStageContainer req = createRequest((String) requestMap.get(UPGRADE_VERSION));
 
     for (StageHolder holder : preUpgrades) {
@@ -367,8 +366,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
 
     req.getRequestStatusResponse();
 
+    entity.setRequestId(req.getId());
+
     req.persist();
 
+    m_upgradeDAO.create(entity);
+
     return entity;
   }
 
@@ -487,7 +490,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     private Set<String> hosts;
   }
 
-
   private RequestStageContainer createRequest(String version) {
     ActionManager actionManager = getManagementController().getActionManager();
 
@@ -524,6 +526,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
       stageId = 1L;
     }
     stage.setStageId(stageId);
+    holder.upgradeItemEntity.setStageId(Long.valueOf(stageId));
 
     // add each host to this stage
     RequestResourceFilter filter = new RequestResourceFilter("", "",
@@ -583,6 +586,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
       stageId = 1L;
     }
     stage.setStageId(stageId);
+    holder.upgradeItemEntity.setStageId(Long.valueOf(stageId));
 
     // !!! TODO verify the action is valid
 

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeEntity.java

@@ -17,7 +17,6 @@
  */
 package org.apache.ambari.server.orm.entities;
 
-import java.util.Collection;
 import java.util.List;
 
 import javax.persistence.CascadeType;
@@ -25,6 +24,7 @@ import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.EnumType;
 import javax.persistence.Enumerated;
+import javax.persistence.FetchType;
 import javax.persistence.GeneratedValue;
 import javax.persistence.GenerationType;
 import javax.persistence.Id;
@@ -64,6 +64,9 @@ public class UpgradeEntity {
   @Column(name = "state", nullable = false)
   private UpgradeState state = UpgradeState.NONE;
 
+  @Column(name="request_id", nullable = false)
+  private Long requestId;
+
   @OneToMany(mappedBy = "upgradeEntity", cascade = { CascadeType.ALL })
   private List<UpgradeItemEntity> upgradeItemEntities;
 
@@ -128,4 +131,18 @@ public class UpgradeEntity {
     upgradeItemEntities = items;
   }
 
+  /**
+   * @return the request id
+   */
+  public Long getRequestId() {
+    return requestId;
+  }
+
+  /**
+   * @param id the request id
+   */
+  public void setRequestId(Long id) {
+    requestId = id;
+  }
+
 }

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/UpgradeItemEntity.java

@@ -67,6 +67,10 @@ public class UpgradeItemEntity {
   @Column(name = "item_text", length = 1024)
   private String itemText = null;
 
+  @Basic
+  @Column(name = "stage_id", nullable = false)
+  private Long stageId = Long.valueOf(0L);
+
   @ManyToOne
   @JoinColumn(name = "upgrade_id", referencedColumnName = "upgrade_id", nullable = false)
   private UpgradeEntity upgradeEntity;
@@ -153,5 +157,13 @@ public class UpgradeItemEntity {
     upgradeEntity = entity;
   }
 
+  public Long getStageId() {
+    return stageId;
+  }
+
+  public void setStageId(Long id) {
+    stageId = id;
+  }
+
 
 }

ambari-server/src/main/java/org/apache/ambari/server/state/UpgradeState.java

@@ -25,6 +25,10 @@ public enum UpgradeState {
    * No upgrade is in progress.
    */
   NONE,
+  /**
+   * Upgrade is complete
+   */
+  COMPLETE,
   /**
    * Upgrade is pending
    */

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java

@@ -180,14 +180,17 @@ public class UpgradeCatalog200 extends AbstractUpgradeCatalog {
     columns = new ArrayList<DBColumnInfo>();
     columns.add(new DBAccessor.DBColumnInfo("upgrade_id", Long.class, null, null, false));
     columns.add(new DBAccessor.DBColumnInfo("cluster_id", Long.class, null, null, false));
+    columns.add(new DBAccessor.DBColumnInfo("request_id", Long.class, null, null, false));
     columns.add(new DBAccessor.DBColumnInfo("state", String.class, 255, UpgradeState.NONE.name(), false));
     dbAccessor.createTable("upgrade", columns, "upgrade_id");
     dbAccessor.addFKConstraint("upgrade", "fk_upgrade_cluster_id", "cluster_id", "clusters", "cluster_id", false);
+    dbAccessor.addFKConstraint("upgrade", "fk_upgrade_request_id", "request_id", "request", "request_id", false);
     dbAccessor.executeQuery("INSERT INTO ambari_sequences(sequence_name, sequence_value) VALUES('upgrade_id_seq', 0)", false);
 
     columns = new ArrayList<DBColumnInfo>();
     columns.add(new DBAccessor.DBColumnInfo("upgrade_item_id", Long.class, null, null, false));
     columns.add(new DBAccessor.DBColumnInfo("upgrade_id", Long.class, null, null, false));
+    columns.add(new DBAccessor.DBColumnInfo("stage_id", Long.class, null, null, false));
     columns.add(new DBAccessor.DBColumnInfo("state", String.class, 255, UpgradeState.NONE.name(), false));
     columns.add(new DBAccessor.DBColumnInfo("hosts", char[].class, 32672, null, true));
     columns.add(new DBAccessor.DBColumnInfo("tasks", char[].class, 32672, null, true));

ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql

@@ -700,14 +700,17 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
 CREATE TABLE upgrade (
   upgrade_id BIGINT NOT NULL,
   cluster_id BIGINT NOT NULL,
+  request_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   PRIMARY KEY (upgrade_id),
-  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id)
+  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
+  FOREIGN KEY (request_id) REFERENCES request(request_id)
 );
 
 CREATE TABLE upgrade_item (
   upgrade_item_id BIGINT NOT NULL,
   upgrade_id BIGINT NOT NULL,
+  stage_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   hosts TEXT,
   tasks TEXT,

ambari-server/src/main/resources/Ambari-DDL-Oracle-CREATE.sql

@@ -690,14 +690,17 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
 CREATE TABLE upgrade (
   upgrade_id BIGINT NOT NULL,
   cluster_id BIGINT NOT NULL,
+  request_id BIGINT NOT NULL,
   state VARCHAR2(255) DEFAULT 'NONE' NOT NULL,
   PRIMARY KEY (upgrade_id),
-  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id)
+  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
+  FOREIGN KEY (request_id) REFERENCES request(request_id)
 );
 
 CREATE TABLE upgrade_item (
   upgrade_item_id BIGINT NOT NULL,
   upgrade_id BIGINT NOT NULL,
+  stage_id BIGINT NOT NULL,
   state VARCHAR2(255) DEFAULT 'NONE' NOT NULL,
   hosts CLOB,
   tasks CLOB,

ambari-server/src/main/resources/Ambari-DDL-Postgres-CREATE.sql

@@ -684,23 +684,26 @@ CREATE INDEX idx_alert_group_name on alert_group(group_name);
 CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
 
 -- upgrade tables
-CREATE TABLE ambari.upgrade (
+CREATE TABLE upgrade (
   upgrade_id BIGINT NOT NULL,
   cluster_id BIGINT NOT NULL,
+  request_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   PRIMARY KEY (upgrade_id),
-  FOREIGN KEY (cluster_id) REFERENCES ambari.clusters(cluster_id)
+  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
+  FOREIGN KEY (request_id) REFERENCES request(request_id)
 );
 
-CREATE TABLE ambari.upgrade_item (
+CREATE TABLE upgrade_item (
   upgrade_item_id BIGINT NOT NULL,
   upgrade_id BIGINT NOT NULL,
+  stage_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   hosts TEXT,
   tasks TEXT,
   item_text VARCHAR(1024),
   PRIMARY KEY (upgrade_item_id),
-  FOREIGN KEY (upgrade_id) REFERENCES ambari.upgrade(upgrade_id)
+  FOREIGN KEY (upgrade_id) REFERENCES upgrade(upgrade_id)
 );
 
 ---------inserting some data-----------

ambari-server/src/main/resources/Ambari-DDL-Postgres-EMBEDDED-CREATE.sql

@@ -770,14 +770,17 @@ CREATE INDEX idx_alert_notice_state on ambari.alert_notice(notify_state);
 CREATE TABLE ambari.upgrade (
   upgrade_id BIGINT NOT NULL,
   cluster_id BIGINT NOT NULL,
+  request_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   PRIMARY KEY (upgrade_id),
-  FOREIGN KEY (cluster_id) REFERENCES ambari.clusters(cluster_id)
+  FOREIGN KEY (cluster_id) REFERENCES ambari.clusters(cluster_id),
+  FOREIGN KEY (request_id) REFERENCES ambari.request(request_id)
 );
 
 CREATE TABLE ambari.upgrade_item (
   upgrade_item_id BIGINT NOT NULL,
   upgrade_id BIGINT NOT NULL,
+  stage_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   hosts TEXT,
   tasks TEXT,

ambari-server/src/main/resources/Ambari-DDL-SQLServer-CREATE.sql

@@ -267,14 +267,17 @@ CREATE INDEX idx_alert_notice_state on alert_notice(notify_state);
 CREATE TABLE upgrade (
   upgrade_id BIGINT NOT NULL,
   cluster_id BIGINT NOT NULL,
+  request_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   PRIMARY KEY (upgrade_id),
-  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id)
+  FOREIGN KEY (cluster_id) REFERENCES clusters(cluster_id),
+  FOREIGN KEY (request_id) REFERENCES request(request_id)
 );
 
 CREATE TABLE upgrade_item (
   upgrade_item_id BIGINT NOT NULL,
   upgrade_id BIGINT NOT NULL,
+  stage_id BIGINT NOT NULL,
   state VARCHAR(255) DEFAULT 'NONE' NOT NULL,
   hosts TEXT,
   tasks TEXT,

ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java

@@ -38,6 +38,7 @@ import org.apache.ambari.server.orm.dao.RepositoryVersionDAO;
 import org.apache.ambari.server.orm.dao.UpgradeDAO;
 import org.apache.ambari.server.orm.entities.RepositoryVersionEntity;
 import org.apache.ambari.server.orm.entities.UpgradeEntity;
+import org.apache.ambari.server.orm.entities.UpgradeItemEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Host;
@@ -45,6 +46,7 @@ import org.apache.ambari.server.state.RepositoryVersionState;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.ServiceComponent;
 import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.UpgradeState;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -153,8 +155,18 @@ public class UpgradeResourceProviderTest {
     List<Long> requests = am.getRequestsByStatus(RequestStatus.IN_PROGRESS, 100, true);
 
     assertEquals(1, requests.size());
+    assertEquals(requests.get(0), entity.getRequestId());
+
+
     List<Stage> stages = am.getRequestStatus(requests.get(0).longValue());
     assertEquals(3, stages.size());
+    for (int i = 0; i < stages.size(); i++) {
+      Stage stage = stages.get(i);
+      UpgradeItemEntity upgradeItem = entity.getUpgradeItems().get(i);
+      assertEquals(stage.getStageId(), upgradeItem.getStageId().longValue());
+      assertEquals(UpgradeState.NONE, upgradeItem.getState());
+    }
+
 
     List<HostRoleCommand> tasks = am.getRequestTasks(requests.get(0).longValue());
     assertEquals(3, tasks.size());

ambari-server/src/test/java/org/apache/ambari/server/orm/dao/UpgradeDAOTest.java

@@ -69,19 +69,21 @@ public class UpgradeDAOTest {
     // create upgrade entities
     UpgradeEntity entity = new UpgradeEntity();
     entity.setClusterId(Long.valueOf(1));
+    entity.setRequestId(Long.valueOf(1));
 
     // create 2 items
     List<UpgradeItemEntity> items = new ArrayList<UpgradeItemEntity>();
     UpgradeItemEntity item = new UpgradeItemEntity();
     item.setState(UpgradeState.IN_PROGRESS);
-    item.setUpgradeEntity(entity);
+    item.setStageId(Long.valueOf(1L));
     items.add(item);
 
     item = new UpgradeItemEntity();
     item.setState(UpgradeState.PENDING);
-    item.setUpgradeEntity(entity);
+    item.setStageId(Long.valueOf(1L));
     items.add(item);
 
+
     entity.setUpgradeItems(items);
     dao.create(entity);
   }

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog200Test.java

@@ -184,8 +184,8 @@ public class UpgradeCatalog200Test {
     assertViewInstancePropertyColumns(valueColumnCapture);
     assertViewInstanceDataColumns(dataValueColumnCapture);
 
-    assertEquals(3, upgradeCapture.getValue().size());
-    assertEquals(6, upgradeItemCapture.getValue().size());
+    assertEquals(4, upgradeCapture.getValue().size());
+    assertEquals(7, upgradeItemCapture.getValue().size());
   }
 
   @Test