AMBARI-10170. Full Delete of Host : Fix UpgradeCatalogs to handle host-related changes in 1.6.1, 1.7.0, and 2.0.0 after refactoring Entity models (alejandro)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java

@@ -91,11 +91,6 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
     return null;
   }
 
-  @Override
-  public String[] getCompatibleVersions() {
-    return null;
-  }
-
   protected static UpgradeCatalog getUpgradeCatalog(String version) {
     return upgradeCatalogMap.get(version);
   }
@@ -229,7 +224,7 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
    * If xml owner service is not in the cluster, the configuration won't be added.
    *
    * @param configType Configuration type. (hdfs-site, etc.)
-   * @param properties Set property names.
+   * @param propertyNames Set property names.
    */
   protected void updateConfigurationPropertiesWithValuesFromXml(String configType,
       Set<String> propertyNames, boolean updateIfExists, boolean createNewConfigType) throws AmbariException {
@@ -376,6 +371,11 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
     executeDDLUpdates();
   }
 
+  @Override
+  public void preUpgradeData() throws AmbariException, SQLException {
+    executePreDMLUpdates();
+  }
+
   @Override
   public void upgradeData() throws AmbariException, SQLException {
     executeDMLUpdates();
@@ -384,6 +384,13 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
 
   protected abstract void executeDDLUpdates() throws AmbariException, SQLException;
 
+  /**
+   * Perform data insertion before running normal upgrade of data, requires started persist service
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  protected abstract void executePreDMLUpdates() throws AmbariException, SQLException;
+
   protected abstract void executeDMLUpdates() throws AmbariException, SQLException;
 
   @Override

ambari-server/src/main/java/org/apache/ambari/server/upgrade/SchemaUpgradeHelper.java

@@ -150,33 +150,6 @@ public class SchemaUpgradeHelper {
     return upgradeCatalogs;
   }
 
-  /**
-   * Determine if the class definitions in the target version allow running the Upgrade Catalogs in the path.
-   * @param upgradeCatalogs Ordered list of upgrade catalogs
-   * @param targetVersion Destination version
-   * @throws AmbariException If invalid path, will throw an exception.
-   */
-  public void validateUpgradePath(List<UpgradeCatalog> upgradeCatalogs, String targetVersion) throws AmbariException {
-    for(UpgradeCatalog upgradeCatalog : upgradeCatalogs) {
-      if (upgradeCatalog.getCompatibleVersions() != null) {
-        boolean validPath = false;
-        for (String version : upgradeCatalog.getCompatibleVersions()) {
-          // Treat dots as dots instead of match-any-char, and * as wildcard
-          String regexVersion = version.replace(".", "\\.").replace("*", ".*");
-          Matcher m = Pattern.compile(regexVersion).matcher(targetVersion);
-          if (m.matches()) {
-            validPath = true;
-            break;
-          }
-        }
-        if (!validPath) {
-          throw new AmbariException("Cannot upgrade because UpgradeCatalog " +
-              upgradeCatalog.getTargetVersion() + " is not supported in Ambari version " + targetVersion + ".");
-        }
-      }
-    }
-  }
-
   /**
    * Extension of main controller module
    */
@@ -220,6 +193,21 @@ public class SchemaUpgradeHelper {
     }
   }
 
+  public void executePreDMLUpdates(List<UpgradeCatalog> upgradeCatalogs) throws AmbariException {
+    LOG.info("Executing Pre-DML changes.");
+
+    if (upgradeCatalogs != null && !upgradeCatalogs.isEmpty()) {
+      for (UpgradeCatalog upgradeCatalog : upgradeCatalogs) {
+        try {
+          upgradeCatalog.preUpgradeData();
+        } catch (Exception e) {
+          LOG.error("Upgrade failed. ", e);
+          throw new AmbariException(e.getMessage(), e);
+        }
+      }
+    }
+  }
+
   public void executeDMLUpdates(List<UpgradeCatalog> upgradeCatalogs) throws AmbariException {
     LOG.info("Executing DML changes.");
 
@@ -288,12 +276,12 @@ public class SchemaUpgradeHelper {
       List<UpgradeCatalog> upgradeCatalogs =
         schemaUpgradeHelper.getUpgradePath(sourceVersion, targetVersion);
 
-      schemaUpgradeHelper.validateUpgradePath(upgradeCatalogs, targetVersion);
-
       schemaUpgradeHelper.executeUpgrade(upgradeCatalogs);
 
       schemaUpgradeHelper.startPersistenceService();
 
+      schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs);
+
       schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs);
 
       schemaUpgradeHelper.executeOnPostUpgrade(upgradeCatalogs);

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog.java

@@ -32,6 +32,13 @@ public interface UpgradeCatalog {
    */
   void upgradeSchema() throws AmbariException, SQLException;
 
+  /**
+   * perform data insertion before running normal upgrade of data, requires started persist service
+   * @throws AmbariException
+   * @throws SQLException
+   */
+  void preUpgradeData() throws AmbariException, SQLException;
+
   /**
    * perform data updates as necessary, requires started persist service
    * @throws AmbariException
@@ -63,10 +70,4 @@ public interface UpgradeCatalog {
    * @return null : default
    */
   String getSourceVersion();
-
-  /**
-   * Returns a list of versions using simplified regex of the Ambari versions that allow running this UpgradeCatalog.
-   * @return null : default
-   */
-  String[] getCompatibleVersions();
 }

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java

@@ -93,14 +93,6 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog {
     return "1.5.0";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"1.5.*", "1.6.*", "1.7.*", "2.0.*"};
-  }
-
   @Override
   public void executeDDLUpdates() throws AmbariException, SQLException {
     LOG.debug("Upgrading schema...");
@@ -423,6 +415,14 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog {
     }
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
+
   @Override
   public void executeDMLUpdates() throws AmbariException, SQLException {
     // Service Config mapping

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog151.java

@@ -44,14 +44,6 @@ public class UpgradeCatalog151 extends AbstractUpgradeCatalog {
     return "1.5.1";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"1.5.*", "1.6.*", "1.7.*", "2.0.*"};
-  }
-
   // ----- Constructors ------------------------------------------------------
 
   @Inject
@@ -146,6 +138,13 @@ public class UpgradeCatalog151 extends AbstractUpgradeCatalog {
 
 
   // ----- UpgradeCatalog ----------------------------------------------------
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
 
   @Override
   public void executeDMLUpdates() throws AmbariException, SQLException {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java

@@ -46,14 +46,6 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
     return "1.6.0";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"1.6.*", "1.7.*", "2.0.*"};
-  }
-
   // ----- Constructors ------------------------------------------------------
 
   @Inject
@@ -193,6 +185,13 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
 
 
   // ----- UpgradeCatalog ----------------------------------------------------
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
 
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog161.java

@@ -57,14 +57,6 @@ public class UpgradeCatalog161 extends AbstractUpgradeCatalog {
     return "1.6.1";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"1.6.*", "1.7.*", "2.0.*"};
-  }
-
   /**
    * Logger.
    */
@@ -273,6 +265,13 @@ public class UpgradeCatalog161 extends AbstractUpgradeCatalog {
 
 
   // ----- UpgradeCatalog ----------------------------------------------------
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
 
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java

@@ -148,14 +148,6 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
     return "1.7.0";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"1.7.*", "2.0.*"};
-  }
-
   /**
    * Logger.
    */
@@ -589,6 +581,13 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
 
 
   // ----- UpgradeCatalog ----------------------------------------------------
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
 
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog200.java

@@ -95,14 +95,6 @@ public class UpgradeCatalog200 extends AbstractUpgradeCatalog {
     return "2.0.0";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] { "2.0.*", "2.1.*" };
-  }
-
   /**
    * Logger.
    */
@@ -316,6 +308,13 @@ public class UpgradeCatalog200 extends AbstractUpgradeCatalog {
   }
 
   // ----- UpgradeCatalog ----------------------------------------------------
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void executePreDMLUpdates() {
+    ;
+  }
 
   /**
    * {@inheritDoc}

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog210.java

@@ -39,6 +39,7 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Service;
 import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.state.stack.OsFamily;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -92,6 +93,9 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
 
   @Inject
   DaoUtils daoUtils;
+
+  @Inject
+  private OsFamily osFamily;
   
   /**
    * {@inheritDoc}
@@ -109,14 +113,6 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
     return "2.1.0";
   }
 
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public String[] getCompatibleVersions() {
-    return new String[] {"*"};
-  }
-
   /**
    * Logger.
    */
@@ -135,6 +131,7 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
     this.injector = injector;
 
     daoUtils = injector.getInstance(DaoUtils.class);
+    osFamily = injector.getInstance(OsFamily.class);
   }
 
   // ----- AbstractUpgradeCatalog --------------------------------------------
@@ -317,6 +314,8 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
         "host_id", HOSTS_TABLE, "host_id", false);
     dbAccessor.addFKConstraint(HOST_COMPONENT_DESIRED_STATE_TABLE, "FK_hcdesiredstate_host_id",
         "host_id", HOSTS_TABLE, "host_id", false);
+    dbAccessor.addFKConstraint(HOST_ROLE_COMMAND_TABLE, "FK_host_role_command_host_id",
+        "host_id", HOSTS_TABLE, "host_id", false);
     dbAccessor.addFKConstraint(HOST_STATE_TABLE, "FK_hoststate_host_id",
         "host_id", HOSTS_TABLE, "host_id", false);
     dbAccessor.addFKConstraint(KERBEROS_PRINCIPAL_HOST_TABLE, "FK_krb_pr_host_id",
@@ -503,7 +502,7 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
   /**
    * Adds the stack table and constraints.
    */
-  protected void executeStackDMLUpdates() throws AmbariException, SQLException {
+  protected void executeStackPreDMLUpdates() throws AmbariException, SQLException {
     Gson gson = new Gson();
 
     injector.getInstance(AmbariMetaInfo.class);
@@ -755,6 +754,14 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
     return found ? constraint : null;
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  protected void executePreDMLUpdates() throws AmbariException, SQLException {
+    executeStackPreDMLUpdates();
+  }
+
   /**
    * {@inheritDoc}
    */
@@ -764,9 +771,6 @@ public class UpgradeCatalog210 extends AbstractUpgradeCatalog {
 
     // Initialize all default widgets and widget layouts
     initializeClusterAndServiceWidgets();
-
-    // populate new stack ID columns
-    executeStackDMLUpdates();
   }
 
   /**

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog210Test.java

@@ -126,6 +126,23 @@ public class UpgradeCatalog210Test {
     viewSectionDDL.verify(dbAccessor);
   }
 
+  @Test
+  public void testExecutePreDMLUpdates() throws Exception {
+    Method executeStackPreDMLUpdates = UpgradeCatalog210.class.getDeclaredMethod("executeStackPreDMLUpdates");
+
+    UpgradeCatalog210 upgradeCatalog210 = createMockBuilder(UpgradeCatalog210.class)
+        .addMockedMethod( executeStackPreDMLUpdates) .createMock();
+
+    upgradeCatalog210.executeStackPreDMLUpdates();
+    expectLastCall().once();
+
+    replay(upgradeCatalog210);
+
+    upgradeCatalog210.executePreDMLUpdates();
+
+    verify(upgradeCatalog210);
+  }
+
   @Test
   public void testExecuteDMLUpdates() throws Exception {
     Method addNewConfigurationsFromXml =
@@ -134,12 +151,9 @@ public class UpgradeCatalog210Test {
     Method initializeClusterAndServiceWidgets =
       UpgradeCatalog210.class.getDeclaredMethod("initializeClusterAndServiceWidgets");
 
-    Method executeStackDMLUpdates = UpgradeCatalog210.class.getDeclaredMethod("executeStackDMLUpdates");
-
     UpgradeCatalog210 upgradeCatalog210 = createMockBuilder(UpgradeCatalog210.class)
       .addMockedMethod(addNewConfigurationsFromXml)
-      .addMockedMethod(initializeClusterAndServiceWidgets)
-      .addMockedMethod( executeStackDMLUpdates) .createMock();
+      .addMockedMethod(initializeClusterAndServiceWidgets).createMock();
 
     upgradeCatalog210.addNewConfigurationsFromXml();
     expectLastCall().once();
@@ -147,9 +161,6 @@ public class UpgradeCatalog210Test {
     upgradeCatalog210.initializeClusterAndServiceWidgets();
     expectLastCall().once();
 
-    upgradeCatalog210.executeStackDMLUpdates();
-    expectLastCall().once();
-
     replay(upgradeCatalog210);
 
     upgradeCatalog210.executeDMLUpdates();

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java

@@ -30,14 +30,11 @@ import org.apache.ambari.server.controller.ClusterRequest;
 import org.apache.ambari.server.controller.ConfigurationRequest;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
-import org.apache.ambari.server.orm.dao.ClusterDAO;
-import org.apache.ambari.server.orm.entities.ClusterEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.junit.After;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.sql.SQLException;
@@ -67,18 +64,18 @@ public class UpgradeCatalogTest {
     }
 
     @Override
-    public void executeDMLUpdates() throws AmbariException, SQLException {
+    public void executePreDMLUpdates() throws AmbariException, SQLException {
 
     }
 
     @Override
-    public String getTargetVersion() {
-      return "1.4.9";
+    public void executeDMLUpdates() throws AmbariException, SQLException {
+
     }
 
     @Override
-    public String[] getCompatibleVersions() {
-      return new String[] {"1.4.9", "1.5.*", "1.7.*", "2.0.*"};
+    public String getTargetVersion() {
+      return "1.4.9";
     }
   }
 
@@ -125,44 +122,6 @@ public class UpgradeCatalogTest {
     Assert.assertEquals(2, upgradeCatalogs.size());
     Assert.assertEquals("1.4.9", upgradeCatalogs.get(0).getTargetVersion());
     Assert.assertEquals("1.5.0", upgradeCatalogs.get(1).getTargetVersion());
-
-    schemaUpgradeHelper.validateUpgradePath(upgradeCatalogs, "1.5.0");
-  }
-
-  @Test
-  public void testValidateUpgradePath() throws Exception {
-    SchemaUpgradeHelper schemaUpgradeHelper = injector.getInstance(SchemaUpgradeHelper.class);
-
-    Set<UpgradeCatalog> upgradeCatalogSet = schemaUpgradeHelper.getAllUpgradeCatalogs();
-
-    Assert.assertNotNull(upgradeCatalogSet);
-    Assert.assertEquals(5, upgradeCatalogSet.size());
-
-    List<UpgradeCatalog> upgradeCatalogs = schemaUpgradeHelper.getUpgradePath(null, "2.1.0");
-
-    Assert.assertNotNull(upgradeCatalogs);
-    Assert.assertEquals(5, upgradeCatalogs.size());
-    Assert.assertEquals("1.4.9", upgradeCatalogs.get(0).getTargetVersion());
-    Assert.assertEquals("1.5.0", upgradeCatalogs.get(1).getTargetVersion());
-    Assert.assertEquals("1.7.0", upgradeCatalogs.get(2).getTargetVersion());
-    Assert.assertEquals("2.0.0", upgradeCatalogs.get(3).getTargetVersion());
-    Assert.assertEquals("2.1.0", upgradeCatalogs.get(4).getTargetVersion());
-
-    try {
-      // This is a valid path, so should not throw exception.
-      schemaUpgradeHelper.validateUpgradePath(upgradeCatalogs, "2.0.0");
-    } catch (Throwable ex) {
-      Assert.assertTrue(false);
-    }
-
-    Throwable e = null;
-    try {
-      // This is an invalid path, so should throw exception.
-      schemaUpgradeHelper.validateUpgradePath(upgradeCatalogs, "2.1.0");
-    } catch (Throwable ex) {
-      e = ex;
-    }
-    Assert.assertTrue(e instanceof AmbariException);
   }
 
   @Test

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java

@@ -213,6 +213,8 @@ public class UpgradeTest {
 
     schemaUpgradeHelper.startPersistenceService();
 
+    schemaUpgradeHelper.executePreDMLUpdates(upgradeCatalogs);
+
     schemaUpgradeHelper.executeDMLUpdates(upgradeCatalogs);
 
     LOG.info("Upgrade successful.");