AMBARI-6746. Ambari Server upgrade to take care of service config versions. (mpapirkovskyy)

ambari-server/src/main/java/org/apache/ambari/server/api/util/StackExtensionHelper.java

@@ -149,6 +149,13 @@ public class StackExtensionHelper {
             parentService.getConfigTypes() != null ?
                 parentService.getConfigTypes() :
                 Collections.<String, Map<String, Map<String, String>>>emptyMap());
+    mergedServiceInfo.setExcludedConfigTypes(
+      childService.getExcludedConfigTypes() != null ?
+        childService.getExcludedConfigTypes() :
+        parentService.getExcludedConfigTypes() != null ?
+          parentService.getExcludedConfigTypes() :
+          Collections.<String>emptySet()
+    );
     
     mergedServiceInfo.setRestartRequiredAfterChange(
             (childService.isRestartRequiredAfterChange() != null) 

ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceConfigApplicationEntity.java

@@ -53,7 +53,7 @@ public class ServiceConfigApplicationEntity {
 
   @Basic
   @Column(name = "user_name")
-  private String user;
+  private String user = "_db";
 
   @ManyToOne
   @JoinColumn(name = "service_config_id", referencedColumnName = "service_config_id")

ambari-server/src/main/java/org/apache/ambari/server/state/ServiceInfo.java

@@ -21,6 +21,7 @@ package org.apache.ambari.server.state;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -70,6 +71,10 @@ public class ServiceInfo {
   @XmlElement(name="config-type")
   private List<String> configDependencies;
 
+  @XmlElementWrapper(name="excluded-config-types")
+  @XmlElement(name="config-type")
+  private Set<String> excludedConfigTypes;
+
   @XmlTransient
   private Map<String, Map<String, Map<String, String>>> configTypes;
 
@@ -452,4 +457,15 @@ public class ServiceInfo {
   public File getAlertsFile() {
     return alertsFile;
   }
+
+  /**
+   * @return config types this service contains configuration for, but which are primarily related to another service
+   */
+  public Set<String> getExcludedConfigTypes() {
+    return excludedConfigTypes;
+  }
+
+  public void setExcludedConfigTypes(Set<String> excludedConfigTypes) {
+    this.excludedConfigTypes = excludedConfigTypes;
+  }
 }

ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java

@@ -235,13 +235,18 @@ public class ClusterImpl implements Cluster {
       LOG.error("Service config versioning disabled due to exception: ", e);
       return serviceConfigTypes;
     }
-    for (String serviceName : serviceInfoMap.keySet()) {
+    for (Entry<String, ServiceInfo> entry : serviceInfoMap.entrySet()) {
+      String serviceName = entry.getKey();
+      ServiceInfo serviceInfo = entry.getValue();
       //collect config types for service
       Set<PropertyInfo> properties = ambariMetaInfo.getProperties(desiredStackVersion.getStackName(), desiredStackVersion.getStackVersion(), serviceName);
       for (PropertyInfo property : properties) {
         int extIndex = property.getFilename().indexOf(AmbariMetaInfo.SERVICE_CONFIG_FILE_NAME_POSTFIX);
         String configType = property.getFilename().substring(0, extIndex);
-        serviceConfigTypes.put(serviceName, configType);
+        if (serviceInfo.getExcludedConfigTypes() == null ||
+          !serviceInfo.getExcludedConfigTypes().contains(configType)) {
+          serviceConfigTypes.put(serviceName, configType);
+        }
       }
     }
 

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog150.java

@@ -312,16 +312,18 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog {
       LOG.error(msg);
       throw new AmbariException(msg);
     } else if (!dbAccessor.tableHasData(tableName)) {
-      String query;
+      String query = null;
       if (dbType.equals(Configuration.POSTGRES_DB_NAME)) {
         query = getPostgresRequestUpgradeQuery();
       } else if (dbType.equals(Configuration.ORACLE_DB_NAME)) {
         query = getOracleRequestUpgradeQuery();
-      } else {
+      } else if (Configuration.MYSQL_DB_NAME.equals(dbType)) {
         query = getMysqlRequestUpgradeQuery();
       }
 
-      dbAccessor.executeQuery(query);
+      if (query != null) {
+        dbAccessor.executeQuery(query);
+      }
     } else {
       LOG.info("Table {} already filled", tableName);
     }
@@ -329,7 +331,8 @@ public class UpgradeCatalog150 extends AbstractUpgradeCatalog {
     // Drop old constraints
     // ========================================================================
     if (Configuration.POSTGRES_DB_NAME.equals(dbType)
-      || Configuration.MYSQL_DB_NAME.equals(dbType)) {
+      || Configuration.MYSQL_DB_NAME.equals(dbType)
+      || Configuration.DERBY_DB_NAME.equals(dbType)) {
 
       //recreate old constraints to sync with oracle
       dbAccessor.dropConstraint("clusterconfigmapping", "FK_clusterconfigmapping_cluster_id");

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog170.java

@@ -206,6 +206,95 @@ public class UpgradeCatalog170 extends AbstractUpgradeCatalog {
     }
 
     addAlertingFrameworkDDL();
+
+    //service config versions changes
+
+    //remove old artifacts (for versions <=1.4.1) which depend on tables changed
+    //actually these had to be dropped in UC150, but some of tables were never used, and others were just cleared
+    if (dbAccessor.tableExists("componentconfigmapping")) {
+      dbAccessor.dropTable("componentconfigmapping");
+    }
+    if (dbAccessor.tableExists("hostcomponentconfigmapping")) {
+      dbAccessor.dropTable("hostcomponentconfigmapping");
+    }
+    if (dbAccessor.tableExists("hcdesiredconfigmapping")) {
+      dbAccessor.dropTable("hcdesiredconfigmapping");
+    }
+    if (dbAccessor.tableExists("serviceconfigmapping")) {
+      dbAccessor.dropTable("serviceconfigmapping");
+    }
+
+    dbAccessor.dropConstraint("confgroupclusterconfigmapping", "FK_confg");
+
+    if (Configuration.ORACLE_DB_NAME.equals(dbType)
+      || Configuration.MYSQL_DB_NAME.equals(dbType)
+      || Configuration.DERBY_DB_NAME.equals(dbType)) {
+      dbAccessor.executeQuery("ALTER TABLE clusterconfig DROP PRIMARY KEY", true);
+    } else if (Configuration.POSTGRES_DB_NAME.equals(dbType)) {
+      dbAccessor.executeQuery("ALTER TABLE clusterconfig DROP CONSTRAINT clusterconfig_pkey CASCADE", true);
+    }
+
+    dbAccessor.addColumn("clusterconfig", new DBColumnInfo("config_id", Long.class, null, null, true));
+
+    if (Configuration.ORACLE_DB_NAME.equals(dbType)) {
+      //sequence looks to be simpler than rownum
+      if (dbAccessor.tableHasData("clusterconfig")) {
+        dbAccessor.executeQuery("CREATE SEQUENCE TEMP_SEQ " +
+          "  START WITH 1 " +
+          "  MAXVALUE 999999999999999999999999999 " +
+          "  MINVALUE 1 " +
+          "  NOCYCLE " +
+          "  NOCACHE " +
+          "  NOORDER");
+        dbAccessor.executeQuery("UPDATE clusterconfig SET config_id = TEMP_SEQ.NEXTVAL");
+        dbAccessor.dropSequence("TEMP_SEQ");
+      }
+    } else if (Configuration.MYSQL_DB_NAME.equals(dbType)) {
+      if (dbAccessor.tableHasData("clusterconfig")) {
+        dbAccessor.executeQuery("UPDATE viewinstance " +
+          "SET config_id = (SELECT @a := @a + 1 FROM (SELECT @a := 1) s)");
+      }
+    } else if (Configuration.POSTGRES_DB_NAME.equals(dbType)) {
+      if (dbAccessor.tableHasData("clusterconfig")) {
+        //window functions like row_number were added in 8.4, workaround for earlier versions (redhat/centos 5)
+        dbAccessor.executeQuery("CREATE SEQUENCE temp_seq START WITH 1");
+        dbAccessor.executeQuery("UPDATE clusterconfig SET config_id = nextval('temp_seq')");
+        dbAccessor.dropSequence("temp_seq");
+      }
+    }
+
+    //upgrade unit test workaround
+    if (Configuration.DERBY_DB_NAME.equals(dbType)) {
+      dbAccessor.executeQuery("ALTER TABLE clusterconfig ALTER COLUMN config_id DEFAULT 0");
+      dbAccessor.executeQuery("ALTER TABLE clusterconfig ALTER COLUMN config_id NOT NULL");
+    }
+
+    dbAccessor.executeQuery("ALTER TABLE clusterconfig ADD PRIMARY KEY (config_id)");
+
+    columns.clear();
+    columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("cluster_id", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("service_name", String.class, null, null, false));
+    columns.add(new DBColumnInfo("version", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("create_timestamp", Long.class, null, null, false));
+    dbAccessor.createTable("serviceconfig", columns, "service_config_id");
+
+    columns.clear();
+    columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("config_id", Long.class, null, null, false));
+    dbAccessor.createTable("serviceconfigmapping", columns, "service_config_id", "config_id");
+
+    columns.clear();
+    columns.add(new DBColumnInfo("apply_id", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("service_config_id", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("apply_timestamp", Long.class, null, null, false));
+    columns.add(new DBColumnInfo("user_name", String.class, null, "_db", false));
+    dbAccessor.createTable("serviceconfigapplication", columns, "apply_id");
+
+    dbAccessor.addFKConstraint("confgroupclusterconfigmapping", "FK_confg",
+      new String[]{"cluster_id", "config_type", "version_tag"}, "clusterconfig",
+      new String[]{"cluster_id", "type_name", "version_tag"}, true);
+
   }
 
 

ambari-server/src/main/resources/Ambari-DDL-MySQL-CREATE.sql

@@ -116,7 +116,7 @@ ALTER TABLE serviceconfigmapping ADD CONSTRAINT FK_scvm_scv FOREIGN KEY (service
 ALTER TABLE serviceconfigmapping ADD CONSTRAINT FK_scvm_config FOREIGN KEY (config_id) REFERENCES clusterconfig(config_id);
 ALTER TABLE serviceconfigapplication ADD CONSTRAINT FK_scva_scv FOREIGN KEY (service_config_id) REFERENCES serviceconfig(service_config_id);
 ALTER TABLE configgroup ADD CONSTRAINT FK_configgroup_cluster_id FOREIGN KEY (cluster_id) REFERENCES clusters (cluster_id);
-ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_confg FOREIGN KEY (version_tag, config_type, cluster_id) REFERENCES clusterconfig (version_tag, type_name, cluster_id);
+ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_confg FOREIGN KEY (cluster_id, config_type, version_tag) REFERENCES clusterconfig (cluster_id, type_name, version_tag);
 ALTER TABLE confgroupclusterconfigmapping ADD CONSTRAINT FK_cgccm_gid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id);
 ALTER TABLE configgrouphostmapping ADD CONSTRAINT FK_cghm_cgid FOREIGN KEY (config_group_id) REFERENCES configgroup (group_id);
 ALTER TABLE configgrouphostmapping ADD CONSTRAINT FK_cghm_hname FOREIGN KEY (host_name) REFERENCES hosts (host_name);

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/metainfo.xml

@@ -181,6 +181,13 @@
         <config-type>hive-env</config-type>
       </configuration-dependencies>
 
+      <excluded-config-types>
+        <config-type>hive-env</config-type>
+        <config-type>hive-site</config-type>
+        <config-type>hive-exec-log4j</config-type>
+        <config-type>hive-log4j</config-type>
+      </excluded-config-types>
+
     </service>
 
   </services>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/metainfo.xml

@@ -193,6 +193,12 @@
         <config-type>hive-site</config-type>
         <config-type>hive-env</config-type>
       </configuration-dependencies>
+      <excluded-config-types>
+        <config-type>hive-env</config-type>
+        <config-type>hive-site</config-type>
+        <config-type>hive-exec-log4j</config-type>
+        <config-type>hive-log4j</config-type>
+      </excluded-config-types>
     </service>
 
   </services>

ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml

@@ -92,6 +92,10 @@
         <config-type>falcon-runtime.properties</config-type>
       </configuration-dependencies>
 
+      <excluded-config-types>
+        <config-type>oozie-site</config-type>
+      </excluded-config-types>
+
     </service>
   </services>
 </metainfo>

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeTest.java

@@ -34,6 +34,8 @@ import org.apache.ambari.server.security.CertificateManager;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.utils.VersionUtils;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -45,6 +47,7 @@ import java.util.*;
 
 import static org.junit.Assert.assertTrue;
 
+@RunWith(Parameterized.class)
 public class UpgradeTest {
   private static final Logger LOG = LoggerFactory.getLogger(UpgradeTest.class);
 
@@ -53,11 +56,14 @@ public class UpgradeTest {
       "1.4.3", "1.4.2", "1.4.1", "1.4.0", "1.2.5", "1.2.4",
       "1.2.3"); //TODO add all
   private static String DROP_DERBY_URL = "jdbc:derby:memory:myDB/ambari;drop=true";
+
+  private final String sourceVersion;
   private  Properties properties = new Properties();
 
   private Injector injector;
 
-  public UpgradeTest() {
+  public UpgradeTest(String sourceVersion) {
+    this.sourceVersion = sourceVersion;
     properties.setProperty(Configuration.SERVER_PERSISTENCE_TYPE_KEY, "remote");
     properties.setProperty(Configuration.SERVER_JDBC_URL_KEY, Configuration.JDBC_IN_MEMORY_URL);
     properties.setProperty(Configuration.SERVER_JDBC_DRIVER_KEY, Configuration.JDBC_IN_MEMROY_DRIVER);
@@ -79,27 +85,15 @@ public class UpgradeTest {
     }
 
     String targetVersion = getLastVersion();
-    List<String> failedVersions = new ArrayList<String>();
-
-    for (String version : VERSIONS) {
-      injector = Guice.createInjector(new ControllerModule(properties));
-
-      try {
-        createSourceDatabase(version);
-
-        performUpgrade(targetVersion);
-
-        testUpgradedSchema();
-      } catch (Exception e) {
-        failedVersions.add(version);
-        e.printStackTrace();
-      }
 
-      dropDatabase();
+    injector = Guice.createInjector(new ControllerModule(properties));
+    LOG.info("Testing upgrade from version {} to {}", sourceVersion, targetVersion);
 
-    }
+    createSourceDatabase(sourceVersion);
+    performUpgrade(targetVersion);
+    testUpgradedSchema();
 
-    assertTrue("Upgrade test failed for version: " + failedVersions, failedVersions.isEmpty());
+    dropDatabase();
 
 
   }
@@ -215,5 +209,14 @@ public class UpgradeTest {
 
   }
 
+  @Parameterized.Parameters
+  public static Collection<Object[]> data() {
+    Collection<Object[]> data = new ArrayList<Object[]>();
+    for (String s : VERSIONS) {
+      data.add(new Object[]{s});
+    }
+    return data;
+  }
+
 
 }