AMBARI-5661. YARN and MR2 failed to start after upgrade. (swagle)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/AbstractUpgradeCatalog.java

@@ -23,18 +23,21 @@ import com.google.inject.Provider;
 import com.google.inject.persist.Transactional;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ConfigurationRequest;
 import org.apache.ambari.server.orm.DBAccessor;
 import org.apache.ambari.server.orm.dao.MetainfoDAO;
 import org.apache.ambari.server.orm.entities.MetainfoEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.utils.VersionUtils;
-import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.persistence.EntityManager;
-import java.io.File;
-import java.io.IOException;
 import java.sql.SQLException;
+import java.text.MessageFormat;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
@@ -168,6 +171,76 @@ public abstract class AbstractUpgradeCatalog implements UpgradeCatalog {
     dbAccessor.executeQuery(String.format("ALTER ROLE %s SET search_path to '%s';", dbUser, dbName));
   }
 
+  /**
+   * Create a new cluster scoped configuration with the new properties added
+   * to the existing set of properties.
+   * @param configType Configuration type. (hdfs-site, etc.)
+   * @param properties Map of key value pairs to add / update.
+   */
+  protected void updateConfigurationProperties(String configType,
+        Map<String, String> properties, boolean updateIfExists) throws
+    AmbariException {
+    AmbariManagementController controller = injector.getInstance(AmbariManagementController.class);
+    String newTag = "version" + System.currentTimeMillis();
+
+    Clusters clusters = controller.getClusters();
+    if (clusters == null) {
+      return;
+    }
+    Map<String, Cluster> clusterMap = clusters.getClusters();
+
+    if (clusterMap != null && !clusterMap.isEmpty()) {
+      for (Cluster cluster : clusterMap.values()) {
+        Config oldConfig = cluster.getDesiredConfigByType(configType);
+
+        if (properties != null) {
+          Map<String, Config> all = cluster.getConfigsByType(configType);
+          if (all == null || !all.containsKey(newTag) || properties.size() > 0) {
+
+            Map<String, String> mergedProperties =
+              mergeProperties(oldConfig.getProperties(), properties, updateIfExists);
+
+            LOG.info("Applying configuration with tag ''{0}'' to " +
+              "cluster ''{1}''", newTag, cluster.getClusterName());
+
+            ConfigurationRequest cr = new ConfigurationRequest();
+            cr.setClusterName(cluster.getClusterName());
+            cr.setVersionTag(newTag);
+            cr.setType(configType);
+            cr.setProperties(mergedProperties);
+            controller.createConfiguration(cr);
+
+            Config baseConfig = cluster.getConfig(cr.getType(), cr.getVersionTag());
+            if (baseConfig != null) {
+              String authName = "ambari-upgrade";
+
+              if (cluster.addDesiredConfig(authName, baseConfig)) {
+                LOG.info("cluster '" + cluster.getClusterName() + "' "
+                  + "changed by: '" + authName + "'; "
+                  + "type='" + baseConfig.getType() + "' "
+                  + "tag='" + baseConfig.getVersionTag() + "'"
+                  + " from='"+ oldConfig.getVersionTag() + "'");
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private Map<String, String> mergeProperties(Map<String, String> originalProperties,
+                               Map<String, String> newProperties,
+                               boolean updateIfExists) {
+
+    Map<String, String> properties = new HashMap<String, String>(originalProperties);
+    for (Map.Entry<String, String> entry : newProperties.entrySet()) {
+      if (!properties.containsKey(entry.getKey()) || updateIfExists) {
+        properties.put(entry.getKey(), entry.getValue());
+      }
+    }
+    return properties;
+  }
+
   @Override
   public void upgradeSchema() throws AmbariException, SQLException {
     if (getDbType().equals(Configuration.POSTGRES_DB_NAME)) {

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog160.java

@@ -26,6 +26,7 @@ import org.apache.ambari.server.orm.DBAccessor;
 
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 /**
@@ -97,7 +98,7 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
   protected void executeDMLUpdates() throws AmbariException, SQLException {
     String dbType = getDbType();
 
-    //add new sequences for view entity
+    // Add new sequences for view entity
     String valueColumnName = "\"value\"";
     if (Configuration.ORACLE_DB_NAME.equals(dbType) || Configuration.MYSQL_DB_NAME.equals(dbType)) {
       valueColumnName = "value";
@@ -105,6 +106,9 @@ public class UpgradeCatalog160 extends AbstractUpgradeCatalog {
 
     dbAccessor.executeQuery("INSERT INTO ambari_sequences(sequence_name, " + valueColumnName + ") " +
         "VALUES('viewentity_id_seq', 0)", true);
+
+    // Add missing property for YARN
+    updateConfigurationProperties("global", Collections.singletonMap("jobhistory_heapsize", "900"), false);
   }
 
   @Override

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog160Test.java

@@ -23,23 +23,32 @@ import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 import org.apache.ambari.server.configuration.Configuration;
+import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
 import org.easymock.Capture;
 import org.junit.Assert;
 import org.junit.Test;
 
 import java.lang.reflect.Field;
+import java.lang.reflect.Method;
 import java.sql.SQLException;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertFalse;
 import static junit.framework.Assert.assertTrue;
 import static junit.framework.Assert.assertNull;
 import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.easymock.EasyMock.createNiceControl;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
 
@@ -79,23 +88,34 @@ public class UpgradeCatalog160Test {
 
   @Test
   public void testExecuteDMLUpdates() throws Exception {
-
     Configuration configuration = createNiceMock(Configuration.class);
+    DBAccessor dbAccessor = createNiceMock(DBAccessor.class);
+
+    Method m = AbstractUpgradeCatalog.class.getDeclaredMethod
+      ("updateConfigurationProperties", String.class, Map.class, boolean.class);
+
+    UpgradeCatalog160 upgradeCatalog = createMockBuilder(UpgradeCatalog160.class)
+      .addMockedMethod(m).createMock();
+
     expect(configuration.getDatabaseUrl()).andReturn(Configuration.JDBC_IN_MEMORY_URL).anyTimes();
 
-    final DBAccessor dbAccessor     = createNiceMock(DBAccessor.class);
-    UpgradeCatalog160 upgradeCatalog = (UpgradeCatalog160) getUpgradeCatalog(dbAccessor);
+    upgradeCatalog.updateConfigurationProperties("global",
+      Collections.singletonMap("jobhistory_heapsize", "900"), false);
+    expectLastCall();
 
-    replay(dbAccessor, configuration);
+    replay(upgradeCatalog, dbAccessor, configuration);
 
     Class<?> c = AbstractUpgradeCatalog.class;
     Field f = c.getDeclaredField("configuration");
     f.setAccessible(true);
     f.set(upgradeCatalog, configuration);
+    f = c.getDeclaredField("dbAccessor");
+    f.setAccessible(true);
+    f.set(upgradeCatalog, dbAccessor);
 
     upgradeCatalog.executeDMLUpdates();
 
-    verify(dbAccessor, configuration);
+    verify(upgradeCatalog, dbAccessor, configuration);
   }
 
   @Test

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalogTest.java

@@ -25,20 +25,34 @@ import com.google.inject.persist.PersistService;
 import junit.framework.Assert;
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.api.services.AmbariMetaInfo;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.ClusterRequest;
+import org.apache.ambari.server.controller.ConfigurationRequest;
 import org.apache.ambari.server.orm.GuiceJpaInitializer;
 import org.apache.ambari.server.orm.InMemoryDefaultTestModule;
+import org.apache.ambari.server.orm.dao.ClusterDAO;
+import org.apache.ambari.server.orm.entities.ClusterEntity;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
 
 import java.sql.SQLException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public class UpgradeCatalogTest {
   private Injector injector;
   private AmbariMetaInfo metaInfo;
+  private final String DESIRED_STACK_VERSION = "{\"stackName\":\"HDP\"," +
+    "\"stackVersion\":\"1.2.0\"}";
 
   private static class UpgradeCatalog149 extends AbstractUpgradeCatalog {
 
@@ -88,11 +102,9 @@ public class UpgradeCatalogTest {
     injector.getInstance(PersistService.class).stop();
   }
 
-
   @Test
   public void testUpgradePath() throws Exception {
-    SchemaUpgradeHelper schemaUpgradeHelper = injector.getInstance
-      (SchemaUpgradeHelper.class);
+    SchemaUpgradeHelper schemaUpgradeHelper = injector.getInstance(SchemaUpgradeHelper.class);
 
     Set<UpgradeCatalog> upgradeCatalogSet = schemaUpgradeHelper.getAllUpgradeCatalogs();
 
@@ -107,4 +119,71 @@ public class UpgradeCatalogTest {
     Assert.assertEquals("1.4.9", upgradeCatalogs.get(0).getTargetVersion());
     Assert.assertEquals("1.5.0", upgradeCatalogs.get(1).getTargetVersion());
   }
+
+  @Test
+  public void testUpdateConfigurationProperties() throws Exception {
+    AmbariManagementController controller = injector.getInstance(AmbariManagementController.class);
+    ClusterRequest r = new ClusterRequest(null, "c1", "HDP-0.1", null);
+    controller.createCluster(r);
+    Clusters clusters = injector.getInstance(Clusters.class);
+    Cluster cluster = clusters.getCluster("c1");
+
+    Assert.assertNotNull(cluster);
+
+    Map<String, String> properties = new HashMap<String, String>() {{
+      put("a", "b");
+    }};
+
+    final ClusterRequest cl = new ClusterRequest(
+      cluster.getClusterId(),
+      cluster.getClusterName(),
+      cluster.getDesiredStackVersion().getStackVersion(),
+      null
+    );
+
+    ConfigurationRequest cr = new ConfigurationRequest();
+    cr.setClusterName(cluster.getClusterName());
+    cr.setType("global");
+    cr.setVersionTag("version1");
+    cr.setProperties(properties);
+
+    cl.setDesiredConfig(cr);
+
+    controller.updateClusters(new HashSet<ClusterRequest>() {{ add(cl); }}, null);
+
+    Config config = cluster.getConfig("global", "version1");
+
+    Assert.assertNotNull(config);
+    Assert.assertEquals(properties, config.getProperties());
+
+    // Add new
+    UpgradeCatalog149 testCatalog = injector.getInstance(UpgradeCatalog149.class);
+    testCatalog.updateConfigurationProperties("global",
+      Collections.singletonMap("x", "y"), false);
+    config = cluster.getDesiredConfigByType("global");
+    String version = config.getVersionTag();
+    Assert.assertNotNull(config);
+    Assert.assertNotSame("version1", version);
+    Assert.assertTrue(config.getProperties().containsKey("x"));
+    Assert.assertEquals("y", config.getProperties().get("x"));
+
+    // Override value
+    testCatalog.updateConfigurationProperties("global",
+      Collections.singletonMap("x", "z"), true);
+    config = cluster.getDesiredConfigByType("global");
+    Assert.assertNotNull(config);
+    Assert.assertNotSame(version, config.getVersionTag());
+    Assert.assertTrue(config.getProperties().containsKey("x"));
+    Assert.assertEquals("z", config.getProperties().get("x"));
+    version = config.getVersionTag();
+
+    // Retain original
+    testCatalog.updateConfigurationProperties("global",
+      Collections.singletonMap("x", "y"), false);
+    config = cluster.getDesiredConfigByType("global");
+    Assert.assertNotNull(config);
+    Assert.assertNotSame(version, config.getVersionTag());
+    Assert.assertTrue(config.getProperties().containsKey("x"));
+    Assert.assertEquals("z", config.getProperties().get("x"));
+  }
 }