AMBARI-15006. Ambari Server Upgrade adds unneeded Atlas properties even though Atlas is not needed, causing forced Hive restart (aonishuk)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog222.java

@@ -28,11 +28,15 @@ import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
+import org.apache.ambari.server.utils.VersionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.sql.SQLException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -49,6 +53,16 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
    */
   private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog222.class);
   private static final String AMS_SITE = "ams-site";
+  private static final String HIVE_SITE_CONFIG = "hive-site";
+  private static final String ATLAS_APPLICATION_PROPERTIES_CONFIG = "application-properties";
+  private static final String ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY = "atlas.hook.hive.minThreads";
+  private static final String ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY = "atlas.hook.hive.maxThreads";
+  private static final String ATLAS_CLUSTER_NAME_PROPERTY = "atlas.cluster.name";
+  private static final String ATLAS_ENABLETLS_PROPERTY = "atlas.enableTLS";
+  private static final String ATLAS_SERVER_HTTP_PORT_PROPERTY = "atlas.server.http.port";
+  private static final String ATLAS_SERVER_HTTPS_PORT_PROPERTY = "atlas.server.https.port";
+  private static final String ATLAS_REST_ADDRESS_PROPERTY = "atlas.rest.address";
+
   private static final String HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER =
     "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
   private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER =
@@ -115,6 +129,7 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     updateAlerts();
     updateStormConfigs();
     updateAMSConfigs();
+    updateHiveConfig();
     updateHostRoleCommands();
   }
 
@@ -285,6 +300,41 @@ public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
     }
   }
 
+  protected void updateHiveConfig() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
+      Config hiveSiteConfig = cluster.getDesiredConfigByType(HIVE_SITE_CONFIG);
+      Config atlasConfig = cluster.getDesiredConfigByType(ATLAS_APPLICATION_PROPERTIES_CONFIG);
+
+      StackId stackId = cluster.getCurrentStackVersion();
+      boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") &&
+        VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
+
+      List<ServiceComponentHost> atlasHost = cluster.getServiceComponentHosts("ATLAS", "ATLAS_SERVER");
+      Map<String, String> updates = new HashMap<String, String>();
+
+      if (isStackNotLess23 && atlasHost.size() != 0 && hiveSiteConfig != null) {
+
+        updates.put(ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY, "1");
+        updates.put(ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY, "1");
+        updates.put(ATLAS_CLUSTER_NAME_PROPERTY, "primary");
+
+        if (atlasConfig != null && atlasConfig.getProperties().containsKey(ATLAS_ENABLETLS_PROPERTY)) {
+          String atlasEnableTLSProperty = atlasConfig.getProperties().get(ATLAS_ENABLETLS_PROPERTY);
+          String atlasScheme = "http";
+          String atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTP_PORT_PROPERTY);
+          if (atlasEnableTLSProperty.toLowerCase().equals("true")) {
+            atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTPS_PORT_PROPERTY);
+            atlasScheme = "https";
+          }
+          updates.put(ATLAS_REST_ADDRESS_PROPERTY, String.format("%s://%s:%s", atlasScheme, atlasHost.get(0).getHostName(), atlasServerHttpPortProperty));
+        }
+        updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, false, false);
+      }
+    }
+  }
+
+
   private String convertToDaysIfInSeconds(String secondsString) {
 
     int seconds = Integer.valueOf(secondsString);

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-site.xml

@@ -434,6 +434,7 @@ limitations under the License.
 
   <property>
     <name>atlas.cluster.name</name>
+    <property-type>DONT_ADD_ON_UPGRADE</property-type>
     <value>primary</value>
     <depends-on>
       <property>

ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-site.xml

@@ -42,6 +42,7 @@ limitations under the License.
 
   <property>
     <name>atlas.hook.hive.minThreads</name>
+    <property-type>DONT_ADD_ON_UPGRADE</property-type>
     <value>1</value>
     <description>
       Minimum number of threads maintained by Atlas hook.
@@ -49,6 +50,7 @@ limitations under the License.
   </property>
 
   <property>
+    <property-type>DONT_ADD_ON_UPGRADE</property-type>
     <name>atlas.hook.hive.maxThreads</name>
     <value>1</value>
     <description>

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog222Test.java

@@ -20,7 +20,9 @@ package org.apache.ambari.server.upgrade;
 
 
 import java.lang.reflect.Method;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import javax.persistence.EntityManager;
@@ -41,6 +43,8 @@ import org.apache.ambari.server.orm.entities.StackEntity;
 import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ServiceComponentHost;
+import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.stack.OsFamily;
 import org.easymock.Capture;
 import org.easymock.EasyMock;
@@ -51,12 +55,14 @@ import org.junit.Test;
 
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
+import com.google.inject.AbstractModule;
 import com.google.inject.Binder;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.google.inject.Provider;
 import com.google.inject.persist.PersistService;
+import org.apache.ambari.server.AmbariException;
 
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.anyString;
@@ -108,6 +114,7 @@ public class UpgradeCatalog222Test {
     Method updateAlerts = UpgradeCatalog222.class.getDeclaredMethod("updateAlerts");
     Method updateStormConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateStormConfigs");
     Method updateAMSConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateAMSConfigs");
+    Method updateHiveConfigs = UpgradeCatalog222.class.getDeclaredMethod("updateHiveConfig");
     Method updateHostRoleCommands = UpgradeCatalog222.class.getDeclaredMethod("updateHostRoleCommands");
 
 
@@ -116,6 +123,7 @@ public class UpgradeCatalog222Test {
             .addMockedMethod(updateAlerts)
             .addMockedMethod(updateStormConfigs)
             .addMockedMethod(updateAMSConfigs)
+            .addMockedMethod(updateHiveConfigs)
             .addMockedMethod(updateHostRoleCommands)
             .createMock();
 
@@ -129,6 +137,8 @@ public class UpgradeCatalog222Test {
     expectLastCall().once();
     upgradeCatalog222.updateHostRoleCommands();
     expectLastCall().once();
+    upgradeCatalog222.updateHiveConfig();
+    expectLastCall().once();
 
     replay(upgradeCatalog222);
 
@@ -137,6 +147,75 @@ public class UpgradeCatalog222Test {
     verify(upgradeCatalog222);
   }
 
+  @Test
+  public void testHiveSiteUpdateConfigs() throws AmbariException {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+    final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+    final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+    final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+
+    final Config hiveSiteConfigs = easyMockSupport.createNiceMock(Config.class);
+    final Config AtlasSiteConfigs = easyMockSupport.createNiceMock(Config.class);
+
+    final ServiceComponentHost atlasHost = easyMockSupport.createNiceMock(ServiceComponentHost.class);
+    final List<ServiceComponentHost> atlasHosts = new ArrayList<>();
+    atlasHosts.add(atlasHost);
+
+    StackId stackId = new StackId("HDP","2.3");
+
+    final Map<String, String> propertiesAtlasSiteConfigs = new HashMap<String, String>() {{
+      put("atlas.enableTLS", "true");
+      put("atlas.server.https.port", "21443");
+    }};
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+        bind(Clusters.class).toInstance(mockClusters);
+        bind(EntityManager.class).toInstance(entityManager);
+        bind(ServiceComponentHost.class).toInstance(atlasHost);
+
+        bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+        bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+      }
+    });
+
+    expect(mockClusterExpected.getCurrentStackVersion()).andReturn(stackId).once();
+    expect(mockClusterExpected.getServiceComponentHosts("ATLAS", "ATLAS_SERVER")).andReturn(atlasHosts).once();
+    expect(atlasHost.getHostName()).andReturn("c6401").once();
+    expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+    expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+      put("normal", mockClusterExpected);
+    }}).atLeastOnce();
+    expect(mockClusterExpected.getDesiredConfigByType("hive-site")).andReturn(hiveSiteConfigs).atLeastOnce();
+    expect(mockClusterExpected.getDesiredConfigByType("application-properties")).andReturn(AtlasSiteConfigs).anyTimes();
+    expect(AtlasSiteConfigs.getProperties()).andReturn(propertiesAtlasSiteConfigs).anyTimes();
+
+    UpgradeCatalog222 upgradeCatalog222 = createMockBuilder(UpgradeCatalog222.class)
+      .withConstructor(Injector.class)
+      .withArgs(mockInjector)
+      .addMockedMethod("updateConfigurationPropertiesForCluster", Cluster.class, String.class,
+        Map.class, boolean.class, boolean.class)
+      .createMock();
+
+    Map<String, String> expectedUpdates = new HashMap<>();
+    expectedUpdates.put("atlas.hook.hive.minThreads", "1");
+    expectedUpdates.put("atlas.hook.hive.maxThreads", "1");
+    expectedUpdates.put("atlas.cluster.name", "primary");
+    expectedUpdates.put("atlas.rest.address", "https://c6401:21443");
+
+    upgradeCatalog222.updateConfigurationPropertiesForCluster(mockClusterExpected, "hive-site", expectedUpdates,
+      false, false);
+    expectLastCall().once();
+
+    easyMockSupport.replayAll();
+    replay(upgradeCatalog222);
+    upgradeCatalog222.updateHiveConfig();
+    easyMockSupport.verifyAll();
+  }
+
+
   @Test
   public void testAmsSiteUpdateConfigs() throws Exception{