AMBARI-18132 : Remove FIFO and Normalizer in ams-env config. (avijayan)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java

@@ -18,9 +18,18 @@
 package org.apache.ambari.server.upgrade;
 
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.controller.AmbariManagementController;
 import org.apache.ambari.server.orm.dao.DaoUtils;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -33,7 +42,7 @@ import com.google.inject.Injector;
 public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
 
   protected static final String HOST_VERSION_TABLE = "host_version";
-
+  private static final String AMS_ENV = "ams-env";
   /**
    * Logger.
    */
@@ -97,6 +106,7 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
    */
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {
+    updateAMSConfigs();
   }
 
   protected void updateHostVersionTable() throws SQLException {
@@ -106,6 +116,56 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
     dbAccessor.addUniqueConstraint(HOST_VERSION_TABLE, "UQ_host_repo", "repo_version_id", "host_id");
   }
 
+  protected void updateAMSConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+
+          Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV);
+          if (amsEnv != null) {
+            Map<String, String> amsEnvProperties = amsEnv.getProperties();
+            String content = amsEnvProperties.get("content");
+            Map<String, String> newProperties = new HashMap<>();
+            newProperties.put("content", updateAmsEnvContent(content));
+            updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true);
+          }
+
+        }
+      }
+    }
+  }
+
+
+  protected String updateAmsEnvContent(String content) {
+    if (content == null) {
+      return null;
+    }
+
+    List<String> toReplaceList = new ArrayList<>();
+    toReplaceList.add("\n# HBase normalizer enabled\n");
+    toReplaceList.add("\n# HBase compaction policy enabled\n");
+    toReplaceList.add("export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n");
+    toReplaceList.add("export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}\n");
+
+    //Because of AMBARI-15331 : AMS HBase FIFO compaction policy and Normalizer settings are not handled correctly
+    toReplaceList.add("export HBASE_NORMALIZATION_ENABLED={{ams_hbase_normalizer_enabled}}\n");
+    toReplaceList.add("export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n");
+
+
+    for (String toReplace : toReplaceList) {
+      if (content.contains(toReplace)) {
+        content = content.replace(toReplace, StringUtils.EMPTY);
+      }
+    }
+
+    return content;
+  }
+
 
 }
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml

@@ -127,12 +127,6 @@ export AMS_HBASE_PID_DIR={{hbase_pid_dir}}
 # AMS Collector heapsize
 export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
 
-# HBase normalizer enabled
-export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}
-
-# HBase compaction policy enabled
-export AMS_HBASE_FIFO_COMPACTION_ENABLED={{ams_hbase_fifo_compaction_enabled}}
-
 # HBase Tables Initialization check enabled
 export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -151,8 +151,6 @@ ams_grafana_cert_key = default("/configurations/ams-grafana-ini/cert_key", '/etc
 
 ams_hbase_home_dir = "/usr/lib/ams-hbase/"
 
-ams_hbase_normalizer_enabled = default("/configurations/ams-hbase-site/hbase.normalizer.enabled", None)
-ams_hbase_fifo_compaction_enabled = default("/configurations/ams-site/timeline.metrics.hbase.fifo.compaction.enabled", None)
 ams_hbase_init_check_enabled = default("/configurations/ams-site/timeline.metrics.hbase.init.check.enabled", True)
 
 #hadoop params

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java

@@ -28,8 +28,21 @@ import static org.easymock.EasyMock.verify;
 
 import javax.persistence.EntityManager;
 
+import com.google.common.collect.Maps;
+import com.google.gson.Gson;
+import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.controller.AmbariManagementController;
+import org.apache.ambari.server.controller.AmbariManagementControllerImpl;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.controller.MaintenanceStateHelper;
 import org.apache.ambari.server.orm.DBAccessor;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.stack.OsFamily;
+import org.easymock.Capture;
+import org.easymock.EasyMock;
+import org.easymock.EasyMockSupport;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -40,6 +53,15 @@ import com.google.inject.Injector;
 import com.google.inject.Module;
 import com.google.inject.Provider;
 
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.anyString;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createMockBuilder;
+import static org.junit.Assert.assertTrue;
 /**
  * {@link UpgradeCatalog250} unit tests.
  */
@@ -86,4 +108,92 @@ public class UpgradeCatalog250Test {
     verify(dbAccessor);
   }
 
+  @Test
+  public void testExecuteDMLUpdates() throws Exception {
+    Method updateAmsConfigs = UpgradeCatalog250.class.getDeclaredMethod("updateAMSConfigs");
+
+    UpgradeCatalog250 upgradeCatalog250 = createMockBuilder(UpgradeCatalog250.class)
+      .addMockedMethod(updateAmsConfigs)
+      .createMock();
+
+    upgradeCatalog250.updateAMSConfigs();
+    expectLastCall().once();
+
+    replay(upgradeCatalog250);
+
+    upgradeCatalog250.executeDMLUpdates();
+
+    verify(upgradeCatalog250);
+  }
+
+  @Test
+  public void testAmsEnvUpdateConfigs() throws Exception{
+
+    Map<String, String> oldPropertiesAmsEnv = new HashMap<String, String>() {
+      {
+        put("content", "\n" +
+          "# AMS Collector heapsize\n" +
+          "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" +
+          "\n" +
+          "# HBase normalizer enabled\n" +
+          "export AMS_HBASE_NORMALIZER_ENABLED={{ams_hbase_normalizer_enabled}}\n" +
+          "\n" +
+          "# HBase compaction policy enabled\n" +
+          "export HBASE_FIFO_COMPACTION_POLICY_ENABLED={{ams_hbase_fifo_compaction_policy_enabled}}\n" +
+          "\n" +
+          "# HBase Tables Initialization check enabled\n" +
+          "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n");
+      }
+    };
+    Map<String, String> newPropertiesAmsEnv = new HashMap<String, String>() {
+      {
+        put("content", "\n" +
+          "# AMS Collector heapsize\n" +
+          "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n" +
+          "\n" +
+          "# HBase Tables Initialization check enabled\n" +
+          "export AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n");
+      }
+    };
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+
+    Clusters clusters = easyMockSupport.createNiceMock(Clusters.class);
+    final Cluster cluster = easyMockSupport.createNiceMock(Cluster.class);
+    Config mockAmsEnv = easyMockSupport.createNiceMock(Config.class);
+
+    expect(clusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+      put("normal", cluster);
+    }}).once();
+    expect(cluster.getDesiredConfigByType("ams-env")).andReturn(mockAmsEnv).atLeastOnce();
+    expect(mockAmsEnv.getProperties()).andReturn(oldPropertiesAmsEnv).anyTimes();
+
+    Injector injector = easyMockSupport.createNiceMock(Injector.class);
+    expect(injector.getInstance(Gson.class)).andReturn(null).anyTimes();
+    expect(injector.getInstance(MaintenanceStateHelper.class)).andReturn(null).anyTimes();
+    expect(injector.getInstance(KerberosHelper.class)).andReturn(createNiceMock(KerberosHelper.class)).anyTimes();
+
+    replay(injector, clusters, mockAmsEnv, cluster);
+
+    AmbariManagementControllerImpl controller = createMockBuilder(AmbariManagementControllerImpl.class)
+      .addMockedMethod("createConfiguration")
+      .addMockedMethod("getClusters", new Class[] { })
+      .addMockedMethod("createConfig")
+      .withConstructor(createNiceMock(ActionManager.class), clusters, injector)
+      .createNiceMock();
+
+    Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
+    Capture<Map> propertiesCapture = EasyMock.newCapture();
+
+    expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
+    expect(controller.getClusters()).andReturn(clusters).anyTimes();
+    expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(),
+      anyObject(Map.class))).andReturn(createNiceMock(Config.class)).once();
+
+    replay(controller, injector2);
+    new UpgradeCatalog250(injector2).updateAMSConfigs();
+    easyMockSupport.verifyAll();
+
+    Map<String, String> updatedProperties = propertiesCapture.getValue();
+    assertTrue(Maps.difference(newPropertiesAmsEnv, updatedProperties).areEqual());
+  }
 }