AMBARI-14542. AMS in distributed mode fails to start with blueprint deploy. (swagle)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog221.java

@@ -56,6 +56,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
   private static final String AMS_ENV = "ams-env";
   private static final String AMS_HBASE_ENV = "ams-hbase-env";
   private static final String ZK_ZNODE_PARENT = "zookeeper.znode.parent";
+  private static final String ZK_CLIENT_PORT = "hbase.zookeeper.property.clientPort";
   private static final String CLUSTER_ENV = "cluster-env";
   private static final String SECURITY_ENABLED = "security_enabled";
 
@@ -200,6 +201,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
 
     return sourceJson.toString();
   }
+
   protected void updateAMSConfigs() throws AmbariException {
     AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
     Clusters clusters = ambariManagementController.getClusters();
@@ -234,9 +236,7 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
             Map<String, String> newProperties = new HashMap<>();
 
             if (!amsHbaseSiteProperties.containsKey(ZK_ZNODE_PARENT)) {
-
               if (StringUtils.isEmpty(znodeParent) || "/hbase".equals(znodeParent)) {
-
                 boolean isSecurityEnabled = false;
                 Config clusterEnv = cluster.getDesiredConfigByType(CLUSTER_ENV);
                 if (clusterEnv != null) {
@@ -252,6 +252,13 @@ public class UpgradeCatalog221 extends AbstractUpgradeCatalog {
               newProperties.put(ZK_ZNODE_PARENT, znodeParent);
 
             }
+            if (amsHbaseSiteProperties.containsKey(ZK_CLIENT_PORT)) {
+              String newValue = "{{zookeeper_clientPort}}";
+              LOG.info("Replacing value of hbase.zookeeper.property.clientPort from " +
+                amsHbaseSiteProperties.get(ZK_CLIENT_PORT) + " to " + newValue);
+
+              newProperties.put(ZK_CLIENT_PORT, newValue);
+            }
             updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_SITE, newProperties, true, true);
           }
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-site.xml

@@ -252,7 +252,7 @@
   </property>
   <property>
     <name>hbase.zookeeper.property.clientPort</name>
-    <value>61181</value>
+    <value>{{zookeeper_clientPort}}</value>
     <depends-on>
       <property>
         <type>zoo.cfg</type>

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -146,8 +146,13 @@ max_open_files_limit = default("/configurations/ams-hbase-env/max_open_files_lim
 
 if not is_hbase_distributed:
   zookeeper_quorum_hosts = 'localhost'
+  zookeeper_clientPort = '61181'
 else:
   zookeeper_quorum_hosts = ",".join(config['clusterHostInfo']['zookeeper_hosts'])
+  if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+    zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+  else:
+    zookeeper_clientPort = '2181'
 
 ams_checkpoint_dir = config['configurations']['ams-site']['timeline.metrics.aggregator.checkpoint.dir']
 hbase_pid_dir = status_params.hbase_pid_dir

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -488,6 +488,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     rootDir = "file:///var/lib/ambari-metrics-collector/hbase"
     tmpDir = "/var/lib/ambari-metrics-collector/hbase-tmp"
     hbaseClusterDistributed = False
+    zk_port_default = []
     if "ams-hbase-site" in services["configurations"]:
       if "hbase.rootdir" in services["configurations"]["ams-hbase-site"]["properties"]:
         rootDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.rootdir"]
@@ -495,12 +496,13 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         tmpDir = services["configurations"]["ams-hbase-site"]["properties"]["hbase.tmp.dir"]
       if "hbase.cluster.distributed" in services["configurations"]["ams-hbase-site"]["properties"]:
         hbaseClusterDistributed = services["configurations"]["ams-hbase-site"]["properties"]["hbase.cluster.distributed"].lower() == 'true'
+      if "hbase.zookeeper.property.clientPort" in services["configurations"]["ams-hbase-site"]["properties"]:
+        zk_port_default = services["configurations"]["ams-hbase-site"]["properties"]["hbase.zookeeper.property.clientPort"]
 
-    if hbaseClusterDistributed:
+    # Skip recommendation item if default value is present
+    if hbaseClusterDistributed and not "{{zookeeper_clientPort}}" in zk_port_default:
       zkPort = self.getZKPort(services)
       putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", zkPort)
-    else:
-      putAmsHbaseSiteProperty("hbase.zookeeper.property.clientPort", "61181")
 
     mountpoints = ["/"]
     for collectorHostName in amsCollectorHosts:

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog221Test.java

@@ -19,9 +19,9 @@
 package org.apache.ambari.server.upgrade;
 
 
-import com.google.inject.AbstractModule;
 import com.google.common.collect.Maps;
 import com.google.gson.Gson;
+import com.google.inject.AbstractModule;
 import com.google.inject.Binder;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -59,6 +59,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.createMockBuilder;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
@@ -80,8 +81,6 @@ public class UpgradeCatalog221Test {
   private UpgradeCatalogHelper upgradeCatalogHelper;
   private StackEntity desiredStackEntity;
 
-
-
   @Before
   public void init() {
     reset(entityManagerProvider);
@@ -280,6 +279,13 @@ public class UpgradeCatalog221Test {
       amsHbaseSecuritySite,
       clusterEnvProperties);
 
+    // Test zookeeper client port set to default
+    amsHbaseSecuritySite.put("hbase.zookeeper.property.clientPort", "61181");
+    newPropertiesAmsHbaseSite.put("hbase.zookeeper.property.clientPort", "{{zookeeper_clientPort}}");
+    testAmsHbaseSiteUpdates(Collections.singletonMap("hbase.zookeeper.property.clientPort", "61181"),
+      newPropertiesAmsHbaseSite,
+      amsHbaseSecuritySite,
+      clusterEnvProperties);
   }
 
   private void testAmsHbaseSiteUpdates(Map<String, String> oldPropertiesAmsHbaseSite,

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -2041,7 +2041,6 @@ class TestHDP22StackAdvisor(TestCase):
           "hbase.regionserver.global.memstore.lowerLimit": "0.3",
           "hbase.regionserver.global.memstore.upperLimit": "0.35",
           "hbase.hregion.memstore.flush.size": "134217728",
-          "hbase.zookeeper.property.clientPort": "61181",
           "hfile.block.cache.size": "0.3",
           "hbase.rootdir": "file:///var/lib/ambari-metrics-collector/hbase",
           "hbase.tmp.dir": "/var/lib/ambari-metrics-collector/hbase-tmp",