AMBARI-15506 Support for multiple Namservices in namenode_ha_utils.py (hkropp via dsen)

ambari-agent/src/test/python/resource_management/TestNamenodeHaUtils.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from unittest import TestCase
+from resource_management.libraries.functions.namenode_ha_utils import \
+  get_nameservice
+
+
+class TestNamenodeHaUtils(TestCase):
+
+  def test_get_nameservice(self):
+    # our cluster is HAA
+
+    # dfs.internal.nameservices in hdfs-site
+    hdfs_site = {
+      "dfs.internal.nameservices": "HAA",
+      "dfs.nameservices": "HAA,HAB",
+      "dfs.ha.namenodes.HAA": "nn1,nn2",
+      "dfs.ha.namenodes.HAB": "nn1,nn2",
+      "dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020",
+      "dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020",
+      "dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020",
+      "dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020",
+    }
+
+    self.assertEqual("HAA", get_nameservice(hdfs_site))
+
+    # dfs.internal.nameservices not in hdfs-site
+    hdfs_site = {
+      "dfs.nameservices": "HAA,HAB",
+      "dfs.ha.namenodes.HAA": "nn1,nn2",
+      "dfs.ha.namenodes.HAB": "nn1,nn2",
+      "dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020",
+      "dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020",
+      "dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020",
+      "dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020",
+    }
+
+    self.assertEqual("HAA", get_nameservice(hdfs_site))
+
+    # Non HA
+    hdfs_site = {}
+
+    self.assertEqual(None, get_nameservice(hdfs_site))

ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py

@@ -26,7 +26,8 @@ from resource_management.core import shell
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.decorator import retry
 
-__all__ = ["get_namenode_states", "get_active_namenode", "get_property_for_active_namenode"]
+__all__ = ["get_namenode_states", "get_active_namenode",
+           "get_property_for_active_namenode", "get_nameservice"]
 
 HDFS_NN_STATE_ACTIVE = 'active'
 HDFS_NN_STATE_STANDBY = 'standby'
@@ -67,7 +68,7 @@ def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
   standby_namenodes = []
   unknown_namenodes = []
   
-  name_service = hdfs_site['dfs.nameservices']
+  name_service = get_nameservice(hdfs_site)
   nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
 
   # now we have something like 'nn1,nn2,nn3,nn4'
@@ -117,7 +118,7 @@ def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
   return active_namenodes, standby_namenodes, unknown_namenodes
 
 def is_ha_enabled(hdfs_site):
-  dfs_ha_nameservices = hdfs_site['dfs.nameservices']
+  dfs_ha_nameservices = get_nameservice(hdfs_site)
   
   if is_empty(dfs_ha_nameservices):
     return False
@@ -151,7 +152,7 @@ def get_property_for_active_namenode(hdfs_site, property_name, security_enabled,
   value = None
   rpc_key = None
   if is_ha_enabled(hdfs_site):
-    name_service = hdfs_site['dfs.nameservices']
+    name_service = get_nameservice(hdfs_site)
     active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
     
     if not len(active_namenodes):
@@ -171,3 +172,31 @@ def get_property_for_active_namenode(hdfs_site, property_name, security_enabled,
       value = value.replace(INADDR_ANY, rpc_host)
 
   return value
+
+def get_nameservice(hdfs_site):
+  """
+  Multiple nameservices can be configured for example to support seamless distcp
+  between two HA clusters. The nameservices are defined as a comma separated
+  list in hdfs_site['dfs.nameservices']. The parameter
+  hdfs['dfs.internal.nameservices'] was introduced in Hadoop 2.6 to denote the
+  nameservice for the current cluster (HDFS-6376).
+
+  This method uses hdfs['dfs.internal.nameservices'] to get the current
+  nameservice, if that parameter is not available it tries to splits the value
+  in hdfs_site['dfs.nameservices'] returning the first string or what is
+  contained in hdfs_site['dfs.namenode.shared.edits.dir'].
+
+  By default hdfs_site['dfs.nameservices'] is returned.
+  :param hdfs_site:
+  :return: string or empty
+  """
+  name_service = hdfs_site.get('dfs.internal.nameservices', None)
+  if not name_service:
+    import re
+    name_service = hdfs_site.get('dfs.nameservices', None)
+    if name_service:
+      for ns in name_service.split(","):
+        if 'dfs.namenode.shared.edits.dir' in hdfs_site and re.match(r'.*%s$' % ns, hdfs_site['dfs.namenode.shared.edits.dir']): # better would be core_site['fs.defaultFS'] but it's not available
+          return ns
+      return name_service.split(",")[0] # default to return the first nameservice
+  return name_service