AMBARI-4914 HBase Service Check failed after adding HBase service (Ivan Kozlov via dsen)

ambari-agent/src/main/python/resource_management/libraries/functions/__init__.py

@@ -26,3 +26,4 @@ from resource_management.libraries.functions.get_kinit_path import *
 from resource_management.libraries.functions.get_unique_id_and_date import *
 from resource_management.libraries.functions.check_process_status import *
 from resource_management.libraries.functions.is_empty import *
+from resource_management.libraries.functions.substitute_vars import *

ambari-agent/src/main/python/resource_management/libraries/functions/substitute_vars.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import re
+
+_MAX_SUBST = 20
+
+def substitute_vars(raw, config):
+  """
+  @param raw: str (e.g '${hbase.tmp.dir}/local')
+  @param config: dict (e.g {'hbase.tmp.dir': '/hadoop/hbase'})
+  """
+  result = raw
+
+  pattern = re.compile("\$\{[^\}\$\x0020]+\}")
+
+  for depth in range(0, _MAX_SUBST - 1):
+    match = pattern.search(result)
+
+    if match:
+      start = match.start()
+      end = match.end()
+
+      name = result[start + 2 : end - 1]
+
+      try:
+        value = config[name]
+      except KeyError:
+        return result
+
+      result = result[:start] + value + result[end:]
+    else:
+      break
+
+  return result

ambari-agent/src/test/python/resource_management/TestSubstituteVars.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from unittest import TestCase, main
+from resource_management.libraries.functions.substitute_vars import substitute_vars
+
+import StringIO, sys
+
+class TestSubstituteVars(TestCase):
+  def setUp(self):
+    # disable stdout
+    out = StringIO.StringIO()
+    sys.stdout = out
+
+  def test_substitute_vars(self):
+    raw_config = {
+      'val.intvar' : '42',
+      'pass.intvar' : '${val.intvar}',
+      'fail.unknown' : 'a${unknown}b',
+      'fail.empty' : '${}',
+      'fail.space' : '${ my.int}',
+      'val.0' : 'will_fail',
+      'fail.digit' : '${val.0}',
+      'val.file' : 'hello',
+      'val.suffix' : '.txt',
+      'pass.seq.depth' : '${val.file}${val.suffix}${pass.intvar}',
+      'fail.seq.depth.a' : '${val.file}${unknown}${pass.intvar}',
+      'fail.seq.depth.b' : '${val.file}${fail.seq.depth.a}${pass.intvar}',
+      'val.name' : 'val.intvar',
+      'pass.name.as.param' : '${${val.name}}',
+      'fail.inf.loop' : '${fail.inf.loop}'
+    }
+    expected_config = {
+      'val.intvar' : '42',
+      'pass.intvar' : '42',
+      'fail.unknown' : 'a${unknown}b',
+      'fail.empty' : '${}',
+      'fail.space' : '${ my.int}',
+      'val.0' : 'will_fail',
+      'fail.digit' : '${val.0}',
+      'val.file' : 'hello',
+      'val.suffix' : '.txt',
+      'pass.seq.depth' : 'hello.txt42',
+      'fail.seq.depth.a' : 'hello${unknown}${pass.intvar}',
+      'fail.seq.depth.b' : 'hellohello${unknown}${pass.intvar}${pass.intvar}',
+      'val.name' : 'val.intvar',
+      'pass.name.as.param' : '42',
+      'fail.inf.loop' : '${fail.inf.loop}'
+    }
+
+    for key in raw_config.keys():
+      actual_value = substitute_vars(raw_config[key], raw_config)
+      expected_value = expected_config[key]
+
+      self.assertEqual(actual_value, expected_value)
+
+  def tearDown(self):
+    # enable stdout
+    sys.stdout = sys.__stdout__

ambari-server/src/main/resources/stacks/HDP/2.0.5/services/HBASE/configuration/hbase-site.xml

@@ -51,6 +51,12 @@
     machine restart).
     </description>
   </property>
+  <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
   <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>

ambari-server/src/main/resources/stacks/HDP/2.0.6.GlusterFS/services/HBASE/configuration/hbase-site.xml

@@ -51,6 +51,12 @@
     machine restart).
     </description>
   </property>
+  <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
   <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-site.xml

@@ -51,6 +51,12 @@
     machine restart).
     </description>
   </property>
+  <property>
+    <name>hbase.local.dir</name>
+    <value>${hbase.tmp.dir}/local</value>
+    <description>Directory on the local filesystem to be used as a local storage
+    </description>
+  </property>
   <property>
     <name>hbase.master.info.bindAddress</name>
     <value></value>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py

@@ -48,6 +48,13 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
              recursive = True
   )
 
+  Directory (os.path.join(params.local_dir, "jars"),
+             owner = params.hbase_user,
+             group = params.user_group,
+             mode=0775,
+             recursive = True
+  )
+
   XmlConfig( "hbase-site.xml",
             conf_dir = params.conf_dir,
             configurations = params.config['configurations']['hbase-site'],

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py

@@ -53,6 +53,7 @@ regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+local_dir = substitute_vars(config['configurations']['hbase-site']['hbase.local.dir'], config['configurations']['hbase-site'])
 
 client_jaas_config_file = default('hbase_client_jaas_config_file', format("{conf_dir}/hbase_client_jaas.conf"))
 master_jaas_config_file = default('hbase_master_jaas_config_file', format("{conf_dir}/hbase_master_jaas.conf"))

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_client.py

@@ -39,6 +39,12 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -96,6 +102,12 @@ class TestHBaseClient(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py

@@ -200,6 +200,12 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -281,6 +287,12 @@ class TestHBaseMaster(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_regionserver.py

@@ -133,6 +133,12 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',
@@ -214,6 +220,12 @@ class TestHbaseRegionServer(RMFTestCase):
       owner = 'hbase',
       recursive = True,
     )
+    self.assertResourceCalled('Directory', '/hadoop/hbase/local/jars',
+      owner = 'hbase',
+      group = 'hadoop',
+      mode=0775,
+      recursive = True,
+    )
     self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
       owner = 'hbase',
       group = 'hadoop',

ambari-server/src/test/python/stacks/2.0.6/configs/default.hbasedecom.json

@@ -372,6 +372,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hbase.hregion.max.filesize": "10737418240", 
             "hfile.block.cache.size": "0.40", 
             "hbase.security.authentication": "simple", 
@@ -609,4 +610,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
+}

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -372,6 +372,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hbase.hregion.max.filesize": "10737418240", 
             "hfile.block.cache.size": "0.40", 
             "hbase.security.authentication": "simple", 
@@ -657,4 +658,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
+}

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -487,6 +487,7 @@
             "hbase.regionserver.global.memstore.upperLimit": "0.4", 
             "zookeeper.session.timeout": "30000", 
             "hbase.tmp.dir": "/hadoop/hbase", 
+            "hbase.local.dir": "${hbase.tmp.dir}/local", 
             "hfile.block.cache.size": "0.40", 
             "hbase.regionserver.kerberos.principal": "hbase/_HOST@EXAMPLE.COM", 
             "hbase.security.authentication": "kerberos", 
@@ -784,4 +785,4 @@
             "c6402.ambari.apache.org"
         ]
     }
-}
+}