AMBARI-4441. Add support for hbase decommisison

ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java

@@ -47,6 +47,7 @@ import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.State;
 import org.apache.ambari.server.state.svccomphost.ServiceComponentHostOpInProgressEvent;
 import org.apache.ambari.server.utils.StageUtils;
+import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -101,6 +102,7 @@ public class AmbariCustomCommandExecutionHelper {
   private static String DECOM_INCLUDED_HOSTS = "included_hosts";
   private static String DECOM_EXCLUDED_HOSTS = "excluded_hosts";
   private static String DECOM_SLAVE_COMPONENT = "slave_type";
+  private static String HBASE_MARK_DRAINING_ONLY = "mark_draining_only";
   @Inject
   private ActionMetadata actionMetadata;
   @Inject
@@ -192,14 +194,15 @@ public class AmbariCustomCommandExecutionHelper {
     } else if (actionRequest.getCommandName().equals("DECOMMISSION")) {
       addDecommissionAction(actionRequest, stage, hostLevelParams);
     } else if (isValidCustomCommand(actionRequest)) {
-      addCustomCommandAction(actionRequest, stage, hostLevelParams);
+      addCustomCommandAction(actionRequest, stage, hostLevelParams, null);
     } else {
       throw new AmbariException("Unsupported action " + actionRequest.getCommandName());
     }
   }
 
   private void addCustomCommandAction(ExecuteActionRequest actionRequest,
-                                      Stage stage, Map<String, String> hostLevelParams)
+                                      Stage stage, Map<String, String> hostLevelParams,
+                                      Map<String, String> additionalCommandParams)
       throws AmbariException {
 
     if (actionRequest.getHosts().isEmpty()) {
@@ -246,6 +249,11 @@ public class AmbariCustomCommandExecutionHelper {
 
       Map<String, String> commandParams = new TreeMap<String, String>();
       commandParams.put(SCHEMA_VERSION, serviceInfo.getSchemaVersion());
+      if (additionalCommandParams != null) {
+        for (String key : additionalCommandParams.keySet()) {
+          commandParams.put(key, additionalCommandParams.get(key));
+        }
+      }
 
       String commandTimeout = configs.getDefaultAgentTaskTimeout();
 
@@ -268,7 +276,7 @@ public class AmbariCustomCommandExecutionHelper {
                   "this service", componentName);
           throw new AmbariException(message);
         }
-        // We don't need package/repo infomation to perform service check
+        // We don't need package/repo information to perform service check
       }
       commandParams.put(COMMAND_TIMEOUT, commandTimeout);
 
@@ -495,9 +503,11 @@ public class AmbariCustomCommandExecutionHelper {
     }
 
     // Set/reset decommissioned flag on all components
+    List<String> listOfExcludedHosts = new ArrayList<String>();
     for (ServiceComponentHost sch : svcComponents.get(slaveCompType).getServiceComponentHosts().values()) {
       if (excludedHosts.contains(sch.getHostName())) {
         sch.setComponentAdminState(HostComponentAdminState.DECOMMISSIONED);
+        listOfExcludedHosts.add(sch.getHostName());
         LOG.info("Adding " + slaveCompType + " host to decommissioned list : " + sch.getHostName());
       }
       if (includedHosts.contains(sch.getHostName())) {
@@ -517,7 +527,19 @@ public class AmbariCustomCommandExecutionHelper {
     // Reset cluster host info as it has changed
     stage.setClusterHostInfo(clusterHostInfoJson);
 
-    addCustomCommandAction(commandRequest, stage, hostLevelParams);
+    Map<String, String> commandParams = null;
+    if (serviceName.equals(Service.Type.HBASE.name()) && listOfExcludedHosts.size() > 0) {
+      commandParams = new HashMap<String, String>();
+      commandParams.put(DECOM_EXCLUDED_HOSTS, StringUtils.join(listOfExcludedHosts, ','));
+      String isDrainOnlyRequest = request.getParameters().get(HBASE_MARK_DRAINING_ONLY);
+      if (isDrainOnlyRequest != null) {
+        if (isDrainOnlyRequest.equals("true") || isDrainOnlyRequest.equals("false")) {
+          commandParams.put(HBASE_MARK_DRAINING_ONLY, isDrainOnlyRequest);
+        }
+      }
+    }
+
+    addCustomCommandAction(commandRequest, stage, hostLevelParams, commandParams);
   }
 
   /**

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/hbase_decommission.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+
+def hbase_decommission(env):
+  import params
+
+  env.set_params(params)
+
+  if params.hbase_drain_only == True:
+    print "TBD: Remove host from draining"
+    pass
+
+  else:
+
+    kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") if params.security_enabled else ""
+
+    hosts = params.hbase_excluded_hosts.split(",")
+    for host in hosts:
+      if host:
+        print "TBD: Add host to draining"
+        regionmover_cmd = format(
+          "{kinit_cmd} {hbase_cmd} --config {conf_dir} org.jruby.Main {region_mover} unload {host}")
+
+        Execute(regionmover_cmd,
+                user=params.hbase_user,
+                logoutput=True
+        )
+      pass
+    pass
+  pass
+
+
+pass

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/hbase_master.py

@@ -23,6 +23,7 @@ from resource_management import *
 
 from hbase import hbase
 from hbase_service import hbase_service
+from hbase_decommission import hbase_decommission
 
          
 class HbaseMaster(Script):
@@ -59,6 +60,12 @@ class HbaseMaster(Script):
     pid_file = format("{pid_dir}/hbase-hbase-master.pid")
     check_process_status(pid_file)
 
+  def decommission(self, env):
+    import params
+    env.set_params(params)
+
+    hbase_decommission(env)
+
 def main():
   command_type = sys.argv[1] if len(sys.argv)>1 else "install"
   print "Running "+command_type

ambari-server/src/main/resources/stacks/HDP/1.3.3/services/HBASE/package/scripts/params.py

@@ -28,6 +28,11 @@ config = Script.get_config()
 
 conf_dir = "/etc/hbase/conf"
 daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
+region_mover = "/usr/lib/hbase/bin/region_mover.rb"
+region_drainer = "/usr/lib/hbase/bin/region_drainer.rb"
+hbase_cmd = "/usr/lib/hbase/bin/hbase"
+hbase_excluded_hosts = default("/commandParams/excluded_hosts", "")
+hbase_drain_only = default("/commandParams/mark_draining_only", "")
 
 hbase_user = config['configurations']['global']['hbase_user']
 smokeuser = config['configurations']['global']['smokeuser']

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/hbase_decommission.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+
+def hbase_decommission(env):
+  import params
+
+  env.set_params(params)
+
+  if params.hbase_drain_only == True:
+    print "TBD: Remove host from draining"
+    pass
+
+  else:
+
+    kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_user};") if params.security_enabled else ""
+
+    hosts = params.hbase_excluded_hosts.split(",")
+    for host in hosts:
+      if host:
+        print "TBD: Add host to draining"
+        regionmover_cmd = format(
+          "{kinit_cmd} {hbase_cmd} --config {conf_dir} org.jruby.Main {region_mover} unload {host}")
+
+        Execute(regionmover_cmd,
+                user=params.hbase_user,
+                logoutput=True
+        )
+      pass
+    pass
+  pass
+
+
+pass

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/hbase_master.py

@@ -23,6 +23,7 @@ from resource_management import *
 
 from hbase import hbase
 from hbase_service import hbase_service
+from hbase_decommission import hbase_decommission
 
          
 class HbaseMaster(Script):
@@ -59,6 +60,12 @@ class HbaseMaster(Script):
     pid_file = format("{pid_dir}/hbase-hbase-master.pid")
     check_process_status(pid_file)
 
+  def decommission(self, env):
+    import params
+    env.set_params(params)
+
+    hbase_decommission(env)
+
 def main():
   command_type = sys.argv[1] if len(sys.argv)>1 else "install"
   print "Running "+command_type

ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HBASE/package/scripts/params.py

@@ -27,6 +27,11 @@ config = Script.get_config()
 
 conf_dir = "/etc/hbase/conf"
 daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
+region_mover = "/usr/lib/hbase/bin/region_mover.rb"
+region_drainer = "/usr/lib/hbase/bin/region_drainer.rb"
+hbase_cmd = "/usr/lib/hbase/bin/hbase"
+hbase_excluded_hosts = default("/commandParams/excluded_hosts", "")
+hbase_drain_only = default("/commandParams/mark_draining_only", "")
 
 hbase_user = config['configurations']['global']['hbase_user']
 smokeuser = config['configurations']['global']['smokeuser']

ambari-server/src/test/python/stacks/1.3.3/HBASE/test_hbase_master.py

@@ -57,7 +57,24 @@ class TestHBaseMaster(RMFTestCase):
       user = 'hbase',
     )
     self.assertNoMoreResources()
-    
+
+  def test_decom_default(self):
+    self.executeScript("2.1.1/services/HBASE/package/scripts/hbase_master.py",
+                       classname = "HbaseMaster",
+                       command = "decommission",
+                       config_file="default.json"
+    )
+
+    self.assertResourceCalled('Execute', ' /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host1',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertResourceCalled('Execute', ' /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host2',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertNoMoreResources()
+
   def test_configure_secured(self):
     self.executeScript("1.3.3/services/HBASE/package/scripts/hbase_master.py",
                    classname = "HbaseMaster",
@@ -95,6 +112,19 @@ class TestHBaseMaster(RMFTestCase):
     )
     self.assertNoMoreResources()
 
+  def test_decom_secure(self):
+    self.executeScript("2.1.1/services/HBASE/package/scripts/hbase_master.py",
+                       classname = "HbaseMaster",
+                       command = "decommission",
+                       config_file="secured.json"
+    )
+
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hbase.headless.keytab hbase; /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host1',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertNoMoreResources()
+
   def assert_configure_default(self):
     self.assertResourceCalled('Directory', '/etc/hbase/conf',
       owner = 'hbase',

ambari-server/src/test/python/stacks/1.3.3/configs/default.json

@@ -28,7 +28,8 @@
         "service_metadata_folder": "HDFS", 
         "script_type": "PYTHON", 
         "schema_version": "2.0", 
-        "script": "scripts/datanode.py"
+        "script": "scripts/datanode.py",
+        "excluded_hosts": "host1,host2"
     }, 
     "taskId": 18, 
     "public_hostname": "c6402.ambari.apache.org", 

ambari-server/src/test/python/stacks/1.3.3/configs/secured.json

@@ -28,7 +28,8 @@
         "service_metadata_folder": "HIVE", 
         "script_type": "PYTHON", 
         "schema_version": "2.0", 
-        "script": "scripts/mysql_server.py"
+        "script": "scripts/mysql_server.py",
+        "excluded_hosts": "host1"
     }, 
     "taskId": 117, 
     "public_hostname": "c6402.ambari.apache.org", 

ambari-server/src/test/python/stacks/2.1.1/HBASE/test_hbase_master.py

@@ -57,7 +57,24 @@ class TestHBaseMaster(RMFTestCase):
       user = 'hbase',
     )
     self.assertNoMoreResources()
-    
+
+  def test_decom_default(self):
+    self.executeScript("2.1.1/services/HBASE/package/scripts/hbase_master.py",
+                       classname = "HbaseMaster",
+                       command = "decommission",
+                       config_file="default.json"
+    )
+
+    self.assertResourceCalled('Execute', ' /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host1',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertResourceCalled('Execute', ' /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host2',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertNoMoreResources()
+
   def test_configure_secured(self):
     self.executeScript("2.1.1/services/HBASE/package/scripts/hbase_master.py",
                    classname = "HbaseMaster",
@@ -95,6 +112,19 @@ class TestHBaseMaster(RMFTestCase):
     )
     self.assertNoMoreResources()
 
+  def test_decom_secure(self):
+    self.executeScript("2.1.1/services/HBASE/package/scripts/hbase_master.py",
+                       classname = "HbaseMaster",
+                       command = "decommission",
+                       config_file="secured.json"
+    )
+
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hbase.headless.keytab hbase; /usr/lib/hbase/bin/hbase --config /etc/hbase/conf org.jruby.Main /usr/lib/hbase/bin/region_mover.rb unload host1',
+                              logoutput = True,
+                              user = 'hbase',
+                              )
+    self.assertNoMoreResources()
+
   def assert_configure_default(self):
     self.assertResourceCalled('Directory', '/etc/hbase/conf',
       owner = 'hbase',

ambari-server/src/test/python/stacks/2.1.1/configs/default.json

@@ -24,8 +24,9 @@
         "service_metadata_folder": "OOZIE", 
         "script_type": "PYTHON", 
         "schema_version": "2.0", 
-        "script": "scripts/service_check.py"
-    }, 
+        "script": "scripts/service_check.py",
+        "excluded_hosts": "host1,host2"
+    },
     "taskId": 152, 
     "public_hostname": "c6401.ambari.apache.org", 
     "configurations": {

ambari-server/src/test/python/stacks/2.1.1/configs/secured.json

@@ -29,7 +29,8 @@
         "service_metadata_folder": "YARN", 
         "script_type": "PYTHON", 
         "schema_version": "2.0", 
-        "script": "scripts/yarn_client.py"
+        "script": "scripts/yarn_client.py",
+        "excluded_hosts": "host1"
     }, 
     "taskId": 186, 
     "public_hostname": "c6401.ambari.apache.org",