AMBARI-15067: Add custom commands for HAWQ to run check and clear cache(mithmatt via jaoki)

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml

@@ -46,6 +46,22 @@
                 <timeout>1200</timeout>
               </commandScript>
             </customCommand>
+            <customCommand>
+              <name>HAWQ_CLEAR_CACHE</name>
+              <commandScript>
+                <script>scripts/hawqmaster.py</script>
+                <scriptType>PYTHON</scriptType>
+                <timeout>1200</timeout>
+              </commandScript>
+            </customCommand>
+            <customCommand>
+              <name>RUN_HAWQ_CHECK</name>
+              <commandScript>
+                <script>scripts/hawqmaster.py</script>
+                <scriptType>PYTHON</scriptType>
+                <timeout>1200</timeout>
+              </commandScript>
+            </customCommand>
           </customCommands>
           <dependencies>
             <dependency>

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawq_constants.py

@@ -47,7 +47,7 @@ sysctl_conf_dir = "/etc/sysctl.d"
 # Files
 hawq_slaves_file = os.path.join(hawq_config_dir, "slaves")
 hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh")
-hawq_hosts_file = "/tmp/hawq_hosts"
+hawq_hosts_file = os.path.join(hawq_config_dir, "hawq_hosts")
 hawq_check_file = os.path.join(hawq_config_dir, "hawq_check.cnf")
 sysctl_suse_file = "/etc/sysctl.conf"
 sysctl_backup_file = "/etc/sysctl.conf.backup.{0}"

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py

@@ -17,7 +17,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 from resource_management import Script
+from resource_management.core.resources.system import Execute
+from resource_management.core.logger import Logger
 from resource_management.libraries.functions.check_process_status import check_process_status
+try:
+    from resource_management.libraries.functions import hdp_select as hadoop_select
+except ImportError:
+    from resource_management.libraries.functions import phd_select as hadoop_select
 
 import master_helper
 import common
@@ -54,5 +60,18 @@ class HawqMaster(Script):
   def immediate_stop_hawq_service(self, env):
     master_helper.stop(hawq_constants.IMMEDIATE, hawq_constants.CLUSTER)
 
+  def hawq_clear_cache(self, env):
+    import params
+    from utils import exec_psql_cmd
+    cmd = "SELECT gp_metadata_cache_clear()"
+    Logger.info("Clearing HAWQ's HDFS Metadata cache ...")
+    exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port)
+
+  def run_hawq_check(self, env):
+    Logger.info("Executing HAWQ Check ...")
+    Execute("source {0} && hawq check -f {1} --hadoop {2} --config {3}".format(hawq_constants.hawq_greenplum_path_file, hawq_constants.hawq_hosts_file, hadoop_select.get_hadoop_dir('home'), hawq_constants.hawq_check_file),
+            user=hawq_constants.hawq_user,
+            timeout=hawq_constants.default_exec_timeout)
+
 if __name__ == "__main__":
   HawqMaster().execute()

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py

@@ -18,7 +18,7 @@ limitations under the License.
 import os
 import sys
 from resource_management.core.resources.system import File, Execute
-from resource_management.core.source import Template
+from resource_management.core.source import InlineTemplate
 from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.format import format
@@ -34,14 +34,14 @@ def __setup_master_specific_conf_files():
   """
   import params
 
-  File(hawq_constants.hawq_check_file, content=params.hawq_check_content, owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
-      mode=0644)
+  params.File(hawq_constants.hawq_check_file,
+              content=params.hawq_check_content)
 
-  File(hawq_constants.hawq_slaves_file, content=Template("slaves.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
-       mode=0644)
+  params.File(hawq_constants.hawq_slaves_file,
+              content=InlineTemplate("{% for host in hawqsegment_hosts %}{{host}}\n{% endfor %}"))
 
-  File(hawq_constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=hawq_constants.hawq_user, group=hawq_constants.hawq_group,
-       mode=0644)
+  params.File(hawq_constants.hawq_hosts_file,
+              content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor %}"))
 
 
 def __setup_passwordless_ssh():
@@ -51,8 +51,6 @@ def __setup_passwordless_ssh():
   import params
   utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}', hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password))
 
-  File(hawq_constants.hawq_hosts_file, action='delete')
-
 
 def configure_master():
   """

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 import functools
 import hawq_constants
 from resource_management import Script
+from resource_management.core.resources.system import File
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.resources.xml_config import XmlConfig
@@ -75,6 +76,11 @@ HdfsResource = functools.partial(HdfsResource,
                                  hdfs_site=hdfs_site,
                                  default_fs=default_fs)
 
+# File partial function
+File = functools.partial(File,
+                         owner=hawq_constants.hawq_user,
+                         group=hawq_constants.hawq_group,
+                         mode=0644)
 
 # XMLConfig partial function
 XmlConfig = functools.partial(XmlConfig,

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2

@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{% for host in hawq_all_hosts %}
-{{host}}
-{% endfor %}

ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2

@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-{% for host in hawqsegment_hosts %}
-{{host}}
-{% endfor %}

ambari-web/app/messages.js

@@ -1690,6 +1690,8 @@ Em.I18n.translations = {
   'services.service.actions.run.immediateStopHawqSegment.context':'Stop HAWQ Segment (Immediate Mode)',
   'services.service.actions.run.resyncHawqStandby.context':'Re-Sync HAWQ Standby',
   'services.service.actions.run.resyncHawqStandby.label':'Re-Synchronize HAWQ Standby',
+  'services.service.actions.run.clearHawqCache.label':'Clear HAWQ\'s HDFS Metadata Cache',
+  'services.service.actions.run.runHawqCheck.label':'Run HAWQ Check',
   'services.service.actions.manage_configuration_groups.short':'Manage Config Groups',
   'services.service.actions.serviceActions':'Service Actions',
 

ambari-web/app/models/host_component.js

@@ -375,6 +375,12 @@ App.HostComponentActionMap = {
         hasSubmenu: ctx.get('controller.isSeveralClients'),
         submenuOptions: ctx.get('controller.clientComponents')
       },
+      DELETE_SERVICE: {
+        action: 'deleteService',
+        context: ctx.get('serviceName'),
+        label: Em.I18n.t('common.delete'),
+        cssClass: 'icon-remove'
+      },
       IMMEDIATE_STOP_HAWQ_SERVICE: {
         action: 'executeHawqCustomCommand',
         customCommand: 'IMMEDIATE_STOP_HAWQ_SERVICE',
@@ -398,18 +404,6 @@ App.HostComponentActionMap = {
         isHidden : App.get('isSingleNode') || !HS ,
         disabled: !((!!HMComponent && HMComponent.get('startedCount') === 1) && (!!HS && HS.get('workStatus') === App.HostComponentStatus.started))
       },
-      MASTER_CUSTOM_COMMAND: {
-        action: 'executeCustomCommand',
-        cssClass: 'icon-play-circle',
-        isHidden: false,
-        disabled: false
-      },
-      DELETE_SERVICE: {
-        action: 'deleteService',
-        context: ctx.get('serviceName'),
-        label: Em.I18n.t('common.delete'),
-        cssClass: 'icon-remove'
-      },
       TOGGLE_ADD_HAWQ_STANDBY: {
         action: 'addHawqStandby',
         label: Em.I18n.t('admin.addHawqStandby.button.enable'),
@@ -424,7 +418,31 @@ App.HostComponentActionMap = {
         cssClass: 'icon-arrow-up',
         isHidden: App.get('isSingleNode') || !HS,
         disabled: false
-       }
+      },
+      HAWQ_CLEAR_CACHE: {
+        action: 'executeHawqCustomCommand',
+        customCommand: 'HAWQ_CLEAR_CACHE',
+        context: Em.I18n.t('services.service.actions.run.clearHawqCache.label'),
+        label: Em.I18n.t('services.service.actions.run.clearHawqCache.label'),
+        cssClass: 'icon-refresh',
+        isHidden : false,
+        disabled: !HM || HM.get('workStatus') != App.HostComponentStatus.started
+      },
+      RUN_HAWQ_CHECK: {
+        action: 'executeHawqCustomCommand',
+        customCommand: 'RUN_HAWQ_CHECK',
+        context: Em.I18n.t('services.service.actions.run.runHawqCheck.label'),
+        label: Em.I18n.t('services.service.actions.run.runHawqCheck.label'),
+        cssClass: 'icon-thumbs-up-alt',
+        isHidden : false,
+        disabled: false
+      },
+      MASTER_CUSTOM_COMMAND: {
+        action: 'executeCustomCommand',
+        cssClass: 'icon-play-circle',
+        isHidden: false,
+        disabled: false
+      }
     };
   }
 };

ambari-web/app/utils/helper.js

@@ -661,16 +661,22 @@ App.format = {
     }
  // HAWQ custom commands on back Ops page.
     if (result === ' Resync Hawq Standby HAWQ Standby Master') {
-        result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label');
-     }
+      result = Em.I18n.t('services.service.actions.run.resyncHawqStandby.label');
+    }
     if (result === ' Immediate Stop Hawq Service HAWQ Master') {
-        result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label');
-     }
+      result = Em.I18n.t('services.service.actions.run.immediateStopHawqService.label');
+    }
     if (result === ' Immediate Stop Hawq Segment HAWQ Segment') {
-        result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label');
-     }
+      result = Em.I18n.t('services.service.actions.run.immediateStopHawqSegment.label');
+    }
     if(result === ' Activate Hawq Standby HAWQ Standby Master') {
-        result = Em.I18n.t('admin.activateHawqStandby.button.enable');
+      result = Em.I18n.t('admin.activateHawqStandby.button.enable');
+    }
+    if(result === ' Hawq Clear Cache HAWQ Master') {
+      result = Em.I18n.t('services.service.actions.run.clearHawqCache.label');
+    }
+    if(result === ' Run Hawq Check HAWQ Master') {
+      result = Em.I18n.t('services.service.actions.run.runHawqCheck.label');
     }
     //<---End HAWQ custom commands--->
     return result;

ambari-web/app/views/main/service/item.js

@@ -35,7 +35,7 @@ App.MainServiceItemView = Em.View.extend({
     'RESOURCEMANAGER': ['DECOMMISSION', 'REFRESHQUEUES'],
     'HBASE_MASTER': ['DECOMMISSION'],
     'KNOX_GATEWAY': ['STARTDEMOLDAP','STOPDEMOLDAP'],
-    'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE'],
+    'HAWQMASTER': ['IMMEDIATE_STOP_HAWQ_SERVICE', 'RUN_HAWQ_CHECK', 'HAWQ_CLEAR_CACHE'],
     'HAWQSEGMENT': ['IMMEDIATE_STOP_HAWQ_SEGMENT'],
     'HAWQSTANDBY' : ['RESYNC_HAWQ_STANDBY','ACTIVATE_HAWQ_STANDBY']
   },