AMBARI-10292. Log information on ports and processes on tasks failure (aonishuk)

ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py

@@ -54,6 +54,7 @@ class CustomServiceOrchestrator():
   IPV4_ADDRESSES_KEY = "all_ipv4_ips"
 
   AMBARI_SERVER_HOST = "ambari_server_host"
+  DONT_DEBUG_FAILURES_FOR_COMMANDS = [COMMAND_NAME_SECURITY_STATUS, COMMAND_NAME_STATUS]
 
   def __init__(self, config, controller):
     self.config = config
@@ -172,11 +173,12 @@ class CustomServiceOrchestrator():
         raise AgentException("Background commands are supported without hooks only")
 
       for py_file, current_base_dir in filtered_py_file_list:
+        log_info_on_failure = not command_name in self.DONT_DEBUG_FAILURES_FOR_COMMANDS
         script_params = [command_name, json_path, current_base_dir]
         ret = self.python_executor.run_file(py_file, script_params,
                                self.exec_tmp_dir, tmpoutfile, tmperrfile, timeout,
                                tmpstrucoutfile, logger_level, self.map_task_to_process,
-                               task_id, override_output_files, handle = handle)
+                               task_id, override_output_files, handle = handle, log_info_on_failure=log_info_on_failure)
         # Next run_file() invocations should always append to current output
         override_output_files = False
         if ret['exitcode'] != 0:

ambari-agent/src/main/python/ambari_agent/PythonExecutor.py

@@ -31,6 +31,7 @@ from ambari_commons.os_check import OSConst, OSCheck
 from Grep import Grep
 import sys
 from ambari_commons import shell
+from ambari_commons.shell import shellRunner
 
 
 logger = logging.getLogger()
@@ -63,7 +64,7 @@ class PythonExecutor:
 
   def run_file(self, script, script_params, tmp_dir, tmpoutfile, tmperrfile,
                timeout, tmpstructedoutfile, logger_level, callback, task_id,
-               override_output_files = True, handle = None):
+               override_output_files = True, handle = None, log_info_on_failure=True):
     """
     Executes the specified python file in a separate subprocess.
     Method returns only when the subprocess is finished.
@@ -93,14 +94,32 @@ class PythonExecutor:
       process.communicate()
       self.event.set()
       thread.join()
-      return self.prepare_process_result(process, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=timeout)
+      result = self.prepare_process_result(process, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=timeout)
+      
+      if log_info_on_failure and result['exitcode']:
+        self.on_failure(pythonCommand, result)
+      
+      return result
     else:
       holder = Holder(pythonCommand, tmpoutfile, tmperrfile, tmpstructedoutfile, handle)
 
       background = BackgroundThread(holder, self)
       background.start()
       return {"exitcode": 777}
-
+    
+  def on_failure(self, pythonCommand, result):
+    """
+    Log some useful information after task failure.
+    """
+    logger.info("Command " + pprint.pformat(pythonCommand) + " failed with exitcode=" + str(result['exitcode']))
+    cmd_list = ["ps faux", "netstat -tulpn"]
+    
+    shell_runner = shellRunner()
+    
+    for cmd in cmd_list:
+      ret = shell_runner.run(cmd)
+      logger.info("Command '{0}' returned {1}. {2}{3}".format(cmd, ret["exitCode"], ret["error"], ret["output"]))
+    
   def prepare_process_result(self, process, tmpoutfile, tmperrfile, tmpstructedoutfile, timeout=None):
     out, error, structured_out = self.read_result_from_files(tmpoutfile, tmperrfile, tmpstructedoutfile)
     # Building results