AMBARI-16199. Improve output of command execution retry logic on agents. (Daniel Gergely via stoader)

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -171,7 +171,7 @@ class ActionQueue(threading.Thread):
                 t.start()
               else:
                 self.process_command(command)
-                break;
+                break
               pass
             pass
         except (Queue.Empty):
@@ -315,12 +315,17 @@ class ActionQueue(threading.Thread):
         if delay > retryDuration:
           delay = retryDuration
         retryDuration -= delay  # allow one last attempt
+        commandresult['stderr'] += "\n\nCommand failed. Retrying command execution ...\n\n"
         logger.info("Retrying command id {cid} after a wait of {delay}".format(cid=taskId, delay=delay))
         time.sleep(delay)
         continue
       else:
         break
 
+    # final result to stdout
+    commandresult['stdout'] += '\n\nCommand completed successfully!\n' if status == self.COMPLETED_STATUS else '\n\nCommand failed after ' + str(numAttempts) + ' tries\n'
+    logger.info('Command {cid} completed successfully!'.format(cid=taskId) if status == self.COMPLETED_STATUS else 'Command {cid} failed after {attempts} tries'.format(cid=taskId, attempts=numAttempts))
+
     roleResult = self.commandStatuses.generate_report_template(command)
     roleResult.update({
       'stdout': commandresult['stdout'],

ambari-agent/src/test/python/ambari_agent/TestActionQueue.py

@@ -366,7 +366,7 @@ class TestActionQueue(TestCase):
     expected = {'status': 'COMPLETED',
                 'configurationTags': {'global': {'tag': 'v123'}},
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n',
                 'clusterName': u'cc',
                 'structuredOut': '""',
                 'roleCommand': u'CUSTOM_COMMAND',
@@ -539,7 +539,7 @@ class TestActionQueue(TestCase):
     configname = os.path.join(tempdir, 'config.json')
     expected = {'status': 'COMPLETED',
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n',
                 'clusterName': u'cc',
                 'structuredOut': '""',
                 'roleCommand': u'INSTALL',
@@ -578,7 +578,7 @@ class TestActionQueue(TestCase):
       # check report
     expected = {'status': 'FAILED',
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n\n\nCommand failed after 1 tries\n',
                 'clusterName': u'cc',
                 'structuredOut': '""',
                 'roleCommand': u'INSTALL',
@@ -610,7 +610,7 @@ class TestActionQueue(TestCase):
     # check report
     expected = {'status': 'COMPLETED',
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n\n\nCommand failed after 1 tries\n\n\nCommand completed successfully!\n',
                 'clusterName': 'clusterName',
                 'structuredOut': '""',
                 'roleCommand': 'UPGRADE',
@@ -654,7 +654,7 @@ class TestActionQueue(TestCase):
     expected = {'status': 'COMPLETED',
                 'configurationTags': {'global': {'tag': 'v123'}},
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n',
                 'clusterName': u'cc',
                 'structuredOut': '""',
                 'roleCommand': u'CUSTOM_COMMAND',
@@ -696,7 +696,7 @@ class TestActionQueue(TestCase):
     expected = {'status': 'COMPLETED',
                 'configurationTags': {'global': {'tag': 'v123'}},
                 'stderr': 'stderr',
-                'stdout': 'out',
+                'stdout': 'out\n\nCommand completed successfully!\n',
                 'clusterName': u'cc',
                 'structuredOut': '""',
                 'roleCommand': u'CUSTOM_COMMAND',
@@ -986,7 +986,7 @@ class TestActionQueue(TestCase):
       'status': 'FAILED'
     }
 
-    times_arr = [8, 10, 14, 18, 22]
+    times_arr = [8, 10, 14, 18, 22, 26]
     if self.logger.isEnabledFor(logging.INFO):
       times_arr.insert(0, 4)
     time_mock.side_effect = times_arr