AMBARI-3305. Ambari agent self restart fails due to Pingport listener. (Vitaly Brodetskyi via swagle)

ambari-agent/conf/unix/ambari-agent

@@ -41,6 +41,7 @@ PIDFILE=/var/run/ambari-agent/$AMBARI_AGENT.pid
 OUTFILE=/var/log/ambari-agent/ambari-agent.out
 LOGFILE=/var/log/ambari-agent/ambari-agent.log
 AGENT_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/main.py
+AMBARI_AGENT_PY_SCRIPT=/usr/lib/python2.6/site-packages/ambari_agent/AmbariAgent.py
 OK=1
 NOTOK=0
 
@@ -53,6 +54,8 @@ if [ "x$PYTHON" == "x" ]; then
   PYTHON=/usr/bin/python
 fi
 
+export PYTHON=$PYTHON
+
 # Trying to read the passphrase from an environment
 if [ ! -z $AMBARI_PASSPHRASE ]; then
   RESOLVED_AMBARI_PASSPHRASE=$AMBARI_PASSPHRASE
@@ -117,7 +120,7 @@ case "$1" in
           fi
         fi
         echo "Starting ambari-agent"
-        nohup $PYTHON $AGENT_SCRIPT "$@" > $OUTFILE 2>&1 &
+        nohup $PYTHON $AMBARI_AGENT_PY_SCRIPT "$@" > $OUTFILE 2>&1 &
         sleep 2
         PID=$!
         echo "Verifying $AMBARI_AGENT process status..."

ambari-agent/src/main/python/ambari_agent/AmbariAgent.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import os
+import sys
+import subprocess
+from Controller import AGENT_AUTO_RESTART_EXIT_CODE
+
+AGENT_SCRIPT = "/usr/lib/python2.6/site-packages/ambari_agent/main.py"
+AGENT_PID_FILE = "/var/run/ambari-agent/ambari-agent.pid"
+# AGENT_AUTO_RESTART_EXIT_CODE = 77 is exit code which we return when restart_agent() is called
+status = AGENT_AUTO_RESTART_EXIT_CODE
+
+def main():
+  global status
+
+  if (os.environ.has_key("PYTHON")):
+    PYTHON = os.environ["PYTHON"]
+  else:
+    print("Key 'PYTHON' is not defined in environment variables")
+    sys.exit(1)
+
+  args = list(sys.argv)
+  del args[0]
+
+  mergedArgs = [PYTHON, AGENT_SCRIPT] + args
+
+  while status == AGENT_AUTO_RESTART_EXIT_CODE:
+    mainProcess = subprocess.Popen(mergedArgs)
+    mainProcess.communicate()
+    status = mainProcess.returncode
+    if os.path.isfile(AGENT_PID_FILE) and status == AGENT_AUTO_RESTART_EXIT_CODE:
+      os.remove(AGENT_PID_FILE)
+
+if __name__ == "__main__":
+    main()

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -22,6 +22,7 @@ import logging
 import signal
 import json
 import sys
+import os
 import time
 import threading
 import urllib2
@@ -41,6 +42,8 @@ import ssl
 
 logger = logging.getLogger()
 
+AGENT_AUTO_RESTART_EXIT_CODE = 77
+
 class Controller(threading.Thread):
 
   def __init__(self, config, range=30):
@@ -240,7 +243,7 @@ class Controller(threading.Thread):
     self.heartbeatWithServer()
 
   def restartAgent(self):
-    ProcessHelper.restartAgent()
+    os._exit(AGENT_AUTO_RESTART_EXIT_CODE)
     pass
 
   def sendRequest(self, url, data):

ambari-agent/src/test/python/TestAmbariAgent.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import unittest
+import subprocess
+import os
+import sys
+from mock.mock import MagicMock, patch, ANY
+from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE
+from ambari_agent import AmbariAgent
+
+
+class TestAmbariAgent(unittest.TestCase):
+
+  @patch.object(subprocess, "Popen")
+  @patch("os.path.isfile")
+  @patch("os.remove")
+  def test_main(self, os_remove_mock, os_path_isfile_mock, subprocess_popen_mock):
+    facter1 = MagicMock()
+    facter2 = MagicMock()
+    subprocess_popen_mock.side_effect = [facter1, facter2]
+    facter1.returncode = 77
+    facter2.returncode = 55
+    os_path_isfile_mock.return_value = True
+    if not (os.environ.has_key("PYTHON")):
+      os.environ['PYTHON'] = "test/python/path"
+    sys.argv[0] = "test data"
+    AmbariAgent.main()
+
+    self.assertTrue(subprocess_popen_mock.called)
+    self.assertTrue(subprocess_popen_mock.call_count == 2)
+    self.assertTrue(facter1.communicate.called)
+    self.assertTrue(facter2.communicate.called)
+    self.assertTrue(os_path_isfile_mock.called)
+    self.assertTrue(os_path_isfile_mock.call_count == 2)
+    self.assertTrue(os_remove_mock.called)

ambari-agent/src/test/python/TestController.py

@@ -26,6 +26,7 @@ from ambari_agent import Controller, ActionQueue
 from  ambari_agent.ActionDependencyManager import ActionDependencyManager
 from ambari_agent import hostname
 import sys
+from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE
 from mock.mock import patch, MagicMock, call, Mock
 import logging
 from threading import Event
@@ -219,11 +220,12 @@ class TestController(unittest.TestCase):
       Controller.Controller.registerWithServer
 
 
-  @patch.object(Controller, "ProcessHelper")
-  def test_restartAgent(self, ProcessHelper_mock):
+  @patch("os._exit")
+  def test_restartAgent(self, os_exit_mock):
 
     self.controller.restartAgent()
-    self.assertTrue(ProcessHelper_mock.restartAgent.called)
+    self.assertTrue(os_exit_mock.called)
+    self.assertTrue(os_exit_mock.call_args[0][0] == AGENT_AUTO_RESTART_EXIT_CODE)
 
 
   @patch("urllib2.Request")