sequentialScriptRunner.php 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. <?php
  2. // all scripts run with base directory as frontend
  3. include_once '../util/Logger.php';
  4. include_once '../conf/Config.inc';
  5. include_once 'localDirs.php';
  6. include_once "../util/lock.php";
  7. include_once '../db/HMCDBAccessor.php';
  8. include_once "../util/HMCTxnUtils.php";
  9. include_once "commandUtils.php";
  10. include_once '../util/util.php';
  11. // this script will write to TransactionStatus table the following:
  12. // 1. txn-id -> describes state sshable, discoverable, bootstrapable
  13. // 2. status_info blob -> specific errors for the state
  14. // initial setup
  15. $logger = new HMCLogger("sequentialScriptExecutor");
  16. $dbHandle = new HMCDBAccessor($GLOBALS["DB_PATH"]);
  17. function updateProgressForStage($clusterName, $rootTxnId, $orchestratorTxnId,
  18. $mySubTxnId, $operationName, $numTotalNodes) {
  19. global $logger, $dbHandle, $stagesInfo;
  20. $clusterDir = getClusterDir($clusterName);
  21. $commandOutputDir = $clusterDir . $operationName . "/";
  22. $numNodesFailed = 0;
  23. $numNodesSucceeded = 0;
  24. $additionalInfo = array();
  25. if (is_dir($commandOutputDir) && $dirHandle = opendir($commandOutputDir)) {
  26. $logger->log_debug($commandOutputDir . " exists finally ");
  27. while (false !== ($entry = readdir($dirHandle))) {
  28. if ($entry == "." || $entry == "..") {
  29. continue;
  30. }
  31. // Only consider .out files
  32. if(!preg_match("/.out/", $entry)) {
  33. continue;
  34. }
  35. $nodeName = basename($entry, ".out");
  36. $doneFile = $commandOutputDir . $nodeName . ".done";
  37. if (file_exists($doneFile)) {
  38. // Read the contents of the done-file
  39. $doneFileContents = file_get_contents($doneFile);
  40. if (trim($doneFileContents) == "0") {
  41. $numNodesSucceeded++;
  42. $additionalInfo[$nodeName] = "Successful with no errors.";
  43. } else {
  44. $numNodesFailed++;
  45. // Let's read the errors
  46. $errorFile = $commandOutputDir . $nodeName . ".err";
  47. $errorFileContents = file_get_contents($errorFile);
  48. $additionalInfo[$nodeName] = "Failed. Reason: $errorFileContents";
  49. }
  50. }
  51. }
  52. } else {
  53. $logger->log_debug( $commandOutputDir . " still doesn't exist");
  54. }
  55. $logger->log_debug(" Current operation is $operationName and commandOutput dir: $commandOutputDir");
  56. $logger->log_debug(" total nodes: $numTotalNodes, succeeded: $numNodesSucceeded, failed: $numNodesFailed");
  57. $progressState = generateNodeActionProgress($numTotalNodes, $numNodesSucceeded, $numNodesFailed, "succeeded");
  58. $logger->log_debug(" Current progressState is ".$progressState);
  59. $state = json_encode($additionalInfo); $description = $stagesInfo[$operationName]["description"]; $progress = $progressState; $subTxnType = "";
  60. $subTransactionReturnValue = $dbHandle->insertOrUpdateSubTransaction($clusterName, $orchestratorTxnId, $mySubTxnId, $orchestratorTxnId, $state, $description, $progress, $subTxnType );
  61. if ($subTransactionReturnValue["result"] != 0 ) {
  62. $logger->log_error("Got error while creating subTxn: ".$subTransactionReturnValue["error"]);
  63. print json_encode($subTransactionReturnValue);
  64. return;
  65. }
  66. }
  67. $clusterName = $argv[1];
  68. $deployUser = $argv[2];
  69. $rootTxnId = $argv[3];
  70. $readFromFile = $argv[4];
  71. $stagesFile = $argv[5];
  72. include_once $stagesFile;
  73. $logger->log_debug("Sequential runner params: $clusterName, $deployUser, $rootTxnId, $readFromFile, $stagesFile");
  74. // Create a sub-txn for the orchestrator
  75. $status = "";
  76. $createTxResult = HMCTxnUtils::createNewTransaction($dbHandle, $clusterName, $status);
  77. if ($createTxResult == FALSE) {
  78. $logger->log_error("Failed to create new transaction " . " in background: $createTxResult\n");
  79. return;
  80. }
  81. $orchestratorTxnId = $createTxResult;
  82. $logger->log_debug("Txn Id: $orchestratorTxnId\n");
  83. $state = ""; $description = "sequentialScriptExecutor"; $progress = ""; $subTxnType = "";
  84. $subTransactionReturnValue = $dbHandle->insertOrUpdateSubTransaction($clusterName, $rootTxnId, $orchestratorTxnId, $rootTxnId, $state, $description, $progress, $subTxnType );
  85. if ($subTransactionReturnValue["result"] != 0 ) {
  86. $logger->log_error("Got error while creating subTxn for sequentialScriptExecutor: ".$subTransactionReturnValue["error"]);
  87. print json_encode($subTransactionReturnValue);
  88. return;
  89. }
  90. $logger->log_debug("sequentialScriptExecutor txnId: $orchestratorTxnId\n");
  91. $startTime = time(0);
  92. $cmd = "";
  93. $currentStage = "";
  94. $statusInfo = "";
  95. $count = 0;
  96. $thisHostName = trim(strtolower(exec('hostname -f')));
  97. // Add the host which runs the server to the list
  98. function addThisHostToList($hosts, $logger, $thisHostName, $readFromFile) {
  99. $result = array();
  100. foreach($hosts as $host) {
  101. array_push($result, $host);
  102. if ($thisHostName == $host) {
  103. $logger->log_debug("Management host $thisHostName exists".
  104. " in the hosts file");
  105. return $hosts;
  106. }
  107. }
  108. $outFd = fopen($readFromFile, "a");
  109. if ($outFd == FALSE) {
  110. $logger->log_error("Failed to add the hmc host to the nodes file");
  111. return $result;
  112. }
  113. fwrite($outFd, "\n".$thisHostName);
  114. fclose($outFd);
  115. array_push($result, $thisHostName);
  116. return $result;
  117. }
  118. $hosts = readHostsFile($readFromFile);
  119. $hosts = convertToLowerCase($hosts);
  120. $logger->log_debug("The hosts after converting to lower case ".print_r($hosts, true));
  121. // Only add this host to list if the db does not have that host already.
  122. $checkThisHostInDB = $dbHandle->getHostInfo($clusterName, $thisHostName);
  123. $logger->log_debug("Host Info in DB ".print_r($checkThisHostInDB, true));
  124. if ($checkThisHostInDB["result"] != 0) {
  125. $logger->log_info($thisHostName . "not found in DB so adding it to the list of hosts");
  126. $hosts = addThisHostToList($hosts, $logger, $thisHostName, $readFromFile);
  127. }
  128. $hosts = convertToLowerCase($hosts);
  129. $logger->log_debug("The hosts that are being sshed to are ".print_r($hosts, true));
  130. $allHosts = $hosts; // Keep a copy in case
  131. foreach ($stagesInfo as $stage => $stageInfo) {
  132. // create a new subtransaction for each stage
  133. $mySubTxnId = 100 + $count;
  134. $state = ""; $description = $stagesInfo[$stage]["description"]; $progress = " ( starting )"; $subTxnType = "";
  135. $subTransactionReturnValue = $dbHandle->insertOrUpdateSubTransaction($clusterName, $orchestratorTxnId, $mySubTxnId, $orchestratorTxnId, $state, $description, $progress, $subTxnType );
  136. if ($subTransactionReturnValue["result"] != 0 ) {
  137. $logger->log_error("Got error while creating subTxn: ".$subTransactionReturnValue["error"]);
  138. print json_encode($subTransactionReturnValue);
  139. return;
  140. }
  141. $currentStage = $stage;
  142. // SubTxn is created. Set start op_status
  143. $updateSubTransactionStatusResult = $dbHandle->updateSubTransactionOpStatus($clusterName, $orchestratorTxnId, $mySubTxnId, $opStatus);
  144. if ($updateSubTransactionStatusResult["result"] != 0 ) {
  145. $logger->log_error("Got error while updating subTxn: ".$updateSubTransactionStatusResult["error"]);
  146. print json_encode($updateSubTransactionStatusResult);
  147. return;
  148. }
  149. // If the host list is empty, say because of failures in previous stage, no point carrying it on..
  150. $hostCount = count($hosts);
  151. if ($hostCount == 0) {
  152. $logger->log_info("Skipping stage " . $stage . " as no valid hosts available");
  153. continue; // so that all stages can get marked as failures
  154. }
  155. $cmd = $GLOBALS["PHP_EXEC_PATH"]." ".$stagesInfo[$stage]["scriptName"];
  156. $args = "$clusterName $deployUser $rootTxnId $mySubTxnId $orchestratorTxnId " . $readFromFile;
  157. $count++;
  158. $execBackgroundResult = HMCTxnUtils::execBackgroundProcess($dbHandle, $clusterName, $rootTxnId, $cmd, $args, "" );
  159. if ($execBackgroundResult == FALSE) {
  160. $logger->log_error("Failed to execute $currentStage".
  161. " in background: $execBackgroundResult\n");
  162. return;
  163. }
  164. $allSubTransactionsInfoResult = $dbHandle->getAllSubTransactionsInfo($clusterName, $orchestratorTxnId);
  165. if ($allSubTransactionsInfoResult["result"] != 0 ) {
  166. $logger->log_error("Got error while getting subTxnInfo: ".$allSubTransactionsInfoResult["error"]);
  167. print json_encode($allSubTransactionsInfoResult);
  168. return;
  169. }
  170. /*
  171. if (count($allSubTransactionsInfoResult["subTxns"]) != $count) {
  172. $logger->log_info("Still waiting ");
  173. }
  174. */
  175. //$mySubTxnId = '"'.$mySubTxnId.'"';
  176. // $logger->log_error(" sequentialScriptExecutors sub txns " . json_encode($allSubTransactionsInfoResult));
  177. $successStatus = "SUCCESS";
  178. $errorStatus = "FAILED";
  179. $totalFailedStatus = "TOTALFAILURE";
  180. $currentStatus = $allSubTransactionsInfoResult["subTxns"][$mySubTxnId]["opStatus"];
  181. while ( $currentStatus != $successStatus && $currentStatus != $errorStatus
  182. && $currentStatus != $totalFailedStatus) {
  183. sleep(1);
  184. $allSubTransactionsInfoResult = $dbHandle->getAllSubTransactionsInfo($clusterName, $orchestratorTxnId);
  185. if ($allSubTransactionsInfoResult["result"] != 0 ) {
  186. $logger->log_error("Got error while getting subTxnInfo: ".$allSubTransactionsInfoResult["error"]);
  187. print json_encode($allSubTransactionsInfoResult);
  188. return;
  189. }
  190. $currentStatus = $allSubTransactionsInfoResult["subTxns"][$mySubTxnId]["opStatus"];
  191. //$logger->log_debug(" sequentialScriptExecutors sub txns " . json_encode($allSubTransactionsInfoResult));
  192. if ($currentStatus != $successStatus && $currentStatus != $errorStatus
  193. && $currentStatus != $totalFailedStatus) {
  194. updateProgressForStage($clusterName, $rootTxnId,
  195. $orchestratorTxnId, $mySubTxnId, $stage, $hostCount);
  196. }
  197. //$logger->log_debug("Status we are seeing: " . $currentStatus . " txnId: " . $orchestratorTxnId . " subTxnId " . $mySubTxnId);
  198. }
  199. // Just in case, the command finished too fast and the while loop is skipped.
  200. updateProgressForStage($clusterName, $rootTxnId, $orchestratorTxnId, $mySubTxnId, $stage, $hostCount);
  201. $logger->log_debug("Came out of the launch for stage " . $currentStage . "\n");
  202. unset($subTxn);
  203. if ($currentStatus == $totalFailedStatus) {
  204. $logger->log_error("Encountered total failure in transaction $mySubTxnId" .
  205. " while running cmd: $cmd with args: $args");
  206. return;
  207. }
  208. ////////// Construct the host list needed for next stage ////////////
  209. $nextStageHosts = array();
  210. foreach ($hosts as $host) {
  211. ////// Get info about this node from db ////
  212. $hostInfo = $dbHandle->getHostInfo($clusterName,$host);
  213. if ($hostInfo["result"] != 0 ) {
  214. $logger->log_error("Got error while getting hostInfo for $host :" .$hostInfo["error"]);
  215. continue;
  216. }
  217. if ($hostInfo["discoveryStatus"] == "SUCCESS") {
  218. $nextStageHosts[] = $host;
  219. }
  220. }
  221. // Change the host list to weed-out bad nodes.
  222. $hosts = $nextStageHosts;
  223. ////////// End of constructructing the host list needed for next stage ////////////
  224. }
  225. ?>