AMBARI-337. Parallelize puppet kick --ping during bootstrap (Contributed by Hitesh)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/ambari-186@1346598 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,8 @@ characters wide.
 
 Release 0.1.x - unreleased
 
+  AMBARI-337. Parallelize puppet kick --ping during bootstrap (Hitesh via Vikram)
+
   AMBARI-335. Redundant downloads even though the artifacts are already installed (Ramya via Vikram)
 
   AMBARI-519. update to fix the ganglia monitor_and_server anchor problem (Rich via Vikram)

hmc/php/frontend/addNodes/finalizeNodes.php

@@ -138,33 +138,100 @@ function sign_and_verify_agent ($hosts, $logger) {
      . ", failedHostsCount=" . $countFailed);
 
   sleep(5);
+
+
+  // Run kick ping in batches of 10 hosts
+  $hostsToKick = array();
+  $index = 0;
+  $counter = 0;
   foreach ($origHosts as $i => $host) {
     $host = trim($host);
     if (array_key_exists ($host , $output)) {
       continue;
     }
+    $counter++;
+    if (!isset($hostsToKick[$index])) {
+      $hostsToKick[$index] = array();
+    }
+    $hostsToKick[$index][] = $host;
+
+    if ($counter == 10) {
+      $index++;
+      $counter = 0;
+    }
+  }
+
+  foreach ($hostsToKick as $idx => $hostKickList)  {
+
+    $hostList = implode(",", $hostKickList);
+
     /* Give puppet kick --ping to check if agent is working */
-    $logger->log_debug("Puppet kick --ping for ".$host."\n");
+    $logger->log_debug("Puppet kick --ping for batch $idx , hosts=".$hostList."\n");
+
+    $hostListStr = "";
+    foreach ($hostKickList as $hostToKick) {
+      $hostListStr .= " --host " . $hostToKick;
+    }
+
     $out_arr = array();
-    $cmd = "puppet kick -f --host $host --ping 2>/dev/null";
+    $cmd = "puppet kick -f --parallel 10 --ping $hostListStr 2>/dev/null";
     exec ($cmd, $out_arr, $err);
-    if ($err == 0 && check_error($out_arr, "status is success", 0) == 0) {
-      // success
-      $logger->log_info("Puppet kick succeeded for host " . $host);
-      $hostsState[$host] = TRUE;
-      if (isset($output[$host])) {
-        unset($output[$host]);
+
+    // TODO do we need to check $err ?
+
+    $pHostOutput = array();
+    $pHostResponse = array();
+    foreach ($out_arr as $line) {
+      foreach ($hostKickList as $host) {
+        if (preg_match ("/$host/", $line)) {
+          if (!isset($pHostOutput[$host])) {
+            $pHostOutput[$host] = array();
+          }
+          $pHostOutput[$host][] = $line;
+          $pattern = $host." finished with exit code (\d+)";
+          $matches = array();
+          if (preg_match("/$pattern/", $line, $matches) > 0) {
+            $retCode = (int)$matches[1];
+            $pHostResponse[$host] = $retCode;
+          }
+        }
       }
-    } else {
-      $logger->log_error("Failed to do puppet kick -ping on host " . $host);
-      if (!isset($output[$host])) {
-        $output[$host] =
-            array ( "discoveryStatus" => "FAILED",
-                  "badHealthReason" => "Puppet kick failed: "
-                      . ", error=" . $err . ", outputLogs="
-                      . implode(";", $out_arr));
+    }
+
+    $logger->log_debug("Output for batch $idx, outputLogs="
+        . print_r($pHostOutput, true) . " , errorCodes="
+        . print_r($pHostResponse, true) );
+
+    foreach ($hostKickList as $host) {
+      if (isset($pHostResponse[$host])
+          && $pHostResponse[$host] == 0) {
+        $logger->log_info("Puppet kick succeeded for host " . $host);
+        $hostsState[$host] = TRUE;
+        if (isset($output[$host])) {
+          unset($output[$host]);
+        }
+      } else {
+        $logger->log_error("Failed to do puppet kick -ping on host " . $host);
+
+        $errorCode = -1;
+        if (isset($pHostResponse[$host])) {
+          $errorCode = $pHostResponse[$host];
+        }
+
+        $errorLogs = "Puppet kick failed";
+        if (isset($pHostOutput[$host])) {
+          $errorLogs = implode(";", $pHostOutput[$host]);
+        }
+
+        if (!isset($output[$host])) {
+          $output[$host] =
+              array ( "discoveryStatus" => "FAILED",
+                      "badHealthReason" => "Puppet kick failed: "
+                          . ", error=" . $errorCode
+                          . ", outputLogs=" . $errorLogs);
+        }
+        $hostsState[$host] = FALSE;
       }
-      $hostsState[$host] = FALSE;
     }
   }
 

hmc/php/orchestrator/Cluster.php

@@ -570,7 +570,7 @@ class Cluster {
       $result = $this->installService($svcTxnMap[$service->name], $service, $dryRun);
       if ($result['result'] !== 0) {
         $this->setState(State::FAILED, $transaction, $dryRun);
-        $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, TRUE);
+        $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, FALSE);
         return $result;
       }
     }
@@ -583,7 +583,7 @@ class Cluster {
         $this->logger->log_error("Installing HDP failed with:  " . $result['error']);
         $this->setState(State::FAILED, $transaction, $dryRun);
         foreach ($services as $service) {
-          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, TRUE);
+          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, FALSE);
         }
         return $result;
       }
@@ -611,7 +611,7 @@ class Cluster {
         $this->logger->log_error("Installing HDP failed with:  " . $result['error']);
         $this->setState(State::FAILED, $transaction, $dryRun);
         foreach ($services as $service) {
-          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, TRUE);
+          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, FALSE);
         }
         return $result;
       }
@@ -621,7 +621,7 @@ class Cluster {
         $this->logger->log_error("Puppet kick failed, no successful nodes");
         $this->setState(State::FAILED, $transaction, $dryRun);
         foreach ($services as $service) {
-          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, TRUE);
+          $service->setState(State::FAILED, $svcTxnMap[$service->name], $dryRun, FALSE);
         }
         return array ( "result" => -3,
                        "error" => "Puppet kick failed on all nodes");

hmc/php/puppet/PuppetInvoker.php

@@ -103,7 +103,7 @@
     }
 
     public function kickPuppet($nodes, $txnObj, $clusterName, $nodesComponents,
-        $globalOptions) {
+        $globalOptions = array()) {
       //Get host config from db
       $txnId = $txnObj->toString();
       $components = array_keys($nodesComponents);