AMBARI-401. Manual config changes for nn get reset on stop/start from hmc (Contributed by Jitendra)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/ambari-186@1347179 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,8 @@ characters wide.
 
 Release 0.1.x - unreleased
 
+  AMBARI-401. Manual config changes for nn get reset on stop/start from hmc (Jitendra via Vikram)
+
   AMBARI-399. Cannot uninstall - the page hangs with the spinning icon (Yusaku via Vikram)
 
   AMBARI-398. if hbase is not installed, nagios sends alerts for "ganglia collector for hbasemaster" being down (Suhas via Vikram)

hmc/ShellScripts/puppet_agent_install.sh

@@ -166,15 +166,6 @@ if [[ "$ret" != "0" ]]; then
   exit 1
 fi
 
-#Download modules and untar
-out=`curl -o /etc/puppet/agent/modules.tgz $master/hmc/modules.tgz && cd /etc/puppet/agent/ &&  tar zxf modules.tgz --strip-components 1 && cd -`
-ret=$?
-if [[ "$ret" != "0" ]]; then
-  echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
-  exit 1
-fi
-out=`rm /etc/puppet/agent/modules.tgz`
-
 #TODO clean this up for better fix. For now make sure we stop puppet agent. The issue here is we do not know if we started this puppet agent during our run or not.
 echo "Stopping puppet agent using service stop command"
 out=`service puppet stop`

hmc/package/rpm/SPECS/hmc.spec

@@ -69,7 +69,6 @@ fi
 %__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
 %__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/
 %__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/manifests
-%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/modules/catalog/files
 %__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__install -d "%{buildroot}%{hmc_db_dir}"
 %__install -d "%{buildroot}%{hmc_log_dir}"
@@ -92,11 +91,11 @@ fi
 %__cp -f yuiCombinator.php $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__cp -rf conf $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__cp -rf puppet/manifestloader $RPM_BUILD_ROOT/%{puppet_master_dir}
-%__cp -rf puppet/modules/stdlib $RPM_BUILD_ROOT/%{puppet_master_dir}/modules
+%__cp -rf puppet/modules $RPM_BUILD_ROOT/%{puppet_master_dir}
+%__mkdir -p $RPM_BUILD_ROOT/%{puppet_master_dir}/modules/catalog/files
 %__cp -f "%{SOURCE2}" $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
 %__install -D -m0755 puppet/reports/get_revision $RPM_BUILD_ROOT/%{web_prefixdir}/bin
 %__cp -rf puppet/reports/hmcreport.rb $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
-%__tar czf $RPM_BUILD_ROOT/%{web_prefixdir}/modules.tgz puppet/modules
 echo "Alias /hdp %{_prefix}/share/hdp" > $RPM_BUILD_ROOT/%{httpd_confdir}/hdp_mon_dashboard.conf
 
 %post

hmc/php/conf/Config.inc

@@ -9,12 +9,12 @@ $GLOBALS["CLUSTERMAIN_PATH"] = $INSTALL_ROOT . "/php/orchestrator/ClusterMain.ph
 $GLOBALS["DB_PATH"] = "/var/db/hmc/data/data.db";
 
 $GLOBALS["puppetMaxParallelKicks"] = 10;
-$GLOBALS["puppetManifestDir"] = "/etc/puppet/master/modules/catalog/files/";
+$GLOBALS["puppetMasterModulesDirectory"] = "/etc/puppet/master/modules";
 $GLOBALS["puppetKickVersionFile"] = "/var/lib/puppet/puppet_kick_version.txt";
 $GLOBALS["puppetReportsDir"] = "/var/lib/puppet/reports";
 $GLOBALS["puppetKickTimeout"] = 120;
-$GLOBALS["puppetModulesDirectory"] = "/etc/puppet/agent/modules";
-$GLOBALS["manifestloaderFile"] = "/etc/puppet/master/manifestloader/site.pp";
+$GLOBALS["puppetAgentModulesDirectory"] = "/etc/puppet/agent/modules";
+$GLOBALS["manifestloaderDir"] = "/etc/puppet/master/manifestloader";
 $GLOBALS["manifestloaderDestinationDir"] = "/etc/puppet/master/manifests";
 $GLOBALS["puppet_install"] = "/usr/share/hmc/ShellScripts/puppet_agent_install.sh";
 $GLOBALS["DRYRUN"] = FALSE;

hmc/php/puppet/PuppetInvoker.php

@@ -41,22 +41,33 @@
 
     private function sendKick($nodes, $txnId, &$failedNodes,
         &$successNodes, &$prevKickRunningNodes) {
-      $cmd = "";
-      $cmd = $cmd . "puppet kick ";
-      foreach ($nodes as $n) {
-        $cmd = $cmd . " --host " . $n;
-      }
-      $p = 10;
-      if (count($nodes) < $p) {
-        $p = count($nodes);
+      //Some nodes may already be done in the meanwhile
+      $doneNodes = array();
+      $this->checkForReportFiles($nodes, $txnId, $doneNodes);
+      if (count($doneNodes) < count($nodes)) {
+        $cmd = "";
+        $cmd = $cmd . "puppet kick ";
+        foreach ($nodes as $n) {
+          $cmd = $cmd . " --host " . $n;
+        }
+        $p = 10;
+        if (count($nodes) < $p) {
+          $p = count($nodes);
+        }
+        $cmd = $cmd . " --parallel " . $p;
+        $this->logger->log_trace("Kick command: " . $cmd);
+        $output = $this->executeAndGetOutput($cmd);
+        $this->logger->log_trace("Kick response begins ===========");
+        $this->logger->log_trace($output);
+        $this->logger->log_trace("Kick response ends ===========");
       }
-      $cmd = $cmd . " --parallel " . $p;
-      $this->logger->log_trace("Kick command: " . $cmd);
-      $output = $this->executeAndGetOutput($cmd);
-      $this->logger->log_trace("Kick response begins ===========");
-      $this->logger->log_trace($output);
-      $this->logger->log_trace("Kick response ends ===========");
       foreach ($nodes as $kNode) {
+        if (isset($doneNodes[$kNode])) {
+          $this->logger->log_debug($kNode . "previous kick has returned, no need to kick again");
+          //Mark as previous kick running, reports will be discovered in waitForResults.
+          $prevKickRunningNodes[] = $kNode;
+          continue;
+        }
         $regExSuccess = "/". $kNode . " .* exit code 0/";
         $regExRunning = "/". $kNode . ".* is already running/";
         if (preg_match($regExSuccess, $output)>0) {
@@ -120,8 +131,9 @@
         }
       }
        
+      $manifestDir = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files";
       $response = $this->genKickWait($nodes, $txnId, $clusterName, $hostInfo,
-          $configInfo, $hostRolesStates, $hostAttributes, $GLOBALS["puppetManifestDir"],
+          $configInfo, $hostRolesStates, $hostAttributes, $manifestDir,
           $GLOBALS["puppetKickVersionFile"], $GLOBALS["DRYRUN"]);
       return $response;
     }
@@ -150,8 +162,9 @@
       $hostInfo = $dbReader->getHostNames($clusterName);
       $configInfo = $dbReader->getAllConfigs($clusterName);
       $hostAttributes = $dbReader->getAllHostAttributes($clusterName);
+      $manifestDir = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files";
       $response = $this->genKickWait($nodesToKick, $txnId, $clusterName, $hostInfo,
-          $configInfo, $hostRolesStates, $hostAttributes, $GLOBALS["puppetManifestDir"],
+          $configInfo, $hostRolesStates, $hostAttributes, $manifestDir,
           $GLOBALS["puppetKickVersionFile"], $GLOBALS["DRYRUN"]);
       return $response;
     }
@@ -201,12 +214,24 @@
       }
 
       //Add manifest loader
-      copy($GLOBALS["manifestloaderFile"], $GLOBALS["manifestloaderDestinationDir"] . "/site.pp");
+      copy($GLOBALS["manifestloaderDir"] . "/site.pp", $GLOBALS["manifestloaderDestinationDir"] . "/site.pp");
 
       //Generate manifest
-      $modulesDir = $GLOBALS["puppetModulesDirectory"];
+      $agentModulesDir = $GLOBALS["puppetAgentModulesDirectory"];
       ManifestGenerator::generateManifest($manifestDir, $hostInfo,
-          $configInfo, $hostRolesStates, $hostAttributes, $modulesDir);
+          $configInfo, $hostRolesStates, $hostAttributes, $agentModulesDir);
+
+      //Tar the modules and catalog
+      $tarCmd = "tar zcf ". $GLOBALS["manifestloaderDir"] . "/modules.tgz" . " " .  $GLOBALS["puppetMasterModulesDirectory"];
+      $this->logger->log_info($tarCmd);
+      exec($tarCmd);
+      $tarLocation = $GLOBALS["puppetMasterModulesDirectory"] . "/catalog/files" ;
+      $removeOldTar = "rm -f " . $tarLocation . "/modules.tgz";
+      $this->logger->log_info($removeOldTar);
+      exec($removeOldTar);
+      $placeNewTarCmd = "mv " . $GLOBALS["manifestloaderDir"] . "/modules.tgz" . " " . $tarLocation;
+      $this->logger->log_info($placeNewTarCmd);
+      exec($placeNewTarCmd);
 
       //Write version file
       $this->writeVersionFile($versionFile, $txnId);
@@ -237,13 +262,25 @@
           sleep(1);
       }
       $sitePPFile = $manifestDir . "/site.pp";
-      system("mv " . $sitePPFile . " " . $sitePPFile ."-".$txnId);
+      system("mv " . $sitePPFile . " " . $GLOBALS["manifestloaderDir"] . "/site.pp-" . $txnId);
       // Delete version file, it will be generated next time.
       unlink($versionFile);
       $response = $this->createGenKickWaitResponse($kickFailedNodes, $failureResponseNodes,
           $timedoutNodes, $successfullNodes, $nodes); 
       return $response;
     }
+    
+    private function checkForReportFiles($nodes, $txnId, &$doneNodes) {
+      foreach ($nodes as $n) {
+        if (isset($doneNodes[$n])) {
+          continue;
+        }
+        $fileName = $this->getReportFilePattern($n, $txnId);
+        if (file_exists($fileName)) {
+          $doneNodes[$n] = 1;
+        }
+      }
+    }
 
     private function waitForResults($nodes, $txnId, &$successfullNodes, 
        &$failureResponseNodes, &$timedoutNodes) {
@@ -252,15 +289,7 @@
       $this->logger->log_info("Waiting for results from "
           . implode(",", $nodes));
       while (true) {
-        foreach ($nodes as $n) {
-          if (isset($doneNodes[$n])) {
-            continue;
-          }
-          $fileName = $this->getReportFilePattern($n, $txnId);
-          if (file_exists($fileName)) {
-            $doneNodes[$n] = 1;
-          }
-        }
+        $this->checkForReportFiles($nodes, $txnId, $doneNodes);
         $this->logger->log_info(count($doneNodes) . " out of " . count($nodes) 
             . " nodes have reported");
         if (count($doneNodes) >= count($nodes)) {

hmc/puppet/manifestloader/site.pp

@@ -1,14 +1,14 @@
 class manifestloader () {
-    file { '/etc/puppet/agent/site.pp':
+    file { '/etc/puppet/agent/modules.tgz':
       ensure => present,
-      source => "puppet:///modules/catalog/site.pp",  
+      source => "puppet:///modules/catalog/modules.tgz",  
       mode => '0755',
     }
 
-    exec {'rm_puppet_apply_log':
-      command   => "rm -f /var/log/puppet_apply.log",
-      path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-    }
+    exec { 'untar_modules':
+      command => "rm -rf /etc/puppet/agent/modules ; tar zxf /etc/puppet/agent/modules.tgz -C /etc/puppet/agent/ --strip-components 3",
+      path    => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
+    } 
 
     exec { 'puppet_apply':
       command   => "sh /etc/puppet/agent/modules/puppetApply.sh",
@@ -17,7 +17,7 @@ class manifestloader () {
       logoutput => "true"
     }
 
-    File['/etc/puppet/agent/site.pp'] -> Exec['rm_puppet_apply_log'] -> Exec['puppet_apply']
+    File['/etc/puppet/agent/modules.tgz'] -> Exec['untar_modules'] -> Exec['puppet_apply']
 }
 
 node default {

hmc/puppet/modules/puppetApply.sh

@@ -1,4 +1,5 @@
-puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush --detailed-exitcodes /etc/puppet/agent/site.pp
+rm -f /var/log/puppet_apply.log
+puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush --detailed-exitcodes /etc/puppet/agent/modules/catalog/files/site.pp
 ret=$?
 cat /var/log/puppet_apply.log
 if [ "$ret" == "0" ] || [ "$ret" == "2" ]; then