AMBARI-353. Passing packages in hdp class for package install optimization. (Contributed by Jitendra)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/ambari-186@1346656 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -6,6 +6,8 @@ characters wide.
 
 Release 0.1.x - unreleased
 
+  AMBARI-353. Passing packages in hdp class for package install optimization. (Jitendra via Vikram)
+
   AMBARI-344. Fix TxnProgressWidget To Not Hide Previously Pending States (Varun via Vikram)
 
   AMBARI-352. Add flow control - force redirects to appropriate pages based on cluster configuration status for better usability (Yusaku via Vikram)

hmc/php/puppet/genmanifest/generateManifest.php

@@ -60,12 +60,93 @@ include "RoleDependencies.php";
            }
          }
        }
+       ManifestGenerator::optimizePackageInstall($n, $nm, $rolesStates, $configInfo, $roleStages);
        $nodeManifestString = $nm->generateNodeManifest();
        fwrite($fh, $nodeManifestString . "\n");
      }
      fclose($fh);
    }
 
+   static function optimizePackageInstall($node, &$manifestObj, $rolesStatesDs, 
+       $configInfo, $roleStages) {
+     //Figure out the state
+     $serviceState = SERVICE_STATE_INSTALLED_AND_CONFIGURED;
+     foreach ($rolesStatesDs as $r => $stateList) {
+       if ((isset($stateList[SERVICE_STATE_KEY])) &&
+           ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+         if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_INSTALLED_AND_CONFIGURED) {
+           $serviceState = NULL;
+           break;
+         }
+       }
+     } 
+     if (!isset($serviceState)) {
+       $serviceState = SERVICE_STATE_UNINSTALLED;
+       //See if it is uninstalled
+       foreach ($rolesStatesDs as $r => $stateList) {
+         if ( (isset($stateList[SERVICE_STATE_KEY])) && 
+           ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_NO_OP )) {
+           if ($stateList[SERVICE_STATE_KEY] != SERVICE_STATE_UNINSTALLED) {
+             $serviceState = NULL;
+             break;
+           }
+         }
+       } 
+     }
+     if (!isset($serviceState)) {
+       //No optimization needed
+       return;
+     }
+     //get list of packages
+     $stages = array();
+     foreach($roleStages as $roleName => $val) {
+        $stages[$val] = $roleName;
+     }
+     ksort($stages, SORT_NUMERIC);
+     $packageList = array();
+     foreach($stages as $theStage => $r) {
+       if (!isset($rolesStatesDs[$r])) {
+         continue;
+       }
+       //Add in the order of the stages
+       $stateList = $rolesStatesDs[$r];
+       if ($stateList[SERVICE_STATE_KEY] != $serviceState) { 
+         continue;
+       }
+       if (isset(self::$rolesToPackageMap[$r])) {
+         $p = self::$rolesToPackageMap[$r];
+         if (!in_array($p, $packageList)) {
+           $packageList[] = $p;
+         }
+       }
+     }
+
+     if (empty($packageList)) {
+       //No packages don't bother
+       return;       
+     }
+
+     //lzo and snappy 
+     $packageList[] = self::$rolesToPackageMap["snappy"];
+     if ($configInfo["lzo_enabled"] == "true") {
+       $packageList[] = self::$rolesToPackageMap["lzo"];
+     }
+
+     $firstP = true;
+     $pList = "\"";
+     foreach ($packageList as $p) {
+       if ($firstP) {
+         $firstP = false;
+       } else {
+         $pList = $pList . " ";
+       }
+       $pList = $pList . $p;
+     }
+     $pList = $pList . "\"";
+     $manifestObj->setRoleState("hdp", SERVICE_STATE_KEY, $serviceState);
+     $manifestObj->setRoleState("hdp", "pre_installed_pkgs", $pList);
+   }
+
    private static function getAllImports($modulesDir) {
      $importString = "";
      $importString = $importString . "import \"" . $modulesDir . "/hdp/manifests/*.pp" ."\"\n";
@@ -75,7 +156,6 @@ include "RoleDependencies.php";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-oozie/manifests/*.pp" ."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-pig/manifests/*.pp" ."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-sqoop/manifests/*.pp" ."\"\n";
-     #$importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp" ."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-templeton/manifests/*.pp" ."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-hive/manifests/*.pp" ."\"\n";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-hcat/manifests/*.pp" ."\"\n";
@@ -83,6 +163,36 @@ include "RoleDependencies.php";
      $importString = $importString . "import \"" . $modulesDir . "/hdp-monitor-webserver/manifests/*.pp" ."\"\n";
      return $importString;
    }
+
+   private static $rolesToPackageMap = array (
+     "hdp-hadoop::namenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+     "hdp-hadoop::snamenode" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+     "hdp-hadoop::jobtracker" => "hadoop hadoop-libhdfs.x86_64 hadoop-native.x86_64 hadoop-pipes.x86_64 hadoop-sbin.x86_64",
+     "hdp-hadoop::client" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+     "hdp-hadoop::datanode" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+     "hdp-hadoop::tasktracker" => "hadoop hadoop-libhdfs.i386 hadoop-native.i386 hadoop-pipes.i386 hadoop-sbin.i386",
+     "hdp-hadoop::zookeeper" => "zookeeper",
+     "hdp-hadoop::zookeeper::client" => "zookeeper",
+     "hdp-hbase::master" => "hbase",
+     "hdp-hbase::regionserver" => "hbase",
+     "hdp-hbase::client" => "hbase",
+     "hdp-pig" => "pig.noarch",
+     "hdp-sqoop" => "sqoop",
+     "hdp-hive::server" => "hive",
+     "hdp-hive::client" => "hive",
+     "hdp-hcat" => "hcatalog",
+     "hdp-oozie::server" => "oozie.noarch",
+     "hdp-oozie::client" => "oozie-client.noarch",
+     "hdp-mysql::mysql" => "mysql mysql-server",
+     "hdp-templeton::server" => "templeton.i386",
+     "hdp-templeton::client" => "templeton.i386",
+     "lzo" => "lzo lzo.i386 lzo-devel lzo-devel.i386",
+     "snappy" => "snappy snappy-devel",
+     "hdp-ganglia::monitor_and_server" => "ganglia-gmetad-3.2.0 ganglia-gmond-3.2.0 gweb hdp_mon_ganglia_addons",
+     "hdp-ganglia::monitor" => "ganglia-gmond-3.2.0 gweb hdp_mon_ganglia_addons",
+     "hdp-nagios::server" => "hdp_mon_nagios_addons nagios-3.2.3 nagios-plugins-1.4.9 fping net-snmp-utils",
+     "hdp-dashboard" => "hdp_mon_dashboard",
+   );
  }
 
 ?>

hmc/php/puppet/genmanifest/nodeManifest.php

@@ -186,20 +186,4 @@
     }
   }
 
-/*** TEST CODE ********
-  $nm = new NodeManifest("x.y.z");
-  $nm->setRoleStage("namenode", "1");
-  $nm->setRoleStage("datanode", "2");
-  $nm->setRoleStage("namenode", "3");
-  $nm->setRoleState("namenode", "service_state", "running");
-  $nm->setRoleOpt("namenode", "deploy", "yes");
-  $nm->setRoleOpt("namenode", "deploy", "no");
-  $nm->setRoleOpt("namenode", "wait", "10");
-  $nm->setRoleOpt("datanode", "running", "yes");
-  $nm->setNamenodeRoleState("stopped", 4);
-  $nm->setDatanodeRoleState("running", 5);
-  $ret = $nm->generateNodeManifest();
-  echo "$ret\n";
-***/
-
 ?>

hmc/puppet/manifestloader/site.pp

@@ -1,38 +1,23 @@
 class manifestloader () {
-    anchor { 'manifestloader::begin': }
     file { '/etc/puppet/agent/site.pp':
       ensure => present,
       source => "puppet:///modules/catalog/site.pp",  
       mode => '0755',
-      require => Anchor['manifestloader::begin'],      
-      notify => Exec['rm_puppet_apply_log']
     }
 
     exec {'rm_puppet_apply_log':
       command   => "rm -f /var/log/puppet_apply.log",
       path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      notify    => Exec['puppet_apply'],
-      require   => File['/etc/puppet/agent/site.pp'],
-      logoutput => "true"
     }
 
     exec { 'puppet_apply':
-      command   => "puppet apply --confdir=/etc/puppet/agent --logdest=/var/log/puppet_apply.log --debug --autoflush /etc/puppet/agent/site.pp",
+      command   => "sh /etc/puppet/agent/modules/puppetApply.sh",
       timeout   => 1200,
       path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      require   => Exec['rm_puppet_apply_log'],
-      notify    => Exec['append'],
       logoutput => "true"
     }
 
-    exec { 'append':
-      command   => "cat /var/log/puppet_apply.log",
-      path      => '/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-      require   => Exec['puppet_apply'],
-      notify   => Anchor['manifestloader::end'],
-      logoutput => "true"
-    }
-    anchor { 'manifestloader::end': }
+    File['/etc/puppet/agent/site.pp'] -> Exec['rm_puppet_apply_log'] -> Exec['puppet_apply']
 }
 
 node default {

hmc/puppet/modules/puppetApply.sh

@@ -0,0 +1,7 @@
+puppet apply --confdir=/etc/puppet/agent --logdest=console --debug --autoflush --detailed-exitcodes /etc/puppet/agent/site.pp
+ret=$?
+if [ "$ret" == "0" ] || [ "$ret" == "2" ]; then
+  exit 0
+else
+  exit 1 
+fi