AMBARI-271. Support for local yum mirror. Contributed by Hitesh

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/ambari-186@1339881 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -2,6 +2,8 @@ Ambari Change log
 
 Release 0.x.x - unreleased
 
+  AMBARI-271. Support for local yum mirror (Hitesh via ramya)
+ 
   AMBARI-270. Puppet cleanup to define all the users in a common 
   location (ramya)
 

hmc/ShellScripts/puppet_agent_install.sh

@@ -25,47 +25,120 @@
 #   * - list of hosts
 #  */
 #set -e
-#set -x 
+#set -x
 trap 'pp_cmd=$ppp_cmd; ppp_cmd=$previous_command; previous_command=$this_command; this_command=$BASH_COMMAND' DEBUG
 #trap 'echo "$host: retcode:[$?] command:[$previous_command], out:[$out]"' EXIT
 #printf 'Argument is __%s__\n' "$@"
 
-master=$1
-repo_name=$2
-repo_desc=$3
-repourl=$4
-gpgkeyurl=$5
+usage() {
+  echo "
+Usage: $0 with the following parameters
+    --puppet-master            Puppet Master
+    --repo-file         Repo File
+    --gpg-key-files     GPG Key files - comma-separated
+  "
+}
 
-host=`hostname -f | tr '[:upper:]' '[:lower:]'`
+OPTS=$(getopt \
+  -n $0 \
+  -o '' \
+  -l 'puppet-master:' \
+  -l 'repo-file:' \
+  -l 'gpg-key-files:' \
+  -l 'help' \
+  -- "$@")
+
+if [ $? != 0 ] ; then
+  usage
+  echo "Invalid args" >&2
+  exit 3
+fi
 
-#echo "$host:_ERROR_:$master, $repo_name, $repo_desc, $repourl, $gpgkeyurl"
+echo "DEBUG: opts ${OPTS}"
 
-repo_file_content=''
-if [[ -z "$gpgkeyurl" ]]; then
-  repo_file_content="[$repo_name]\nname=$repo_desc\nbaseurl=$repourl\nenabled=1\ngpgcheck=0"
-else 
-  repo_file_content="[$repo_name]\nname=$repo_desc\nbaseurl=$repourl\nenabled=1\ngpgcheck=1\ngpgkey=$gpgkeyurl"
+eval set -- "${OPTS}"
+while true ; do
+  case "$1" in
+    --puppet-master)
+      MASTER=$2 ; shift 2
+      ;;
+    --repo-file)
+      REPOFILE=$2 ; shift 2
+      ;;
+    --gpg-key-files)
+      GPGKEYFILESTR=$2 ; shift 2
+      ;;
+    --help)
+      usage ;
+      exit 0
+      ;;
+    --)
+      shift ; break
+      ;;
+    *)
+      echo "Unknown option: $1" >&2
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+if [[ "x" == "x${MASTER}" ]]; then
+  echo "Error: Puppet master not specified" >&2
+  exit 3
 fi
 
-out=`echo -e $repo_file_content > /etc/yum.repos.d/$repo_name.repo`
-ret=$?
-if [[ "$ret" != "0" ]]; then
-  echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
-  exit 1
+if [[ "x" == "x${REPOFILE}" ]]; then
+  echo "Error: Repo file not specified" >&2
+  exit 3
+fi
+
+if [[ "x" != "x${GPGKEYFILESTR}" ]]; then
+  GPGKEYFILES=$(echo ${GPGKEYFILESTR} | tr "," " ")
 fi
 
+master=${MASTER}
+repoFile=${REPOFILE}
+gpgKeyFiles=${GPGKEYFILES}
 
-if [[ ! -z "$gpgkeyurl" ]]; then
-  out=`rpm --import $gpgkeyurl`
-  ret=$?
-  if [[ "$ret" != "0" ]]; then
-    echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
-    exit 1
-  fi
+echo "DEBUG: Puppet Master: ${master}"
+echo "DEBUG: Repo File: ${repoFile}"
+echo "DEBUG: GPG Key File Locations: ${gpgKeyFiles}"
+
+if [[ ! -f ${repoFile} ]]; then
+  echo "Error: Repo file ${repoFile} does not exist" >&2
+  exit 3
+else
+  echo "Copying $repoFile to /etc/yum.repos.d/"
+  cp -f $repoFile /etc/yum.repos.d/
 fi
 
-out=`rpm -Uvh http://dl.fedoraproject.org/pub/epel/5/i386/epel-release-5-4.noarch.rpm 2>&1`
+repoFileName=`basename $repoFile`
+if [[ ! -f "/etc/yum.repos.d/${repoFileName}" ]]; then
+  echo "Error: Repo file ${repoFile} not copied over to /etc/yum.repos.d/" >&2
+  exit 3
+fi
+
+for gpgKeyFile in ${gpgKeyFiles}
+do
+  if [[ ! -f ${gpgKeyFile} ]]; then
+    echo "Error: Specified GPG key file ${gpgKeyFile} does not exist" >&2
+    exit 3
+  fi
+  echo "Copying ${gpgKeyFile} to /etc/pki/rpm-gpg/"
+  cp -f ${gpgKeyFile} /etc/pki/rpm-gpg/
+  gpgKeyFileName=`basename ${gpgKeyFile}`
+  if [[ ! -f "/etc/pki/rpm-gpg/${gpgKeyFileName}" ]]; then
+    echo "Error: GPG key file ${gpgKeyFile} not copied over to /etc/pki/rpm-gpg/" >&2
+    exit 3
+  fi
+done
+
+host=`hostname -f | tr '[:upper:]' '[:lower:]'`
+
 out=`/etc/init.d/iptables stop 1>/dev/null`
+
+echo "Installing puppet using yum"
 out=`yum install -y puppet-2.7.9-2`
 ret=$?
 if [[ "$ret" != "0" ]]; then
@@ -76,15 +149,20 @@ out=`mkdir -p /etc/puppet/agent 2>&1`
 agent_auth_conf="path /run\nauth any\nallow $master\n\npath /\nauth any"
 out=`echo -e $agent_auth_conf > /etc/puppet/agent/auth.conf`
 out=`touch /etc/puppet/agent/namespaceauth.conf`
+
 out=`cp -f /etc/puppet/puppet.conf /etc/puppet/agent/ 2>&1`
 ret=$?
 if [[ "$ret" != "0" ]]; then
   echo "$host:_ERROR_:retcode:[$ret], CMD:[$pp_cmd]: OUT:[$out]" >&2
   exit 1
 fi
+
 #TODO clean this up for better fix. For now make sure we stop puppet agent. The issue here is we do not know if we started this puppet agent during our run or not.
+echo "Stopping puppet agent using service stop command"
 out=`service puppet stop`
 ret=$?
+
+echo "Starting puppet agent for HMC"
 out=`puppet agent --verbose --confdir=/etc/puppet/agent --listen --runinterval 5 --server $master --report --no-client --waitforcert 10 --configtimeout 600 --debug --logdest=/var/log/puppet_agent.log --httplog /var/log/puppet_agent_http.log --autoflush 2>&1`
 ret=$?
 if [[ "$ret" != "0" ]]; then

hmc/db/schema.dump

@@ -256,6 +256,11 @@ INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, de
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "nagios_web_password", "admin", "Nagios Admin password", "", "NAGIOS", "SECRET", '{ "isPassword": true, "noDisplay": false, "reconfigurable": false, "displayType": "text" }' );
 INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "nagios_contact", "", "Nagios Admin email", "", "NAGIOS", "", '{ "isPassword": false, "noDisplay": false, "reconfigurable": true, "displayType": "text" }' );
 
+-- Configuration for local yum mirror support
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "yum_repo_file", "/usr/share/hmc/yum_repo/hmc_hdp.repo", "Path to YUM Repo file", "Path to YUM Repo file", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "apache_artifacts_download_url", "", "Apache Artifacts Download URL", "URL form where to download HDP artifacts", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+INSERT OR REPLACE INTO "ConfigProperties" ( key, default_value, display_name, description, service_name, display_type, display_attributes ) VALUES ( "gpl_artifacts_download_url", "", "GPL Artifacts Download URL", "URL form where to download GPL artifacts", "MISCELLANEOUS", "NODISPLAY", '{ "isPassword": false, "noDisplay": true, "reconfigurable": false, "displayType": "text" }' );
+
 --                                         gsCluster.properties keys
 
 -- maps to hadoop_heap_size in gsCluster.properties in MB

hmc/html/initializeCluster.php

@@ -7,7 +7,7 @@
     <link type="text/css" rel="stylesheet" href="../css/bootstrap.css" media="screen"/>
     <link type="text/css" rel="stylesheet" href="../css/common.css" media="screen"/>
     <link type="text/css" rel="stylesheet" href="../css/common2.css" media="screen"/>
-    <link type="text/css" rel="stylesheet" href="../css/common3.css" media="screen"/>    
+    <link type="text/css" rel="stylesheet" href="../css/common3.css" media="screen"/>
     <link type="text/css" rel="stylesheet" href="../css/selectHosts.css" media="screen"/>
     <!-- End CSS -->
   </head>
@@ -128,7 +128,31 @@
                     <label for="clusterHostsFileId">Newline-delimited list of node hostnames</label>
                     <input type="file" name="clusterHostsFile" id="clusterHostsFileId" value="" placeholder="">
                     </p>
-
+                    <br/>
+                    <div id="yumMirrorSupportFormId">
+                      <div id="yumMirrorSupportFormButtonWrapperId">
+                        <p>
+                        <label for="yumMirrorSupportFormButtonId">Use local yum mirror instead of HMC defaults?</label>
+                        <input type="checkbox" name="YumMirrorSupportFormButton" id="yumMirrorSupportFormButtonId" value="" placeholder="">
+                        </p>
+                      </div>
+                      <div id="yumMirrorSupportFormFieldsId" style="display:none">
+                        <p>
+                        <label for="yumRepoFilePathId">YUM Repo File Path</label>
+                        <input type="text" name="YumRepoFilePath" id="yumRepoFilePathId" value="" placeholder="">
+                        </p>
+                        <br/>
+                        <p>
+                        <label for="hmcArtifactsDownloadUrlId">URL from where to download Apache Artifacts</label>
+                        <input type="text" name="HmcArtifactsDownloadUrl" id="hmcArtifactsDownloadUrlId" value="" placeholder="">
+                        </p>
+                        <br/>
+                        <p>
+                        <label for="hmcGplArtifactsDownloadUrlId">URL from where to download GPL Artifacts</label>
+                        <input type="text" name="HmcGplArtifactsDownloadUrl" id="hmcGplArtifactsDownloadUrlId" value="" placeholder="">
+                        </p>
+                      </div>
+                    </div>
                     <div id="fileUploadWrapperDivId">
                       <iframe name="fileUploadTarget" id="fileUploadTargetId" src="about:blank" style="display:none"></iframe>
                     </div>

hmc/js/addNodes.js

@@ -3,19 +3,40 @@ InstallationWizard.AddNodes = {
   renderData:
     {},
 
-  render: 
+  render:
     function (addNodesRenderData) {
 
       /* Always update the object's renderData first. */
       InstallationWizard.AddNodes.renderData = addNodesRenderData;
 
-      /* Since this screen is completely statically rendered, nothing else 
-       * needs to be done here. 
+      /* Since this screen is completely statically rendered, nothing else
+       * needs to be done here.
        */
+      if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+        globalYui.one("#yumRepoFilePathId").set('value', '');
+        globalYui.one("#hmcArtifactsDownloadUrlId").set('value', '');
+        globalYui.one("#hmcGplArtifactsDownloadUrlId").set('value', '');
+      }
       globalYui.one("#addNodesCoreDivId").setStyle('display', 'block');
+
+      if (globalYui.one("#yumMirrorSupportFormButtonId") && addNodesRenderData.yumRepo) {
+        globalYui.one("#yumRepoFilePathId").set('value', addNodesRenderData.yumRepo.yumRepoFilePath);
+        globalYui.one("#hmcArtifactsDownloadUrlId").set('value', addNodesRenderData.yumRepo.hdpArtifactsDownloadUrl);
+        globalYui.one("#hmcGplArtifactsDownloadUrlId").set('value', addNodesRenderData.yumRepo.gplArtifactsDownloadUrl);
+      }
     }
 };
 
+if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+  globalYui.one("#yumMirrorSupportFormButtonId").on('click', function(e) {
+    if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+      globalYui.one('#yumMirrorSupportFormFieldsId').setStyle('display', 'block');
+    } else {
+      globalYui.one('#yumMirrorSupportFormFieldsId').setStyle('display', 'none');
+    }
+  });
+}
+
 globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
 
   var focusId = '';
@@ -40,7 +61,7 @@ globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
     }
     if (message != '') {
       message += ',';
-    } 
+    }
     message += 'User Identity file not specified';
     globalYui.one("#clusterDeployUserIdentityFileId").addClass('formInputError');
   } else {
@@ -55,7 +76,7 @@ globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
     }
     if (message != '') {
       message += ',';
-    } 
+    }
     message += 'Hosts file not specified';
     globalYui.one("#clusterHostsFileId").addClass('formInputError');
   } else {
@@ -68,6 +89,51 @@ globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
     return;
   }
 
+  if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+    if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+      // local yum mirror support
+      var repoFile = globalYui.Lang.trim(globalYui.one("#yumRepoFilePathId").get('value'));
+      var artifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcArtifactsDownloadUrlId").get('value'));
+      var gplArtifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcGplArtifactsDownloadUrlId").get('value'));
+
+      if (repoFile = '') {
+        errCount++;
+        if (focusId == '') {
+          focusId = '#yumRepoFilePathId';
+        }
+        if (message != '') {
+          message += ',';
+        }
+        message += 'Yum Repo file not specified';
+        globalYui.one("#yumRepoFilePathId").addClass('formInputError');
+      }
+
+      if (artifactsUrl = '') {
+        errCount++;
+        if (focusId == '') {
+          focusId = '#hmcArtifactsDownloadUrlId';
+        }
+        if (message != '') {
+          message += ',';
+        }
+        message += 'HDP Artifacts Download URL not specified';
+        globalYui.one("#hmcArtifactsDownloadUrlId").addClass('formInputError');
+      }
+
+      if (artifactsUrl = '') {
+        errCount++;
+        if (focusId == '') {
+          focusId = '#hmcGplArtifactsDownloadUrlId';
+        }
+        if (message != '') {
+          message += ',';
+        }
+        message += 'GPL Artifacts Download URL not specified';
+        globalYui.one("#hmcGplArtifactsDownloadUrlId").addClass('formInputError');
+      }
+    }
+  }
+
   clearFormStatus();
 
   showLoadingImg();
@@ -77,19 +143,19 @@ globalYui.one('#addNodesSubmitButtonId').on('click',function (e) {
 
   var addNodesFilesForm = globalYui.one("#addNodesFilesFormId");
 
-  addNodesFilesForm.set('action', '../php/frontend/addNodes.php?clusterName=' + 
+  addNodesFilesForm.set('action', '../php/frontend/addNodes.php?clusterName=' +
     InstallationWizard.AddNodes.renderData.clusterName + "&freshInstall=" + InstallationWizard.AddNodes.renderData.freshInstall);
 
-  /* Set the target of the first form's upload to be a hidden iframe 
-   * on the page so as not to redirect to the PHP page we're POSTing 
+  /* Set the target of the first form's upload to be a hidden iframe
+   * on the page so as not to redirect to the PHP page we're POSTing
    * to.
    *
-   * See http://www.openjs.com/articles/ajax/ajax_file_upload/ for 
+   * See http://www.openjs.com/articles/ajax/ajax_file_upload/ for
    * more on this.
    */
   addNodesFilesForm.set('target', 'fileUploadTarget');
 
-  /* And then programmatically submit the first of the 2 forms. */ 
+  /* And then programmatically submit the first of the 2 forms. */
   addNodesFilesForm.submit();
   globalYui.log("Files submitted to server.");
 
@@ -102,8 +168,24 @@ globalYui.one("#fileUploadTargetId").on('load', function (e) {
 
     globalYui.log("File upload finished");
 
+    var repoFile = '';
+    var artifactsUrl = '';
+    var gplArtifactsUrl = '';
+
+    if (globalYui.one("#yumMirrorSupportFormButtonId")) {
+      if (globalYui.one("#yumMirrorSupportFormButtonId").get('checked')) {
+        // local yum mirror support
+        repoFile = globalYui.Lang.trim(globalYui.one("#yumRepoFilePathId").get('value'));
+        artifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcArtifactsDownloadUrlId").get('value'));
+        gplArtifactsUrl = globalYui.Lang.trim(globalYui.one("#hmcGplArtifactsDownloadUrlId").get('value'));
+      }
+    }
+
     var addNodesRequestData = {
-      "ClusterDeployUser" : globalYui.Lang.trim(globalYui.one("#clusterDeployUserId").get('value'))
+      "ClusterDeployUser" : globalYui.Lang.trim(globalYui.one("#clusterDeployUserId").get('value')),
+      "yumRepoFilePath": repoFile,
+      "hdpArtifactsDownloadUrl" : artifactsUrl,
+      "gplArtifactsDownloadUrl": gplArtifactsUrl
     }
 
     // Trigger the execution of setting up nodes
@@ -125,7 +207,7 @@ globalYui.one("#fileUploadTargetId").on('load', function (e) {
           globalYui.log("PARSED DATA: " + globalYui.Lang.dump(setupNodesJson));
           if (setupNodesJson.result != 0) {
             // Error!
-            alert("Got error!" + setupNodesJson.error); 
+            alert("Got error!" + setupNodesJson.error);
             return;
           }
           setupNodesJson = setupNodesJson.response;

hmc/package/rpm/SOURCES/hmc_hdp.repo

@@ -0,0 +1,14 @@
+[epel]
+name=Extra Packages for Enterprise Linux 5 - $basearch
+#baseurl=http://download.fedoraproject.org/pub/epel/5/$basearch
+mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-5&arch=$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL
+
+[Puppet]
+name=Puppet
+baseurl = http://yum.puppetlabs.com/el/5/products/x86_64/
+enabled=1
+gpgcheck=0

hmc/package/rpm/SPECS/hmc.spec

@@ -19,19 +19,20 @@
 # */
 
 #
-# RPM Spec file for HMC 
+# RPM Spec file for HMC
 #
 
 Summary: HMC
 Name: hmc
 Version: 0.0.1
 URL: http://hortonworks.com
-Release: 2%{?dist} 
+Release: 2%{?dist}
 License: Apache License, Version 2.0
 Vendor: Hortonworks <hmc-dev-group@hortonworks.com>
 Group: System Environment/Base
 Source: %{name}-%{version}.tar.gz
 Source1: hmc.init.in
+Source2: hmc_hdp.repo
 Buildroot: %{_tmppath}/%{name}-%{version}-buildroot
 Requires: php >= 5, sqlite >= 3, php-pdo, php-pecl-json, httpd, puppet = 2.7.9, pdsh, httpd-devel, ruby-devel, rubygems, mod_passenger, mod_ssl
 %define web_prefixdir %{_prefix}/share/hmc
@@ -57,6 +58,7 @@ This package provides a Management Console for Hadoop Cluster.
 %__mkdir -p $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
 %__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/bin/
+%__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
 %__mkdir -p $RPM_BUILD_ROOT/%{puppet_dir}/
 %__mkdir -p $RPM_BUILD_ROOT/%{puppet_dir}/manifests
 %__mkdir -p $RPM_BUILD_ROOT/%{web_prefixdir}/
@@ -80,6 +82,7 @@ This package provides a Management Console for Hadoop Cluster.
 %__cp -f yuiCombinator.php $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__cp -rf conf $RPM_BUILD_ROOT/%{web_prefixdir}/
 %__cp -rf puppet/modules $RPM_BUILD_ROOT/%{puppet_dir}
+%__cp -f "%{SOURCE2}" $RPM_BUILD_ROOT/%{web_prefixdir}/yum_repo/
 %__install -D -m0755 puppet/reports/get_revision $RPM_BUILD_ROOT/%{web_prefixdir}/bin
 %__cp -rf puppet/reports/hmcreport.rb $RPM_BUILD_ROOT/usr/lib/ruby/site_ruby/1.8/puppet/reports/
 echo "Alias /hdp %{_prefix}/share/hdp" > $RPM_BUILD_ROOT/%{httpd_confdir}/hdp_mon_dashboard.conf

hmc/php/frontend/addNodes/bootstrap.php

@@ -9,6 +9,7 @@ include_once '../db/HMCDBAccessor.php';
 
 include_once "../util/HMCTxnUtils.php";
 include_once 'commandUtils.php';
+include_once "../util/YumRepoConfigParser.php";
 
   $logger = new HMCLogger("BootStrap");
   $dbAccessor = new HMCDBAccessor($GLOBALS["DB_PATH"]);
@@ -18,24 +19,45 @@ include_once 'commandUtils.php';
     global $logger, $dbAccessoar, $hosts, $readFromFile;
 
     $master=strtolower(exec('hostname -f'));
-    $repo_name = $repo['name'];
-    $repo_desc = $repo['desc'];
-    $repo_url = $repo['url'];
-    $repo_gpgkey = $repo['gpgkeyurl'];
+    $repoFile = $repo['yumRepoFilePath'];
+    $gpgKeyFiles = $repo['gpgKeyFiles'];
+
     exec ("/etc/init.d/iptables stop");
     $logger->log_debug("List of hosts to BootStrap ".json_encode($hosts));
     $logger->log_debug("Run script for pdsh ".$rscript);
     $scpCmd = "scp -o StrictHostKeyChecking=no ";
+
     foreach ($hosts as $host) {
       $host = trim($host);
-      $cmd = "$scpCmd -i $sshkey $rscript $user@$host:/tmp";
-      exec ("$scpCmd -i $sshkey $rscript $user@$host:/tmp");
+
+      $filesToCopy = array_merge ( array ($rscript, $repoFile), $gpgKeyFiles);
+
+      /* Copy puppet run script to all nodes */
+      // Copy repo file to each node
+      // Copy gpg keys to each node
+      if (!empty($filesToCopy)) {
+        $cmd = "$scpCmd -i $sshkey " . implode(" ", $filesToCopy)
+            . " $user@$host:/tmp/ ";
+        $logger->log_debug("Running scp command $cmd");
+        exec($cmd);
+      }
+    }
+
+    $remoteRepoFilePath = trim("/tmp/" . basename(trim($repoFile)));
+    $remoteGpgKeyPaths = "";
+    foreach ($gpgKeyFiles as $kFile) {
+      $dFile = trim("/tmp/" . basename(trim($kFile)));
+      if ($remoteGpgKeyPaths != "") {
+        $remoteGpgKeyPaths .= ",";
+      }
+      $remoteGpgKeyPaths .= $dFile;
     }
 
-    /* Copy puppet run script to all nodes */
-    $logger->log_debug("/tmp/puppet_agent_install.sh $master $repo_name $repo_desc $repo_url $repo_gpgkey");
+    $rcmd = "/tmp/puppet_agent_install.sh --puppet-master=" . $master
+        . " --repo-file=" . $remoteRepoFilePath
+        . " --gpg-key-files=" . $remoteGpgKeyPaths;
+    $logger->log_info("Running $rcmd to bootstrap each node");
 
-    $rcmd = "/tmp/puppet_agent_install.sh $master $repo_name $repo_desc $repo_url $repo_gpgkey";
     runPdsh($clusterName, "bootstrapNodes", $user, $readFromFile, $rcmd);
 
     $result = parseAndUpdateNodeInfo ($clusterName, "bootstrapNodes", $logger);
@@ -79,10 +101,71 @@ $sshkey = getSshKeyFilePath($clusterName);
 $rscript = realpath("../../ShellScripts/puppet_agent_install.sh");
 
 $repository=array();
-$repository['name']="hmc_puppet";
-$repository['desc']="puppetlabs";
-$repository['url']="http://yum.puppetlabs.com/el/5/products/x86_64/";
-$repository['gpgkeyurl']="http://yum.puppetlabs.com/RPM-GPG-KEY-puppetlabs";
+$configs = $dbAccessor->getServiceConfig($clusterName);
+if ($configs["result"] != 0) {
+  $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+  $logger->log_error("Got error when trying to retrieve configs from DB");
+  return;
+}
+
+$repoFile = $configs["properties"]["yum_repo_file"];
+$gpgKeyLocations = getEnabledGpgKeyLocations($repoFile);
+if ($gpgKeyLocations === FALSE) {
+  $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+  $logger->log_error("Got error when trying to parse yum repo config");
+  return;
+}
+
+$tmpDir = "/tmp/hmcDownloads-".time()."/";
+$retVal = 0;
+$output = array();
+exec("mkdir -p ".$tmpDir, $output, $retVal);
+if ($retVal != 0) {
+  $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+  $logger->log_error("Got error when trying to create tmp download dir"
+      . ", dir=" . $tmpDir . ", output=" . print_r($output, true));
+  return;
+}
+
+$gpgKeyFiles = array();
+
+foreach ($gpgKeyLocations as $repoId => $gpgInfo) {
+  if (!isset($gpgInfo["gpgkey"])) {
+    continue;
+  }
+  $loc = $gpgInfo["gpgkey"];
+  $logger->log_info("Fetching gpg key for $repoId from location $loc");
+  $info = parse_url($loc);
+  if ($info === FALSE || !isset($info["path"])) {
+    $logger->log_error("Skipping invalid url $loc");
+    continue;
+  }
+  $fileName = basename($info["path"]);
+
+  $destFilePath = $tmpDir . "/" . $fileName;
+
+  $fetchCurlCmd = "curl --connect-timeout 30 --fail -s -o "
+      . $destFilePath . " " . $loc;
+
+  $logger->log_info("Fetching gpg key for $repoId from location $loc using "
+      . $fetchCurlCmd);
+
+  $retVal = 0;
+  $output = array();
+  exec($fetchCurlCmd, $output, $retVal);
+
+  if ($retVal != 0) {
+    $subTransactionReturnValue = $dbAccessor->updateSubTransactionOpStatus($clusterName, $parentSubTxnId, $mySubTxnId, "TOTALFAILURE");
+    $logger->log_error("Error when trying to download gpg key using "
+        . $fetchCurlCmd . ", output=" . print_r($output, true));
+    return;
+  }
+  array_push($gpgKeyFiles, $destFilePath);
+}
+
+$repository = array( "yumRepoFilePath" => $repoFile,
+                     "gpgKeyFiles" => $gpgKeyFiles);
+
 $logger->log_debug("BootStrapping with puppet");
 $boot_result = bootstrap($clusterName, $deployUser, $rscript,
                            $sshkey, $repository);

hmc/php/frontend/addNodes/finalizeNodes.php

@@ -157,11 +157,13 @@ function sign_and_verify_agent ($hosts, $logger) {
       }
     } else {
       $logger->log_error("Failed to do puppet kick -ping on host " . $host);
-      $output[$host] =
-          array ( "discoveryStatus" => "FAILED",
+      if (!isset($output[$host])) {
+        $output[$host] =
+            array ( "discoveryStatus" => "FAILED",
                   "badHealthReason" => "Puppet kick failed: "
                       . ", error=" . $err . ", outputLogs="
                       . implode(";", $out_arr));
+      }
       $hostsState[$host] = FALSE;
     }
   }

hmc/php/frontend/createCluster.php

@@ -123,11 +123,22 @@ if (!is_dir($clusterDir) && !mkdir($clusterDir, 0700, true)) {
   return;
 }
 
+$propertiesArr = $dbAccessor->getConfigPropertiesMetaInfo();
+if ($propertiesArr["result"] != 0) {
+  print json_encode(array( "result" => 1, "error" => "Error in config properties meta info"));
+  return;
+}
+
 $output = array(
                  "result" => 0,
                  "error" => "",
                  "response" => array(
-                                 "clusterName" => $response["clusterName"]
+                                 "clusterName" => $response["clusterName"],
+                                 "yumRepo" => array (
+                                   "yumRepoFilePath" => $propertiesArr["configs"]["yum_repo_file"]["value"],
+                                   "hdpArtifactsDownloadUrl" => $propertiesArr["configs"]["apache_artifacts_download_url"]["value"],
+                                   "gplArtifactsDownloadUrl" => $propertiesArr["configs"]["gpl_artifacts_download_url"]["value"]
+                        )
                    ),
               );
 

hmc/php/frontend/nodesAction.php

@@ -16,8 +16,87 @@ $clusterName = $_GET['clusterName'];
 $action = $_GET['action'];
 $deployUser = $_POST['ClusterDeployUser'];
 
+$propertiesArr = $dbAccessor->getConfigPropertiesMetaInfo();
+if ($propertiesArr["result"] != 0) {
+  print json_encode(array( "result" => 1, "error" => "Error in config properties meta info"));
+  return;
+}
+
+// Use meta info defaults
+// Override with current svc configs
+// Override with POST params
+
+$repoFilePath = $propertiesArr["configs"]["yum_repo_file"]["value"];
+$hdpArtifactsDownloadUrl = $propertiesArr["configs"]["apache_artifacts_download_url"]["value"];
+$gplArtifactsDownloadUrl = $propertiesArr["configs"]["gpl_artifacts_download_url"]["value"];
+
+$currentConfigs = $dbAccessor->getServiceConfig($clusterName);
+if ($currentConfigs["result"] != 0) {
+  print json_encode(array( "result" => 1, "error" => "Could not get configs from DB"));
+  return;
+}
+
+if (isset($currentConfigs["properties"]["yum_repo_file"])
+    && $currentConfigs["properties"]["yum_repo_file"] != "") {
+  $repoFilePath = $currentConfigs["properties"]["yum_repo_file"];
+}
+
+if (isset($currentConfigs["properties"]["apache_artifacts_download_url"])
+    && $currentConfigs["properties"]["apache_artifacts_download_url"] != "") {
+  $hdpArtifactsDownloadUrl = $currentConfigs["properties"]["apache_artifacts_download_url"];
+}
+
+if (isset($currentConfigs["properties"]["gpl_artifacts_download_url"])
+    && $currentConfigs["properties"]["gpl_artifacts_download_url"] != "") {
+  $gplArtifactsDownloadUrl = $currentConfigs["properties"]["gpl_artifacts_download_url"];
+}
+
+if (isset($_POST['yumRepoFilePath'])
+    && $_POST['yumRepoFilePath'] != "") {
+  $repoFilePath = $_POST['yumRepoFilePath'];
+}
+
+if (isset($_POST['hdpArtifactsDownloadUrl'])
+    && $_POST['hdpArtifactsDownloadUrl'] != "") {
+  $hdpArtifactsDownloadUrl = $_POST['hdpArtifactsDownloadUrl'];
+}
+
+if (isset($_POST['gplArtifactsDownloadUrl'])
+    && $_POST['gplArtifactsDownloadUrl'] != "") {
+  $gplArtifactsDownloadUrl = $_POST['gplArtifactsDownloadUrl'];
+}
+
 header("Content-type: application/json");
 
+if (!file_exists($repoFilePath)) {
+  print (json_encode(array(
+      "result" => 1,
+      "error" => "Invalid repo file path specified"
+  )
+  ));
+  return;
+}
+
+// TODO - error checks for download urls
+/*
+if (parse_url($hdpArtifactsDownloadUrl) === FALSE
+    || parse_url($gplArtifactsDownloadUrl) === FALSE) {
+  print (json_encode(array(
+        "result" => 1,
+        "error" => "Invalid download urls specified")));
+  return;
+}
+*/
+
+$configs =  array ( "yum_repo_file" => $repoFilePath,
+                    "apache_artifacts_download_url" => $hdpArtifactsDownloadUrl,
+                    "gpl_artifacts_download_url" => $gplArtifactsDownloadUrl);
+$dbResponse = $dbAccessor->updateServiceConfigs($clusterName, $configs);
+if ($dbResponse["result"] != 0) {
+  $logger->log_error("Got error while persisting configs: ".$dbResponse["error"]);
+  return $dbResponse;
+}
+
 $stagesFiles = "";
 if ($action == "addNodes") {
   $stagesFile = "./addNodes/stages.php";

hmc/php/util/YumRepoConfigParser.php

@@ -18,6 +18,15 @@
  * limitations under the License.
  */
 
+/**
+ * Parse repo file and get all enabled gpg keys
+ * @param string $repoFile
+ * @return mixed
+ *   array (
+ *      $currentRepoId = array ( "gpgkey" => $currentGpgLocation),
+ *      ....
+ *      )
+ */
 function getEnabledGpgKeyLocations($repoFile) {
   $logger = new HMCLogger("YumRepoConfigParser");
 
@@ -64,6 +73,8 @@ function getEnabledGpgKeyLocations($repoFile) {
           && (($currentGpgCheck == -1 && $globalGpgCheck == 1)
               || ($currentGpgCheck == 1))) {
         if ($currentGpgLocation != "") {
+          $logger->log_debug("Adding gpgkey $currentGpgLocation for repo"
+              .", id=" . $currentRepoId);
           $response[$currentRepoId] = array ( "gpgkey" => $currentGpgLocation);
         }
       } else if ($currentRepoId != ""
@@ -118,5 +129,4 @@ function getEnabledGpgKeyLocations($repoFile) {
   return $response;
 }
 
-
 ?>