#!/bin/sh #/* # * Licensed to the Apache Software Foundation (ASF) under one # * or more contributor license agreements. See the NOTICE file # * distributed with this work for additional information # * regarding copyright ownership. The ASF licenses this file # * to you under the Apache License, Version 2.0 (the # * "License"); you may not use this file except in compliance # * with the License. You may obtain a copy of the License at # * # * http://www.apache.org/licenses/LICENSE-2.0 # * # * Unless required by applicable law or agreed to in writing, software # * distributed under the License is distributed on an "AS IS" BASIS, # * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # * See the License for the specific language governing permissions and # * limitations under the License. # */ baseDir=`dirname ${0}`; source ${baseDir}/easyInstallerLib.sh; source ${baseDir}/dbLib.sh; source ${baseDir}/servicesLib.sh; ########################### ### FILE-SCOPED GLOBALS ### ########################### # TODO XXX Take this in from the UI in the future (and likely store it as a # marker on disk, to really be of any use). clusterName="MyHDPCluster"; ############################# ### FILE-SCOPED FUNCTIONS ### ############################# function getServiceHost { local serviceHost=""; local serviceName="${1}"; if [ "x" != "x${serviceName}" ] then serviceHostFile=`fetchDbFileForCluster "${clusterName}" "${serviceName}"`; serviceHost=`getFirstWordFromFile "${serviceHostFile}"`; fi echo "${serviceHost}"; } function generateServiceConfigChoiceXml { local serviceConfigChoiceXml=""; local serviceHostFileName="${1}"; local serviceConfigKey="${2}"; if [ "x" != "x${serviceHostFileName}" ] && [ "x" != "x${serviceConfigKey}" ] then local serviceHostValue=`getServiceHost "${serviceHostFileName}"`; serviceConfigChoiceXml="\ ${serviceConfigKey}\ \ \ \ ${serviceHostValue}\ \ \ "; fi echo "${serviceConfigChoiceXml}"; } function generateOptionalServicesConfigChoicesXml { local optionalServicesConfigChoicesXml=""; local isHBaseInstalled="${1}"; local isHCatalogInstalled="${2}"; local isTempletonInstalled="${3}"; local isOozieInstalled="${4}"; if [ "xyes" == "x${isHBaseInstalled}" ] then hBaseConfigChoicesXml=`generateServiceConfigChoiceXml "hbasemaster" "HDPHBaseMasterHost"`; optionalServicesConfigChoicesXml="${optionalServicesConfigChoicesXml}${hBaseConfigChoicesXml}"; fi if [ "xyes" == "x${isHCatalogInstalled}" ] then hCatalogConfigChoicesXml=`generateServiceConfigChoiceXml "hcatserver" "HDPHCatalogServerHost"`; optionalServicesConfigChoicesXml="${optionalServicesConfigChoicesXml}${hCatalogConfigChoicesXml}"; fi if [ "xyes" == "x${isTempletonInstalled}" ] then templetonConfigChoicesXml=`generateServiceConfigChoiceXml "templetonnode" "HDPTempletonNodeHost"`; optionalServicesConfigChoicesXml="${optionalServicesConfigChoicesXml}${templetonConfigChoicesXml}"; fi if [ "xyes" == "x${isOozieInstalled}" ] then oozieConfigChoicesXml=`generateServiceConfigChoiceXml "oozieserver" "HDPOozieServerHost"`; optionalServicesConfigChoicesXml="${optionalServicesConfigChoicesXml}${oozieConfigChoicesXml}"; fi echo "${optionalServicesConfigChoicesXml}"; } ############# ### SETUP ### ############# # XXX XXX RESUME FROM HERE ###checkpointedStageNumber=`fetchDbCookieForCluster "${clusterName}" "${CHECKPOINTED_STAGE_NUMBER_COOKIE_NAME}"`; ### ###currentStageName="1-HDP-Initialize-Cluster"; ###currentStageNumber=`getStageNumberFromName "${currentStageName}"`; ### ###currentStageCanRun=`isStageTransitionAdmissable "${checkpointedStageNumber}" "${currentStageNumber}"`; ### ###if [ "${currentStageCanRun}" != "1" ] ###then ### echo "Inadmissable stage transition attempted - bailing out."; ### exit 1; ###fi # Create our per-build workspace - it's a shame Jenkins doesn't provide this for us. perBuildWorkspace=${WORKSPACE}/${BUILD_NUMBER}; cmd="mkdir ${perBuildWorkspace}"; echo "${cmd}"; eval "${cmd}"; ###################### ### PRE-PROCESSING ### ###################### # Do nothing. ###################### ### THE MAIN EVENT ### ###################### # Do nothing. ####################### ### POST-PROCESSING ### ####################### # Store the data we've gotten from the user into our DB. read -r -d '' gsClusterConf << EOConf s!@HDPHadoopHeapSize@!${HDPHadoopHeapSize}!g s!@HDPNameNodeHeapSize@!${HDPNameNodeHeapSize}!g s!@HDPFSInMemorySize@!${HDPFSInMemorySize}!g s!@HDPNameNodeOptNewSize@!${HDPNameNodeOptNewSize}!g s!@HDPDataNodeDuReserved@!${HDPDataNodeDuReserved}!g s!@HDPDataNodeHeapSize@!${HDPDataNodeHeapSize}!g s!@HDPJobTrackerOptNewSize@!${HDPJobTrackerOptNewSize}!g s!@HDPJobTrackerOptMaxNewSize@!${HDPJobTrackerOptMaxNewSize}!g s!@HDPJobTrackerHeapSize@!${HDPJobTrackerHeapSize}!g s!@HDPMapRedMapTasksMax@!${HDPMapRedMapTasksMax}!g s!@HDPMapRedReduceTasksMax@!${HDPMapRedReduceTasksMax}!g s!@HDPMapRedClusterMapMemoryMB@!${HDPMapRedClusterMapMemoryMB}!g s!@HDPMapRedClusterReduceMemoryMB@!${HDPMapRedClusterReduceMemoryMB}!g s!@HDPMapRedClusterMaxMapMemoryMB@!${HDPMapRedClusterMaxMapMemoryMB}!g s!@HDPMapRedClusterMaxReduceMemoryMB@!${HDPMapRedClusterMaxReduceMemoryMB}!g s!@HDPMapRedJobMapMemoryMB@!${HDPMapRedJobMapMemoryMB}!g s!@HDPMapRedJobReduceMemoryMB@!${HDPMapRedJobReduceMemoryMB}!g s!@HDPMapRedChildJavaOptsSize@!${HDPMapRedChildJavaOptsSize}!g s!@HDPIoSortMB@!${HDPIoSortMB}!g s!@HDPIoSortSpillPercent@!${HDPIoSortSpillPercent}!g s!@HDPMapReduceUserLogRetainHours@!${HDPMapReduceUserLogRetainHours}!g s!@HDPMaxTasksPerJob@!${HDPMaxTasksPerJob}!g s!@HDPDFSDataNodeFailedVolumeTolerated@!${HDPDFSDataNodeFailedVolumeTolerated}!g s!@HDPHBaseMasterHeapSize@!${HDPHBaseMasterHeapSize:-1024m}!g s!@HDPHBaseRegionServerHeapSize@!${HDPHBaseRegionServerHeapSize:-1024m}!g EOConf # The escaped quotes around ${gsClusterConf} are important because # we're passing in a multi-line blob as a string, so the command-line # invocation needs to have the blob quoted so storeDbConfForCluster can treat # it as a single string. cmd="storeDbConfForCluster ${clusterName} ${GSCLUSTER_PROPERTIES_CONF_NAME} \"${gsClusterConf}\""; echo "${cmd}"; eval "${cmd}"; # Fetch the next stage's configuration nextJobName="5-HDP-Deploy"; nextJobConfigFile="${perBuildWorkspace}/${nextJobName}.config.xml"; cmd="fetchJobConfigTemplate ${nextJobName} ${nextJobConfigFile}"; echo "${cmd}"; eval "${cmd}"; # Modify this fetched config to take into account the output of all the jobs till now. isHBaseInstalled=`isServiceInstalledForCluster "${clusterName}" "HBase"`; isPigInstalled=`isServiceInstalledForCluster "${clusterName}" "Pig"`; isHCatalogInstalled=`isServiceInstalledForCluster "${clusterName}" "HCatalog"`; isTempletonInstalled=`isServiceInstalledForCluster "${clusterName}" "Templeton"`; isOozieInstalled=`isServiceInstalledForCluster "${clusterName}" "Oozie"`; isSqoopInstalled=`isServiceInstalledForCluster "${clusterName}" "Sqoop"`; nameNodeHost=`getServiceHost "namenode"`; secondaryNameNodeHost=`getServiceHost "snamenode"`; jobTrackerHost=`getServiceHost "jobtracker"`; gangliaCollectorHost=`getServiceHost "gangliaserver"`; nagiosServerHost=`getServiceHost "nagiosserver"`; optionalServicesConfigChoicesXml=`generateOptionalServicesConfigChoicesXml \ "${isHBaseInstalled}" "${isHCatalogInstalled}" "${isTempletonInstalled}" "${isOozieInstalled}"`; cmd="sed -i.prev \ -e \"s!@HDPInstallHBaseChoice@!${isHBaseInstalled}!g\" \ -e \"s!@HDPInstallPigChoice@!${isPigInstalled}!g\" \ -e \"s!@HDPInstallHCatalogChoice@!${isHCatalogInstalled}!g\" \ -e \"s!@HDPInstallTempletonChoice@!${isTempletonInstalled}!g\" \ -e \"s!@HDPInstallOozieChoice@!${isOozieInstalled}!g\" \ -e \"s!@HDPInstallSqoopChoice@!${isSqoopInstalled}!g\" \ -e \"s!@HDPNameNodeHostChoice@!${nameNodeHost}!g\" \ -e \"s!@HDPSecondaryNameNodeHostChoice@!${secondaryNameNodeHost}!g\" \ -e \"s!@HDPJobTrackerHostChoice@!${jobTrackerHost}!g\" \ -e \"s!@HDPGangliaCollectorHostChoice@!${gangliaCollectorHost}!g\" \ -e \"s!@HDPNagiosServerHostChoice@!${nagiosServerHost}!g\" \ -e \"s!@HDPOptionalServicesConfigChoices@!${optionalServicesConfigChoicesXml}!g\" \ ${nextJobConfigFile}"; echo "${cmd}"; eval "${cmd}"; # "Pipe" the output of this job into the next one's configuration. cmd="curl -H \"Content-Type: text/xml\" -d @${nextJobConfigFile} ${JENKINS_URL}job/${nextJobName}/config.xml"; echo "${cmd}"; eval "${cmd}"; # XXX TODO Advance the stage disk cookie.