AMBARI-868. Clean up site.pp generation on the agent and remove the imports in the sample site.pp. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1398674 13f79535-47bb-0310-9956-ffa450edef68

AMBARI-666-CHANGES.txt

@@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes)
 
   NEW FEATURES
 
+  AMBARI-868. Clean up site.pp generation on the agent and remove the imports
+  in the sample site.pp. (mahadev)
+
   AMBARI-862. API query against /clusters doesn't return any data.
   (John Speidel via mahadev)
 

ambari-agent/src/main/python/ambari_agent/manifestGenerator.py

@@ -1,184 +1,184 @@
-#!/usr/bin/env python2.6
-
-'''
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-'''
-
-import json
-import os.path
-import logging
-
-logger = logging.getLogger()
-
-  #read static imports from file and write them to manifest
-def writeImports(outputFile, inputFileName='imports.txt'):
-  inputFile = open(inputFileName, 'r')
-  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
-  logger.info("Modules dir is " + modulesdir)
-  for line in inputFile:
-    modulename = line.rstrip('\n')
-    line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"
-    outputFile.write(line)
-    
-  inputFile.close()
-
-def generateManifest(inputJsonStr):
-#reading json
-  parsedJson = json.loads(inputJsonStr)
-  hostname = parsedJson['hostname']
-  clusterHostInfo = parsedJson['clusterHostInfo']
-  params = parsedJson['params']
-  configurations = parsedJson['configurations']
-  #hostAttributes = parsedJson['hostAttributes']
-  roles = parsedJson['roleCommands']
-  
-#writing manifest
-  manifest = open('site.pp', 'w')
-
-  #writing imports from external static file
-  writeImports(manifest)
-  
-  #writing nodes
-  writeNodes(manifest, clusterHostInfo)
-  
-  #writing params from map
-  writeParams(manifest, params)
-  
-  #writing config maps
-  writeConfigurations(manifest, configurations)
-
-  #writing host attributes
-  #writeHostAttributes(manifest, hostAttributes)
-
-  #writing task definitions 
-  writeTasks(manifest, roles)
-     
-  manifest.close()
-    
-  
-  #read dictionary
-def readDict(file, separator='='):
-  result = dict()
-  
-  for line in file :
-    dictTuple = line.partition(separator)
-    result[dictTuple[0].strip()] = dictTuple[2].strip()
-  
-  return result
-  
-
-  #write nodes
-def writeNodes(outputFile, clusterHostInfo):
-  for node in clusterHostInfo.iterkeys():
-    outputFile.write('$' + node + '= [')
-    coma = ''
-    
-    for value in clusterHostInfo[node]:
-      outputFile.write(coma + '\'' + value + '\'')
-      coma = ', '
-
-    outputFile.write(']\n')
-
-#write params
-def writeParams(outputFile, params):
-  for param in params.iterkeys():
-    outputFile.write('$' +  param + '="' + params[param] + '"\n')
-
-#write host attributes
-def writeHostAttributes(outputFile, hostAttributes):
-  outputFile.write('$hostAttributes={\n')
-
-  coma = ''
-  for attribute in hostAttributes.iterkeys():
-    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')
-    coma = ',\n'
-
-  outputFile.write('}\n')
-
-#write configurations
-def writeConfigurations(outputFile, configs):
-  outputFile.write('$configuration =  {\n')
-
-  for configName in configs.iterkeys():
-    outputFile.write(configName + '=> {\n')
-    config = configs[configName]
-    
-    coma = ''
-    for configParam in config.iterkeys():
-      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
-      coma = ',\n'
-
-    outputFile.write('\n},\n')
-    
-  outputFile.write('\n}\n')
-
-#write node tasks
-def writeTasks(outputFile, roles):
-  #reading dictionaries
-  rolesToClassFile = open('rolesToClass.dict', 'r')
-  rolesToClass = readDict(rolesToClassFile)
-  rolesToClassFile.close()
-
-  serviceStatesFile =  open('serviceStates.dict', 'r')
-  serviceStates = readDict(serviceStatesFile)
-  serviceStatesFile.close()
-
-  outputFile.write('node /default/ {\n ')
-  writeStages(outputFile, len(roles))
-  stageNum = 1
-
-  for role in roles :
-    rolename = role['role']
-    command = role['cmd']
-    taskParams = role['roleParams']
-    taskParamsNormalized = normalizeTaskParams(taskParams)
-    taskParamsPostfix = ''
-    
-    if len(taskParamsNormalized) > 0 :
-      taskParamsPostfix = ', ' + taskParamsNormalized
-    
-    className = rolesToClass[rolename]
-    serviceState = serviceStates[command]
-    
-    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 
-                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
-    stageNum = stageNum + 1
-  outputFile.write('}\n')
-def normalizeTaskParams(taskParams):
-  result = ''
-  coma = ''
-  
-  for paramName in taskParams.iterkeys():
-    result = coma + result + paramName + ' => ' + taskParams[paramName]
-    coma = ','
-    
-  return result
-  
-def writeStages(outputFile, numStages):
-  arrow = ''
-  
-  for i in range(numStages):
-    outputFile.write(arrow + 'stage{' + str(i) + ' :}')
-    arrow = ' -> '
-  
-  outputFile.write('\n')
-    
-logging.basicConfig(level=logging.DEBUG)    
-#test code
-jsonFile = open('test.json', 'r')
-jsonStr = jsonFile.read() 
-generateManifest(jsonStr)
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os.path
+import logging
+
+logger = logging.getLogger()
+
+  #read static imports from file and write them to manifest
+def writeImports(outputFile, inputFileName='imports.txt'):
+  inputFile = open(inputFileName, 'r')
+  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
+  logger.info("Modules dir is " + modulesdir)
+  for line in inputFile:
+    modulename = line.rstrip('\n')
+    line = "import '" + modulesdir + "/" + modulename + "'\n"
+    outputFile.write(line)
+    
+  inputFile.close()
+
+def generateManifest(inputJsonStr):
+#reading json
+  parsedJson = json.loads(inputJsonStr)
+  hostname = parsedJson['hostname']
+  clusterHostInfo = parsedJson['clusterHostInfo']
+  params = parsedJson['params']
+  configurations = parsedJson['configurations']
+  #hostAttributes = parsedJson['hostAttributes']
+  roles = parsedJson['roleCommands']
+  
+#writing manifest
+  manifest = open('site.pp', 'w')
+
+  #writing imports from external static file
+  writeImports(manifest)
+  
+  #writing nodes
+  writeNodes(manifest, clusterHostInfo)
+  
+  #writing params from map
+  writeParams(manifest, params)
+  
+  #writing config maps
+  writeConfigurations(manifest, configurations)
+
+  #writing host attributes
+  #writeHostAttributes(manifest, hostAttributes)
+
+  #writing task definitions 
+  writeTasks(manifest, roles)
+     
+  manifest.close()
+    
+  
+  #read dictionary
+def readDict(file, separator='='):
+  result = dict()
+  
+  for line in file :
+    dictTuple = line.partition(separator)
+    result[dictTuple[0].strip()] = dictTuple[2].strip()
+  
+  return result
+  
+
+  #write nodes
+def writeNodes(outputFile, clusterHostInfo):
+  for node in clusterHostInfo.iterkeys():
+    outputFile.write('$' + node + '= [')
+    coma = ''
+    
+    for value in clusterHostInfo[node]:
+      outputFile.write(coma + '\'' + value + '\'')
+      coma = ', '
+
+    outputFile.write(']\n')
+
+#write params
+def writeParams(outputFile, params):
+  for param in params.iterkeys():
+    outputFile.write('$' +  param + '="' + params[param] + '"\n')
+
+#write host attributes
+def writeHostAttributes(outputFile, hostAttributes):
+  outputFile.write('$hostAttributes={\n')
+
+  coma = ''
+  for attribute in hostAttributes.iterkeys():
+    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] + '"}')
+    coma = ',\n'
+
+  outputFile.write('}\n')
+
+#write configurations
+def writeConfigurations(outputFile, configs):
+  outputFile.write('$configuration =  {\n')
+
+  for configName in configs.iterkeys():
+    outputFile.write(configName + '=> {\n')
+    config = configs[configName]
+    
+    coma = ''
+    for configParam in config.iterkeys():
+      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
+      coma = ',\n'
+
+    outputFile.write('\n},\n')
+    
+  outputFile.write('\n}\n')
+
+#write node tasks
+def writeTasks(outputFile, roles):
+  #reading dictionaries
+  rolesToClassFile = open('rolesToClass.dict', 'r')
+  rolesToClass = readDict(rolesToClassFile)
+  rolesToClassFile.close()
+
+  serviceStatesFile =  open('serviceStates.dict', 'r')
+  serviceStates = readDict(serviceStatesFile)
+  serviceStatesFile.close()
+
+  outputFile.write('node /default/ {\n ')
+  writeStages(outputFile, len(roles))
+  stageNum = 1
+
+  for role in roles :
+    rolename = role['role']
+    command = role['cmd']
+    taskParams = role['roleParams']
+    taskParamsNormalized = normalizeTaskParams(taskParams)
+    taskParamsPostfix = ''
+    
+    if len(taskParamsNormalized) > 0 :
+      taskParamsPostfix = ', ' + taskParamsNormalized
+    
+    className = rolesToClass[rolename]
+    serviceState = serviceStates[command]
+    
+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 
+                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
+    stageNum = stageNum + 1
+  outputFile.write('}\n')
+def normalizeTaskParams(taskParams):
+  result = ''
+  coma = ''
+  
+  for paramName in taskParams.iterkeys():
+    result = coma + result + paramName + ' => ' + taskParams[paramName]
+    coma = ','
+    
+  return result
+  
+def writeStages(outputFile, numStages):
+  arrow = ''
+  
+  for i in range(numStages):
+    outputFile.write(arrow + 'stage{' + str(i) + ' :}')
+    arrow = ' -> '
+  
+  outputFile.write('\n')
+    
+logging.basicConfig(level=logging.DEBUG)    
+#test code
+jsonFile = open('test.json', 'r')
+jsonStr = jsonFile.read() 
+generateManifest(jsonStr)

ambari-agent/src/main/python/ambari_agent/site.pp

@@ -1,27 +1,15 @@
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
-import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
 $NAMENODE= ['h2.hortonworks.com']
 $DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
 $hdfs_user="hdfs"
 $jdk_location="lah/blah"
 $configuration =  {
-$hdfs_site=> {
+hdfs_site=> {
 "dfs.block.size" => "256000000",
 "dfs.replication" => "1"
-}
-$core_site=> {
+},
+core_site=> {
 "fs.default.name" => "hrt8n36.cc1.ygridcore.net"
-}
+},
 
 }
 node /default/ {