AMBARI-831. Move manifest generation into the ambari agent directory. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1395859 13f79535-47bb-0310-9956-ffa450edef68

AMBARI-666-CHANGES.txt

@@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes)
 
   NEW FEATURES
 
+  AMBARI-831. Move manifest generation into the ambari agent directory.
+  (mahadev)
+
   AMBARI-828. Manifest generation for various actions from the server.
   (mahadev)
 

ambari-agent/src/main/python/ambari_agent/imports.txt

@@ -0,0 +1,12 @@
+hdp/manifests/*.pp
+hdp-hadoop/manifests/*.pp
+hdp-hbase/manifests/*.pp
+hdp-zookeeper/manifests/*.pp
+hdp-oozie/manifests/*.pp
+hdp-pig/manifests/*.pp
+hdp-sqoop/manifests/*.pp
+hdp-templeton/manifests/*.pp
+hdp-hive/manifests/*.pp
+hdp-hcat/manifests/*.pp
+hdp-mysql/manifests/*.pp
+hdp-monitor-webserver/manifests/*.pp

ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py → ambari-agent/src/main/python/ambari_agent/manifestGenerator.py

@@ -1,10 +1,37 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
 import json
+import os.path
+import logging
+
+logger = logging.getLogger()
 
   #read static imports from file and write them to manifest
 def writeImports(outputFile, inputFileName='imports.txt'):
   inputFile = open(inputFileName, 'r')
-  
+  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
+  logger.info("Modules dir is " + modulesdir)
   for line in inputFile:
+    modulename = line.rstrip('\n')
+    line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"
     outputFile.write(line)
     
   inputFile.close()
@@ -17,7 +44,7 @@ def generateManifest(inputJsonStr):
   params = parsedJson['params']
   configurations = parsedJson['configurations']
   #hostAttributes = parsedJson['hostAttributes']
-  roles = parsedJson['roles']
+  roles = parsedJson['roleCommands']
   
 #writing manifest
   manifest = open('site.pp', 'w')
@@ -60,8 +87,8 @@ def writeNodes(outputFile, clusterHostInfo):
     outputFile.write('$' + node + '= [')
     coma = ''
     
-    for host in node:
-      outputFile.write(coma + '\'' + host + '\'')
+    for value in clusterHostInfo[node]:
+      outputFile.write(coma + '\'' + value + '\'')
       coma = ', '
 
     outputFile.write(']\n')
@@ -87,7 +114,7 @@ def writeConfigurations(outputFile, configs):
   outputFile.write('$configuration =  {\n')
 
   for configName in configs.iterkeys():
-    outputFile.write('$' + configName + '=> {\n')
+    outputFile.write(configName + '=> {\n')
     config = configs[configName]
     
     coma = ''
@@ -95,7 +122,7 @@ def writeConfigurations(outputFile, configs):
       outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
       coma = ',\n'
 
-    outputFile.write('\n}\n')
+    outputFile.write('\n},\n')
     
   outputFile.write('\n}\n')
 
@@ -110,14 +137,14 @@ def writeTasks(outputFile, roles):
   serviceStates = readDict(serviceStatesFile)
   serviceStatesFile.close()
 
-  outputFile.write('node /<toBeDefined>/ {\n ')
+  outputFile.write('node /default/ {\n ')
   writeStages(outputFile, len(roles))
   stageNum = 1
 
   for role in roles :
     rolename = role['role']
-    command = role['roleCommand']
-    taskParams = role['params']
+    command = role['cmd']
+    taskParams = role['roleParams']
     taskParamsNormalized = normalizeTaskParams(taskParams)
     taskParamsPostfix = ''
     
@@ -127,9 +154,10 @@ def writeTasks(outputFile, roles):
     className = rolesToClass[rolename]
     serviceState = serviceStates[command]
     
-    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) + 
+                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
     stageNum = stageNum + 1
-    
+  outputFile.write('}\n')
 def normalizeTaskParams(taskParams):
   result = ''
   coma = ''
@@ -149,9 +177,8 @@ def writeStages(outputFile, numStages):
   
   outputFile.write('\n')
     
-    
+logging.basicConfig(level=logging.DEBUG)    
 #test code
 jsonFile = open('test.json', 'r')
 jsonStr = jsonFile.read() 
-print '>>>JSON STRING: ' + jsonStr
 generateManifest(jsonStr)

ambari-agent/src/main/python/ambari_agent/site.pp

@@ -0,0 +1,31 @@
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
+$NAMENODE= ['h2.hortonworks.com']
+$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
+$hdfs_user="hdfs"
+$jdk_location="lah/blah"
+$configuration =  {
+$hdfs_site=> {
+"dfs.block.size" => "256000000",
+"dfs.replication" => "1"
+}
+$core_site=> {
+"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+}
+
+}
+node /default/ {
+ stage{0 :} -> stage{1 :}
+class {'hdp-hadoop::datanode': stage => 1, service_state => running}
+class {'hdp-hadoop::namenode': stage => 2, service_state => installed_and_configured}
+}

ambari-agent/src/main/python/manifestGenerator/test.json → ambari-agent/src/main/python/ambari_agent/test.json

@@ -16,18 +16,17 @@
 "hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
 "core_site" :  { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" }
 },
-"roles": [
+"roleCommands": [
 {
 "role" : "DATANODE", 
-"roleCommand": "START",
-"params" : {
-"myid" : "23"
+"cmd": "START",
+"roleParams" : {
 }
 },
 {
 "role": "NAMENODE",
-"roleCommand": "INSTALL",
-"params" : {}
+"cmd": "INSTALL",
+"roleParams" : {}
 }
 ]
 }

ambari-agent/src/main/python/manifestGenerator/imports.txt

@@ -1,12 +0,0 @@
-import "/etc/puppet/agent/modules/hdp/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hadoop/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hbase/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-zookeeper/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-oozie/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-pig/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-sqoop/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-templeton/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hive/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-hcat/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-mysql/manifests/*.pp"
-import "/etc/puppet/agent/modules/hdp-monitor-webserver/manifests/*.pp"