AMBARI-905. Fix puppet site creation with flattening of execution commands send from the server. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/branches/AMBARI-666@1401568 13f79535-47bb-0310-9956-ffa450edef68

AMBARI-666-CHANGES.txt

@@ -352,6 +352,9 @@ AMBARI-666 branch (unreleased changes)
 
   BUG FIXES
 
+  AMBARI-905. Fix puppet site creation with flattening of execution commands
+  send from the server. (mahadev)
+
   AMBARI-885. Fix miscellaneous issues related to Ambari Web. (yusaku)
 
   AMBARI-879. Installer skips Confirm Hosts page of the wizard when testing

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -119,6 +119,10 @@ class ActionQueue(threading.Thread):
     configurations = command['configurations']
     result = []
     commandresult = self.executor.runCommand(command)
+    status = "COMPLETED"
+    if (commandresult['exitcode'] != 0):
+      status = "FAILED"
+      
     # assume some puppet pluing to run these commands
     roleResult = {'role' : command['role'],
                   'actionId' : commandId,
@@ -127,7 +131,7 @@ class ActionQueue(threading.Thread):
                   'stderr' : commandresult['stderr'],
                   'exitCode' : commandresult['exitcode'],
                   'serviceName' : serviceName,
-                  'status' : "COMPLETED"}
+                  'status' : status}
     result.append(roleResult)
     pass
     return result

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -31,7 +31,7 @@ url=https://localhost:4080
 secured_url=https://localhost:8443
 
 [agent]
-prefix=/tmp/ambari
+prefix=/tmp/ambari-agent
 
 [stack]
 installprefix=/var/ambari/

ambari-agent/src/main/python/ambari_agent/manifestGenerator.py

@@ -24,6 +24,13 @@ import logging
 
 logger = logging.getLogger()
 
+xml_configurations_keys= ["hdfs_site", "hdfs_site", "core_site", 
+                          "mapred_queue_acls",
+                             "hadoop_policy", "mapred_site", 
+                             "capacity_scheduler", "hbase_site",
+                             "hbase_policy", "hive_site", "oozie_site", 
+                             "templeton_site"]
+
 #read static imports from file and write them to manifest
 def writeImports(outputFile, modulesdir, inputFileName='imports.txt'):
   inputFile = open(inputFileName, 'r')
@@ -39,12 +46,13 @@ def generateManifest(parsedJson, fileName, modulesdir):
 #reading json
   hostname = parsedJson['hostname']
   clusterHostInfo = parsedJson['clusterHostInfo']
-  params = parsedJson['params']
+  params = parsedJson['hostLevelParams']
   configurations = parsedJson['configurations']
-  xmlConfigurationsKeys = parsedJson['xml_configurations_keys']
+  xmlConfigurationsKeys = xml_configurations_keys
   #hostAttributes = parsedJson['hostAttributes']
-  roles = parsedJson['roleCommands']
-  
+  roles = [{'role' : parsedJson['role'],
+            'cmd' : parsedJson['roleCommand'],
+            'roleParams' : parsedJson['roleParams']}]
   #writing manifest
   manifest = open(fileName, 'w')
 

ambari-agent/src/main/python/ambari_agent/puppetExecutor.py

@@ -59,12 +59,12 @@ class puppetExecutor:
       
   def runCommand(self, command):
     result = {}
-    taskId = "0";
+    taskId = 0;
     if command.has_key("taskId"):
       taskId = command['taskId']
       
     puppetEnv = os.environ
-    siteppFileName = os.path.join(self.tmpDir, "site-" + taskId + ".pp") 
+    siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp") 
     generateManifest(command, siteppFileName, self.puppetModule + "/modules")
     puppetcommand = self.puppetCommand(siteppFileName)
     """ Run the command and make sure the output gets propagated"""

ambari-agent/src/main/python/ambari_agent/site.pp

@@ -1,15 +1,15 @@
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
-import '/home/centos/ambari/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-zookeeper/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-oozie/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-pig/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-sqoop/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-templeton/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hive/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-hcat/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-mysql/manifests/*.pp'
+import '/media/sf_/home/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/modules/hdp-monitor-webserver/manifests/*.pp'
 $NAMENODE= ['h2.hortonworks.com']
 $DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
 $hdfs_user="hdfs"
@@ -66,13 +66,6 @@ $task_bin_exe = "ls"
 $hadoop_piddirprefix = "/tmp"
 $ganglia_server_host = "localhost"
 node /default/ {
- stage{1 :} -> stage{2 :} -> stage{3 :} -> stage{4 :} -> stage{5 :} -> stage{6 :} -> stage{7 :} -> stage{8 :}
+ stage{1 :}
 class {'hdp-hadoop::namenode': stage => 1, service_state => installed_and_configured}
-class {'hdp-hadoop::datanode': stage => 2, service_state => installed_and_configured}
-class {'hdp-hbase::master': stage => 3, service_state => installed_and_configured}
-class {'hdp-hive::server': stage => 4, service_state => installed_and_configured}
-class {'hdp-hive::client': stage => 5, service_state => installed_and_configured}
-class {'hdp-oozie::server': stage => 6, service_state => installed_and_configured}
-class {'hdp-templeton::server': stage => 7, service_state => installed_and_configured}
-class {'hdp-templeton::client': stage => 8, service_state => installed_and_configured}
 }

ambari-agent/src/main/python/ambari_agent/test.json

@@ -7,7 +7,7 @@
 "NAMENODE": ["h2.hortonworks.com"],
 "DATANODE": ["h1.hortonworks.com", "h2.hortonworks.com"]
 },
-"params": 
+"hostLevelParams": 
 {
 "hdfs_user" : "hdfs",
 "jdk_location" : "http://hdp1/downloads",
@@ -17,9 +17,6 @@
 "jdk_bins" :  { "32" : "jdk-6u31-linux-x64.bin", "64" : "jdk-6u31-linux-x64.bin" }
 },
 
-"xml_configurations_keys" : ["hdfs_site", "hdfs_site", "core_site", "mapred_queue_acls", 
-                             "hadoop_policy", "mapred_site", "capacity_scheduler", "hbase_site",
-                             "hbase_policy", "hive_site", "oozie_site", "templeton_site"],
 
 "configurations" : {
 "hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
@@ -43,46 +40,8 @@
 "oozie_site" : {"oozie.service.ActionService.executor.ext.classes" : "org.apache.oozie.action.hadoop.HiveActionExecutor, org.apache.oozie.action.hadoop.SqoopActionExecutor,org.apache.oozie.action.email.EmailActionExecutor,"},
 "templeton_site" : {"templeton.override.enabled" : "true"}
 },
-"roleCommands": [
-{
-  "role": "NAMENODE",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "DATANODE",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "HBASE_MASTER",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "HIVE_SERVER",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "HIVE_CLIENT",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "OOZIE_SERVER",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "TEMPLETON_SERVER",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-},
-{
-  "role": "TEMPLETON_CLIENT",
-  "cmd": "INSTALL",
-  "roleParams" : {}
-}
-]
+
+"role": "NAMENODE",
+"roleCommand": "INSTALL",
+"roleParams" : {}
 }

ambari-server/src/main/java/org/apache/ambari/server/agent/CommandReport.java

@@ -17,8 +17,11 @@
  */
 package org.apache.ambari.server.agent;
 
+import org.codehaus.jackson.annotate.JsonProperty;
+
 
 public class CommandReport {
+
   String role;
   String actionId;
   String stdout;
@@ -27,67 +30,94 @@ public class CommandReport {
   int exitCode;
   private String clusterName;
   private String serviceName;
+  private long taskId;
+  
+  @JsonProperty("taskId")
+  public long getTaskId() {
+    return taskId;
+  }
   
+  @JsonProperty("taskId")
+  public void setTaskId(long taskId) {
+    this.taskId = taskId;
+  }
+  
+  @JsonProperty("clusterName")
   public void setClusterName(String clusterName) {
     this.clusterName = clusterName;
   }
   
+  @JsonProperty("clusterName") 
   public String getClusterName() {
     return this.clusterName;
   }
-  
+
+  @JsonProperty("actionId")
   public String getActionId() {
     return this.actionId;
   }
-
+  
+  @JsonProperty("actionId")
   public void setActionId(String actionId) {
     this.actionId = actionId;
   }
-
+  
+  @JsonProperty("stderr")
   public String getStdErr() {
     return this.stderr;
   }
-
+  
+  @JsonProperty("stderr")
   public void setStdErr(String stderr) {
     this.stderr = stderr;
   }
-
+  
+  @JsonProperty("exitcode")
   public int getExitCode() {
     return this.exitCode;
   }
-
+  
+  @JsonProperty("exitcode")
   public void setExitCode(int exitCode) {
     this.exitCode = exitCode;
   }
-
+  
+  @JsonProperty("stdout")
   public String getStdOut() {
     return this.stdout;
   }
-
+  
+  @JsonProperty("stdout")
   public void setStdOut(String stdout) {
     this.stdout = stdout;
   }
 
+  @JsonProperty("role")
   public String getRole() {
     return role;
   }
-
+  
+  @JsonProperty("role")
   public void setRole(String role) {
     this.role = role;
   }
-
+  
+  @JsonProperty("status")
   public String getStatus() {
     return status;
   }
-
+  
+  @JsonProperty("status")
   public void setStatus(String status) {
     this.status = status;
   }
-
+  
+  @JsonProperty("serviceName")
   public String getServiceName() {
     return serviceName;
   }
-
+  
+  @JsonProperty("serviceName")
   public void setServiceName(String serviceName) {
     this.serviceName = serviceName;
   }

ambari-server/src/main/java/org/apache/ambari/server/state/ServiceImpl.java

@@ -328,9 +328,9 @@ public class ServiceImpl implements Service {
         + ", clusterId=" + cluster.getClusterId()
         + ", desiredStackVersion=" + getDesiredStackVersion()
         + ", desiredState=" + getDesiredState().toString()
-        + ", configs = " + getConfigVersions()
+        + ", configs = " + getConfigVersions() 
         + ", components=[ ");
-
+    
     boolean first = true;
     for(ServiceComponent sc : components.values()) {
       if (!first) {