AMBARI-1233. Directory permissions on httpd /var/www/cgi-bin should not be touched by Ambari. (mahadev)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1437734 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -166,6 +166,13 @@ Trunk (unreleased changes):
  AMBARI-1243. Remove unwanted import causing the builds to fail on linux.
  (mahadev)
 
+ AMBARI-1233.  Directory permissions on httpd /var/www/cgi-bin should not be
+ touched by Ambari. (mahadev)
+
+ AMBARI-1170. For live status checks we should only look at the run
+ directories that we get from the server (only for hadoop and its eco system)
+ and not all. (mahadev)
+
 AMBARI-1.2.0 branch:
 
  INCOMPATIBLE CHANGES

ambari-agent/src/main/puppet/modules/hdp-ganglia/manifests/server.pp

@@ -130,7 +130,8 @@ class hdp-ganglia::server::files(
 
   $rrd_py_path = $hdp::params::rrd_py_path [$hdp::params::hdp_os_type]
   hdp::directory_recursive_create{$rrd_py_path:
-    ensure => "directory"  
+    ensure => "directory", 
+    override_owner => false 
   }
 
   $rrd_py_file_path = "${rrd_py_path}/rrd.py"

ambari-agent/src/main/puppet/modules/hdp-nagios/files/check_name_dir_status.php

@@ -36,8 +36,8 @@
   $json_array = json_decode($json_string, true);
   $object = $json_array['beans'][0];
   if ($object['NameDirStatuses'] == "") {
-    echo "UNKNOWN: Namenode directory status not available via http://<nn_host>:port/jmx url" . "\n";
-    exit(3);
+    echo "WARNING: Namenode directory status not available via http://".$host.":".$port."/jmx url" . "\n";
+    exit(1);
   }
   $NameDirStatuses = json_decode($object['NameDirStatuses'], true);
   $failed_dir_count = count($NameDirStatuses['failed']);

ambari-agent/src/main/puppet/modules/hdp/manifests/init.pp

@@ -183,7 +183,8 @@ define hdp::directory(
   $mode  = undef,
   $ensure = directory,
   $force = undef,
-  $service_state = 'running'
+  $service_state = 'running',
+  $override_owner = false
   )
 {
  if (($service_state == 'uninstalled') and ($wipeoff_data == true)) {
@@ -195,13 +196,21 @@ define hdp::directory(
     force  => $force
    }
   } elsif ($service_state != 'uninstalled') {
-  file { $name :
-    ensure => present,
-    owner  => $owner,
-    group  => $group,
-    mode   => $mode,
-    force  => $force
-   }
+    if $override_owner == true {
+      file { $name :
+      ensure => present,
+      owner  => $owner,
+      group  => $group,
+      mode   => $mode,
+      force  => $force
+     }
+    } else {
+      file { $name :
+      ensure => present,
+      mode   => $mode,
+      force  => $force
+     }
+    }
   }
 }
 #TODO: check on -R flag and use of recurse
@@ -212,7 +221,8 @@ define hdp::directory_recursive_create(
   $context_tag = undef,
   $ensure = directory,
   $force = undef,
-  $service_state = 'running'
+  $service_state = 'running',
+  $override_owner = true
   )
 {
 
@@ -227,7 +237,8 @@ define hdp::directory_recursive_create(
     mode  => $mode,
     ensure => $ensure,
     force => $force,
-    service_state => $service_state
+    service_state => $service_state,
+    override_owner => $override_owner
   }
   Hdp::Exec["mkdir -p ${name}"] -> Hdp::Directory[$name]
 }

ambari-agent/src/main/python/ambari_agent/ActionQueue.py

@@ -109,8 +109,9 @@ class ActionQueue(threading.Thread):
           cluster = command['clusterName']
           service = command['serviceName']
           component = command['componentName']
+          globalConfig = command['configurations']['global']
           try:
-            livestatus = LiveStatus(cluster, service, component)
+            livestatus = LiveStatus(cluster, service, component, globalConfig)
             result = livestatus.build()
             logger.info("Got live status for component " + component + " of service " + str(service) +\
                         " of cluster " + str(cluster) + "\n" + pprint.pformat(result))

ambari-agent/src/main/python/ambari_agent/AmbariConfig.py

@@ -35,7 +35,6 @@ secured_url_port=8441
 prefix=/tmp/ambari-agent
 
 [services]
-pidLookupPath=/var/run/
 
 [stack]
 installprefix=/tmp
@@ -148,6 +147,28 @@ servicesToPidNames = {
   'MYSQL_SERVER': 'mysqld.pid'
 }
 
+pidPathesVars = [
+  {'var' : 'hadoop_pid_dir_prefix',
+   'defaultValue' : '/var/run/hadoop'},
+  {'var' : 'hadoop_pid_dir_prefix',
+   'defaultValue' : '/var/run/hadoop'},                 
+  {'var' : 'ganglia_runtime_dir',
+   'defaultValue' : '/var/run/ganglia/hdp'},                 
+  {'var' : 'hbase_pid_dir',
+   'defaultValue' : '/var/run/hbase'},                
+  {'var' : '',
+   'defaultValue' : '/var/run/nagios'},                    
+  {'var' : 'zk_pid_dir',
+   'defaultValue' : '/var/run/zookeeper'},             
+  {'var' : 'oozie_pid_dir',
+   'defaultValue' : '/var/run/oozie'},             
+  {'var' : 'hcat_pid_dir',
+   'defaultValue' : '/var/run/webhcat'},                       
+  {'var' : 'hive_pid_dir',
+   'defaultValue' : '/var/run/hive'},                      
+   {'var' : 'mysqld_pid_dir',
+   'defaultValue' : '/var/run/mysqld'}
+]
 
 class AmbariConfig:
   def getConfig(self):
@@ -170,6 +191,10 @@ class AmbariConfig:
     global servicesToPidNames
     return servicesToPidNames
 
+  def getPidPathesVars(self):
+    global pidPathesVars
+    return pidPathesVars
+
 
 def setConfig(customConfig):
   global config

ambari-agent/src/main/python/ambari_agent/LiveStatus.py

@@ -91,10 +91,11 @@ class LiveStatus:
   LIVE_STATUS = "STARTED"
   DEAD_STATUS = "INSTALLED"
 
-  def __init__(self, cluster, service, component):
+  def __init__(self, cluster, service, component, globalConfig):
     self.cluster = cluster
     self.service = service
     self.component = component
+    self.globalConfig = globalConfig
 
 
   def belongsToService(self, component):
@@ -104,9 +105,7 @@ class LiveStatus:
   # Live status was stripped from heartbeat after revision e1718dd
   def build(self):
     global SERVICES, COMPONENTS, LIVE_STATUS, DEAD_STATUS
-    pidLookupPath = AmbariConfig.config.get('services','pidLookupPath')
-
-    statusCheck = StatusCheck(pidLookupPath, AmbariConfig.servicesToPidNames)
+    statusCheck = StatusCheck(AmbariConfig.servicesToPidNames, AmbariConfig.pidPathesVars, self.globalConfig)
     livestatus = None
     for component in self.COMPONENTS:
       if component["serviceName"] == self.service and component["componentName"] == self.component:

ambari-agent/src/main/python/ambari_agent/StatusCheck.py

@@ -28,15 +28,9 @@ import re
 logger = logging.getLogger()
 
 
-
-
-
 class StatusCheck:
-
-  def get_pair(self, line):
-    key, sep, value = line.strip().partition("=")
-    return key, value
-
+    
+    
   def listFiles(self, dir):
     basedir = dir
     logger.debug("Files in " + os.path.abspath(dir) + ": ")
@@ -45,9 +39,9 @@ class StatusCheck:
       if os.path.isdir(dir):
         for item in os.listdir(dir):
             if os.path.isfile(item) and item.endswith('.pid'):
-              self.pidFilesDict[item.split(os.sep).pop()] = item
+              self.pidFilesDict[item.split(os.sep).pop()] = os.getcwd() + os.sep + item
             else:
-                subdirlist.append(os.path.join(basedir, item))
+              subdirlist.append(os.path.join(basedir, item))
         for subdir in subdirlist:
             self.listFiles(subdir)
       else:
@@ -55,19 +49,35 @@ class StatusCheck:
           self.pidFilesDict[dir.split(os.sep).pop()] = dir
     except OSError as e:
       logger.info(e.strerror + ' to ' + e.filename)
+      
+  def fillDirValues(self):
+    try:
+      for pidVar in self.pidPathesVars:
+        pidVarName = pidVar['var']
+        pidDefaultvalue = pidVar['defaultValue']
+        if self.globalConfig.has_key(pidVarName):
+          self.pidPathes.append(self.globalConfig[pidVarName])
+        else:
+          self.pidPathes.append(pidDefaultvalue)
+    except Exception as e:
+        logger.error("Error while filling directories values " + str(e))
+        
+  def __init__(self, serviceToPidDict, pidPathesVars, globalConfig):
 
-  def __init__(self, path, serviceToPidDict):
-    if not os.path.isdir(path):
-      raise ValueError("Path argument must be valid directory")
-
-    self.path = path
     self.serToPidDict = serviceToPidDict
+    self.pidPathesVars = pidPathesVars
+    self.pidPathes = []
     self.sh = shellRunner()
     self.pidFilesDict = {}
-    self.listFiles(self.path)
-
+    self.globalConfig = globalConfig
+    
+    self.fillDirValues()
+    
+    for pidPath in self.pidPathes:
+      self.listFiles(pidPath)
 
   def getIsLive(self, pidPath):
+
     if not pidPath:
       return False
 
@@ -92,7 +102,7 @@ class StatusCheck:
     try:
       pidPath = None
       pidPattern = self.serToPidDict[serviceCode]
-      logger.info( 'pidPattern: ' + pidPattern)
+      logger.info('pidPattern: ' + pidPattern)
     except KeyError as e:
       logger.warn('There is no mapping for ' + serviceCode)
       return None

ambari-agent/src/test/python/TestActionQueue.py

@@ -63,7 +63,8 @@ class TestActionQueue(TestCase):
       'clusterHostInfo': "clusterHostInfo",
       'roleCommand': "roleCommand",
       'configurations': "configurations",
-      'commandType': "EXECUTION_COMMAND"
+      'commandType': "EXECUTION_COMMAND",
+      'configurations':{'global' : {}}
     }
     actionQueue.put(command)
 
@@ -121,4 +122,4 @@ class FakeExecutor():
       "exitcode": 0,
       "stdout": "returned stdout",
       "stderr": "returned stderr"
-    }
+    }

ambari-agent/src/test/python/TestHeartbeat.py

@@ -29,7 +29,6 @@ import time
 
 class TestHeartbeat(TestCase):
 
-
   def test_build(self):
     actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
     heartbeat = Heartbeat(actionQueue)
@@ -55,7 +54,8 @@ class TestHeartbeat(TestCase):
       "serviceName" : 'HDFS',
       "commandType" : "STATUS_COMMAND",
       "clusterName" : "",
-      "componentName" : "DATANODE"
+      "componentName" : "DATANODE",
+      'configurations':{'global' : {}}
     }
     actionQueue.put(statusCommand)
     actionQueue.start()
@@ -76,7 +76,8 @@ class TestHeartbeat(TestCase):
         "serviceName" : 'HDFS',
         "commandType" : "STATUS_COMMAND",
         "clusterName" : "",
-        "componentName" : "DATANODE"
+        "componentName" : "DATANODE",
+        'configurations':{'global' : {}}
       }
       actionQueue.put(statusCommand)
       time.sleep(0.1)
@@ -102,7 +103,8 @@ class TestHeartbeat(TestCase):
       'stderr' : 'none',
       'exitCode' : 777,
       'serviceName' : "serviceName",
-      'status' : 'IN_PROGRESS'
+      'status' : 'IN_PROGRESS',
+      'configurations':{'global' : {}}
     }
     heartbeat = Heartbeat(actionQueue)
     result = heartbeat.build(100)

ambari-agent/src/test/python/TestLiveStatus.py

@@ -27,7 +27,7 @@ import os
 class TestLiveStatus(TestCase):
   def test_build(self):
     for component in LiveStatus.COMPONENTS:
-      livestatus = LiveStatus('', component['serviceName'], component['componentName'])
+      livestatus = LiveStatus('', component['serviceName'], component['componentName'], {})
       result = livestatus.build()
       print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
       self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')

ambari-agent/src/test/python/TestStatusCheck.py

@@ -48,8 +48,15 @@ class TestStatusCheck(TestCase):
   def setUp(self):
 
     self.tmpdir = tempfile.mkdtemp()
-    self.tmpdict = { COMPONENT_LIVE : COMPONENT_LIVE_PID,
-                     COMPONENT_DEAD : COMPONENT_DEAD_PID}
+    self.serviceToPidDict = {
+      COMPONENT_LIVE : COMPONENT_LIVE_PID,
+      COMPONENT_DEAD : COMPONENT_DEAD_PID
+    }
+
+    self.pidPathesVars = [
+      {'var' : '',
+      'defaultValue' : self.tmpdir}
+    ]
 
     self.sh = shellRunner()
     
@@ -70,11 +77,7 @@ class TestStatusCheck(TestCase):
     dead_pid_file.close()
 
     #Init status checker
-    self.statusCheck = StatusCheck(self.tmpdir, self.tmpdict)
-
-  # Ensure that status checker throws exceptions on invalid params
-  def test_exceptions(self):
-    self.assertRaises(ValueError,StatusCheck,"tmp","tmp")
+    self.statusCheck = StatusCheck(self.serviceToPidDict,self.pidPathesVars,{})
 
   # Ensure that status checker return True for running process
   def test_live(self):

ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java

@@ -18,7 +18,10 @@
 package org.apache.ambari.server.agent;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 import org.apache.ambari.server.AmbariException;
 import org.apache.ambari.server.actionmanager.ActionManager;
@@ -145,10 +148,23 @@ public class HeartbeatMonitor implements Runnable {
           LOG.debug("Live status will include status of service " + serviceName +
                 " of cluster " + cl.getClusterName());
         }
+        
+        Map<String, Config> configs = sch.getDesiredConfigs();
+        
+        Map<String, Map<String, String>> configurations =
+            new TreeMap<String, Map<String, String>>();
+        
+        for (Config config : configs.values()) {
+          if (config.getType().equals("global"))
+            configurations.put(config.getType(),
+              config.getProperties());
+        }
+        
         StatusCommand statusCmd = new StatusCommand();
         statusCmd.setClusterName(cl.getClusterName());
         statusCmd.setServiceName(serviceName);
         statusCmd.setComponentName(sch.getServiceComponentName());
+        statusCmd.setConfigurations(configurations);			
         cmds.add(statusCmd);
       }
     }

ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java

@@ -18,6 +18,7 @@
 package org.apache.ambari.server.agent;
 
 import java.util.List;
+import java.util.Map;
 
 import org.codehaus.jackson.annotate.JsonProperty;
 
@@ -33,6 +34,7 @@ public class StatusCommand extends AgentCommand {
   private String clusterName;
   private String serviceName;
   private String componentName;
+  private Map<String, Map<String, String>> configurations;
 
   @JsonProperty("clusterName")
   public String getClusterName() {
@@ -63,4 +65,14 @@ public class StatusCommand extends AgentCommand {
   public void setComponentName(String componentName) {
     this.componentName = componentName;
   }
+  
+  @JsonProperty("configurations")
+  public Map<String, Map<String, String>> getConfigurations() {
+    return configurations;
+  }
+
+  @JsonProperty("configurations")
+  public void setConfigurations(Map<String, Map<String, String>> configurations) {
+    this.configurations = configurations;
+  }
 }