AMBARI-5358. Make sure a disk goes R/O for datanodes the datanodes are still started. (aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_datanode.py

@@ -33,12 +33,16 @@ def datanode(action=None):
     for data_dir in params.dfs_data_dir.split(","):
       Directory(os.path.dirname(data_dir),
                 recursive=True,
-                mode=0755)
+                mode=0755,
+                ignore_failures=True
+      )
       Directory(data_dir,
                 recursive=False,
                 mode=0750,
                 owner=params.hdfs_user,
-                group=params.user_group)
+                group=params.user_group,
+                ignore_failures=True
+      )
 
   elif action == "start" or action == "stop":
     service(

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py

@@ -64,7 +64,8 @@ def mapreduce(name=None):
   Directory(params.mapred_local_dir.split(','),
             owner=params.mapred_user,
             mode=0755,
-            recursive=True
+            recursive=True,
+            ignore_failures=True
   )
 
   File(params.exclude_file_path,

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_datanode.py

@@ -34,7 +34,9 @@ def datanode(action=None):
                 recursive=True,
                 mode=0755,
                 owner=params.hdfs_user,
-                group=params.user_group)
+                group=params.user_group,
+                ignore_failures=True
+      )
 
   elif action == "start" or action == "stop":
     service(

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py

@@ -71,17 +71,10 @@ def yarn(name = None):
             group=params.user_group,
             recursive=True
   )
-  Directory(params.nm_local_dirs.split(','),
+  Directory(params.nm_local_dirs.split(',')+params.nm_log_dirs.split(',')+[params.yarn_log_dir_prefix],
             owner=params.yarn_user,
-            recursive=True
-  )
-  Directory(params.nm_log_dirs.split(','),
-            owner=params.yarn_user,
-            recursive=True
-  )
-  Directory(params.yarn_log_dir_prefix,
-            owner=params.yarn_user,
-            recursive=True
+            recursive=True,
+            ignore_failures=True,
   )
 
   XmlConfig("core-site.xml",

ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py

@@ -140,12 +140,14 @@ class TestDatanode(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/hdfs',
                               mode = 0755,
                               recursive = True,
+                              ignore_failures=True,
                               )
     self.assertResourceCalled('Directory', '/hadoop/hdfs/data',
                               owner = 'hdfs',
                               group = 'hadoop',
                               mode = 0750,
                               recursive = False,
+                              ignore_failures=True,
                               )
 
   def assert_configure_secured(self):
@@ -158,10 +160,12 @@ class TestDatanode(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/hdfs',
                               mode = 0755,
                               recursive = True,
+                              ignore_failures=True,
                               )
     self.assertResourceCalled('Directory', '/hadoop/hdfs/data',
                               owner = 'hdfs',
                               group = 'hadoop',
                               mode = 0750,
                               recursive = False,
+                              ignore_failures=True,
                               )

ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py

@@ -42,11 +42,13 @@ class TestMapreduceClient(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('Directory', '/hadoop/mapred1',
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',
@@ -79,6 +81,7 @@ class TestMapreduceClient(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',

ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_historyserver.py

@@ -178,11 +178,13 @@ class TestHistoryServer(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('Directory', '/hadoop/mapred1',
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',
@@ -265,6 +267,7 @@ class TestHistoryServer(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',

ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py

@@ -232,11 +232,13 @@ class TestJobtracker(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('Directory', '/hadoop/mapred1',
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',
@@ -319,6 +321,7 @@ class TestJobtracker(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',

ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_tasktracker.py

@@ -121,11 +121,13 @@ class TestTasktracker(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('Directory', '/hadoop/mapred1',
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',
@@ -151,6 +153,7 @@ class TestTasktracker(RMFTestCase):
       owner = 'mapred',
       recursive = True,
       mode = 0755,
+      ignore_failures=True,
     )
     self.assertResourceCalled('File', '/etc/hadoop/conf/mapred.exclude',
       owner = 'mapred',

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_datanode.py

@@ -163,6 +163,7 @@ class TestDatanode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              ignore_failures=True,
                               )
 
   def assert_configure_secured(self):
@@ -177,4 +178,5 @@ class TestDatanode(RMFTestCase):
                               group = 'hadoop',
                               mode = 0755,
                               recursive = True,
+                              ignore_failures=True,
                               )

ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py

@@ -193,22 +193,27 @@ class TestHistoryServer(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -341,14 +346,17 @@ class TestHistoryServer(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py

@@ -52,22 +52,27 @@ class TestMapReduce2Client(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -147,14 +152,17 @@ class TestMapReduce2Client(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py

@@ -193,22 +193,27 @@ class TestNodeManager(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -341,14 +346,17 @@ class TestNodeManager(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py

@@ -133,22 +133,27 @@ class TestResourceManager(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -226,14 +231,17 @@ class TestResourceManager(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py

@@ -53,22 +53,27 @@ class TestYarnClient(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -148,14 +153,17 @@ class TestYarnClient(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',
@@ -245,22 +253,27 @@ class TestYarnClient(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
       owner = 'yarn',
       recursive = True,
+      ignore_failures = True,
     )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
       owner = 'hdfs',

ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py

@@ -87,22 +87,27 @@ class TestAppTimelineServer(RMFTestCase):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
                               owner = 'yarn',
                               recursive = True,
+                              ignore_failures = True,
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/local1',
                               owner = 'yarn',
                               recursive = True,
+                              ignore_failures = True,
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log',
                               owner = 'yarn',
                               recursive = True,
+                              ignore_failures = True,
                               )
     self.assertResourceCalled('Directory', '/hadoop/yarn/log1',
                               owner = 'yarn',
                               recursive = True,
+                              ignore_failures = True,
                               )
     self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
                               owner = 'yarn',
                               recursive = True,
+                              ignore_failures = True,
                               )
     self.assertResourceCalled('XmlConfig', 'core-site.xml',
                               owner = 'hdfs',