test_mapreduce2_client.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396
  1. #!/usr/bin/env python
  2. '''
  3. Licensed to the Apache Software Foundation (ASF) under one
  4. or more contributor license agreements. See the NOTICE file
  5. distributed with this work for additional information
  6. regarding copyright ownership. The ASF licenses this file
  7. to you under the Apache License, Version 2.0 (the
  8. "License"); you may not use this file except in compliance
  9. with the License. You may obtain a copy of the License at
  10. http://www.apache.org/licenses/LICENSE-2.0
  11. Unless required by applicable law or agreed to in writing, software
  12. distributed under the License is distributed on an "AS IS" BASIS,
  13. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. See the License for the specific language governing permissions and
  15. limitations under the License.
  16. '''
  17. import json
  18. from mock.mock import MagicMock, call, patch
  19. from stacks.utils.RMFTestCase import *
  20. import os
  21. origin_exists = os.path.exists
  22. @patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
  23. @patch.object(os.path, "exists", new=MagicMock(
  24. side_effect=lambda *args: origin_exists(args[0])
  25. if args[0][-2:] == "j2" else True))
  26. class TestMapReduce2Client(RMFTestCase):
  27. COMMON_SERVICES_PACKAGE_DIR = "YARN/2.1.0.2.0/package"
  28. STACK_VERSION = "2.0.6"
  29. def test_configure_default(self):
  30. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
  31. classname = "MapReduce2Client",
  32. command = "configure",
  33. config_file="default.json",
  34. hdp_stack_version = self.STACK_VERSION,
  35. target = RMFTestCase.TARGET_COMMON_SERVICES
  36. )
  37. self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
  38. owner = 'yarn',
  39. group = 'hadoop',
  40. recursive = True,
  41. cd_access = 'a',
  42. )
  43. self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn',
  44. owner = 'yarn',
  45. group = 'hadoop',
  46. recursive = True,
  47. cd_access = 'a',
  48. )
  49. self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn',
  50. owner = 'yarn',
  51. group = 'hadoop',
  52. recursive = True,
  53. cd_access = 'a',
  54. )
  55. self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce',
  56. owner = 'mapred',
  57. group = 'hadoop',
  58. recursive = True,
  59. cd_access = 'a',
  60. )
  61. self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred',
  62. owner = 'mapred',
  63. group = 'hadoop',
  64. recursive = True,
  65. cd_access = 'a',
  66. )
  67. self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce',
  68. owner = 'mapred',
  69. group = 'hadoop',
  70. recursive = True,
  71. cd_access = 'a',
  72. )
  73. self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred',
  74. owner = 'mapred',
  75. group = 'hadoop',
  76. recursive = True,
  77. cd_access = 'a',
  78. )
  79. self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
  80. owner = 'yarn',
  81. recursive = True,
  82. ignore_failures = True,
  83. cd_access = 'a',
  84. )
  85. self.assertResourceCalled('XmlConfig', 'core-site.xml',
  86. owner = 'hdfs',
  87. group = 'hadoop',
  88. mode = 0644,
  89. conf_dir = '/etc/hadoop/conf',
  90. configurations = self.getConfig()['configurations']['core-site'],
  91. configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
  92. )
  93. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  94. owner = 'yarn',
  95. group = 'hadoop',
  96. mode = 0644,
  97. conf_dir = '/etc/hadoop/conf',
  98. configurations = self.getConfig()['configurations']['mapred-site'],
  99. configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site']
  100. )
  101. self.assertResourceCalled('XmlConfig', 'yarn-site.xml',
  102. owner = 'yarn',
  103. group = 'hadoop',
  104. mode = 0644,
  105. conf_dir = '/etc/hadoop/conf',
  106. configurations = self.getConfig()['configurations']['yarn-site'],
  107. configuration_attributes = self.getConfig()['configuration_attributes']['yarn-site']
  108. )
  109. self.assertResourceCalled('XmlConfig', 'capacity-scheduler.xml',
  110. owner = 'yarn',
  111. group = 'hadoop',
  112. mode = 0644,
  113. conf_dir = '/etc/hadoop/conf',
  114. configurations = self.getConfig()['configurations']['capacity-scheduler'],
  115. configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
  116. )
  117. self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
  118. owner = 'yarn',
  119. group = 'hadoop',
  120. )
  121. self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
  122. content = Template('yarn.conf.j2'),
  123. mode = 0644,
  124. )
  125. self.assertResourceCalled('File', '/etc/security/limits.d/mapreduce.conf',
  126. content = Template('mapreduce.conf.j2'),
  127. mode = 0644,
  128. )
  129. self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh',
  130. content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']),
  131. owner = 'yarn',
  132. group = 'hadoop',
  133. mode = 0755,
  134. )
  135. self.assertResourceCalled('File', '/usr/lib/hadoop-yarn/bin/container-executor',
  136. group = 'hadoop',
  137. mode = 06050,
  138. )
  139. self.assertResourceCalled('File', '/etc/hadoop/conf/container-executor.cfg',
  140. content = Template('container-executor.cfg.j2'),
  141. group = 'hadoop',
  142. mode = 0644,
  143. )
  144. self.assertResourceCalled('Directory', '/cgroups_test/cpu',
  145. group = 'hadoop',
  146. recursive = True,
  147. mode = 0755,
  148. cd_access="a"
  149. )
  150. self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
  151. content = InlineTemplate(self.getConfig()['configurations']['mapred-env']['content']),
  152. owner = 'hdfs',
  153. )
  154. self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
  155. content = Template('taskcontroller.cfg.j2'),
  156. owner = 'hdfs',
  157. )
  158. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  159. owner = 'mapred',
  160. group = 'hadoop',
  161. conf_dir = '/etc/hadoop/conf',
  162. configurations = self.getConfig()['configurations']['mapred-site'],
  163. configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site']
  164. )
  165. self.assertResourceCalled('XmlConfig', 'capacity-scheduler.xml',
  166. owner = 'hdfs',
  167. group = 'hadoop',
  168. conf_dir = '/etc/hadoop/conf',
  169. configurations = self.getConfig()['configurations']['capacity-scheduler'],
  170. configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
  171. )
  172. self.assertResourceCalled('File', '/etc/hadoop/conf/fair-scheduler.xml',
  173. owner = 'mapred',
  174. group = 'hadoop',
  175. )
  176. self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-client.xml.example',
  177. owner = 'mapred',
  178. group = 'hadoop',
  179. )
  180. self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example',
  181. owner = 'mapred',
  182. group = 'hadoop',
  183. )
  184. self.assertNoMoreResources()
  185. def test_configure_secured(self):
  186. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
  187. classname = "MapReduce2Client",
  188. command = "configure",
  189. config_file="secured.json",
  190. hdp_stack_version = self.STACK_VERSION,
  191. target = RMFTestCase.TARGET_COMMON_SERVICES
  192. )
  193. self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
  194. owner = 'yarn',
  195. group = 'hadoop',
  196. recursive = True,
  197. cd_access = 'a',
  198. )
  199. self.assertResourceCalled('Directory', '/var/run/hadoop-yarn/yarn',
  200. owner = 'yarn',
  201. group = 'hadoop',
  202. recursive = True,
  203. cd_access = 'a',
  204. )
  205. self.assertResourceCalled('Directory', '/var/log/hadoop-yarn/yarn',
  206. owner = 'yarn',
  207. group = 'hadoop',
  208. recursive = True,
  209. cd_access = 'a',
  210. )
  211. self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce',
  212. owner = 'mapred',
  213. group = 'hadoop',
  214. recursive = True,
  215. cd_access = 'a',
  216. )
  217. self.assertResourceCalled('Directory', '/var/run/hadoop-mapreduce/mapred',
  218. owner = 'mapred',
  219. group = 'hadoop',
  220. recursive = True,
  221. cd_access = 'a',
  222. )
  223. self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce',
  224. owner = 'mapred',
  225. group = 'hadoop',
  226. recursive = True,
  227. cd_access = 'a',
  228. )
  229. self.assertResourceCalled('Directory', '/var/log/hadoop-mapreduce/mapred',
  230. owner = 'mapred',
  231. group = 'hadoop',
  232. recursive = True,
  233. cd_access = 'a',
  234. )
  235. self.assertResourceCalled('Directory', '/var/log/hadoop-yarn',
  236. owner = 'yarn',
  237. recursive = True,
  238. ignore_failures = True,
  239. cd_access = 'a',
  240. )
  241. self.assertResourceCalled('XmlConfig', 'core-site.xml',
  242. owner = 'hdfs',
  243. group = 'hadoop',
  244. mode = 0644,
  245. conf_dir = '/etc/hadoop/conf',
  246. configurations = self.getConfig()['configurations']['core-site'],
  247. configuration_attributes = self.getConfig()['configuration_attributes']['core-site']
  248. )
  249. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  250. owner = 'yarn',
  251. group = 'hadoop',
  252. mode = 0644,
  253. conf_dir = '/etc/hadoop/conf',
  254. configurations = self.getConfig()['configurations']['mapred-site'],
  255. configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site']
  256. )
  257. self.assertResourceCalled('XmlConfig', 'yarn-site.xml',
  258. owner = 'yarn',
  259. group = 'hadoop',
  260. mode = 0644,
  261. conf_dir = '/etc/hadoop/conf',
  262. configurations = self.getConfig()['configurations']['yarn-site'],
  263. configuration_attributes = self.getConfig()['configuration_attributes']['yarn-site']
  264. )
  265. self.assertResourceCalled('XmlConfig', 'capacity-scheduler.xml',
  266. owner = 'yarn',
  267. group = 'hadoop',
  268. mode = 0644,
  269. conf_dir = '/etc/hadoop/conf',
  270. configurations = self.getConfig()['configurations']['capacity-scheduler'],
  271. configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
  272. )
  273. self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
  274. owner = 'yarn',
  275. group = 'hadoop',
  276. )
  277. self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
  278. content = Template('yarn.conf.j2'),
  279. mode = 0644,
  280. )
  281. self.assertResourceCalled('File', '/etc/security/limits.d/mapreduce.conf',
  282. content = Template('mapreduce.conf.j2'),
  283. mode = 0644,
  284. )
  285. self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh',
  286. content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']),
  287. owner = 'yarn',
  288. group = 'hadoop',
  289. mode = 0755,
  290. )
  291. self.assertResourceCalled('File', '/usr/lib/hadoop-yarn/bin/container-executor',
  292. group = 'hadoop',
  293. mode = 06050,
  294. )
  295. self.assertResourceCalled('File', '/etc/hadoop/conf/container-executor.cfg',
  296. content = Template('container-executor.cfg.j2'),
  297. group = 'hadoop',
  298. mode = 0644,
  299. )
  300. self.assertResourceCalled('Directory', '/cgroups_test/cpu',
  301. group = 'hadoop',
  302. recursive = True,
  303. mode = 0755,
  304. cd_access="a"
  305. )
  306. self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
  307. content = InlineTemplate(self.getConfig()['configurations']['mapred-env']['content']),
  308. owner = 'root',
  309. )
  310. self.assertResourceCalled('File', '/usr/lib/hadoop/sbin/task-controller',
  311. owner = 'root',
  312. group = 'hadoop',
  313. mode = 06050,
  314. )
  315. self.assertResourceCalled('File', '/etc/hadoop/conf/taskcontroller.cfg',
  316. content = Template('taskcontroller.cfg.j2'),
  317. owner = 'root',
  318. group = 'hadoop',
  319. mode = 0644,
  320. )
  321. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  322. owner = 'mapred',
  323. group = 'hadoop',
  324. conf_dir = '/etc/hadoop/conf',
  325. configurations = self.getConfig()['configurations']['mapred-site'],
  326. configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site']
  327. )
  328. self.assertResourceCalled('XmlConfig', 'capacity-scheduler.xml',
  329. owner = 'hdfs',
  330. group = 'hadoop',
  331. conf_dir = '/etc/hadoop/conf',
  332. configurations = self.getConfig()['configurations']['capacity-scheduler'],
  333. configuration_attributes = self.getConfig()['configuration_attributes']['capacity-scheduler']
  334. )
  335. self.assertResourceCalled('File', '/etc/hadoop/conf/fair-scheduler.xml',
  336. owner = 'mapred',
  337. group = 'hadoop',
  338. )
  339. self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-client.xml.example',
  340. owner = 'mapred',
  341. group = 'hadoop',
  342. )
  343. self.assertResourceCalled('File', '/etc/hadoop/conf/ssl-server.xml.example',
  344. owner = 'mapred',
  345. group = 'hadoop',
  346. )
  347. self.assertNoMoreResources()
  348. def test_upgrade(self):
  349. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
  350. classname = "MapReduce2Client",
  351. command = "restart",
  352. config_file="client-upgrade.json",
  353. hdp_stack_version = self.STACK_VERSION,
  354. target = RMFTestCase.TARGET_COMMON_SERVICES
  355. )
  356. # for now, it's enough that hdp-select is confirmed
  357. self.assertResourceCalled("Execute", "hdp-select set hadoop-client 2.2.1.0-2067")
  358. def test_pre_rolling_restart_23(self):
  359. config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/client-upgrade.json"
  360. with open(config_file, "r") as f:
  361. json_content = json.load(f)
  362. version = '2.3.0.0-1234'
  363. json_content['commandParams']['version'] = version
  364. mocks_dict = {}
  365. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/mapreduce2_client.py",
  366. classname = "MapReduce2Client",
  367. command = "pre_rolling_restart",
  368. config_dict = json_content,
  369. hdp_stack_version = self.STACK_VERSION,
  370. target = RMFTestCase.TARGET_COMMON_SERVICES,
  371. call_mocks = [(0, None), (0, None)],
  372. mocks_dict = mocks_dict)
  373. self.assertResourceCalled('Execute', 'hdp-select set hadoop-client {0}'.format(version))
  374. self.assertNoMoreResources()
  375. self.assertEquals(2, mocks_dict['call'].call_count)
  376. self.assertEquals(
  377. "conf-select create-conf-dir --package hadoop --stack-version 2.3.0.0-1234 --conf-version 0",
  378. mocks_dict['call'].call_args_list[0][0][0])
  379. self.assertEquals(
  380. "conf-select set-conf-dir --package hadoop --stack-version 2.3.0.0-1234 --conf-version 0",
  381. mocks_dict['call'].call_args_list[1][0][0])