test_hive_server.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625
  1. #!/usr/bin/env python
  2. '''
  3. Licensed to the Apache Software Foundation (ASF) under one
  4. or more contributor license agreements. See the NOTICE file
  5. distributed with this work for additional information
  6. regarding copyright ownership. The ASF licenses this file
  7. to you under the Apache License, Version 2.0 (the
  8. "License"); you may not use this file except in compliance
  9. with the License. You may obtain a copy of the License at
  10. http://www.apache.org/licenses/LICENSE-2.0
  11. Unless required by applicable law or agreed to in writing, software
  12. distributed under the License is distributed on an "AS IS" BASIS,
  13. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. See the License for the specific language governing permissions and
  15. limitations under the License.
  16. '''
  17. import socket
  18. import subprocess
  19. from mock.mock import MagicMock, patch
  20. from resource_management.core import shell
  21. from resource_management.libraries.functions import dynamic_variable_interpretation
  22. from stacks.utils.RMFTestCase import *
  23. class TestHiveServer(RMFTestCase):
  24. COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package"
  25. STACK_VERSION = "2.0.6"
  26. UPGRADE_STACK_VERSION = "2.2"
  27. def test_configure_default(self):
  28. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  29. classname = "HiveServer",
  30. command = "configure",
  31. config_file="default.json",
  32. hdp_stack_version = self.STACK_VERSION,
  33. target = RMFTestCase.TARGET_COMMON_SERVICES
  34. )
  35. self.assert_configure_default()
  36. self.assertNoMoreResources()
  37. @patch.object(shell, "call", new=MagicMock(return_value=(0, '')))
  38. @patch.object(subprocess,"Popen")
  39. @patch("socket.socket")
  40. @patch.object(dynamic_variable_interpretation, "copy_tarballs_to_hdfs", new=MagicMock())
  41. def test_start_default(self, socket_mock, popen_mock):
  42. s = socket_mock.return_value
  43. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  44. classname = "HiveServer",
  45. command = "start",
  46. config_file="default.json",
  47. hdp_stack_version = self.STACK_VERSION,
  48. target = RMFTestCase.TARGET_COMMON_SERVICES
  49. )
  50. self.assert_configure_default()
  51. self.assertResourceCalled('Execute', 'hive --config /etc/hive/conf.server --service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020/apps/hive/warehouse ',
  52. environment = {'PATH' : "/bin:/usr/lib/hive/bin:/usr/bin"},
  53. user = 'hive',
  54. )
  55. self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive',
  56. not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1',
  57. environment = {'HADOOP_HOME' : '/usr', 'JAVA_HOME':'/usr/jdk64/jdk1.7.0_45'},
  58. path = ["/bin:/usr/lib/hive/bin:/usr/bin"],
  59. user = 'hive'
  60. )
  61. self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/lib/hive/lib//mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
  62. path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
  63. )
  64. self.assertNoMoreResources()
  65. self.assertTrue(socket_mock.called)
  66. self.assertTrue(s.close.called)
  67. @patch("socket.socket")
  68. @patch.object(dynamic_variable_interpretation, "copy_tarballs_to_hdfs", new=MagicMock())
  69. def test_stop_default(self, socket_mock):
  70. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  71. classname = "HiveServer",
  72. command = "stop",
  73. config_file="default.json",
  74. hdp_stack_version = self.STACK_VERSION,
  75. target = RMFTestCase.TARGET_COMMON_SERVICES
  76. )
  77. self.assertResourceCalled('Execute', 'ambari-sudo.sh kill `cat /var/run/hive/hive-server.pid`',
  78. not_if = '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1)',
  79. )
  80. self.assertResourceCalled('Execute', 'ambari-sudo.sh kill -9 `cat /var/run/hive/hive-server.pid`',
  81. not_if = '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1) || ( sleep 5 && ! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1) )',
  82. )
  83. self.assertResourceCalled('Execute', '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1)',
  84. tries = 20,
  85. try_sleep = 3,
  86. )
  87. self.assertResourceCalled('File', '/var/run/hive/hive-server.pid',
  88. action = ['delete'],
  89. )
  90. self.assertNoMoreResources()
  91. self.assertFalse(socket_mock.called)
  92. def test_configure_secured(self):
  93. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  94. classname = "HiveServer",
  95. command = "configure",
  96. config_file="secured.json",
  97. hdp_stack_version = self.STACK_VERSION,
  98. target = RMFTestCase.TARGET_COMMON_SERVICES
  99. )
  100. self.assert_configure_secured()
  101. self.assertNoMoreResources()
  102. @patch("hive_service.check_fs_root")
  103. @patch("socket.socket")
  104. def test_start_secured(self, socket_mock, check_fs_root_mock):
  105. s = socket_mock.return_value
  106. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  107. classname = "HiveServer",
  108. command = "start",
  109. config_file="secured.json",
  110. hdp_stack_version = self.STACK_VERSION,
  111. target = RMFTestCase.TARGET_COMMON_SERVICES
  112. )
  113. self.assert_configure_secured()
  114. self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hive.service.keytab hive/c6401.ambari.apache.org@EXAMPLE.COM; ',
  115. user = 'hive',
  116. )
  117. self.assertResourceCalled('Execute', '/tmp/start_hiveserver2_script /var/log/hive/hive-server2.out /var/log/hive/hive-server2.log /var/run/hive/hive-server.pid /etc/hive/conf.server /var/log/hive',
  118. not_if = 'ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1',
  119. environment = {'HADOOP_HOME' : '/usr', 'JAVA_HOME': '/usr/jdk64/jdk1.7.0_45'},
  120. path = ["/bin:/usr/lib/hive/bin:/usr/bin"],
  121. user = 'hive'
  122. )
  123. self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/lib/hive/lib//mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
  124. path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
  125. )
  126. self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa@EXAMPLE.COM; ',
  127. user = 'ambari-qa',
  128. )
  129. self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6401.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'",
  130. path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
  131. user = 'ambari-qa',
  132. timeout = 30,
  133. )
  134. self.assertNoMoreResources()
  135. self.assertTrue(check_fs_root_mock.called)
  136. @patch("socket.socket")
  137. def test_stop_secured(self, socket_mock):
  138. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  139. classname = "HiveServer",
  140. command = "stop",
  141. config_file="secured.json",
  142. hdp_stack_version = self.STACK_VERSION,
  143. target = RMFTestCase.TARGET_COMMON_SERVICES
  144. )
  145. self.assertResourceCalled('Execute', 'ambari-sudo.sh kill `cat /var/run/hive/hive-server.pid`',
  146. not_if = '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1)',
  147. )
  148. self.assertResourceCalled('Execute', 'ambari-sudo.sh kill -9 `cat /var/run/hive/hive-server.pid`',
  149. not_if = '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1) || ( sleep 5 && ! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1) )',
  150. )
  151. self.assertResourceCalled('Execute', '! (ls /var/run/hive/hive-server.pid >/dev/null 2>&1 && ps -p `cat /var/run/hive/hive-server.pid` >/dev/null 2>&1)',
  152. tries = 20,
  153. try_sleep = 3,
  154. )
  155. self.assertResourceCalled('File', '/var/run/hive/hive-server.pid',
  156. action = ['delete'],
  157. )
  158. self.assertNoMoreResources()
  159. self.assertFalse(socket_mock.called)
  160. def assert_configure_default(self):
  161. self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse',
  162. security_enabled = False,
  163. keytab = UnknownConfigurationMock(),
  164. conf_dir = '/etc/hadoop/conf',
  165. hdfs_user = 'hdfs',
  166. kinit_path_local = '/usr/bin/kinit',
  167. mode = 0777,
  168. owner = 'hive',
  169. bin_dir = '/usr/bin',
  170. action = ['create_delayed'],
  171. )
  172. self.assertResourceCalled('HdfsDirectory', '/user/hive',
  173. security_enabled = False,
  174. keytab = UnknownConfigurationMock(),
  175. conf_dir = '/etc/hadoop/conf',
  176. hdfs_user = 'hdfs',
  177. kinit_path_local = '/usr/bin/kinit',
  178. mode = 0700,
  179. owner = 'hive',
  180. bin_dir = '/usr/bin',
  181. action = ['create_delayed'],
  182. )
  183. self.assertResourceCalled('HdfsDirectory', None,
  184. security_enabled = False,
  185. keytab = UnknownConfigurationMock(),
  186. conf_dir = '/etc/hadoop/conf',
  187. hdfs_user = 'hdfs',
  188. kinit_path_local = '/usr/bin/kinit',
  189. action = ['create'],
  190. bin_dir = '/usr/bin',
  191. )
  192. self.assertResourceCalled('Directory', '/etc/hive',
  193. mode = 0755,
  194. )
  195. self.assertResourceCalled('Directory', '/etc/hive/conf',
  196. owner = 'hive',
  197. group = 'hadoop',
  198. recursive = True,
  199. )
  200. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  201. group = 'hadoop',
  202. conf_dir = '/etc/hive/conf',
  203. mode = 0644,
  204. configuration_attributes = {u'final': {u'mapred.healthChecker.script.path': u'true',
  205. u'mapreduce.jobtracker.staging.root.dir': u'true'}},
  206. owner = 'hive',
  207. configurations = self.getConfig()['configurations']['mapred-site'],
  208. )
  209. self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template',
  210. owner = 'hive',
  211. group = 'hadoop',
  212. )
  213. self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template',
  214. owner = 'hive',
  215. group = 'hadoop',
  216. )
  217. self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties',
  218. content = 'log4jproperties\nline2',
  219. owner = 'hive',
  220. group = 'hadoop',
  221. mode = 0644,
  222. )
  223. self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties',
  224. content = 'log4jproperties\nline2',
  225. owner = 'hive',
  226. group = 'hadoop',
  227. mode = 0644,
  228. )
  229. self.assertResourceCalled('XmlConfig', 'hive-site.xml',
  230. group = 'hadoop',
  231. conf_dir = '/etc/hive/conf.server',
  232. mode = 0644,
  233. configuration_attributes = {u'final': {u'hive.optimize.bucketmapjoin.sortedmerge': u'true',
  234. u'javax.jdo.option.ConnectionDriverName': u'true',
  235. u'javax.jdo.option.ConnectionPassword': u'true'}},
  236. owner = 'hive',
  237. configurations = self.getConfig()['configurations']['hive-site'],
  238. )
  239. self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh',
  240. content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']),
  241. owner = 'hive',
  242. group = 'hadoop',
  243. )
  244. self.assertResourceCalled('Directory', '/etc/security/limits.d',
  245. owner = 'root',
  246. group = 'root',
  247. recursive = True,
  248. )
  249. self.assertResourceCalled('File', '/etc/security/limits.d/hive.conf',
  250. content = Template('hive.conf.j2'),
  251. owner = 'root',
  252. group = 'root',
  253. mode = 0644,
  254. )
  255. self.assertResourceCalled('Execute', ('cp',
  256. '--remove-destination',
  257. '/usr/share/java/mysql-connector-java.jar',
  258. '/usr/lib/hive/lib//mysql-connector-java.jar'),
  259. path = ['/bin', '/usr/bin/'],
  260. sudo = True,
  261. )
  262. self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar',
  263. mode = 0644,
  264. )
  265. self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
  266. content = DownloadSource('http://c6401.ambari.apache.org:8080/resources'
  267. '/DBConnectionVerification.jar'),
  268. )
  269. self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
  270. content = Template('startHiveserver2.sh.j2'),
  271. mode = 0755,
  272. )
  273. self.assertResourceCalled('Directory', '/var/run/hive',
  274. owner = 'hive',
  275. mode = 0755,
  276. group = 'hadoop',
  277. recursive = True,
  278. cd_access = 'a',
  279. )
  280. self.assertResourceCalled('Directory', '/var/log/hive',
  281. owner = 'hive',
  282. mode = 0755,
  283. group = 'hadoop',
  284. recursive = True,
  285. cd_access = 'a',
  286. )
  287. self.assertResourceCalled('Directory', '/var/lib/hive',
  288. owner = 'hive',
  289. mode = 0755,
  290. group = 'hadoop',
  291. recursive = True,
  292. cd_access = 'a',
  293. )
  294. def assert_configure_secured(self):
  295. self.assertResourceCalled('HdfsDirectory', '/apps/hive/warehouse',
  296. security_enabled = True,
  297. keytab = '/etc/security/keytabs/hdfs.headless.keytab',
  298. conf_dir = '/etc/hadoop/conf',
  299. hdfs_user = 'hdfs',
  300. kinit_path_local = '/usr/bin/kinit',
  301. mode = 0777,
  302. owner = 'hive',
  303. bin_dir = '/usr/bin',
  304. action = ['create_delayed'],
  305. )
  306. self.assertResourceCalled('HdfsDirectory', '/user/hive',
  307. security_enabled = True,
  308. keytab = '/etc/security/keytabs/hdfs.headless.keytab',
  309. conf_dir = '/etc/hadoop/conf',
  310. hdfs_user = 'hdfs',
  311. kinit_path_local = '/usr/bin/kinit',
  312. mode = 0700,
  313. owner = 'hive',
  314. bin_dir = '/usr/bin',
  315. action = ['create_delayed'],
  316. )
  317. self.assertResourceCalled('HdfsDirectory', None,
  318. security_enabled = True,
  319. keytab = '/etc/security/keytabs/hdfs.headless.keytab',
  320. conf_dir = '/etc/hadoop/conf',
  321. hdfs_user = 'hdfs',
  322. kinit_path_local = '/usr/bin/kinit',
  323. action = ['create'],
  324. bin_dir = '/usr/bin',
  325. )
  326. self.assertResourceCalled('Directory', '/etc/hive',
  327. mode = 0755,
  328. )
  329. self.assertResourceCalled('Directory', '/etc/hive/conf',
  330. owner = 'hive',
  331. group = 'hadoop',
  332. recursive = True,
  333. )
  334. self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
  335. group = 'hadoop',
  336. conf_dir = '/etc/hive/conf',
  337. mode = 0644,
  338. configuration_attributes = {u'final': {u'mapred.healthChecker.script.path': u'true',
  339. u'mapreduce.jobtracker.staging.root.dir': u'true'}},
  340. owner = 'hive',
  341. configurations = self.getConfig()['configurations']['mapred-site'],
  342. )
  343. self.assertResourceCalled('File', '/etc/hive/conf/hive-default.xml.template',
  344. owner = 'hive',
  345. group = 'hadoop',
  346. )
  347. self.assertResourceCalled('File', '/etc/hive/conf/hive-env.sh.template',
  348. owner = 'hive',
  349. group = 'hadoop',
  350. )
  351. self.assertResourceCalled('File', '/etc/hive/conf/hive-exec-log4j.properties',
  352. content = 'log4jproperties\nline2',
  353. owner = 'hive',
  354. group = 'hadoop',
  355. mode = 0644,
  356. )
  357. self.assertResourceCalled('File', '/etc/hive/conf/hive-log4j.properties',
  358. content = 'log4jproperties\nline2',
  359. owner = 'hive',
  360. group = 'hadoop',
  361. mode = 0644,
  362. )
  363. self.assertResourceCalled('XmlConfig', 'hive-site.xml',
  364. group = 'hadoop',
  365. conf_dir = '/etc/hive/conf.server',
  366. mode = 0644,
  367. configuration_attributes = {u'final': {u'hive.optimize.bucketmapjoin.sortedmerge': u'true',
  368. u'javax.jdo.option.ConnectionDriverName': u'true',
  369. u'javax.jdo.option.ConnectionPassword': u'true'}},
  370. owner = 'hive',
  371. configurations = self.getConfig()['configurations']['hive-site'],
  372. )
  373. self.assertResourceCalled('File', '/etc/hive/conf.server/hive-env.sh',
  374. content = InlineTemplate(self.getConfig()['configurations']['hive-env']['content']),
  375. owner = 'hive',
  376. group = 'hadoop',
  377. )
  378. self.assertResourceCalled('Directory', '/etc/security/limits.d',
  379. owner = 'root',
  380. group = 'root',
  381. recursive = True,
  382. )
  383. self.assertResourceCalled('File', '/etc/security/limits.d/hive.conf',
  384. content = Template('hive.conf.j2'),
  385. owner = 'root',
  386. group = 'root',
  387. mode = 0644,
  388. )
  389. self.assertResourceCalled('Execute', ('cp',
  390. '--remove-destination',
  391. '/usr/share/java/mysql-connector-java.jar',
  392. '/usr/lib/hive/lib//mysql-connector-java.jar'),
  393. path = ['/bin', '/usr/bin/'],
  394. sudo = True,
  395. )
  396. self.assertResourceCalled('File', '/usr/lib/hive/lib//mysql-connector-java.jar',
  397. mode = 0644,
  398. )
  399. self.assertResourceCalled('File', '/usr/lib/ambari-agent/DBConnectionVerification.jar',
  400. content = DownloadSource('http://c6401.ambari.apache.org:8080/resources/DBConnectionVerification.jar'),
  401. )
  402. self.assertResourceCalled('File', '/tmp/start_hiveserver2_script',
  403. content = Template('startHiveserver2.sh.j2'),
  404. mode = 0755,
  405. )
  406. self.assertResourceCalled('Directory', '/var/run/hive',
  407. owner = 'hive',
  408. group = 'hadoop',
  409. mode = 0755,
  410. recursive = True,
  411. cd_access = 'a',
  412. )
  413. self.assertResourceCalled('Directory', '/var/log/hive',
  414. owner = 'hive',
  415. group = 'hadoop',
  416. mode = 0755,
  417. recursive = True,
  418. cd_access = 'a',
  419. )
  420. self.assertResourceCalled('Directory', '/var/lib/hive',
  421. owner = 'hive',
  422. group = 'hadoop',
  423. mode = 0755,
  424. recursive = True,
  425. cd_access = 'a',
  426. )
  427. @patch("hive_service.check_fs_root")
  428. @patch("time.time")
  429. @patch("socket.socket")
  430. def test_socket_timeout(self, socket_mock, time_mock, check_fs_root_mock):
  431. s = socket_mock.return_value
  432. s.connect = MagicMock()
  433. s.connect.side_effect = socket.error("")
  434. time_mock.side_effect = [0, 1000, 2000, 3000, 4000]
  435. try:
  436. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  437. classname = "HiveServer",
  438. command = "start",
  439. config_file="default.json",
  440. hdp_stack_version = self.STACK_VERSION,
  441. target = RMFTestCase.TARGET_COMMON_SERVICES
  442. )
  443. self.fail("Script failure due to socket error was expected")
  444. except:
  445. self.assert_configure_default()
  446. self.assertFalse(socket_mock.called)
  447. self.assertFalse(s.close.called)
  448. @patch("hive_server.HiveServer.pre_rolling_restart")
  449. @patch("hive_server.HiveServer.start")
  450. @patch.object(shell, "call", new=MagicMock(return_value=(0,"hive-server2 - 2.2.0.0-2041")))
  451. def test_stop_during_upgrade(self, hive_server_start_mock,
  452. hive_server_pre_rolling_mock):
  453. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  454. classname = "HiveServer", command = "restart", config_file = "hive-upgrade.json",
  455. hdp_stack_version = self.UPGRADE_STACK_VERSION,
  456. target = RMFTestCase.TARGET_COMMON_SERVICES )
  457. self.assertResourceCalled('Execute', 'hive --service hiveserver2 --deregister 2.2.0.0-2041',
  458. path=['/bin:/usr/hdp/current/hive-server2/bin:/usr/hdp/current/hadoop-client/bin'],
  459. tries=1, user='hive')
  460. self.assertResourceCalled('Execute', 'hdp-select set hive-server2 2.2.1.0-2065',)
  461. @patch("hive_server.HiveServer.pre_rolling_restart")
  462. @patch("hive_server.HiveServer.start")
  463. @patch.object(shell, "call", new=MagicMock(return_value=(0,"BAD VERSION")))
  464. def test_stop_during_upgrade_bad_hive_version(self, hive_server_start_mock,
  465. hive_server_pre_rolling_mock):
  466. try:
  467. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  468. classname = "HiveServer", command = "restart", config_file = "hive-upgrade.json",
  469. hdp_stack_version = self.UPGRADE_STACK_VERSION,
  470. target = RMFTestCase.TARGET_COMMON_SERVICES )
  471. self.fail("Invalid hive version should have caused an exception")
  472. except:
  473. pass
  474. self.assertNoMoreResources()
  475. @patch("resource_management.libraries.functions.security_commons.build_expectations")
  476. @patch("resource_management.libraries.functions.security_commons.get_params_from_filesystem")
  477. @patch("resource_management.libraries.functions.security_commons.validate_security_config_properties")
  478. @patch("resource_management.libraries.functions.security_commons.cached_kinit_executor")
  479. @patch("resource_management.libraries.script.Script.put_structured_out")
  480. def test_security_status(self, put_structured_out_mock, cached_kinit_executor_mock, validate_security_config_mock, get_params_mock, build_exp_mock):
  481. # Test that function works when is called with correct parameters
  482. security_params = {
  483. 'hive-site': {
  484. "hive.server2.authentication": "KERBEROS",
  485. "hive.metastore.sasl.enabled": "true",
  486. "hive.security.authorization.enabled": "true",
  487. "hive.server2.authentication.kerberos.keytab": "path/to/keytab",
  488. "hive.server2.authentication.kerberos.principal": "principal",
  489. "hive.server2.authentication.spnego.keytab": "path/to/spnego_keytab",
  490. "hive.server2.authentication.spnego.principal": "spnego_principal"
  491. }
  492. }
  493. result_issues = []
  494. props_value_check = {"hive.server2.authentication": "KERBEROS",
  495. "hive.metastore.sasl.enabled": "true",
  496. "hive.security.authorization.enabled": "true"}
  497. props_empty_check = ["hive.server2.authentication.kerberos.keytab",
  498. "hive.server2.authentication.kerberos.principal",
  499. "hive.server2.authentication.spnego.principal",
  500. "hive.server2.authentication.spnego.keytab"]
  501. props_read_check = ["hive.server2.authentication.kerberos.keytab",
  502. "hive.server2.authentication.spnego.keytab"]
  503. get_params_mock.return_value = security_params
  504. validate_security_config_mock.return_value = result_issues
  505. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  506. classname = "HiveServer",
  507. command = "security_status",
  508. config_file="../../2.1/configs/secured.json",
  509. hdp_stack_version = self.STACK_VERSION,
  510. target = RMFTestCase.TARGET_COMMON_SERVICES
  511. )
  512. get_params_mock.assert_called_with('/etc/hive/conf', {'hive-site.xml': "XML"})
  513. build_exp_mock.assert_called_with('hive-site', props_value_check, props_empty_check, props_read_check)
  514. put_structured_out_mock.assert_called_with({"securityState": "SECURED_KERBEROS"})
  515. self.assertTrue(cached_kinit_executor_mock.call_count, 2)
  516. cached_kinit_executor_mock.assert_called_with('/usr/bin/kinit',
  517. self.config_dict['configurations']['hive-env']['hive_user'],
  518. security_params['hive-site']['hive.server2.authentication.spnego.keytab'],
  519. security_params['hive-site']['hive.server2.authentication.spnego.principal'],
  520. self.config_dict['hostname'],
  521. '/tmp')
  522. # Testing that the exception throw by cached_executor is caught
  523. cached_kinit_executor_mock.reset_mock()
  524. cached_kinit_executor_mock.side_effect = Exception("Invalid command")
  525. try:
  526. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  527. classname = "HiveServer",
  528. command = "security_status",
  529. config_file="../../2.1/configs/secured.json",
  530. hdp_stack_version = self.STACK_VERSION,
  531. target = RMFTestCase.TARGET_COMMON_SERVICES
  532. )
  533. except:
  534. self.assertTrue(True)
  535. # Testing with a security_params which doesn't contains startup
  536. empty_security_params = {}
  537. cached_kinit_executor_mock.reset_mock()
  538. get_params_mock.reset_mock()
  539. put_structured_out_mock.reset_mock()
  540. get_params_mock.return_value = empty_security_params
  541. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  542. classname = "HiveServer",
  543. command = "security_status",
  544. config_file="../../2.1/configs/secured.json",
  545. hdp_stack_version = self.STACK_VERSION,
  546. target = RMFTestCase.TARGET_COMMON_SERVICES
  547. )
  548. put_structured_out_mock.assert_called_with({"securityIssuesFound": "Keytab file or principal are not set property."})
  549. # Testing with not empty result_issues
  550. result_issues_with_params = {}
  551. result_issues_with_params['hive-site']="Something bad happened"
  552. validate_security_config_mock.reset_mock()
  553. get_params_mock.reset_mock()
  554. validate_security_config_mock.return_value = result_issues_with_params
  555. get_params_mock.return_value = security_params
  556. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  557. classname = "HiveServer",
  558. command = "security_status",
  559. config_file="../../2.1/configs/secured.json",
  560. hdp_stack_version = self.STACK_VERSION,
  561. target = RMFTestCase.TARGET_COMMON_SERVICES
  562. )
  563. put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
  564. # Testing with security_enable = false
  565. self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
  566. classname = "HiveServer",
  567. command = "security_status",
  568. config_file="../../2.1/configs/default.json",
  569. hdp_stack_version = self.STACK_VERSION,
  570. target = RMFTestCase.TARGET_COMMON_SERVICES
  571. )
  572. put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})