init.pp 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391
  1. #
  2. #
  3. # Licensed to the Apache Software Foundation (ASF) under one
  4. # or more contributor license agreements. See the NOTICE file
  5. # distributed with this work for additional information
  6. # regarding copyright ownership. The ASF licenses this file
  7. # to you under the Apache License, Version 2.0 (the
  8. # "License"); you may not use this file except in compliance
  9. # with the License. You may obtain a copy of the License at
  10. #
  11. # http://www.apache.org/licenses/LICENSE-2.0
  12. #
  13. # Unless required by applicable law or agreed to in writing,
  14. # software distributed under the License is distributed on an
  15. # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  16. # KIND, either express or implied. See the License for the
  17. # specific language governing permissions and limitations
  18. # under the License.
  19. #
  20. #
  21. class hdp(
  22. $service_state = undef,
  23. $pre_installed_pkgs = undef
  24. )
  25. {
  26. import 'params.pp'
  27. include hdp::params
  28. Exec { logoutput => 'on_failure' }
  29. group { $hdp::params::user_group :
  30. ensure => present
  31. }
  32. group { $hdp::params::mapred_group :
  33. ensure => present
  34. }
  35. ## Port settings
  36. if has_key($configuration, 'hdfs-site') {
  37. $hdfs-site = $configuration['hdfs-site']
  38. $namenode_port = hdp_get_port_from_url($hdfs-site["dfs.http.address"])
  39. $snamenode_port = hdp_get_port_from_url($hdfs-site["dfs.secondary.http.address"])
  40. $datanode_port = hdp_get_port_from_url($hdfs-site["dfs.datanode.http.address"])
  41. } else {
  42. $namenode_port = "50070"
  43. $snamenode_port = "50090"
  44. $datanode_port = "50075"
  45. }
  46. if has_key($configuration, 'mapred-site') {
  47. $mapred-site = $configuration['mapred-site']
  48. $jtnode_port = hdp_get_port_from_url($mapred-site["mapred.job.tracker.http.address"],"50030")
  49. $tasktracker_port = hdp_get_port_from_url($mapred-site["mapred.task.tracker.http.address"],"50060")
  50. $jobhistory_port = hdp_get_port_from_url($mapred-site["mapreduce.history.server.http.address"],"51111")
  51. }
  52. $hbase_master_port = hdp_default("hbase-site/hbase.master.info.port","60010")
  53. $hbase_rs_port = hdp_default("hbase-site/hbase.regionserver.info.port","60030")
  54. $ganglia_port = hdp_default("ganglia_port","8651")
  55. $ganglia_collector_slaves_port = hdp_default("ganglia_collector_slaves_port","8660")
  56. $ganglia_collector_namenode_port = hdp_default("ganglia_collector_namenode_port","8661")
  57. $ganglia_collector_jobtracker_port = hdp_default("ganglia_collector_jobtracker_port","8662")
  58. $ganglia_collector_hbase_port = hdp_default("ganglia_collector_hbase_port","8663")
  59. $oozie_server_port = hdp_default("oozie_server_port","11000")
  60. $templeton_port = hdp_default("webhcat-site/templeton.port","50111")
  61. $namenode_metadata_port = hdp_default("namenode_metadata_port","8020")
  62. #TODO: think not needed and also there seems to be a puppet bug around this and ldap
  63. class { 'hdp::snmp': service_state => 'running'}
  64. class { 'hdp::create_smoke_user': }
  65. if ($pre_installed_pkgs != undef) {
  66. class { 'hdp::pre_install_pkgs': }
  67. }
  68. #turns off selinux
  69. class { 'hdp::set_selinux': }
  70. if ($service_state != 'uninstalled') {
  71. if ($hdp::params::lzo_enabled == true) {
  72. @hdp::lzo::package{ 32:}
  73. @hdp::lzo::package{ 64:}
  74. }
  75. }
  76. #TODO: treat consistently
  77. if ($service_state != 'uninstalled') {
  78. if ($hdp::params::snappy_enabled == true) {
  79. include hdp::snappy::package
  80. }
  81. }
  82. Hdp::Package<|title == 'hadoop 32'|> -> Hdp::Package<|title == 'hbase'|>
  83. Hdp::Package<|title == 'hadoop 64'|> -> Hdp::Package<|title == 'hbase'|>
  84. #TODO: just for testing
  85. class{ 'hdp::iptables':
  86. ensure => stopped,
  87. }
  88. hdp::package{ 'glibc':
  89. ensure => 'present',
  90. size => $size,
  91. java_needed => false,
  92. lzo_needed => false
  93. }
  94. }
  95. class hdp::pre_install_pkgs
  96. {
  97. if ($service_state == 'installed_and_configured') {
  98. hdp::exec{ 'yum install $pre_installed_pkgs':
  99. command => "yum install -y $pre_installed_pkgs"
  100. }
  101. } elsif ($service_state == 'uninstalled') {
  102. hdp::exec{ 'yum erase $pre_installed_pkgs':
  103. command => "yum erase -y $pre_installed_pkgs"
  104. }
  105. }
  106. }
  107. class hdp::create_smoke_user()
  108. {
  109. $smoke_group = $hdp::params::smoke_user_group
  110. $smoke_user = $hdp::params::smokeuser
  111. $security_enabled = $hdp::params::security_enabled
  112. if ( $smoke_group != $proxyuser_group) {
  113. group { $smoke_group :
  114. ensure => present
  115. }
  116. }
  117. if ($hdp::params::user_group != $proxyuser_group) {
  118. group { $proxyuser_group :
  119. ensure => present
  120. }
  121. }
  122. hdp::user { $smoke_user:
  123. gid => $hdp::params::user_group,
  124. groups => ["$proxyuser_group"]
  125. }
  126. ## Set smoke user uid to > 1000 for enable security feature
  127. $secure_uid = $hdp::params::smoketest_user_secure_uid
  128. $changeUid_path = "/tmp/changeUid.sh"
  129. $smoke_user_dirs = "/tmp/hadoop-${smoke_user},/tmp/hsperfdata_${smoke_user},/home/${smoke_user}"
  130. $cmd_set_uid = "$changeUid_path ${smoke_user} ${secure_uid} ${smoke_user_dirs}"
  131. $cmd_set_uid_check = "test $(id -u ${smoke_user}) -gt 1000"
  132. file { $changeUid_path :
  133. ensure => present,
  134. source => "puppet:///modules/hdp/changeToSecureUid.sh",
  135. mode => '0755'
  136. }
  137. hdp::exec{ $cmd_set_uid:
  138. command => $cmd_set_uid,
  139. unless => $cmd_set_uid_check,
  140. require => File[$changeUid_path]
  141. }
  142. Group<|title == $smoke_group or title == $proxyuser_group|> ->
  143. Hdp::User[$smoke_user] -> Hdp::Exec[$cmd_set_uid]
  144. }
  145. class hdp::set_selinux()
  146. {
  147. $cmd = "/bin/echo 0 > /selinux/enforce"
  148. hdp::exec{ $cmd:
  149. command => $cmd,
  150. unless => "head -n 1 /selinux/enforce | grep ^0$",
  151. onlyif => "test -f /selinux/enforce"
  152. }
  153. }
  154. define hdp::user(
  155. $gid = $hdp::params::user_group,
  156. $just_validate = undef,
  157. $groups = undef
  158. )
  159. {
  160. $user_info = $hdp::params::user_info[$name]
  161. if ($just_validate != undef) {
  162. $just_val = $just_validate
  163. } elsif (($user_info == undef) or ("|${user_info}|" == '||')){ #tests for different versions of Puppet
  164. $just_val = false
  165. } else {
  166. $just_val = $user_info[just_validate]
  167. }
  168. if ($just_val == true) {
  169. exec { "user ${name} exists":
  170. command => "su - ${name} -c 'ls /dev/null' >/dev/null 2>&1",
  171. path => ['/bin']
  172. }
  173. } else {
  174. user { $name:
  175. ensure => present,
  176. managehome => true,
  177. gid => $gid, #TODO either remove this to support LDAP env or fix it
  178. shell => '/bin/bash',
  179. groups => $groups
  180. }
  181. }
  182. }
  183. define hdp::directory(
  184. $owner = undef,
  185. $group = $hdp::params::user_group,
  186. $mode = undef,
  187. $ensure = directory,
  188. $force = undef,
  189. $service_state = 'running',
  190. $override_owner = false
  191. )
  192. {
  193. if (($service_state == 'uninstalled') and ($wipeoff_data == true)) {
  194. file { $name :
  195. ensure => absent,
  196. owner => $owner,
  197. group => $group,
  198. mode => $mode,
  199. force => $force
  200. }
  201. } elsif ($service_state != 'uninstalled') {
  202. if $override_owner == true {
  203. file { $name :
  204. ensure => present,
  205. owner => $owner,
  206. group => $group,
  207. mode => $mode,
  208. force => $force
  209. }
  210. } else {
  211. file { $name :
  212. ensure => present,
  213. mode => $mode,
  214. force => $force
  215. }
  216. }
  217. }
  218. }
  219. #TODO: check on -R flag and use of recurse
  220. define hdp::directory_recursive_create(
  221. $owner = undef,
  222. $group = $hdp::params::user_group,
  223. $mode = undef,
  224. $context_tag = undef,
  225. $ensure = directory,
  226. $force = undef,
  227. $service_state = 'running',
  228. $override_owner = true
  229. )
  230. {
  231. hdp::exec {"mkdir -p ${name}" :
  232. command => "mkdir -p ${name}",
  233. creates => $name
  234. }
  235. #to take care of setting ownership and mode
  236. hdp::directory { $name :
  237. owner => $owner,
  238. group => $group,
  239. mode => $mode,
  240. ensure => $ensure,
  241. force => $force,
  242. service_state => $service_state,
  243. override_owner => $override_owner
  244. }
  245. Hdp::Exec["mkdir -p ${name}"] -> Hdp::Directory[$name]
  246. }
  247. define hdp::directory_recursive_create_ignore_failure(
  248. $owner = undef,
  249. $group = $hdp::params::user_group,
  250. $mode = undef,
  251. $context_tag = undef,
  252. $ensure = directory,
  253. $force = undef,
  254. $service_state = 'running'
  255. )
  256. {
  257. hdp::exec {"mkdir -p ${name} ; exit 0" :
  258. command => "mkdir -p ${name} ; exit 0",
  259. creates => $name
  260. }
  261. hdp::exec {"chown ${owner}:${group} ${name}; exit 0" :
  262. command => "chown ${owner}:${group} ${name}; exit 0"
  263. }
  264. hdp::exec {"chmod ${mode} ${name} ; exit 0" :
  265. command => "chmod ${mode} ${name} ; exit 0"
  266. }
  267. Hdp::Exec["mkdir -p ${name} ; exit 0"] -> Hdp::Exec["chown ${owner}:${group} ${name}; exit 0"] -> Hdp::Exec["chmod ${mode} ${name} ; exit 0"]
  268. }
  269. ### helper to do exec
  270. define hdp::exec(
  271. $command,
  272. $refreshonly = undef,
  273. $unless = undef,
  274. $onlyif = undef,
  275. $path = $hdp::params::exec_path,
  276. $user = undef,
  277. $creates = undef,
  278. $tries = 1,
  279. $timeout = 300,
  280. $try_sleep = undef,
  281. $initial_wait = undef,
  282. $logoutput = 'on_failure',
  283. $cwd = undef
  284. )
  285. {
  286. if (($initial_wait != undef) and ($initial_wait != "undef")) {
  287. #passing in creates and unless so dont have to wait if condition has been acheived already
  288. hdp::wait { "service ${name}" :
  289. wait_time => $initial_wait,
  290. creates => $creates,
  291. unless => $unless,
  292. onlyif => $onlyif,
  293. path => $path
  294. }
  295. }
  296. exec { $name :
  297. command => $command,
  298. refreshonly => $refreshonly,
  299. path => $path,
  300. user => $user,
  301. creates => $creates,
  302. unless => $unless,
  303. onlyif => $onlyif,
  304. tries => $tries,
  305. timeout => $timeout,
  306. try_sleep => $try_sleep,
  307. logoutput => $logoutput,
  308. cwd => $cwd
  309. }
  310. anchor{ "hdp::exec::${name}::begin":} -> Exec[$name] -> anchor{ "hdp::exec::${name}::end":}
  311. if (($initial_wait != undef) and ($initial_wait != "undef")) {
  312. Anchor["hdp::exec::${name}::begin"] -> Hdp::Wait["service ${name}"] -> Exec[$name]
  313. }
  314. }
  315. #### utilities for waits
  316. define hdp::wait(
  317. $wait_time,
  318. $creates = undef,
  319. $unless = undef,
  320. $onlyif = undef,
  321. $path = undef #used for unless
  322. )
  323. {
  324. exec { "wait ${name} ${wait_time}" :
  325. command => "/bin/sleep ${wait_time}",
  326. creates => $creates,
  327. unless => $unless,
  328. onlyif => $onlyif,
  329. path => $path
  330. }
  331. }
  332. ##### temp
  333. class hdp::iptables($ensure)
  334. {
  335. #TODO: just temp so not considering things like saving firewall rules
  336. service { 'iptables':
  337. ensure => $ensure
  338. }
  339. }