capacity-scheduler_default.js 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. module.exports = [
  19. {
  20. name: 'mapred.capacity-scheduler.maximum-system-jobs',
  21. value: 3000,
  22. description: 'Maximum number of jobs in the system which can be initialized, concurrently, by the CapacityScheduler.'
  23. },
  24. {
  25. name: 'mapred.capacity-scheduler.queue.default.capacity',
  26. value: 100,
  27. description: 'Percentage of the number of slots in the cluster that are to be available for jobs in this queue.'
  28. },
  29. {
  30. name: 'mapred.capacity-scheduler.queue.default.maximum-capacity',
  31. value: -1,
  32. description: 'maximum-capacity defines a limit beyond which a queue cannot use the capacity of the cluster.' +
  33. 'This provides a means to limit how much excess capacity a queue can use. By default, there is no limit.' +
  34. 'The maximum-capacity of a queue can only be greater than or equal to its minimum capacity.' +
  35. 'Default value of -1 implies a queue can use complete capacity of the cluster.' +
  36. '\n' +
  37. 'This property could be to curtail certain jobs which are long running in nature from occupying more than a' +
  38. 'certain percentage of the cluster, which in the absence of pre-emption, could lead to capacity guarantees of ' +
  39. 'other queues being affected.' +
  40. '\n' +
  41. 'One important thing to note is that maximum-capacity is a percentage , so based on the cluster\'s capacity' +
  42. 'the max capacity would change. So if large no of nodes or racks get added to the cluster , max Capacity in' +
  43. 'absolute terms would increase accordingly.'
  44. },
  45. {
  46. name: 'mapred.capacity-scheduler.queue.default.supports-priority',
  47. value: false,
  48. description: 'If true, priorities of jobs will be taken into account in scheduling decisions.'
  49. },
  50. {
  51. name: 'mapred.capacity-scheduler.queue.default.minimum-user-limit-percent',
  52. value: 100,
  53. description: 'Each queue enforces a limit on the percentage of resources' +
  54. 'allocated to a user at any given time, if there is competition for them.' +
  55. 'This user limit can vary between a minimum and maximum value. The former' +
  56. 'depends on the number of users who have submitted jobs, and the latter is' +
  57. 'set to this property value.'
  58. },
  59. {
  60. name: 'mapred.capacity-scheduler.queue.default.user-limit-factor',
  61. value: 1,
  62. description: 'The multiple of the queue capacity which can be configured to' +
  63. 'allow a single user to acquire more slots.'
  64. },
  65. {
  66. name: 'mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks',
  67. value: 200000,
  68. description: 'The maximum number of tasks, across all jobs in the queue,' +
  69. 'which can be initialized concurrently. Once the queue\'s jobs exceed this' +
  70. 'limit they will be queued on disk. '
  71. },
  72. {
  73. name: 'mapred.capacity-scheduler.queue.default.maximum-initialized-active-tasks-per-user',
  74. value: 100000,
  75. description: 'The maximum number of tasks per-user, across all the of the' +
  76. 'user\'s jobs in the queue, which can be initialized concurrently. Once the' +
  77. 'user\'s jobs exceed this limit they will be queued on disk.'
  78. },
  79. {
  80. name: 'mapred.capacity-scheduler.queue.default.init-accept-jobs-factor',
  81. value: 10,
  82. description: 'The multipe of (maximum-system-jobs * queue-capacity) used to' +
  83. 'determine the number of jobs which are accepted by the scheduler.'
  84. },
  85. {
  86. name: 'mapred.capacity-scheduler.default-supports-priority',
  87. value: false,
  88. description: 'If true, priorities of jobs will be taken into' +
  89. 'account in scheduling decisions by default in a job queue.'
  90. },
  91. {
  92. name: 'mapred.capacity-scheduler.default-minimum-user-limit-percent',
  93. value: 100,
  94. description: 'The percentage of the resources limited to a particular user' +
  95. 'for the job queue at any given point of time by default.'
  96. },
  97. {
  98. name: 'mapred.capacity-scheduler.default-user-limit-factor',
  99. value: 1,
  100. description: 'The default multiple of queue-capacity which is used to' +
  101. 'determine the amount of slots a single user can consume concurrently.'
  102. },
  103. {
  104. name: 'mapred.capacity-scheduler.default-user-limit-factor',
  105. value: 1,
  106. description: 'The default multiple of queue-capacity which is used to' +
  107. 'determine the amount of slots a single user can consume concurrently.'
  108. },
  109. {
  110. name: 'mapred.capacity-scheduler.default-maximum-active-tasks-per-queue',
  111. value: 200000,
  112. description: 'The default maximum number of tasks, across all jobs in the' +
  113. 'queue, which can be initialized concurrently. Once the queue\'s jobs exceed' +
  114. 'this limit they will be queued on disk.'
  115. },
  116. {
  117. name: 'mapred.capacity-scheduler.default-maximum-active-tasks-per-user',
  118. value: 100000,
  119. description: 'The default maximum number of tasks per-user, across all the of' +
  120. 'the user\'s jobs in the queue, which can be initialized concurrently. Once' +
  121. 'the user\'s jobs exceed this limit they will be queued on disk.'
  122. },
  123. {
  124. name: 'mapred.capacity-scheduler.default-init-accept-jobs-factor',
  125. value: 10,
  126. description: 'The default multipe of (maximum-system-jobs * queue-capacity)' +
  127. 'used to determine the number of jobs which are accepted by the scheduler.'
  128. },
  129. {
  130. name: 'mapred.capacity-scheduler.init-poll-interval',
  131. value: 5000,
  132. description: 'The amount of time in milliseconds which is used to poll' +
  133. 'the job queues for jobs to initialize.'
  134. },
  135. {
  136. name: 'mapred.capacity-scheduler.init-worker-threads',
  137. value: 5,
  138. description: 'Number of worker threads which would be used by' +
  139. 'Initialization poller to initialize jobs in a set of queue.' +
  140. 'If number mentioned in property is equal to number of job queues' +
  141. 'then a single thread would initialize jobs in a queue. If lesser' +
  142. 'then a thread would get a set of queues assigned. If the number' +
  143. 'is greater then number of threads would be equal to number of' +
  144. 'job queues.'
  145. }
  146. ];