hdfs-rbf-default.xml 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045
  1. <?xml version="1.0"?>
  2. <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
  3. <!--
  4. Licensed to the Apache Software Foundation (ASF) under one or more
  5. contributor license agreements. See the NOTICE file distributed with
  6. this work for additional information regarding copyright ownership.
  7. The ASF licenses this file to You under the Apache License, Version 2.0
  8. (the "License"); you may not use this file except in compliance with
  9. the License. You may obtain a copy of the License at
  10. http://www.apache.org/licenses/LICENSE-2.0
  11. Unless required by applicable law or agreed to in writing, software
  12. distributed under the License is distributed on an "AS IS" BASIS,
  13. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. See the License for the specific language governing permissions and
  15. limitations under the License.
  16. -->
  17. <!-- Do not modify this file directly. Instead, copy entries that you -->
  18. <!-- wish to modify from this file into hdfs-rbf-site.xml and change them -->
  19. <!-- there. If hdfs-rbf-site.xml does not already exist, create it. -->
  20. <configuration>
  21. <property>
  22. <name>dfs.federation.router.default.nameserviceId</name>
  23. <value></value>
  24. <description>
  25. Nameservice identifier of the default subcluster to monitor.
  26. </description>
  27. </property>
  28. <property>
  29. <name>dfs.federation.router.default.nameservice.enable</name>
  30. <value>true</value>
  31. <description>
  32. The default subcluster is enabled to read and write files.
  33. </description>
  34. </property>
  35. <property>
  36. <name>dfs.federation.router.rpc.enable</name>
  37. <value>true</value>
  38. <description>
  39. If true, the RPC service to handle client requests in the router is
  40. enabled.
  41. </description>
  42. </property>
  43. <property>
  44. <name>dfs.federation.router.async.rpc.enable</name>
  45. <value>false</value>
  46. <description>
  47. If true, router will process the RPC request asynchronously.
  48. </description>
  49. </property>
  50. <property>
  51. <name>dfs.federation.router.async.rpc.ns.handler.count</name>
  52. <value></value>
  53. <description>
  54. The number of asynchronous handlers per nameservice, separated by commas, internally separated by colons.
  55. The identifier of nameservice is in dfs.nameservices configuration entry.
  56. Such as: ns1:count1,ns2:count2,ns3:count3.
  57. </description>
  58. </property>
  59. <property>
  60. <name>dfs.federation.router.async.rpc.responder.count</name>
  61. <value>10</value>
  62. <description>
  63. For those nameservices not in dfs.federation.router.async.rpc.ns.handler.count configuration entry,
  64. use this value as the asynchronous handler thread counts.
  65. </description>
  66. </property>
  67. <property>
  68. <name>dfs.federation.router.async.rpc.responder.count</name>
  69. <value>10</value>
  70. <description>
  71. The thread counts of async responder executor.
  72. </description>
  73. </property>
  74. <property>
  75. <name>dfs.federation.router.rpc-address</name>
  76. <value>0.0.0.0:8888</value>
  77. <description>
  78. RPC address that handles all clients requests.
  79. The value of this property will take the form of router-host1:rpc-port.
  80. </description>
  81. </property>
  82. <property>
  83. <name>dfs.federation.router.rpc-bind-host</name>
  84. <value></value>
  85. <description>
  86. The actual address the RPC server will bind to. If this optional address is
  87. set, it overrides only the hostname portion of
  88. dfs.federation.router.rpc-address. This is useful for making the name node
  89. listen on all interfaces by setting it to 0.0.0.0.
  90. </description>
  91. </property>
  92. <property>
  93. <name>dfs.federation.router.handler.count</name>
  94. <value>10</value>
  95. <description>
  96. The number of server threads for the router to handle RPC requests from
  97. clients.
  98. </description>
  99. </property>
  100. <property>
  101. <name>dfs.federation.router.handler.queue.size</name>
  102. <value>100</value>
  103. <description>
  104. The size of the queue for the number of handlers to handle RPC client requests.
  105. </description>
  106. </property>
  107. <property>
  108. <name>dfs.federation.router.reader.count</name>
  109. <value>1</value>
  110. <description>
  111. The number of readers for the router to handle RPC client requests.
  112. </description>
  113. </property>
  114. <property>
  115. <name>dfs.federation.router.reader.queue.size</name>
  116. <value>100</value>
  117. <description>
  118. The size of the queue for the number of readers for the router to handle RPC client requests.
  119. </description>
  120. </property>
  121. <property>
  122. <name>dfs.federation.router.async.rpc.handler.count</name>
  123. <value>10</value>
  124. <description>
  125. The number of async handler for the router to handle RPC client requests.
  126. </description>
  127. </property>
  128. <property>
  129. <name>dfs.federation.router.async.rpc.responder.count</name>
  130. <value>10</value>
  131. <description>
  132. The number of async responder for the router to handle responses.
  133. </description>
  134. </property>
  135. <property>
  136. <name>dfs.federation.router.async.rpc.max.asynccall.permit</name>
  137. <value>20000</value>
  138. <description>
  139. Maximum number of asynchronous RPC requests the Router can send to
  140. one downstream nameservice.
  141. </description>
  142. </property>
  143. <property>
  144. <name>dfs.federation.router.connection.creator.queue-size</name>
  145. <value>100</value>
  146. <description>
  147. Size of async connection creator queue.
  148. </description>
  149. </property>
  150. <property>
  151. <name>dfs.federation.router.connection.pool-size</name>
  152. <value>1</value>
  153. <description>
  154. Size of the pool of connections from the router to namenodes.
  155. </description>
  156. </property>
  157. <property>
  158. <name>dfs.federation.router.connection.min-active-ratio</name>
  159. <value>0.5f</value>
  160. <description>
  161. Minimum active ratio of connections from the router to namenodes.
  162. </description>
  163. </property>
  164. <property>
  165. <name>dfs.federation.router.connection.clean.ms</name>
  166. <value>10000</value>
  167. <description>
  168. Time interval, in milliseconds, to check if the connection pool should
  169. remove unused connections.
  170. </description>
  171. </property>
  172. <property>
  173. <name>dfs.federation.router.enable.multiple.socket</name>
  174. <value>false</value>
  175. <description>
  176. If enable multiple downstream socket or not. If true, ConnectionPool
  177. will use a new socket when creating a new connection for the same user,
  178. and RouterRPCClient will get a better throughput. It's best used with
  179. dfs.federation.router.max.concurrency.per.connection together to get
  180. a better throughput with fewer sockets. Such as enable
  181. dfs.federation.router.enable.multiple.socket and
  182. set dfs.federation.router.max.concurrency.per.connection = 20.
  183. </description>
  184. </property>
  185. <property>
  186. <name>dfs.federation.router.max.concurrency.per.connection</name>
  187. <value>1</value>
  188. <description>
  189. The maximum number of requests that a connection can handle concurrently.
  190. When the number of requests being processed by a socket is less than this value,
  191. new request will be processed by this socket. When enable
  192. dfs.federation.router.enable.multiple.socket, it's best
  193. set this value greater than 1, such as 20, to avoid frequent
  194. creation and idle sockets in the case of a NS with jitter requests.
  195. </description>
  196. </property>
  197. <property>
  198. <name>dfs.federation.router.connection.pool.clean.ms</name>
  199. <value>60000</value>
  200. <description>
  201. Time interval, in milliseconds, to check if the connection manager should
  202. remove unused connection pools.
  203. </description>
  204. </property>
  205. <property>
  206. <name>dfs.federation.router.metrics.enable</name>
  207. <value>true</value>
  208. <description>
  209. If the metrics in the router are enabled.
  210. </description>
  211. </property>
  212. <property>
  213. <name>dfs.federation.router.dn-report.time-out</name>
  214. <value>1000</value>
  215. <description>
  216. Time out, in milliseconds for getDatanodeReport.
  217. </description>
  218. </property>
  219. <property>
  220. <name>dfs.federation.router.dn-report.cache-expire</name>
  221. <value>10s</value>
  222. <description>
  223. Expiration time in seconds for datanodereport.
  224. </description>
  225. </property>
  226. <property>
  227. <name>dfs.federation.router.enable.get.dn.usage</name>
  228. <value>true</value>
  229. <description>
  230. If true, the getNodeUsage method in RBFMetrics will return an up-to-date
  231. result collecting from downstream nameservices. But it will take a long
  232. time and take up thread resources. If false, it will return a mock result with all 0.
  233. </description>
  234. </property>
  235. <property>
  236. <name>dfs.federation.router.metrics.class</name>
  237. <value>org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor</value>
  238. <description>
  239. Class to monitor the RPC system in the router. It must implement the
  240. RouterRpcMonitor interface.
  241. </description>
  242. </property>
  243. <property>
  244. <name>dfs.federation.router.admin.enable</name>
  245. <value>true</value>
  246. <description>
  247. If true, the RPC admin service to handle client requests in the router is
  248. enabled.
  249. </description>
  250. </property>
  251. <property>
  252. <name>dfs.federation.router.admin-address</name>
  253. <value>0.0.0.0:8111</value>
  254. <description>
  255. RPC address that handles the admin requests.
  256. The value of this property will take the form of router-host1:rpc-port.
  257. </description>
  258. </property>
  259. <property>
  260. <name>dfs.federation.router.admin-bind-host</name>
  261. <value></value>
  262. <description>
  263. The actual address the RPC admin server will bind to. If this optional
  264. address is set, it overrides only the hostname portion of
  265. dfs.federation.router.admin-address. This is useful for making the name
  266. node listen on all interfaces by setting it to 0.0.0.0.
  267. </description>
  268. </property>
  269. <property>
  270. <name>dfs.federation.router.admin.handler.count</name>
  271. <value>1</value>
  272. <description>
  273. The number of server threads for the router to handle RPC requests from
  274. admin.
  275. </description>
  276. </property>
  277. <property>
  278. <name>dfs.federation.router.admin.mount.check.enable</name>
  279. <value>false</value>
  280. <description>
  281. If true, add/update mount table will include a destination check to make
  282. sure the file exists in downstream namenodes, and changes to mount table
  283. will fail if the file doesn't exist in any of the destination namenode.
  284. </description>
  285. </property>
  286. <property>
  287. <name>dfs.federation.router.http-address</name>
  288. <value>0.0.0.0:50071</value>
  289. <description>
  290. HTTP address that handles the web requests to the Router.
  291. The value of this property will take the form of router-host1:http-port.
  292. </description>
  293. </property>
  294. <property>
  295. <name>dfs.federation.router.http-bind-host</name>
  296. <value></value>
  297. <description>
  298. The actual address the HTTP server will bind to. If this optional
  299. address is set, it overrides only the hostname portion of
  300. dfs.federation.router.http-address. This is useful for making the name
  301. node listen on all interfaces by setting it to 0.0.0.0.
  302. </description>
  303. </property>
  304. <property>
  305. <name>dfs.federation.router.https-address</name>
  306. <value>0.0.0.0:50072</value>
  307. <description>
  308. HTTPS address that handles the web requests to the Router.
  309. The value of this property will take the form of router-host1:https-port.
  310. </description>
  311. </property>
  312. <property>
  313. <name>dfs.federation.router.https-bind-host</name>
  314. <value></value>
  315. <description>
  316. The actual address the HTTPS server will bind to. If this optional
  317. address is set, it overrides only the hostname portion of
  318. dfs.federation.router.https-address. This is useful for making the name
  319. node listen on all interfaces by setting it to 0.0.0.0.
  320. </description>
  321. </property>
  322. <property>
  323. <name>dfs.federation.router.http.enable</name>
  324. <value>true</value>
  325. <description>
  326. If the HTTP service to handle client requests in the router is enabled.
  327. </description>
  328. </property>
  329. <property>
  330. <name>dfs.federation.router.fs-limits.max-component-length</name>
  331. <value>0</value>
  332. <description>
  333. Defines the maximum number of bytes in UTF-8 encoding in each
  334. component of a path at Router side. A value of 0 will disable the check.
  335. Support multiple size unit suffix(case insensitive). It is act as
  336. configuration dfs.namenode.fs-limits.max-component-length at NameNode
  337. side.
  338. </description>
  339. </property>
  340. <property>
  341. <name>dfs.federation.router.file.resolver.client.class</name>
  342. <value>org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver</value>
  343. <description>
  344. Class to resolve files to subclusters. To enable multiple subclusters for a mount point,
  345. set to org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver.
  346. </description>
  347. </property>
  348. <property>
  349. <name>dfs.federation.router.namenode.resolver.client.class</name>
  350. <value>org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver</value>
  351. <description>
  352. Class to resolve the namenode for a subcluster.
  353. </description>
  354. </property>
  355. <property>
  356. <name>dfs.federation.router.store.enable</name>
  357. <value>true</value>
  358. <description>
  359. If true, the Router connects to the State Store.
  360. </description>
  361. </property>
  362. <property>
  363. <name>dfs.federation.router.store.serializer</name>
  364. <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl</value>
  365. <description>
  366. Class to serialize State Store records.
  367. </description>
  368. </property>
  369. <property>
  370. <name>dfs.federation.router.store.driver.class</name>
  371. <value>org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl</value>
  372. <description>
  373. Class to implement the State Store. There are three implementation classes currently
  374. being supported:
  375. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl,
  376. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl,
  377. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreMySQLImpl and
  378. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
  379. These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively.
  380. By default it uses the ZooKeeper as the default State Store.
  381. </description>
  382. </property>
  383. <property>
  384. <name>dfs.federation.router.store.driver.async.override.max.threads</name>
  385. <value>-1</value>
  386. <description>
  387. Number of threads used by StateStoreDriver to overwrite and delete records asynchronously.
  388. Only used by MembershipStore and RouterStore. Non-positive values will make StateStoreDriver
  389. run in sync mode.
  390. </description>
  391. </property>
  392. <property>
  393. <name>dfs.federation.router.store.connection.test</name>
  394. <value>60000</value>
  395. <description>
  396. How often to check for the connection to the State Store in milliseconds.
  397. </description>
  398. </property>
  399. <property>
  400. <name>dfs.federation.router.store.driver.zk.address</name>
  401. <description>
  402. Host:Port of the ZooKeeper for StateStoreZooKeeperImpl.
  403. </description>
  404. </property>
  405. <property>
  406. <name>dfs.federation.router.store.driver.zk.parent-path</name>
  407. <value>/hdfs-federation</value>
  408. <description>
  409. The parent path of zookeeper for StateStoreZooKeeperImpl.
  410. </description>
  411. </property>
  412. <property>
  413. <name>dfs.federation.router.store.driver.zk.async.max.threads</name>
  414. <value>-1</value>
  415. <description>
  416. Max threads number of StateStoreZooKeeperImpl in async mode.
  417. The only class currently being supported:
  418. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl.
  419. Default value is -1, which means StateStoreZooKeeperImpl is working in sync mode.
  420. Use positive integer value to enable async mode.
  421. </description>
  422. </property>
  423. <property>
  424. <name>dfs.federation.router.cache.ttl</name>
  425. <value>1m</value>
  426. <description>
  427. How often to refresh the State Store caches in milliseconds. This setting
  428. supports multiple time unit suffixes as described in
  429. dfs.heartbeat.interval. If no suffix is specified then milliseconds is
  430. assumed.
  431. </description>
  432. </property>
  433. <property>
  434. <name>dfs.federation.router.store.membership.expiration</name>
  435. <value>300000</value>
  436. <description>
  437. Expiration time in milliseconds for a membership record.
  438. </description>
  439. </property>
  440. <property>
  441. <name>dfs.federation.router.store.membership.expiration.deletion</name>
  442. <value>-1</value>
  443. <description>
  444. Deletion time in milliseconds for a membership record. If an expired
  445. membership record exists beyond this time, it will be deleted. If this
  446. value is negative, the deletion is disabled.
  447. </description>
  448. </property>
  449. <property>
  450. <name>dfs.federation.router.heartbeat.enable</name>
  451. <value>true</value>
  452. <description>
  453. If true, the Router heartbeats into the State Store.
  454. </description>
  455. </property>
  456. <property>
  457. <name>dfs.federation.router.heartbeat.interval</name>
  458. <value>5000</value>
  459. <description>
  460. How often the Router should heartbeat into the State Store in milliseconds.
  461. </description>
  462. </property>
  463. <property>
  464. <name>dfs.federation.router.health.monitor.timeout</name>
  465. <value>30s</value>
  466. <description>
  467. Time out for Router to obtain HAServiceStatus from NameNode.
  468. </description>
  469. </property>
  470. <property>
  471. <name>dfs.federation.router.heartbeat-state.interval</name>
  472. <value>5s</value>
  473. <description>
  474. How often the Router should heartbeat its state into the State Store in
  475. milliseconds. This setting supports multiple time unit suffixes as
  476. described in dfs.federation.router.quota-cache.update.interval.
  477. </description>
  478. </property>
  479. <property>
  480. <name>dfs.federation.router.namenode.heartbeat.enable</name>
  481. <value></value>
  482. <description>
  483. If true, get namenode heartbeats and send into the State Store.
  484. If not explicitly specified takes the same value as for
  485. dfs.federation.router.heartbeat.enable.
  486. </description>
  487. </property>
  488. <property>
  489. <name>dfs.federation.router.namenode.heartbeat.jmx.interval</name>
  490. <value>0</value>
  491. <description>
  492. How often the Router should request JMX reports from the Namenode in miliseconds.
  493. If this value is 0, it will request JMX reports every time a Namenode report is requested.
  494. If this value is negative, it will disable JMX reports from the Namenode.
  495. </description>
  496. </property>
  497. <property>
  498. <name>dfs.federation.router.store.router.expiration</name>
  499. <value>5m</value>
  500. <description>
  501. Expiration time in milliseconds for a router state record. This setting
  502. supports multiple time unit suffixes as described in
  503. dfs.federation.router.quota-cache.update.interval.
  504. </description>
  505. </property>
  506. <property>
  507. <name>dfs.federation.router.store.router.expiration.deletion</name>
  508. <value>-1</value>
  509. <description>
  510. Deletion time in milliseconds for a router state record. If an expired
  511. router state record exists beyond this time, it will be deleted. If this
  512. value is negative, the deletion is disabled.
  513. </description>
  514. </property>
  515. <property>
  516. <name>dfs.federation.router.safemode.enable</name>
  517. <value>true</value>
  518. <description>
  519. </description>
  520. </property>
  521. <property>
  522. <name>dfs.federation.router.safemode.extension</name>
  523. <value>30s</value>
  524. <description>
  525. Time after startup that the Router is in safe mode. This setting
  526. supports multiple time unit suffixes as described in
  527. dfs.heartbeat.interval. If no suffix is specified then milliseconds is
  528. assumed.
  529. </description>
  530. </property>
  531. <property>
  532. <name>dfs.federation.router.safemode.expiration</name>
  533. <value>3m</value>
  534. <description>
  535. Time without being able to reach the State Store to enter safe mode. This
  536. setting supports multiple time unit suffixes as described in
  537. dfs.heartbeat.interval. If no suffix is specified then milliseconds is
  538. assumed.
  539. </description>
  540. </property>
  541. <property>
  542. <name>dfs.federation.router.safemode.checkperiod</name>
  543. <value>5s</value>
  544. <description>
  545. How often the Router should check safe mode. This
  546. setting supports multiple time unit suffixes as described in
  547. dfs.heartbeat.interval. If no suffix is specified then milliseconds is
  548. assumed.
  549. </description>
  550. </property>
  551. <property>
  552. <name>dfs.federation.router.monitor.namenode</name>
  553. <value></value>
  554. <description>
  555. The identifier of the namenodes to monitor and heartbeat.
  556. </description>
  557. </property>
  558. <property>
  559. <name>dfs.federation.router.monitor.namenode.nameservice.resolution-enabled</name>
  560. <value>false</value>
  561. <description>
  562. Determines if the given monitored namenode address is a domain name which needs to
  563. be resolved.
  564. This is used by router to resolve namenodes.
  565. </description>
  566. </property>
  567. <property>
  568. <name>dfs.federation.router.monitor.namenode.nameservice.resolver.impl</name>
  569. <value></value>
  570. <description>
  571. Nameservice resolver implementation used by router.
  572. Effective with
  573. dfs.federation.router.monitor.namenode.nameservices.resolution-enabled on.
  574. </description>
  575. </property>
  576. <property>
  577. <name>dfs.federation.router.monitor.localnamenode.enable</name>
  578. <value>true</value>
  579. <description>
  580. If true, the Router should monitor the namenode in the local machine.
  581. </description>
  582. </property>
  583. <property>
  584. <name>dfs.federation.router.mount-table.max-cache-size</name>
  585. <value>10000</value>
  586. <description>
  587. Maximum number of mount table cache entries to have.
  588. By default, remove cache entries if we have more than 10k.
  589. </description>
  590. </property>
  591. <property>
  592. <name>dfs.federation.router.mount-table.cache.enable</name>
  593. <value>true</value>
  594. <description>
  595. Set to true to enable mount table cache (Path to Remote Location cache).
  596. Disabling the cache is recommended when a large amount of unique paths are queried.
  597. </description>
  598. </property>
  599. <property>
  600. <name>dfs.federation.router.quota.enable</name>
  601. <value>false</value>
  602. <description>
  603. Set to true to enable quota system in Router. When it's enabled, setting
  604. or clearing sub-cluster's quota directly is not recommended since Router
  605. Admin server will override sub-cluster's quota with global quota.
  606. </description>
  607. </property>
  608. <property>
  609. <name>dfs.federation.router.quota-cache.update.interval</name>
  610. <value>60s</value>
  611. <description>
  612. Interval time for updating quota usage cache in Router.
  613. This property is used only if the value of
  614. dfs.federation.router.quota.enable is true.
  615. This setting supports multiple time unit suffixes as described
  616. in dfs.heartbeat.interval. If no suffix is specified then milliseconds
  617. is assumed.
  618. </description>
  619. </property>
  620. <property>
  621. <name>dfs.federation.router.client.thread-size</name>
  622. <value>32</value>
  623. <description>
  624. Max threads size for the RouterClient to execute concurrent
  625. requests.
  626. </description>
  627. </property>
  628. <property>
  629. <name>dfs.federation.router.client.retry.max.attempts</name>
  630. <value>3</value>
  631. <description>
  632. Max retry attempts for the RouterClient talking to the Router.
  633. </description>
  634. </property>
  635. <property>
  636. <name>dfs.federation.router.client.reject.overload</name>
  637. <value>false</value>
  638. <description>
  639. Set to true to reject client requests when we run out of RPC client
  640. threads.
  641. </description>
  642. </property>
  643. <property>
  644. <name>dfs.federation.router.client.allow-partial-listing</name>
  645. <value>true</value>
  646. <description>
  647. If the Router can return a partial list of files in a multi-destination mount point when one of the subclusters is unavailable.
  648. True may return a partial list of files if a subcluster is down.
  649. False will fail the request if one is unavailable.
  650. </description>
  651. </property>
  652. <property>
  653. <name>dfs.federation.router.client.mount-status.time-out</name>
  654. <value>1s</value>
  655. <description>
  656. Set a timeout for the Router when listing folders containing mount
  657. points. In this process, the Router checks the mount table and then it
  658. checks permissions in the subcluster. After the time out, we return the
  659. default values.
  660. </description>
  661. </property>
  662. <property>
  663. <name>dfs.federation.router.connect.max.retries.on.timeouts</name>
  664. <value>0</value>
  665. <description>
  666. Maximum number of retries for the IPC Client when connecting to the
  667. subclusters. By default, it doesn't let the IPC retry and the Router
  668. handles it.
  669. </description>
  670. </property>
  671. <property>
  672. <name>dfs.federation.router.connect.timeout</name>
  673. <value>2s</value>
  674. <description>
  675. Time out for the IPC client connecting to the subclusters. This should be
  676. short as the Router has knowledge of the state of the Routers.
  677. </description>
  678. </property>
  679. <property>
  680. <name>dfs.federation.router.keytab.file</name>
  681. <value></value>
  682. <description>
  683. The keytab file used by router to login as its
  684. service principal. The principal name is configured with
  685. dfs.federation.router.kerberos.principal.
  686. </description>
  687. </property>
  688. <property>
  689. <name>dfs.federation.router.kerberos.principal</name>
  690. <value></value>
  691. <description>
  692. The Router service principal. This is typically set to
  693. router/_HOST@REALM.TLD. Each Router will substitute _HOST with its
  694. own fully qualified hostname at startup. The _HOST placeholder
  695. allows using the same configuration setting on both Router
  696. in an HA setup.
  697. </description>
  698. </property>
  699. <property>
  700. <name>dfs.federation.router.kerberos.principal.hostname</name>
  701. <value></value>
  702. <description>
  703. Optional. The hostname for the Router containing this
  704. configuration file. Will be different for each machine.
  705. Defaults to current hostname.
  706. </description>
  707. </property>
  708. <property>
  709. <name>dfs.federation.router.kerberos.internal.spnego.principal</name>
  710. <value>${dfs.web.authentication.kerberos.principal}</value>
  711. <description>
  712. The server principal used by the Router for web UI SPNEGO
  713. authentication when Kerberos security is enabled. This is
  714. typically set to HTTP/_HOST@REALM.TLD The SPNEGO server principal
  715. begins with the prefix HTTP/ by convention.
  716. If the value is '*', the web server will attempt to login with
  717. every principal specified in the keytab file
  718. dfs.web.authentication.kerberos.keytab.
  719. </description>
  720. </property>
  721. <property>
  722. <name>dfs.federation.router.mount-table.cache.update</name>
  723. <value>false</value>
  724. <description>Set true to enable MountTableRefreshService. This service
  725. updates mount table cache immediately after adding, modifying or
  726. deleting the mount table entries. If this service is not enabled
  727. mount table cache are refreshed periodically by
  728. StateStoreCacheUpdateService
  729. </description>
  730. </property>
  731. <property>
  732. <name>dfs.federation.router.mount-table.cache.update.timeout</name>
  733. <value>1m</value>
  734. <description>This property defines how long to wait for all the
  735. admin servers to finish their mount table cache update. This setting
  736. supports multiple time unit suffixes as described in
  737. dfs.federation.router.safemode.extension.
  738. </description>
  739. </property>
  740. <property>
  741. <name>dfs.federation.router.mount-table.cache.update.client.max.time
  742. </name>
  743. <value>5m</value>
  744. <description>Remote router mount table cache is updated through
  745. RouterClient(RPC call). To improve performance, RouterClient
  746. connections are cached but it should not be kept in cache forever.
  747. This property defines the max time a connection can be cached. This
  748. setting supports multiple time unit suffixes as described in
  749. dfs.federation.router.safemode.extension.
  750. </description>
  751. </property>
  752. <property>
  753. <name>dfs.federation.router.secret.manager.class</name>
  754. <value>org.apache.hadoop.hdfs.server.federation.router.security.token.ZKDelegationTokenSecretManagerImpl</value>
  755. <description>
  756. Class to implement state store to delegation tokens.
  757. Default implementation uses zookeeper as the backend to store delegation tokens.
  758. </description>
  759. </property>
  760. <property>
  761. <name>dfs.federation.router.top.num.token.realowners</name>
  762. <value>10</value>
  763. <description>
  764. The number of top real owners by tokens count to report in the JMX metrics.
  765. Real owners are the effective users whose cretential are used to generate
  766. the tokens.
  767. </description>
  768. </property>
  769. <property>
  770. <name>dfs.federation.router.fairness.policy.controller.class</name>
  771. <value>org.apache.hadoop.hdfs.server.federation.fairness.NoRouterRpcFairnessPolicyController</value>
  772. <description>
  773. No fairness policy handler by default, for fairness
  774. StaticFairnessPolicyController should be configured.
  775. </description>
  776. </property>
  777. <property>
  778. <name>dfs.federation.router.fairness.handler.count.EXAMPLENAMESERVICE</name>
  779. <value></value>
  780. <description>
  781. Dedicated handler count for nameservice EXAMPLENAMESERVICE. The handler
  782. (configed by dfs.federation.router.handler.count)resource is controlled
  783. internally by Semaphore permits. Two requirements have to be satisfied.
  784. 1) all downstream nameservices need this config otherwise no permit will
  785. be given thus not proxy will happen. 2) if a special *concurrent*
  786. nameservice is specified, the sum of all configured values is smaller or
  787. equal to the total number of router handlers; if the special *concurrent*
  788. is not specified, the sum of all configured values must be strictly
  789. smaller than the router handlers thus the left will be allocated to the
  790. concurrent calls.
  791. </description>
  792. </property>
  793. <property>
  794. <name>dfs.federation.router.fairness.acquire.timeout</name>
  795. <value>1s</value>
  796. <description>
  797. The maximum time to wait for a permit.
  798. </description>
  799. </property>
  800. <property>
  801. <name>dfs.federation.router.fairness.handler.proportion.EXAMPLENAMESERVICE</name>
  802. <value>0.1</value>
  803. <description>
  804. Dedicated handler proportion for nameservice EXAMPLENAMESERVICE.
  805. The range of this value is [0, 1], and the data type is float.
  806. If this value is configured as x, and the total number of handlers
  807. (configed by dfs.federation.router.handler.count) of the router is y,
  808. then the maximum number of handlers for the EXAMPLENAMESERVICE is z=(int) x*y;
  809. If z is 0, z is reset to 1, ensuring the ns has at least one handler.
  810. </description>
  811. </property>
  812. <property>
  813. <name>dfs.federation.router.federation.rename.bandwidth</name>
  814. <value>10</value>
  815. <description>
  816. Specify bandwidth per map in MB.
  817. </description>
  818. </property>
  819. <property>
  820. <name>dfs.federation.router.federation.rename.map</name>
  821. <value>10</value>
  822. <description>
  823. Max number of concurrent maps to use for copy.
  824. </description>
  825. </property>
  826. <property>
  827. <name>dfs.federation.router.federation.rename.delay</name>
  828. <value>1000</value>
  829. <description>
  830. Specify the delayed duration(millie seconds) when the job needs to retry.
  831. </description>
  832. </property>
  833. <property>
  834. <name>dfs.federation.router.federation.rename.diff</name>
  835. <value>0</value>
  836. <description>
  837. Specify the threshold of the diff entries that used in incremental copy
  838. stage.
  839. </description>
  840. </property>
  841. <property>
  842. <name>dfs.federation.router.federation.rename.option</name>
  843. <value>NONE</value>
  844. <description>
  845. Specify the action when rename across namespaces. The option can be NONE
  846. and DISTCP.
  847. </description>
  848. </property>
  849. <property>
  850. <name>dfs.federation.router.federation.rename.force.close.open.file</name>
  851. <value>true</value>
  852. <description>
  853. Force close all open files when there is no diff in the DIFF_DISTCP stage.
  854. </description>
  855. </property>
  856. <property>
  857. <name>dfs.federation.router.federation.rename.trash</name>
  858. <value>trash</value>
  859. <description>
  860. This options has 3 values: trash (move the source path to trash), delete
  861. (delete the source path directly) and skip (skip both trash and deletion).
  862. </description>
  863. </property>
  864. <property>
  865. <name>dfs.federation.router.observer.read.default</name>
  866. <value>false</value>
  867. <description>
  868. Whether observer reads are enabled. This is a default for all nameservices.
  869. The default can be inverted for individual namespace by adding them to
  870. dfs.federation.router.observer.read.overrides.
  871. </description>
  872. </property>
  873. <property>
  874. <name>dfs.federation.router.observer.read.overrides</name>
  875. <value/>
  876. <description>
  877. Commas separated list of namespaces for which to invert the default configuration,
  878. dfs.federation.router.observer.read.default, for whether to enable observer reads.
  879. </description>
  880. </property>
  881. <property>
  882. <name>dfs.federation.router.observer.federated.state.propagation.maxsize</name>
  883. <value>5</value>
  884. <description>
  885. The maximum size of the federated state to send in the RPC header. Sending the federated
  886. state removes the need to msync on every read call, but at the expense of having a larger
  887. header. The cost tradeoff between the larger header and always msync'ing depends on the number
  888. of namespaces in use and the latency of the msync requests.
  889. </description>
  890. </property>
  891. <property>
  892. <name>dfs.federation.router.observer.state.id.refresh.period</name>
  893. <value>15s</value>
  894. <description>
  895. Period to refresh namespace stateID using active namenode. This ensures the
  896. namespace stateID is refresh even when an observer is trailing behind.
  897. If this is below 0, the auto-refresh is disabled.
  898. </description>
  899. </property>
  900. <property>
  901. <name>dfs.federation.router.store.driver.file.async.threads</name>
  902. <value>0</value>
  903. <description>
  904. Max threads used by StateStoreFileImpl to access state store files concurrently.
  905. The only class currently being supported:
  906. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl.
  907. Default value is 0, which means StateStoreFileImpl would work in sync mode, meaning it
  908. would access one file at a time.
  909. Use positive integer value to enable concurrent files access.
  910. </description>
  911. </property>
  912. <property>
  913. <name>dfs.federation.router.store.driver.fs.async.threads</name>
  914. <value>0</value>
  915. <description>
  916. Max threads used by StateStoreFileSystemImpl to access state store files from the given
  917. filesystem concurrently.
  918. The only class currently being supported:
  919. org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl.
  920. Default value is 0, which means StateStoreFileSystemImpl would work in sync mode, meaning it
  921. would access one file from the filesystem at a time.
  922. Use positive integer value to enable concurrent files access from the given filesystem.
  923. </description>
  924. </property>
  925. <property>
  926. <name>dfs.federation.router.heartbeat.with.ip.enable</name>
  927. <description>
  928. Make router use IP instead of host when communicating with router state state store.
  929. </description>
  930. <value>false</value>
  931. </property>
  932. </configuration>