DatanodeProtocol.proto 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /**
  19. * These .proto interfaces are private and stable.
  20. * Please see https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html
  21. * for what changes are allowed for a *stable* .proto interface.
  22. */
  23. // This file contains protocol buffers that are used throughout HDFS -- i.e.
  24. // by the client, server, and data transfer protocols.
  25. syntax = "proto2";
  26. option java_package = "org.apache.hadoop.hdfs.protocol.proto";
  27. option java_outer_classname = "DatanodeProtocolProtos";
  28. option java_generic_services = true;
  29. option java_generate_equals_and_hash = true;
  30. package hadoop.hdfs.datanode;
  31. import "hdfs.proto";
  32. import "erasurecoding.proto";
  33. import "HdfsServer.proto";
  34. /**
  35. * Information to identify a datanode to a namenode
  36. */
  37. message DatanodeRegistrationProto {
  38. required DatanodeIDProto datanodeID = 1; // Datanode information
  39. required StorageInfoProto storageInfo = 2; // Node information
  40. required ExportedBlockKeysProto keys = 3; // Block keys
  41. required string softwareVersion = 4; // Software version of the DN, e.g. "2.0.0"
  42. }
  43. /**
  44. * Commands sent from namenode to the datanodes
  45. */
  46. message DatanodeCommandProto {
  47. enum Type {
  48. BalancerBandwidthCommand = 0;
  49. BlockCommand = 1;
  50. BlockRecoveryCommand = 2;
  51. FinalizeCommand = 3;
  52. KeyUpdateCommand = 4;
  53. RegisterCommand = 5;
  54. UnusedUpgradeCommand = 6;
  55. NullDatanodeCommand = 7;
  56. BlockIdCommand = 8;
  57. BlockECReconstructionCommand = 9;
  58. }
  59. required Type cmdType = 1; // Type of the command
  60. // One of the following command is available when the corresponding
  61. // cmdType is set
  62. optional BalancerBandwidthCommandProto balancerCmd = 2;
  63. optional BlockCommandProto blkCmd = 3;
  64. optional BlockRecoveryCommandProto recoveryCmd = 4;
  65. optional FinalizeCommandProto finalizeCmd = 5;
  66. optional KeyUpdateCommandProto keyUpdateCmd = 6;
  67. optional RegisterCommandProto registerCmd = 7;
  68. optional BlockIdCommandProto blkIdCmd = 8;
  69. optional BlockECReconstructionCommandProto blkECReconstructionCmd = 9;
  70. }
  71. /**
  72. * Command sent from namenode to datanode to set the
  73. * maximum bandwidth to be used for balancing.
  74. */
  75. message BalancerBandwidthCommandProto {
  76. // Maximum bandwidth to be used by datanode for balancing
  77. required uint64 bandwidth = 1;
  78. }
  79. /**
  80. * Command to instruct datanodes to perform certain action
  81. * on the given set of blocks.
  82. */
  83. message BlockCommandProto {
  84. enum Action {
  85. TRANSFER = 1; // Transfer blocks to another datanode
  86. INVALIDATE = 2; // Invalidate blocks
  87. SHUTDOWN = 3; // Shutdown the datanode
  88. }
  89. required Action action = 1;
  90. required string blockPoolId = 2;
  91. repeated BlockProto blocks = 3;
  92. repeated DatanodeInfosProto targets = 4;
  93. repeated StorageUuidsProto targetStorageUuids = 5;
  94. repeated StorageTypesProto targetStorageTypes = 6;
  95. }
  96. /**
  97. * Command to instruct datanodes to perform certain action
  98. * on the given set of block IDs.
  99. */
  100. message BlockIdCommandProto {
  101. enum Action {
  102. CACHE = 1;
  103. UNCACHE = 2;
  104. }
  105. required Action action = 1;
  106. required string blockPoolId = 2;
  107. repeated uint64 blockIds = 3 [packed=true];
  108. }
  109. /**
  110. * List of blocks to be recovered by the datanode
  111. */
  112. message BlockRecoveryCommandProto {
  113. repeated RecoveringBlockProto blocks = 1;
  114. }
  115. /**
  116. * Finalize the upgrade at the datanode
  117. */
  118. message FinalizeCommandProto {
  119. required string blockPoolId = 1; // Block pool to be finalized
  120. }
  121. /**
  122. * Update the block keys at the datanode
  123. */
  124. message KeyUpdateCommandProto {
  125. required ExportedBlockKeysProto keys = 1;
  126. }
  127. /**
  128. * Instruct datanode to register with the namenode
  129. */
  130. message RegisterCommandProto {
  131. // void
  132. }
  133. /**
  134. * Block Erasure coding reconstruction command
  135. */
  136. message BlockECReconstructionCommandProto {
  137. repeated BlockECReconstructionInfoProto blockECReconstructioninfo = 1;
  138. }
  139. /**
  140. * registration - Information of the datanode registering with the namenode
  141. */
  142. message RegisterDatanodeRequestProto {
  143. required DatanodeRegistrationProto registration = 1; // Datanode info
  144. }
  145. /**
  146. * registration - Update registration of the datanode that successfully
  147. * registered. StorageInfo will be updated to include new
  148. * storage ID if the datanode did not have one in the request.
  149. */
  150. message RegisterDatanodeResponseProto {
  151. required DatanodeRegistrationProto registration = 1; // Datanode info
  152. }
  153. /**
  154. * failedStorageLocations - storage locations that have failed
  155. * lastVolumeFailureDate - date/time of last volume failure
  156. * estimatedCapacityLost - estimate of total capacity lost due to volume failures
  157. */
  158. message VolumeFailureSummaryProto {
  159. repeated string failedStorageLocations = 1;
  160. required uint64 lastVolumeFailureDate = 2;
  161. required uint64 estimatedCapacityLostTotal = 3;
  162. }
  163. /**
  164. * registration - datanode registration information
  165. * capacity - total storage capacity available at the datanode
  166. * dfsUsed - storage used by HDFS
  167. * remaining - remaining storage available for HDFS
  168. * blockPoolUsed - storage used by the block pool
  169. * xmitsInProgress - number of transfers from this datanode to others
  170. * xceiverCount - number of active transceiver threads
  171. * failedVolumes - number of failed volumes. This is redundant with the
  172. * information included in volumeFailureSummary, but the field is retained
  173. * for backwards compatibility.
  174. * cacheCapacity - total cache capacity available at the datanode
  175. * cacheUsed - amount of cache used
  176. * volumeFailureSummary - info about volume failures
  177. * slowPeers - info about peer DataNodes that are suspected to be slow.
  178. * slowDisks - info about DataNode disks that are suspected to be slow.
  179. * blksMovementResults - status of the scheduled blocks movements
  180. */
  181. message HeartbeatRequestProto {
  182. required DatanodeRegistrationProto registration = 1; // Datanode info
  183. repeated StorageReportProto reports = 2;
  184. optional uint32 xmitsInProgress = 3 [ default = 0 ];
  185. optional uint32 xceiverCount = 4 [ default = 0 ];
  186. optional uint32 failedVolumes = 5 [ default = 0 ];
  187. optional uint64 cacheCapacity = 6 [ default = 0 ];
  188. optional uint64 cacheUsed = 7 [default = 0 ];
  189. optional VolumeFailureSummaryProto volumeFailureSummary = 8;
  190. optional bool requestFullBlockReportLease = 9 [ default = false ];
  191. repeated SlowPeerReportProto slowPeers = 10;
  192. repeated SlowDiskReportProto slowDisks = 11;
  193. }
  194. /**
  195. * cmds - Commands from namenode to datanode.
  196. * haStatus - Status (from an HA perspective) of the NN sending this response
  197. */
  198. message HeartbeatResponseProto {
  199. repeated DatanodeCommandProto cmds = 1; // Returned commands can be null
  200. required NNHAStatusHeartbeatProto haStatus = 2;
  201. optional RollingUpgradeStatusProto rollingUpgradeStatus = 3;
  202. optional RollingUpgradeStatusProto rollingUpgradeStatusV2 = 4;
  203. optional uint64 fullBlockReportLeaseId = 5 [ default = 0 ];
  204. optional bool isSlownode = 6 [ default = false ];
  205. }
  206. /**
  207. * registration - datanode registration information
  208. * blockPoolID - block pool ID of the reported blocks
  209. * blocks - each block is represented as multiple longs in the array.
  210. * first long represents block ID
  211. * second long represents length
  212. * third long represents gen stamp
  213. * fourth long (if under construction) represents replica state
  214. * context - An optional field containing information about the context
  215. * of this block report.
  216. */
  217. message BlockReportRequestProto {
  218. required DatanodeRegistrationProto registration = 1;
  219. required string blockPoolId = 2;
  220. repeated StorageBlockReportProto reports = 3;
  221. optional BlockReportContextProto context = 4;
  222. }
  223. message BlockReportContextProto {
  224. // The total number of RPCs this block report is broken into.
  225. required int32 totalRpcs = 1;
  226. // The index of the current RPC (zero-based)
  227. required int32 curRpc = 2;
  228. // The unique 64-bit ID of this block report
  229. required int64 id = 3;
  230. // The block report lease ID, or 0 if we are sending without a lease to
  231. // bypass rate-limiting.
  232. optional uint64 leaseId = 4 [ default = 0 ];
  233. // for compatibility, field number 5 should not be reused, see HDFS-13671.
  234. // True if the reported blocks are sorted by increasing block IDs
  235. // optional bool sorted = 5 [default = false];
  236. }
  237. /**
  238. * Report of blocks in a storage
  239. */
  240. message StorageBlockReportProto {
  241. required DatanodeStorageProto storage = 1; // Storage
  242. repeated uint64 blocks = 2 [packed=true];
  243. optional uint64 numberOfBlocks = 3;
  244. repeated bytes blocksBuffers = 4;
  245. }
  246. /**
  247. * cmd - Command from namenode to the datanode
  248. */
  249. message BlockReportResponseProto {
  250. optional DatanodeCommandProto cmd = 1;
  251. }
  252. /**
  253. * registration - datanode registration information
  254. * blockPoolId - block pool ID of the reported blocks
  255. * blocks - representation of blocks as longs for efficiency reasons
  256. */
  257. message CacheReportRequestProto {
  258. required DatanodeRegistrationProto registration = 1;
  259. required string blockPoolId = 2;
  260. repeated uint64 blocks = 3 [packed=true];
  261. }
  262. message CacheReportResponseProto {
  263. optional DatanodeCommandProto cmd = 1;
  264. }
  265. /**
  266. * Data structure to send received or deleted block information
  267. * from datanode to namenode.
  268. */
  269. message ReceivedDeletedBlockInfoProto {
  270. enum BlockStatus {
  271. RECEIVING = 1; // block being created
  272. RECEIVED = 2; // block creation complete
  273. DELETED = 3;
  274. }
  275. required BlockProto block = 1;
  276. required BlockStatus status = 3;
  277. optional string deleteHint = 2;
  278. }
  279. /**
  280. * List of blocks received and deleted for a storage.
  281. */
  282. message StorageReceivedDeletedBlocksProto {
  283. required string storageUuid = 1 [ deprecated = true ];
  284. repeated ReceivedDeletedBlockInfoProto blocks = 2;
  285. optional DatanodeStorageProto storage = 3; // supersedes storageUuid.
  286. }
  287. /**
  288. * registration - datanode registration information
  289. * blockPoolID - block pool ID of the reported blocks
  290. * blocks - Received/deleted block list
  291. */
  292. message BlockReceivedAndDeletedRequestProto {
  293. required DatanodeRegistrationProto registration = 1;
  294. required string blockPoolId = 2;
  295. repeated StorageReceivedDeletedBlocksProto blocks = 3;
  296. }
  297. /**
  298. * void response
  299. */
  300. message BlockReceivedAndDeletedResponseProto {
  301. }
  302. /**
  303. * registartion - Datanode reporting the error
  304. * errorCode - error code indicating the error
  305. * msg - Free text description of the error
  306. */
  307. message ErrorReportRequestProto {
  308. enum ErrorCode {
  309. NOTIFY = 0; // Error report to be logged at the namenode
  310. DISK_ERROR = 1; // DN has disk errors but still has valid volumes
  311. INVALID_BLOCK = 2; // Command from namenode has invalid block ID
  312. FATAL_DISK_ERROR = 3; // No valid volumes left on datanode
  313. }
  314. required DatanodeRegistrationProto registartion = 1; // Registartion info
  315. required uint32 errorCode = 2; // Error code
  316. required string msg = 3; // Error message
  317. }
  318. /**
  319. * void response
  320. */
  321. message ErrorReportResponseProto {
  322. }
  323. /**
  324. * blocks - list of blocks that are reported as corrupt
  325. */
  326. message ReportBadBlocksRequestProto {
  327. repeated LocatedBlockProto blocks = 1;
  328. }
  329. /**
  330. * void response
  331. */
  332. message ReportBadBlocksResponseProto {
  333. }
  334. /**
  335. * Commit block synchronization request during lease recovery
  336. */
  337. message CommitBlockSynchronizationRequestProto {
  338. required ExtendedBlockProto block = 1;
  339. required uint64 newGenStamp = 2;
  340. required uint64 newLength = 3;
  341. required bool closeFile = 4;
  342. required bool deleteBlock = 5;
  343. repeated DatanodeIDProto newTaragets = 6;
  344. repeated string newTargetStorages = 7;
  345. }
  346. /**
  347. * void response
  348. */
  349. message CommitBlockSynchronizationResponseProto {
  350. }
  351. /**
  352. * Information about a single slow peer that may be reported by
  353. * the DataNode to the NameNode as part of the heartbeat request.
  354. * The message includes the peer's DataNodeId and its
  355. * aggregate packet latency as observed by the reporting DataNode.
  356. * (DataNodeId must be transmitted as a string for protocol compability
  357. * with earlier versions of Hadoop).
  358. *
  359. * The exact choice of the aggregate is opaque to the NameNode but it
  360. * _should_ be chosen consistenly by all DataNodes in the cluster.
  361. * Examples of aggregates are 90th percentile (good) and mean (not so
  362. * good).
  363. */
  364. message SlowPeerReportProto {
  365. optional string dataNodeId = 1;
  366. optional double aggregateLatency = 2;
  367. optional double median = 3;
  368. optional double mad = 4;
  369. optional double upperLimitLatency = 5;
  370. }
  371. /**
  372. * Information about a single slow disk that may be reported by
  373. * the DataNode to the NameNode as part of the heartbeat request.
  374. * The message includes the disk's basePath, mean metadata op latency,
  375. * mean read io latency and mean write io latency as observed by the DataNode.
  376. */
  377. message SlowDiskReportProto {
  378. optional string basePath = 1;
  379. optional double meanMetadataOpLatency = 2;
  380. optional double meanReadIoLatency = 3;
  381. optional double meanWriteIoLatency = 4;
  382. }
  383. /**
  384. * Protocol used from datanode to the namenode
  385. * See the request and response for details of rpc call.
  386. */
  387. service DatanodeProtocolService {
  388. /**
  389. * Register a datanode at a namenode
  390. */
  391. rpc registerDatanode(RegisterDatanodeRequestProto)
  392. returns(RegisterDatanodeResponseProto);
  393. /**
  394. * Send heartbeat from datanode to namenode
  395. */
  396. rpc sendHeartbeat(HeartbeatRequestProto) returns(HeartbeatResponseProto);
  397. /**
  398. * Report blocks at a given datanode to the namenode
  399. */
  400. rpc blockReport(BlockReportRequestProto) returns(BlockReportResponseProto);
  401. /**
  402. * Report cached blocks at a datanode to the namenode
  403. */
  404. rpc cacheReport(CacheReportRequestProto) returns(CacheReportResponseProto);
  405. /**
  406. * Incremental block report from the DN. This contains info about recently
  407. * received and deleted blocks, as well as when blocks start being
  408. * received.
  409. */
  410. rpc blockReceivedAndDeleted(BlockReceivedAndDeletedRequestProto)
  411. returns(BlockReceivedAndDeletedResponseProto);
  412. /**
  413. * Report from a datanode of an error to the active namenode.
  414. * Used for debugging.
  415. */
  416. rpc errorReport(ErrorReportRequestProto) returns(ErrorReportResponseProto);
  417. /**
  418. * Request the version
  419. */
  420. rpc versionRequest(VersionRequestProto) returns(VersionResponseProto);
  421. /**
  422. * Report corrupt blocks at the specified location
  423. */
  424. rpc reportBadBlocks(ReportBadBlocksRequestProto) returns(ReportBadBlocksResponseProto);
  425. /**
  426. * Commit block synchronization during lease recovery.
  427. */
  428. rpc commitBlockSynchronization(CommitBlockSynchronizationRequestProto)
  429. returns(CommitBlockSynchronizationResponseProto);
  430. }