hdfs.proto 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /**
  19. * These .proto interfaces are private and stable.
  20. * Please see http://wiki.apache.org/hadoop/Compatibility
  21. * for what changes are allowed for a *stable* .proto interface.
  22. */
  23. // This file contains protocol buffers that are used throughout HDFS -- i.e.
  24. // by the client, server, and data transfer protocols.
  25. option java_package = "org.apache.hadoop.hdfs.protocol.proto";
  26. option java_outer_classname = "HdfsProtos";
  27. option java_generate_equals_and_hash = true;
  28. package hadoop.hdfs;
  29. import "Security.proto";
  30. /**
  31. * Extended block idenfies a block
  32. */
  33. message ExtendedBlockProto {
  34. required string poolId = 1; // Block pool id - gloablly unique across clusters
  35. required uint64 blockId = 2; // the local id within a pool
  36. required uint64 generationStamp = 3;
  37. optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid
  38. // here for historical reasons
  39. }
  40. /**
  41. * Identifies a Datanode
  42. */
  43. message DatanodeIDProto {
  44. required string ipAddr = 1; // IP address
  45. required string hostName = 2; // hostname
  46. required string datanodeUuid = 3; // UUID assigned to the Datanode. For
  47. // upgraded clusters this is the same
  48. // as the original StorageID of the
  49. // Datanode.
  50. required uint32 xferPort = 4; // data streaming port
  51. required uint32 infoPort = 5; // datanode http port
  52. required uint32 ipcPort = 6; // ipc server port
  53. optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
  54. optional uint32 containerPort = 8 [default = 0]; // Ozone container protocol
  55. }
  56. /**
  57. * Datanode local information
  58. */
  59. message DatanodeLocalInfoProto {
  60. required string softwareVersion = 1;
  61. required string configVersion = 2;
  62. required uint64 uptime = 3;
  63. }
  64. /**
  65. * Datanode volume information
  66. */
  67. message DatanodeVolumeInfoProto {
  68. required string path = 1;
  69. required StorageTypeProto storageType = 2;
  70. required uint64 usedSpace = 3;
  71. required uint64 freeSpace = 4;
  72. required uint64 reservedSpace = 5;
  73. required uint64 reservedSpaceForReplicas = 6;
  74. required uint64 numBlocks = 7;
  75. }
  76. /**
  77. * DatanodeInfo array
  78. */
  79. message DatanodeInfosProto {
  80. repeated DatanodeInfoProto datanodes = 1;
  81. }
  82. /**
  83. * The status of a Datanode
  84. */
  85. message DatanodeInfoProto {
  86. required DatanodeIDProto id = 1;
  87. optional uint64 capacity = 2 [default = 0];
  88. optional uint64 dfsUsed = 3 [default = 0];
  89. optional uint64 remaining = 4 [default = 0];
  90. optional uint64 blockPoolUsed = 5 [default = 0];
  91. optional uint64 lastUpdate = 6 [default = 0];
  92. optional uint32 xceiverCount = 7 [default = 0];
  93. optional string location = 8;
  94. optional uint64 nonDfsUsed = 9;
  95. enum AdminState {
  96. NORMAL = 0;
  97. DECOMMISSION_INPROGRESS = 1;
  98. DECOMMISSIONED = 2;
  99. ENTERING_MAINTENANCE = 3;
  100. IN_MAINTENANCE = 4;
  101. }
  102. optional AdminState adminState = 10 [default = NORMAL];
  103. optional uint64 cacheCapacity = 11 [default = 0];
  104. optional uint64 cacheUsed = 12 [default = 0];
  105. optional uint64 lastUpdateMonotonic = 13 [default = 0];
  106. optional string upgradeDomain = 14;
  107. optional uint64 lastBlockReportTime = 15 [default = 0];
  108. optional uint64 lastBlockReportMonotonic = 16 [default = 0];
  109. }
  110. /**
  111. * Represents a storage available on the datanode
  112. */
  113. message DatanodeStorageProto {
  114. enum StorageState {
  115. NORMAL = 0;
  116. READ_ONLY_SHARED = 1;
  117. }
  118. required string storageUuid = 1;
  119. optional StorageState state = 2 [default = NORMAL];
  120. optional StorageTypeProto storageType = 3 [default = DISK];
  121. }
  122. message StorageReportProto {
  123. required string storageUuid = 1 [ deprecated = true ];
  124. optional bool failed = 2 [ default = false ];
  125. optional uint64 capacity = 3 [ default = 0 ];
  126. optional uint64 dfsUsed = 4 [ default = 0 ];
  127. optional uint64 remaining = 5 [ default = 0 ];
  128. optional uint64 blockPoolUsed = 6 [ default = 0 ];
  129. optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
  130. optional uint64 nonDfsUsed = 8;
  131. }
  132. /**
  133. * Summary of a file or directory
  134. */
  135. message ContentSummaryProto {
  136. required uint64 length = 1;
  137. required uint64 fileCount = 2;
  138. required uint64 directoryCount = 3;
  139. required uint64 quota = 4;
  140. required uint64 spaceConsumed = 5;
  141. required uint64 spaceQuota = 6;
  142. optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
  143. optional uint64 snapshotLength = 8;
  144. optional uint64 snapshotFileCount = 9;
  145. optional uint64 snapshotDirectoryCount = 10;
  146. optional uint64 snapshotSpaceConsumed = 11;
  147. }
  148. /**
  149. * Summary of quota usage of a directory
  150. */
  151. message QuotaUsageProto {
  152. required uint64 fileAndDirectoryCount = 1;
  153. required uint64 quota = 2;
  154. required uint64 spaceConsumed = 3;
  155. required uint64 spaceQuota = 4;
  156. optional StorageTypeQuotaInfosProto typeQuotaInfos = 5;
  157. }
  158. /**
  159. * Storage type quota and usage information of a file or directory
  160. */
  161. message StorageTypeQuotaInfosProto {
  162. repeated StorageTypeQuotaInfoProto typeQuotaInfo = 1;
  163. }
  164. message StorageTypeQuotaInfoProto {
  165. required StorageTypeProto type = 1;
  166. required uint64 quota = 2;
  167. required uint64 consumed = 3;
  168. }
  169. /**
  170. * Contains a list of paths corresponding to corrupt files and a cookie
  171. * used for iterative calls to NameNode.listCorruptFileBlocks.
  172. *
  173. */
  174. message CorruptFileBlocksProto {
  175. repeated string files = 1;
  176. required string cookie = 2;
  177. }
  178. /**
  179. * File or Directory permision - same spec as posix
  180. */
  181. message FsPermissionProto {
  182. required uint32 perm = 1; // Actually a short - only 16bits used
  183. }
  184. /**
  185. * Types of recognized storage media.
  186. */
  187. enum StorageTypeProto {
  188. DISK = 1;
  189. SSD = 2;
  190. ARCHIVE = 3;
  191. RAM_DISK = 4;
  192. }
  193. /**
  194. * Types of recognized blocks.
  195. */
  196. enum BlockTypeProto {
  197. CONTIGUOUS = 0;
  198. STRIPED = 1;
  199. }
  200. /**
  201. * A list of storage types.
  202. */
  203. message StorageTypesProto {
  204. repeated StorageTypeProto storageTypes = 1;
  205. }
  206. /**
  207. * Block replica storage policy.
  208. */
  209. message BlockStoragePolicyProto {
  210. required uint32 policyId = 1;
  211. required string name = 2;
  212. // a list of storage types for storing the block replicas when creating a
  213. // block.
  214. required StorageTypesProto creationPolicy = 3;
  215. // A list of storage types for creation fallback storage.
  216. optional StorageTypesProto creationFallbackPolicy = 4;
  217. optional StorageTypesProto replicationFallbackPolicy = 5;
  218. }
  219. /**
  220. * A LocatedBlock gives information about a block and its location.
  221. */
  222. message LocatedBlockProto {
  223. required ExtendedBlockProto b = 1;
  224. required uint64 offset = 2; // offset of first byte of block in the file
  225. repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip
  226. required bool corrupt = 4; // true if all replicas of a block are corrupt, else false
  227. // If block has few corrupt replicas, they are filtered and
  228. // their locations are not part of this object
  229. required hadoop.common.TokenProto blockToken = 5;
  230. repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
  231. repeated StorageTypeProto storageTypes = 7;
  232. repeated string storageIDs = 8;
  233. // striped block related fields
  234. optional bytes blockIndices = 9; // used for striped block to indicate block index for each storage
  235. repeated hadoop.common.TokenProto blockTokens = 10; // each internal block has a block token
  236. }
  237. message DataEncryptionKeyProto {
  238. required uint32 keyId = 1;
  239. required string blockPoolId = 2;
  240. required bytes nonce = 3;
  241. required bytes encryptionKey = 4;
  242. required uint64 expiryDate = 5;
  243. optional string encryptionAlgorithm = 6;
  244. }
  245. /**
  246. * Cipher suite.
  247. */
  248. enum CipherSuiteProto {
  249. UNKNOWN = 1;
  250. AES_CTR_NOPADDING = 2;
  251. }
  252. /**
  253. * Crypto protocol version used to access encrypted files.
  254. */
  255. enum CryptoProtocolVersionProto {
  256. UNKNOWN_PROTOCOL_VERSION = 1;
  257. ENCRYPTION_ZONES = 2;
  258. }
  259. /**
  260. * Encryption information for a file.
  261. */
  262. message FileEncryptionInfoProto {
  263. required CipherSuiteProto suite = 1;
  264. required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  265. required bytes key = 3;
  266. required bytes iv = 4;
  267. required string keyName = 5;
  268. required string ezKeyVersionName = 6;
  269. }
  270. /**
  271. * Encryption information for an individual
  272. * file within an encryption zone
  273. */
  274. message PerFileEncryptionInfoProto {
  275. required bytes key = 1;
  276. required bytes iv = 2;
  277. required string ezKeyVersionName = 3;
  278. }
  279. /**
  280. * Encryption information for an encryption
  281. * zone
  282. */
  283. message ZoneEncryptionInfoProto {
  284. required CipherSuiteProto suite = 1;
  285. required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  286. required string keyName = 3;
  287. }
  288. /**
  289. * Cipher option
  290. */
  291. message CipherOptionProto {
  292. required CipherSuiteProto suite = 1;
  293. optional bytes inKey = 2;
  294. optional bytes inIv = 3;
  295. optional bytes outKey = 4;
  296. optional bytes outIv = 5;
  297. }
  298. /**
  299. * A set of file blocks and their locations.
  300. */
  301. message LocatedBlocksProto {
  302. required uint64 fileLength = 1;
  303. repeated LocatedBlockProto blocks = 2;
  304. required bool underConstruction = 3;
  305. optional LocatedBlockProto lastBlock = 4;
  306. required bool isLastBlockComplete = 5;
  307. optional FileEncryptionInfoProto fileEncryptionInfo = 6;
  308. // Optional field for erasure coding
  309. optional ErasureCodingPolicyProto ecPolicy = 7;
  310. }
  311. /**
  312. * ECSchema options entry
  313. */
  314. message ECSchemaOptionEntryProto {
  315. required string key = 1;
  316. required string value = 2;
  317. }
  318. /**
  319. * ECSchema for erasurecoding
  320. */
  321. message ECSchemaProto {
  322. required string codecName = 1;
  323. required uint32 dataUnits = 2;
  324. required uint32 parityUnits = 3;
  325. repeated ECSchemaOptionEntryProto options = 4;
  326. }
  327. message ErasureCodingPolicyProto {
  328. optional string name = 1;
  329. optional ECSchemaProto schema = 2;
  330. optional uint32 cellSize = 3;
  331. required uint32 id = 4; // Actually a byte - only 8 bits used
  332. }
  333. message AddECPolicyResponseProto {
  334. required ErasureCodingPolicyProto policy = 1;
  335. required bool succeed = 2;
  336. optional string errorMsg = 3;
  337. }
  338. /**
  339. * Status of a file, directory or symlink
  340. * Optionally includes a file's block locations if requested by client on the rpc call.
  341. */
  342. message HdfsFileStatusProto {
  343. enum FileType {
  344. IS_DIR = 1;
  345. IS_FILE = 2;
  346. IS_SYMLINK = 3;
  347. }
  348. required FileType fileType = 1;
  349. required bytes path = 2; // local name of inode encoded java UTF8
  350. required uint64 length = 3;
  351. required FsPermissionProto permission = 4;
  352. required string owner = 5;
  353. required string group = 6;
  354. required uint64 modification_time = 7;
  355. required uint64 access_time = 8;
  356. // Optional fields for symlink
  357. optional bytes symlink = 9; // if symlink, target encoded java UTF8
  358. // Optional fields for file
  359. optional uint32 block_replication = 10 [default = 0]; // only 16bits used
  360. optional uint64 blocksize = 11 [default = 0];
  361. optional LocatedBlocksProto locations = 12; // suppled only if asked by client
  362. // Optional field for fileId
  363. optional uint64 fileId = 13 [default = 0]; // default as an invalid id
  364. optional int32 childrenNum = 14 [default = -1];
  365. // Optional field for file encryption
  366. optional FileEncryptionInfoProto fileEncryptionInfo = 15;
  367. optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id
  368. // Optional field for erasure coding
  369. optional ErasureCodingPolicyProto ecPolicy = 17;
  370. }
  371. /**
  372. * Checksum algorithms/types used in HDFS
  373. * Make sure this enum's integer values match enum values' id properties defined
  374. * in org.apache.hadoop.util.DataChecksum.Type
  375. */
  376. enum ChecksumTypeProto {
  377. CHECKSUM_NULL = 0;
  378. CHECKSUM_CRC32 = 1;
  379. CHECKSUM_CRC32C = 2;
  380. }
  381. /**
  382. * HDFS Server Defaults
  383. */
  384. message FsServerDefaultsProto {
  385. required uint64 blockSize = 1;
  386. required uint32 bytesPerChecksum = 2;
  387. required uint32 writePacketSize = 3;
  388. required uint32 replication = 4; // Actually a short - only 16 bits used
  389. required uint32 fileBufferSize = 5;
  390. optional bool encryptDataTransfer = 6 [default = false];
  391. optional uint64 trashInterval = 7 [default = 0];
  392. optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
  393. optional string keyProviderUri = 9;
  394. optional uint32 policyId = 10 [default = 0];
  395. }
  396. /**
  397. * Directory listing
  398. */
  399. message DirectoryListingProto {
  400. repeated HdfsFileStatusProto partialListing = 1;
  401. required uint32 remainingEntries = 2;
  402. }
  403. /**
  404. * Status of a snapshottable directory: besides the normal information for
  405. * a directory status, also include snapshot quota, number of snapshots, and
  406. * the full path of the parent directory.
  407. */
  408. message SnapshottableDirectoryStatusProto {
  409. required HdfsFileStatusProto dirStatus = 1;
  410. // Fields specific for snapshottable directory
  411. required uint32 snapshot_quota = 2;
  412. required uint32 snapshot_number = 3;
  413. required bytes parent_fullpath = 4;
  414. }
  415. /**
  416. * Snapshottable directory listing
  417. */
  418. message SnapshottableDirectoryListingProto {
  419. repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
  420. }
  421. /**
  422. * Snapshot diff report entry
  423. */
  424. message SnapshotDiffReportEntryProto {
  425. required bytes fullpath = 1;
  426. required string modificationLabel = 2;
  427. optional bytes targetPath = 3;
  428. }
  429. /**
  430. * Snapshot diff report
  431. */
  432. message SnapshotDiffReportProto {
  433. // full path of the directory where snapshots were taken
  434. required string snapshotRoot = 1;
  435. required string fromSnapshot = 2;
  436. required string toSnapshot = 3;
  437. repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
  438. }
  439. /**
  440. * Block information
  441. *
  442. * Please be wary of adding additional fields here, since INodeFiles
  443. * need to fit in PB's default max message size of 64MB.
  444. * We restrict the max # of blocks per file
  445. * (dfs.namenode.fs-limits.max-blocks-per-file), but it's better
  446. * to avoid changing this.
  447. */
  448. message BlockProto {
  449. required uint64 blockId = 1;
  450. required uint64 genStamp = 2;
  451. optional uint64 numBytes = 3 [default = 0];
  452. }
  453. /**
  454. * Information related to a snapshot
  455. * TODO: add more information
  456. */
  457. message SnapshotInfoProto {
  458. required string snapshotName = 1;
  459. required string snapshotRoot = 2;
  460. required FsPermissionProto permission = 3;
  461. required string owner = 4;
  462. required string group = 5;
  463. required string createTime = 6;
  464. // TODO: do we need access time?
  465. }
  466. /**
  467. * Rolling upgrade status
  468. */
  469. message RollingUpgradeStatusProto {
  470. required string blockPoolId = 1;
  471. optional bool finalized = 2 [default = false];
  472. }
  473. /**
  474. * A list of storage IDs.
  475. */
  476. message StorageUuidsProto {
  477. repeated string storageUuids = 1;
  478. }
  479. /**
  480. * File access permissions mode.
  481. */
  482. enum AccessModeProto {
  483. READ = 1;
  484. WRITE = 2;
  485. COPY = 3;
  486. REPLACE = 4;
  487. }
  488. /**
  489. * Secret information for the BlockKeyProto. This is not sent on the wire as
  490. * such but is used to pack a byte array and encrypted and put in
  491. * BlockKeyProto.bytes
  492. * When adding further fields, make sure they are optional as they would
  493. * otherwise not be backwards compatible.
  494. *
  495. * Note: As part of the migration from WritableUtils based tokens (aka "legacy")
  496. * to Protocol Buffers, we use the first byte to determine the type. If the
  497. * first byte is <=0 then it is a legacy token. This means that when using
  498. * protobuf tokens, the the first field sent must have a `field_number` less
  499. * than 16 to make sure that the first byte is positive. Otherwise it could be
  500. * parsed as a legacy token. See HDFS-11026 for more discussion.
  501. */
  502. message BlockTokenSecretProto {
  503. optional uint64 expiryDate = 1;
  504. optional uint32 keyId = 2;
  505. optional string userId = 3;
  506. optional string blockPoolId = 4;
  507. optional uint64 blockId = 5;
  508. repeated AccessModeProto modes = 6;
  509. repeated StorageTypeProto storageTypes = 7;
  510. repeated string storageIds = 8;
  511. }