hdfs.proto 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445
  1. /**
  2. * Licensed to the Apache Software Foundation (ASF) under one
  3. * or more contributor license agreements. See the NOTICE file
  4. * distributed with this work for additional information
  5. * regarding copyright ownership. The ASF licenses this file
  6. * to you under the Apache License, Version 2.0 (the
  7. * "License"); you may not use this file except in compliance
  8. * with the License. You may obtain a copy of the License at
  9. *
  10. * http://www.apache.org/licenses/LICENSE-2.0
  11. *
  12. * Unless required by applicable law or agreed to in writing, software
  13. * distributed under the License is distributed on an "AS IS" BASIS,
  14. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15. * See the License for the specific language governing permissions and
  16. * limitations under the License.
  17. */
  18. /**
  19. * These .proto interfaces are private and stable.
  20. * Please see http://wiki.apache.org/hadoop/Compatibility
  21. * for what changes are allowed for a *stable* .proto interface.
  22. */
  23. // This file contains protocol buffers that are used throughout HDFS -- i.e.
  24. // by the client, server, and data transfer protocols.
  25. option java_package = "org.apache.hadoop.hdfs.protocol.proto";
  26. option java_outer_classname = "HdfsProtos";
  27. option java_generate_equals_and_hash = true;
  28. package hadoop.hdfs;
  29. import "Security.proto";
  30. /**
  31. * Extended block idenfies a block
  32. */
  33. message ExtendedBlockProto {
  34. required string poolId = 1; // Block pool id - gloablly unique across clusters
  35. required uint64 blockId = 2; // the local id within a pool
  36. required uint64 generationStamp = 3;
  37. optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid
  38. // here for historical reasons
  39. }
  40. /**
  41. * Identifies a Datanode
  42. */
  43. message DatanodeIDProto {
  44. required string ipAddr = 1; // IP address
  45. required string hostName = 2; // hostname
  46. required string storageID = 3; // unique storage id
  47. required uint32 xferPort = 4; // data streaming port
  48. required uint32 infoPort = 5; // info server port
  49. required uint32 ipcPort = 6; // ipc server port
  50. }
  51. /**
  52. * DatanodeInfo array
  53. */
  54. message DatanodeInfosProto {
  55. repeated DatanodeInfoProto datanodes = 1;
  56. }
  57. /**
  58. * The status of a Datanode
  59. */
  60. message DatanodeInfoProto {
  61. required DatanodeIDProto id = 1;
  62. optional uint64 capacity = 2 [default = 0];
  63. optional uint64 dfsUsed = 3 [default = 0];
  64. optional uint64 remaining = 4 [default = 0];
  65. optional uint64 blockPoolUsed = 5 [default = 0];
  66. optional uint64 lastUpdate = 6 [default = 0];
  67. optional uint32 xceiverCount = 7 [default = 0];
  68. optional string location = 8;
  69. enum AdminState {
  70. NORMAL = 0;
  71. DECOMMISSION_INPROGRESS = 1;
  72. DECOMMISSIONED = 2;
  73. }
  74. optional AdminState adminState = 10 [default = NORMAL];
  75. }
  76. /**
  77. * Summary of a file or directory
  78. */
  79. message ContentSummaryProto {
  80. required uint64 length = 1;
  81. required uint64 fileCount = 2;
  82. required uint64 directoryCount = 3;
  83. required uint64 quota = 4;
  84. required uint64 spaceConsumed = 5;
  85. required uint64 spaceQuota = 6;
  86. }
  87. /**
  88. * Contains a list of paths corresponding to corrupt files and a cookie
  89. * used for iterative calls to NameNode.listCorruptFileBlocks.
  90. *
  91. */
  92. message CorruptFileBlocksProto {
  93. repeated string files = 1;
  94. required string cookie = 2;
  95. }
  96. /**
  97. * File or Directory permision - same spec as posix
  98. */
  99. message FsPermissionProto {
  100. required uint32 perm = 1; // Actually a short - only 16bits used
  101. }
  102. /**
  103. * Types of recognized storage media.
  104. */
  105. enum StorageTypeProto {
  106. DISK = 1;
  107. SSD = 2;
  108. }
  109. /**
  110. * A LocatedBlock gives information about a block and its location.
  111. */
  112. message LocatedBlockProto {
  113. required ExtendedBlockProto b = 1;
  114. required uint64 offset = 2; // offset of first byte of block in the file
  115. repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip
  116. required bool corrupt = 4; // true if all replicas of a block are corrupt, else false
  117. // If block has few corrupt replicas, they are filtered and
  118. // their locations are not part of this object
  119. required hadoop.common.TokenProto blockToken = 5;
  120. repeated StorageTypeProto storageTypes = 6;
  121. }
  122. message DataEncryptionKeyProto {
  123. required uint32 keyId = 1;
  124. required string blockPoolId = 2;
  125. required bytes nonce = 3;
  126. required bytes encryptionKey = 4;
  127. required uint64 expiryDate = 5;
  128. optional string encryptionAlgorithm = 6;
  129. }
  130. /**
  131. * A set of file blocks and their locations.
  132. */
  133. message LocatedBlocksProto {
  134. required uint64 fileLength = 1;
  135. repeated LocatedBlockProto blocks = 2;
  136. required bool underConstruction = 3;
  137. optional LocatedBlockProto lastBlock = 4;
  138. required bool isLastBlockComplete = 5;
  139. }
  140. /**
  141. * Status of a file, directory or symlink
  142. * Optionally includes a file's block locations if requested by client on the rpc call.
  143. */
  144. message HdfsFileStatusProto {
  145. enum FileType {
  146. IS_DIR = 1;
  147. IS_FILE = 2;
  148. IS_SYMLINK = 3;
  149. }
  150. required FileType fileType = 1;
  151. required bytes path = 2; // local name of inode encoded java UTF8
  152. required uint64 length = 3;
  153. required FsPermissionProto permission = 4;
  154. required string owner = 5;
  155. required string group = 6;
  156. required uint64 modification_time = 7;
  157. required uint64 access_time = 8;
  158. // Optional fields for symlink
  159. optional bytes symlink = 9; // if symlink, target encoded java UTF8
  160. // Optional fields for file
  161. optional uint32 block_replication = 10 [default = 0]; // only 16bits used
  162. optional uint64 blocksize = 11 [default = 0];
  163. optional LocatedBlocksProto locations = 12; // suppled only if asked by client
  164. // Optional field for fileId
  165. optional uint64 fileId = 13 [default = 0]; // default as an invalid id
  166. optional int32 childrenNum = 14 [default = -1];
  167. }
  168. /**
  169. * Checksum algorithms/types used in HDFS
  170. * Make sure this enum's integer values match enum values' id properties defined
  171. * in org.apache.hadoop.util.DataChecksum.Type
  172. */
  173. enum ChecksumTypeProto {
  174. CHECKSUM_NULL = 0;
  175. CHECKSUM_CRC32 = 1;
  176. CHECKSUM_CRC32C = 2;
  177. }
  178. /**
  179. * HDFS Server Defaults
  180. */
  181. message FsServerDefaultsProto {
  182. required uint64 blockSize = 1;
  183. required uint32 bytesPerChecksum = 2;
  184. required uint32 writePacketSize = 3;
  185. required uint32 replication = 4; // Actually a short - only 16 bits used
  186. required uint32 fileBufferSize = 5;
  187. optional bool encryptDataTransfer = 6 [default = false];
  188. optional uint64 trashInterval = 7 [default = 0];
  189. optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
  190. }
  191. /**
  192. * Directory listing
  193. */
  194. message DirectoryListingProto {
  195. repeated HdfsFileStatusProto partialListing = 1;
  196. required uint32 remainingEntries = 2;
  197. }
  198. /**
  199. * Status of a snapshottable directory: besides the normal information for
  200. * a directory status, also include snapshot quota, number of snapshots, and
  201. * the full path of the parent directory.
  202. */
  203. message SnapshottableDirectoryStatusProto {
  204. required HdfsFileStatusProto dirStatus = 1;
  205. // Fields specific for snapshottable directory
  206. required uint32 snapshot_quota = 2;
  207. required uint32 snapshot_number = 3;
  208. required bytes parent_fullpath = 4;
  209. }
  210. /**
  211. * Snapshottable directory listing
  212. */
  213. message SnapshottableDirectoryListingProto {
  214. repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
  215. }
  216. /**
  217. * Snapshot diff report entry
  218. */
  219. message SnapshotDiffReportEntryProto {
  220. required bytes fullpath = 1;
  221. required string modificationLabel = 2;
  222. }
  223. /**
  224. * Snapshot diff report
  225. */
  226. message SnapshotDiffReportProto {
  227. // full path of the directory where snapshots were taken
  228. required string snapshotRoot = 1;
  229. required string fromSnapshot = 2;
  230. required string toSnapshot = 3;
  231. repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
  232. }
  233. /**
  234. * Common node information shared by all the nodes in the cluster
  235. */
  236. message StorageInfoProto {
  237. required uint32 layoutVersion = 1; // Layout version of the file system
  238. required uint32 namespceID = 2; // File system namespace ID
  239. required string clusterID = 3; // ID of the cluster
  240. required uint64 cTime = 4; // File system creation time
  241. }
  242. /**
  243. * Information sent by a namenode to identify itself to the primary namenode.
  244. */
  245. message NamenodeRegistrationProto {
  246. required string rpcAddress = 1; // host:port of the namenode RPC address
  247. required string httpAddress = 2; // host:port of the namenode http server
  248. enum NamenodeRoleProto {
  249. NAMENODE = 1;
  250. BACKUP = 2;
  251. CHECKPOINT = 3;
  252. }
  253. required StorageInfoProto storageInfo = 3; // Node information
  254. optional NamenodeRoleProto role = 4 [default = NAMENODE]; // Namenode role
  255. }
  256. /**
  257. * Unique signature to identify checkpoint transactions.
  258. */
  259. message CheckpointSignatureProto {
  260. required string blockPoolId = 1;
  261. required uint64 mostRecentCheckpointTxId = 2;
  262. required uint64 curSegmentTxId = 3;
  263. required StorageInfoProto storageInfo = 4;
  264. }
  265. /**
  266. * Command sent from one namenode to another namenode.
  267. */
  268. message NamenodeCommandProto {
  269. enum Type {
  270. NamenodeCommand = 0; // Base command
  271. CheckPointCommand = 1; // Check point command
  272. }
  273. required uint32 action = 1;
  274. required Type type = 2;
  275. optional CheckpointCommandProto checkpointCmd = 3;
  276. }
  277. /**
  278. * Command returned from primary to checkpointing namenode.
  279. * This command has checkpoint signature that identifies
  280. * checkpoint transaction and is needed for further
  281. * communication related to checkpointing.
  282. */
  283. message CheckpointCommandProto {
  284. // Unique signature to identify checkpoint transation
  285. required CheckpointSignatureProto signature = 1;
  286. // If true, return transfer image to primary upon the completion of checkpoint
  287. required bool needToReturnImage = 2;
  288. }
  289. /**
  290. * Block information
  291. */
  292. message BlockProto {
  293. required uint64 blockId = 1;
  294. required uint64 genStamp = 2;
  295. optional uint64 numBytes = 3 [default = 0];
  296. }
  297. /**
  298. * Block and datanodes where is it located
  299. */
  300. message BlockWithLocationsProto {
  301. required BlockProto block = 1; // Block
  302. repeated string storageIDs = 2; // Datanodes with replicas of the block
  303. }
  304. /**
  305. * List of block with locations
  306. */
  307. message BlocksWithLocationsProto {
  308. repeated BlockWithLocationsProto blocks = 1;
  309. }
  310. /**
  311. * Editlog information with available transactions
  312. */
  313. message RemoteEditLogProto {
  314. required uint64 startTxId = 1; // Starting available edit log transaction
  315. required uint64 endTxId = 2; // Ending available edit log transaction
  316. optional bool isInProgress = 3 [default = false];
  317. }
  318. /**
  319. * Enumeration of editlogs available on a remote namenode
  320. */
  321. message RemoteEditLogManifestProto {
  322. repeated RemoteEditLogProto logs = 1;
  323. }
  324. /**
  325. * Namespace information that describes namespace on a namenode
  326. */
  327. message NamespaceInfoProto {
  328. required string buildVersion = 1; // Software revision version (e.g. an svn or git revision)
  329. required uint32 unused = 2; // Retained for backward compatibility
  330. required string blockPoolID = 3; // block pool used by the namespace
  331. required StorageInfoProto storageInfo = 4;// Node information
  332. required string softwareVersion = 5; // Software version number (e.g. 2.0.0)
  333. }
  334. /**
  335. * Block access token information
  336. */
  337. message BlockKeyProto {
  338. required uint32 keyId = 1; // Key identifier
  339. required uint64 expiryDate = 2; // Expiry time in milliseconds
  340. optional bytes keyBytes = 3; // Key secret
  341. }
  342. /**
  343. * Current key and set of block keys at the namenode.
  344. */
  345. message ExportedBlockKeysProto {
  346. required bool isBlockTokenEnabled = 1;
  347. required uint64 keyUpdateInterval = 2;
  348. required uint64 tokenLifeTime = 3;
  349. required BlockKeyProto currentKey = 4;
  350. repeated BlockKeyProto allKeys = 5;
  351. }
  352. /**
  353. * State of a block replica at a datanode
  354. */
  355. enum ReplicaStateProto {
  356. FINALIZED = 0; // State of a replica when it is not modified
  357. RBW = 1; // State of replica that is being written to
  358. RWR = 2; // State of replica that is waiting to be recovered
  359. RUR = 3; // State of replica that is under recovery
  360. TEMPORARY = 4; // State of replica that is created for replication
  361. }
  362. /**
  363. * Block that needs to be recovered with at a given location
  364. */
  365. message RecoveringBlockProto {
  366. required uint64 newGenStamp = 1; // New genstamp post recovery
  367. required LocatedBlockProto block = 2; // Block to be recovered
  368. }
  369. /**
  370. * void request
  371. */
  372. message VersionRequestProto {
  373. }
  374. /**
  375. * Version response from namenode.
  376. */
  377. message VersionResponseProto {
  378. required NamespaceInfoProto info = 1;
  379. }
  380. /**
  381. * Information related to a snapshot
  382. * TODO: add more information
  383. */
  384. message SnapshotInfoProto {
  385. required string snapshotName = 1;
  386. required string snapshotRoot = 2;
  387. required FsPermissionProto permission = 3;
  388. required string owner = 4;
  389. required string group = 5;
  390. required string createTime = 6;
  391. // TODO: do we need access time?
  392. }