hadoop/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * These .proto interfaces are private and stable.
 * Please see http://wiki.apache.org/hadoop/Compatibility
 * for what changes are allowed for a *stable* .proto interface.
 */

// This file contains protocol buffers that are used throughout HDFS -- i.e.
// by the client, server, and data transfer protocols.


option java_package = "org.apache.hadoop.hdfs.protocol.proto";
option java_outer_classname = "HdfsProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdfs;

import "Security.proto";

/**
 * Extended block idenfies a block
 */
message ExtendedBlockProto {
  required string poolId = 1;   // Block pool id - gloablly unique across clusters
  required uint64 blockId = 2;  // the local id within a pool
  required uint64 generationStamp = 3;
  optional uint64 numBytes = 4 [default = 0];  // len does not belong in ebid 
                                               // here for historical reasons
}

/**
 * Identifies a Datanode
 */
message DatanodeIDProto {
  required string ipAddr = 1;    // IP address
  required string hostName = 2;  // hostname
  required string datanodeUuid = 3;     // UUID assigned to the Datanode. For
                                        // upgraded clusters this is the same
                                        // as the original StorageID of the
                                        // Datanode.
  required uint32 xferPort = 4;  // data streaming port
  required uint32 infoPort = 5;  // datanode http port
  required uint32 ipcPort = 6;   // ipc server port
  optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
}

/**
 * Datanode local information
 */
message DatanodeLocalInfoProto {
  required string softwareVersion = 1;
  required string configVersion = 2;
  required uint64 uptime = 3;
}

/**
 * DatanodeInfo array
 */
message DatanodeInfosProto {
  repeated DatanodeInfoProto datanodes = 1;
}

/**
 * The status of a Datanode
 */
message DatanodeInfoProto {
  required DatanodeIDProto id = 1;
  optional uint64 capacity = 2 [default = 0];
  optional uint64 dfsUsed = 3 [default = 0];
  optional uint64 remaining = 4 [default = 0];
  optional uint64 blockPoolUsed = 5 [default = 0];
  optional uint64 lastUpdate = 6 [default = 0];
  optional uint32 xceiverCount = 7 [default = 0];
  optional string location = 8;
  enum AdminState {
    NORMAL = 0;
    DECOMMISSION_INPROGRESS = 1;
    DECOMMISSIONED = 2;
  }

  optional AdminState adminState = 10 [default = NORMAL];
  optional uint64 cacheCapacity = 11 [default = 0];
  optional uint64 cacheUsed = 12 [default = 0];
  optional uint64 lastUpdateMonotonic = 13 [default = 0];
}

/**
 * Represents a storage available on the datanode
 */
message DatanodeStorageProto {
  enum StorageState {
    NORMAL = 0;
    READ_ONLY_SHARED = 1;
  }

  required string storageUuid = 1;
  optional StorageState state = 2 [default = NORMAL];
  optional StorageTypeProto storageType = 3 [default = DISK];
}

message StorageReportProto {
  required string storageUuid = 1 [ deprecated = true ];
  optional bool failed = 2 [ default = false ];
  optional uint64 capacity = 3 [ default = 0 ];
  optional uint64 dfsUsed = 4 [ default = 0 ];
  optional uint64 remaining = 5 [ default = 0 ];
  optional uint64 blockPoolUsed = 6 [ default = 0 ];
  optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
}

/**
 * Summary of a file or directory
 */
message ContentSummaryProto {
  required uint64 length = 1;
  required uint64 fileCount = 2;
  required uint64 directoryCount = 3;
  required uint64 quota = 4;
  required uint64 spaceConsumed = 5;
  required uint64 spaceQuota = 6;
  optional StorageTypeQuotaInfosProto typeQuotaInfos = 7;
}

/**
 * Storage type quota and usage information of a file or directory
 */
message StorageTypeQuotaInfosProto {
  repeated StorageTypeQuotaInfoProto typeQuotaInfo = 1;
}

message StorageTypeQuotaInfoProto {
  required StorageTypeProto type = 1;
  required uint64 quota = 2;
  required uint64 consumed = 3;
}

/**
 * Contains a list of paths corresponding to corrupt files and a cookie
 * used for iterative calls to NameNode.listCorruptFileBlocks.
 *
 */
message CorruptFileBlocksProto {
 repeated string files = 1;
 required string   cookie = 2;
}

/**
 * File or Directory permision - same spec as posix
 */
message FsPermissionProto {
  required uint32 perm = 1;       // Actually a short - only 16bits used
}

/**
 * Types of recognized storage media.
 */
enum StorageTypeProto {
  DISK = 1;
  SSD = 2;
  ARCHIVE = 3;
  RAM_DISK = 4;
}

/**
 * A list of storage types. 
 */
message StorageTypesProto {
  repeated StorageTypeProto storageTypes = 1;
}

/**
 * Block replica storage policy.
 */
message BlockStoragePolicyProto {
  required uint32 policyId = 1;
  required string name = 2;
  // a list of storage types for storing the block replicas when creating a
  // block.
  required StorageTypesProto creationPolicy = 3;
  // A list of storage types for creation fallback storage.
  optional StorageTypesProto creationFallbackPolicy = 4;
  optional StorageTypesProto replicationFallbackPolicy = 5;
}

/**
 * A list of storage IDs. 
 */
message StorageUuidsProto {
  repeated string storageUuids = 1;
}

/**
 * A LocatedBlock gives information about a block and its location.
 */ 
message LocatedBlockProto {
  required ExtendedBlockProto b  = 1;
  required uint64 offset = 2;           // offset of first byte of block in the file
  repeated DatanodeInfoProto locs = 3;  // Locations ordered by proximity to client ip
  required bool corrupt = 4;            // true if all replicas of a block are corrupt, else false
                                        // If block has few corrupt replicas, they are filtered and 
                                        // their locations are not part of this object

  required hadoop.common.TokenProto blockToken = 5;
  repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
  repeated StorageTypeProto storageTypes = 7;
  repeated string storageIDs = 8;
  repeated uint32 blockIndex = 9; // used for striped block to indicate block index for each storage
}

message DataEncryptionKeyProto {
  required uint32 keyId = 1;
  required string blockPoolId = 2;
  required bytes nonce = 3;
  required bytes encryptionKey = 4;
  required uint64 expiryDate = 5;
  optional string encryptionAlgorithm = 6;
}

/**
 * Cipher suite.
 */
enum CipherSuiteProto {
    UNKNOWN = 1;
    AES_CTR_NOPADDING = 2;
}

/**
 * Crypto protocol version used to access encrypted files.
 */
enum CryptoProtocolVersionProto {
    UNKNOWN_PROTOCOL_VERSION = 1;
    ENCRYPTION_ZONES = 2;
}

/**
 * Encryption information for a file.
 */
message FileEncryptionInfoProto {
  required CipherSuiteProto suite = 1;
  required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  required bytes key = 3;
  required bytes iv = 4;
  required string keyName = 5;
  required string ezKeyVersionName = 6;
}

/**
 * Encryption information for an individual
 * file within an encryption zone
 */
message PerFileEncryptionInfoProto {
  required bytes key = 1;
  required bytes iv = 2;
  required string ezKeyVersionName = 3;
}

/**
 * Encryption information for an encryption
 * zone
 */
message ZoneEncryptionInfoProto {
  required CipherSuiteProto suite = 1;
  required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
  required string keyName = 3;
}

/**
 * Cipher option
 */
message CipherOptionProto {
  required CipherSuiteProto suite = 1;
  optional bytes inKey = 2;
  optional bytes inIv = 3;
  optional bytes outKey = 4;
  optional bytes outIv = 5;
}

/**
 * A set of file blocks and their locations.
 */
message LocatedBlocksProto {
  required uint64 fileLength = 1;
  repeated LocatedBlockProto blocks = 2;
  required bool underConstruction = 3;
  optional LocatedBlockProto lastBlock = 4;
  required bool isLastBlockComplete = 5;
  optional FileEncryptionInfoProto fileEncryptionInfo = 6;

  // Optional field for erasure coding
  optional ECSchemaProto eCSchema = 7;
  optional uint32 stripeCellSize = 8;
}

/**
 * ECSchema options entry
 */
message ECSchemaOptionEntryProto {
  required string key = 1;
  required string value = 2;
}

/**
 * ECSchema for erasurecoding
 */
message ECSchemaProto {
  required string schemaName = 1;
  required string codecName = 2;
  required uint32 dataUnits = 3;
  required uint32 parityUnits = 4;
  repeated ECSchemaOptionEntryProto options = 5;
}

/**
 * Status of a file, directory or symlink
 * Optionally includes a file's block locations if requested by client on the rpc call.
 */
message HdfsFileStatusProto {
  enum FileType {
    IS_DIR = 1;
    IS_FILE = 2;
    IS_SYMLINK = 3;
  }
  required FileType fileType = 1;
  required bytes path = 2;          // local name of inode encoded java UTF8
  required uint64 length = 3;
  required FsPermissionProto permission = 4;
  required string owner = 5;
  required string group = 6;
  required uint64 modification_time = 7;
  required uint64 access_time = 8;

  // Optional fields for symlink
  optional bytes symlink = 9;             // if symlink, target encoded java UTF8 

  // Optional fields for file
  optional uint32 block_replication = 10 [default = 0]; // only 16bits used
  optional uint64 blocksize = 11 [default = 0];
  optional LocatedBlocksProto locations = 12;  // suppled only if asked by client

  // Optional field for fileId
  optional uint64 fileId = 13 [default = 0]; // default as an invalid id
  optional int32 childrenNum = 14 [default = -1];
  // Optional field for file encryption
  optional FileEncryptionInfoProto fileEncryptionInfo = 15;

  optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id

  // Optional field for erasure coding
  optional ECSchemaProto ecSchema = 17;
  optional uint32 stripeCellSize = 18;
}

/**
 * Checksum algorithms/types used in HDFS
 * Make sure this enum's integer values match enum values' id properties defined
 * in org.apache.hadoop.util.DataChecksum.Type
 */
enum ChecksumTypeProto {
  CHECKSUM_NULL = 0;
  CHECKSUM_CRC32 = 1;
  CHECKSUM_CRC32C = 2;
}

/**
 * HDFS Server Defaults
 */
message FsServerDefaultsProto {
  required uint64 blockSize = 1;
  required uint32 bytesPerChecksum = 2;
  required uint32 writePacketSize = 3;
  required uint32 replication = 4; // Actually a short - only 16 bits used
  required uint32 fileBufferSize = 5;
  optional bool encryptDataTransfer = 6 [default = false];
  optional uint64 trashInterval = 7 [default = 0];
  optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
}


/**
 * Directory listing
 */
message DirectoryListingProto {
  repeated HdfsFileStatusProto partialListing = 1;
  required uint32 remainingEntries  = 2;
}

/**
 * Status of a snapshottable directory: besides the normal information for 
 * a directory status, also include snapshot quota, number of snapshots, and
 * the full path of the parent directory. 
 */
message SnapshottableDirectoryStatusProto {
  required HdfsFileStatusProto dirStatus = 1;

  // Fields specific for snapshottable directory
  required uint32 snapshot_quota = 2;
  required uint32 snapshot_number = 3;
  required bytes parent_fullpath = 4;
}

/**
 * Snapshottable directory listing
 */
message SnapshottableDirectoryListingProto {
  repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
}

/**
 * Snapshot diff report entry
 */
message SnapshotDiffReportEntryProto {
  required bytes fullpath = 1;
  required string modificationLabel = 2;
  optional bytes targetPath = 3;
}

/**
 * Snapshot diff report
 */
message SnapshotDiffReportProto {
  // full path of the directory where snapshots were taken
  required string snapshotRoot = 1;
  required string fromSnapshot = 2;
  required string toSnapshot = 3;
  repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
}

/**
 * Common node information shared by all the nodes in the cluster
 */
message StorageInfoProto {
  required uint32 layoutVersion = 1; // Layout version of the file system
  required uint32 namespceID = 2;    // File system namespace ID
  required string clusterID = 3;     // ID of the cluster
  required uint64 cTime = 4;         // File system creation time
}

/**
 * Information sent by a namenode to identify itself to the primary namenode.
 */
message NamenodeRegistrationProto {
  required string rpcAddress = 1;    // host:port of the namenode RPC address
  required string httpAddress = 2;   // host:port of the namenode http server
  enum NamenodeRoleProto {
    NAMENODE = 1;
    BACKUP = 2;
    CHECKPOINT = 3;
  }
  required StorageInfoProto storageInfo = 3;  // Node information
  optional NamenodeRoleProto role = 4 [default = NAMENODE];        // Namenode role
}

/**
 * Unique signature to identify checkpoint transactions.
 */
message CheckpointSignatureProto {
  required string blockPoolId = 1;
  required uint64 mostRecentCheckpointTxId = 2;
  required uint64 curSegmentTxId = 3;
  required StorageInfoProto storageInfo = 4;
}

/**
 * Command sent from one namenode to another namenode.
 */
message NamenodeCommandProto {
  enum Type {
    NamenodeCommand = 0;      // Base command
    CheckPointCommand = 1;    // Check point command
  }
  required uint32 action = 1;
  required Type type = 2;
  optional CheckpointCommandProto checkpointCmd = 3;
}

/**
 * Command returned from primary to checkpointing namenode.
 * This command has checkpoint signature that identifies
 * checkpoint transaction and is needed for further
 * communication related to checkpointing.
 */
message CheckpointCommandProto {
  // Unique signature to identify checkpoint transation
  required CheckpointSignatureProto signature = 1; 

  // If true, return transfer image to primary upon the completion of checkpoint
  required bool needToReturnImage = 2;
}

/**
 * Block information
 *
 * Please be wary of adding additional fields here, since INodeFiles
 * need to fit in PB's default max message size of 64MB.
 * We restrict the max # of blocks per file
 * (dfs.namenode.fs-limits.max-blocks-per-file), but it's better
 * to avoid changing this.
 */
message BlockProto {
  required uint64 blockId = 1;
  required uint64 genStamp = 2;
  optional uint64 numBytes = 3 [default = 0];
}

/**
 * Block and datanodes where is it located
 */
message BlockWithLocationsProto {
  required BlockProto block = 1;   // Block
  repeated string datanodeUuids = 2; // Datanodes with replicas of the block
  repeated string storageUuids = 3;  // Storages with replicas of the block
  repeated StorageTypeProto storageTypes = 4;

  optional bytes indices = 5;
  optional uint32 dataBlockNum = 6;
}

/**
 * List of block with locations
 */
message BlocksWithLocationsProto {
  repeated BlockWithLocationsProto blocks = 1;
}

/**
 * Editlog information with available transactions
 */
message RemoteEditLogProto {
  required uint64 startTxId = 1;  // Starting available edit log transaction
  required uint64 endTxId = 2;    // Ending available edit log transaction
  optional bool isInProgress = 3 [default = false];
}

/**
 * Enumeration of editlogs available on a remote namenode
 */
message RemoteEditLogManifestProto {
  repeated RemoteEditLogProto logs = 1;
}

/**
 * Namespace information that describes namespace on a namenode
 */
message NamespaceInfoProto {
  required string buildVersion = 1;         // Software revision version (e.g. an svn or git revision)
  required uint32 unused = 2;               // Retained for backward compatibility
  required string blockPoolID = 3;          // block pool used by the namespace
  required StorageInfoProto storageInfo = 4;// Node information
  required string softwareVersion = 5;      // Software version number (e.g. 2.0.0)
  optional uint64 capabilities = 6 [default = 0]; // feature flags
}

/**
 * Block access token information
 */
message BlockKeyProto {
  required uint32 keyId = 1;      // Key identifier
  required uint64 expiryDate = 2; // Expiry time in milliseconds
  optional bytes keyBytes = 3;    // Key secret
}

/**
 * Current key and set of block keys at the namenode.
 */
message ExportedBlockKeysProto {
  required bool isBlockTokenEnabled = 1;
  required uint64 keyUpdateInterval = 2;
  required uint64 tokenLifeTime = 3;
  required BlockKeyProto currentKey = 4;
  repeated BlockKeyProto allKeys = 5;
}

/**
 * State of a block replica at a datanode
 */
enum ReplicaStateProto {
  FINALIZED = 0;  // State of a replica when it is not modified
  RBW = 1;        // State of replica that is being written to
  RWR = 2;        // State of replica that is waiting to be recovered
  RUR = 3;        // State of replica that is under recovery
  TEMPORARY = 4;  // State of replica that is created for replication
}

/**
 * Block that needs to be recovered with at a given location
 */
message RecoveringBlockProto {
  required uint64 newGenStamp = 1;        // New genstamp post recovery
  required LocatedBlockProto block = 2;   // Block to be recovered
  optional BlockProto truncateBlock = 3;  // New block for recovery (truncate)
}

/**
 * void request
 */
message VersionRequestProto {
}

/**
 * Version response from namenode.
 */
message VersionResponseProto {
  required NamespaceInfoProto info = 1;
}

/**
 * Information related to a snapshot
 * TODO: add more information
 */
message SnapshotInfoProto {
  required string snapshotName = 1;
  required string snapshotRoot = 2;
  required FsPermissionProto permission = 3;
  required string owner = 4;
  required string group = 5;
  required string createTime = 6;
  // TODO: do we need access time?
}

/**
 * Rolling upgrade status
 */
message RollingUpgradeStatusProto {
  required string blockPoolId = 1;
  optional bool finalized = 2 [default = false];
}