hadoop/hadoop-hdds/common/src/main/proto/DatanodeContainerProtocol.proto

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
 * These .proto interfaces are private and Unstable.
 * Please see http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html
 * for what changes are allowed for a *Unstable* .proto interface.
 */

// This file contains protocol buffers that are used to transfer data
// to and from the datanode.
syntax = "proto2";
option java_package = "org.apache.hadoop.hdds.protocol.datanode.proto";
option java_outer_classname = "ContainerProtos";
option java_generate_equals_and_hash = true;
package hadoop.hdds.datanode;

/**
 * Commands that are used to manipulate the state of containers on a datanode.
 *
 * These commands allow us to work against the datanode - from
 * StorageContainer Manager as well as clients.
 *
 *  1. CreateContainer - This call is usually made by Storage Container
 *     manager, when we need to create a new container on a given datanode.
 *
 *  2. ReadContainer - Allows end user to stat a container. For example
 *     this allows us to return the metadata of a container.
 *
 *  3. UpdateContainer - Updates a container metadata.

 *  4. DeleteContainer - This call is made to delete a container.
 *
 *  5. ListContainer - Returns the list of containers on this
 *     datanode. This will be used by tests and tools.
 *
 *  6. PutKey - Given a valid container, creates a key.
 *
 *  7. GetKey - Allows user to read the metadata of a Key.
 *
 *  8. DeleteKey - Deletes a given key.
 *
 *  9. ListKey - Returns a list of keys that are present inside
 *      a given container.
 *
 *  10. ReadChunk - Allows us to read a chunk.
 *
 *  11. DeleteChunk - Delete an unused chunk.
 *
 *  12. WriteChunk - Allows us to write a chunk
 *
 *  13. ListChunk - Given a Container/Key returns the list of Chunks.
 *
 *  14. CompactChunk - Re-writes a chunk based on Offsets.
 *
 *  15. PutSmallFile - A single RPC that combines both putKey and WriteChunk.
 *
 *  16. GetSmallFile - A single RPC that combines both getKey and ReadChunk.
 *
 *  17. CloseContainer - Closes an open container and makes it immutable.
 *
 *  18. CopyContainer - Copies a container from a remote machine.
 */

enum Type {
  CreateContainer = 1;
  ReadContainer = 2;
  UpdateContainer = 3;
  DeleteContainer = 4;
  ListContainer = 5;

  PutKey = 6;
  GetKey = 7;
  DeleteKey = 8;
  ListKey = 9;

  ReadChunk = 10;
  DeleteChunk = 11;
  WriteChunk = 12;
  ListChunk = 13;
  CompactChunk = 14;

  /** Combines Key and Chunk Operation into Single RPC. */
  PutSmallFile = 15;
  GetSmallFile = 16;
  CloseContainer = 17;

}


enum Result {
  SUCCESS = 1;
  UNSUPPORTED_REQUEST = 2;
  MALFORMED_REQUEST = 3;
  CONTAINER_INTERNAL_ERROR = 4;
  INVALID_CONFIG = 5;
  INVALID_FILE_HASH_FOUND = 6;
  CONTAINER_EXISTS = 7;
  NO_SUCH_ALGORITHM = 8;
  CONTAINER_NOT_FOUND = 9;
  IO_EXCEPTION = 10;
  UNABLE_TO_READ_METADATA_DB = 11;
  NO_SUCH_KEY = 12;
  OVERWRITE_FLAG_REQUIRED = 13;
  UNABLE_TO_FIND_DATA_DIR = 14;
  INVALID_WRITE_SIZE = 15;
  CHECKSUM_MISMATCH = 16;
  UNABLE_TO_FIND_CHUNK = 17;
  PROTOC_DECODING_ERROR = 18;
  INVALID_ARGUMENT = 19;
  PUT_SMALL_FILE_ERROR = 20;
  GET_SMALL_FILE_ERROR = 21;
  CLOSED_CONTAINER_IO = 22;
  ERROR_CONTAINER_NOT_EMPTY = 23;
  ERROR_IN_COMPACT_DB = 24;
  UNCLOSED_CONTAINER_IO = 25;
  DELETE_ON_OPEN_CONTAINER = 26;
  CLOSED_CONTAINER_RETRY = 27;
  INVALID_CONTAINER_STATE = 28;
}

/**
 * Block ID that uniquely identify a block in Datanode.
 */
message DatanodeBlockID {
  required int64 containerID = 1;
  required int64 localID = 2;
}

message KeyValue {
  required string key = 1;
  optional string value = 2;
}

/**
 * Lifecycle states of a container in Datanode.
 */
enum ContainerLifeCycleState {
    OPEN = 1;
    CLOSING = 2;
    CLOSED = 3;
    INVALID = 4;
}

message ContainerCommandRequestProto {
  required Type cmdType = 1; // Type of the command

  // A string that identifies this command, we generate  Trace ID in Ozone
  // frontend and this allows us to trace that command all over ozone.
  optional string traceID = 2;

  // One of the following command is available when the corresponding
  // cmdType is set. At the protocol level we allow only
  // one command in each packet.
  // TODO : Upgrade to Protobuf 2.6 or later.
  optional   CreateContainerRequestProto createContainer = 3;
  optional   ReadContainerRequestProto readContainer = 4;
  optional   UpdateContainerRequestProto updateContainer = 5;
  optional   DeleteContainerRequestProto deleteContainer = 6;
  optional   ListContainerRequestProto listContainer = 7;

  optional   PutKeyRequestProto putKey = 8;
  optional   GetKeyRequestProto getKey = 9;
  optional   DeleteKeyRequestProto deleteKey = 10;
  optional   ListKeyRequestProto listKey = 11;

  optional   ReadChunkRequestProto readChunk = 12;
  optional   WriteChunkRequestProto writeChunk = 13;
  optional   DeleteChunkRequestProto deleteChunk = 14;
  optional   ListChunkRequestProto listChunk = 15;

  optional   PutSmallFileRequestProto putSmallFile = 16;
  optional   GetSmallFileRequestProto getSmallFile = 17;
  optional   CloseContainerRequestProto closeContainer = 18;
  required   string datanodeUuid = 19;
}

message ContainerCommandResponseProto {
  required Type cmdType = 1;
  optional string traceID = 2;

  optional   CreateContainerResponseProto createContainer = 3;
  optional   ReadContainerResponseProto readContainer = 4;
  optional   UpdateContainerResponseProto updateContainer = 5;
  optional   DeleteContainerResponseProto deleteContainer = 6;
  optional   ListContainerResponseProto listContainer = 7;

  optional   PutKeyResponseProto putKey = 8;
  optional   GetKeyResponseProto getKey = 9;
  optional   DeleteKeyResponseProto deleteKey = 10;
  optional   ListKeyResponseProto listKey = 11;

  optional  WriteChunkResponseProto writeChunk = 12;
  optional  ReadChunkResponseProto readChunk = 13;
  optional  DeleteChunkResponseProto deleteChunk = 14;
  optional  ListChunkResponseProto listChunk = 15;

  required Result result = 17;
  optional string message = 18;

  optional PutSmallFileResponseProto putSmallFile = 19;
  optional GetSmallFileResponseProto getSmallFile = 20;
  optional CloseContainerResponseProto closeContainer = 21;

}

message ContainerData {
  required int64 containerID = 1;
  repeated KeyValue metadata = 2;
  optional string dbPath = 3;
  optional string containerPath = 4;
  optional int64 bytesUsed = 6;
  optional int64 size = 7;
  optional int64 keyCount = 8;
  optional ContainerLifeCycleState state = 9 [default = OPEN];
  optional ContainerType containerType = 10 [default = KeyValueContainer];
  optional string containerDBType = 11;
}

enum ContainerType {
  KeyValueContainer = 1;
}


// Container Messages.
message  CreateContainerRequestProto {
  required ContainerData containerData = 1;
}

message  CreateContainerResponseProto {
}

message  ReadContainerRequestProto {
  required int64 containerID = 1;
}

message  ReadContainerResponseProto {
  optional ContainerData containerData = 1;
}

message  UpdateContainerRequestProto {
  required ContainerData containerData = 1;
  optional bool forceUpdate = 2 [default = false];
}

message  UpdateContainerResponseProto {
}

message  DeleteContainerRequestProto {
  required int64 containerID = 1;
  optional bool forceDelete = 2 [default = false];
}

message  DeleteContainerResponseProto {
}

message  ListContainerRequestProto {
  required int64 startContainerID = 1;
  optional uint32 count = 2; // Max Results to return
}

message  ListContainerResponseProto {
  repeated ContainerData containerData = 1;
}

message CloseContainerRequestProto {
  required int64 containerID = 1;
}

message CloseContainerResponseProto {
  optional string hash = 1;
  optional int64 containerID = 2;
}

message KeyData {
  required DatanodeBlockID blockID = 1;
  optional int64 flags = 2; // for future use.
  repeated KeyValue metadata = 3;
  repeated ChunkInfo chunks = 4;
}

// Key Messages.
message  PutKeyRequestProto {
  required KeyData keyData = 1;
}

message  PutKeyResponseProto {
}

message  GetKeyRequestProto  {
  required KeyData keyData = 1;
}

message  GetKeyResponseProto  {
  required KeyData keyData = 1;
}


message  DeleteKeyRequestProto {
  required DatanodeBlockID blockID = 1;
}

message   DeleteKeyResponseProto {
}

message  ListKeyRequestProto {
  required int64 containerID = 1;
  optional int64 startLocalID = 2;
  required uint32 count = 3;

}

message  ListKeyResponseProto {
  repeated KeyData keyData = 1;
}

// Chunk Operations

message ChunkInfo {
  required string chunkName = 1;
  required uint64 offset = 2;
  required uint64 len = 3;
  optional string checksum = 4;
  repeated KeyValue metadata = 5;
}

enum Stage {
    WRITE_DATA = 1;
    COMMIT_DATA = 2;
    COMBINED = 3;
}

message  WriteChunkRequestProto  {
  required DatanodeBlockID blockID = 1;
  required ChunkInfo chunkData = 2;
  optional bytes data = 3;
  optional Stage stage = 4 [default = COMBINED];
}

message  WriteChunkResponseProto {
}

message  ReadChunkRequestProto  {
  required DatanodeBlockID blockID = 1;
  required ChunkInfo chunkData = 2;
}

message  ReadChunkResponseProto {
  required DatanodeBlockID blockID = 1;
  required ChunkInfo chunkData = 2;
  required bytes data = 3;
}

message  DeleteChunkRequestProto {
  required DatanodeBlockID blockID = 1;
  required ChunkInfo chunkData = 2;
}

message  DeleteChunkResponseProto {
}

message  ListChunkRequestProto {
  required DatanodeBlockID blockID = 1;
  required string prevChunkName = 2;
  required uint32 count = 3;
}

message  ListChunkResponseProto {
  repeated ChunkInfo chunkData = 1;
}

/** For small file access combines write chunk and putKey into a single
RPC */

message PutSmallFileRequestProto {
  required PutKeyRequestProto key = 1;
  required ChunkInfo chunkInfo = 2;
  required bytes data = 3;
}


message PutSmallFileResponseProto {

}

message GetSmallFileRequestProto {
  required GetKeyRequestProto key = 1;
}

message GetSmallFileResponseProto {
  required ReadChunkResponseProto data = 1;
}

message CopyContainerRequestProto {
  required int64 containerID = 1;
  required uint64 readOffset = 2;
  optional uint64 len = 3;
}

message CopyContainerResponseProto {
  required string archiveName = 1;
  required uint64 readOffset = 2;
  required uint64 len = 3;
  required bool eof = 4;
  repeated bytes data = 5;
  optional int64 checksum = 6;
}

service XceiverClientProtocolService {
  // A client-to-datanode RPC to send container commands
  rpc send(stream ContainerCommandRequestProto) returns
    (stream ContainerCommandResponseProto) {}
}