HDDS-694. Plugin new Pipeline management code in SCM.

Contributed by Lokesh Jain.

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServi
 import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc.XceiverClientProtocolServiceStub;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.util.Time;
 import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.UUID;
@@ -84,9 +85,9 @@ public class XceiverClientGrpc extends XceiverClientSpi {
   public void connect() throws Exception {
 
     // leader by default is the 1st datanode in the datanode list of pipleline
-    DatanodeDetails leader = this.pipeline.getLeader();
+    DatanodeDetails dn = this.pipeline.getFirstNode();
     // just make a connection to the 1st datanode at the beginning
-    connectToDatanode(leader);
+    connectToDatanode(dn);
   }
 
   private void connectToDatanode(DatanodeDetails dn) {
@@ -148,18 +149,16 @@ public class XceiverClientGrpc extends XceiverClientSpi {
 
   public ContainerCommandResponseProto sendCommandWithRetry(
       ContainerCommandRequestProto request) throws IOException {
-    int size = pipeline.getMachines().size();
     ContainerCommandResponseProto responseProto = null;
-    DatanodeDetails dn = null;
 
     // In case of an exception or an error, we will try to read from the
     // datanodes in the pipeline in a round robin fashion.
 
     // TODO: cache the correct leader info in here, so that any subsequent calls
     // should first go to leader
-    for (int dnIndex = 0; dnIndex < size; dnIndex++) {
+    List<DatanodeDetails> dns = pipeline.getNodes();
+    for (DatanodeDetails dn : dns) {
       try {
-        dn = pipeline.getMachines().get(dnIndex);
         LOG.debug("Executing command " + request + " on datanode " + dn);
         // In case the command gets retried on a 2nd datanode,
         // sendCommandAsyncCall will create a new channel and async stub
@@ -201,7 +200,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
   public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
       ContainerCommandRequestProto request)
       throws IOException, ExecutionException, InterruptedException {
-    return sendCommandAsync(request, pipeline.getLeader());
+    return sendCommandAsync(request, pipeline.getFirstNode());
   }
 
   private CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java

@@ -25,7 +25,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.RemovalListener;
 import com.google.common.cache.RemovalNotification;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 
 import java.io.Closeable;
@@ -115,8 +115,8 @@ public class XceiverClientManager implements Closeable {
   public XceiverClientSpi acquireClient(Pipeline pipeline)
       throws IOException {
     Preconditions.checkNotNull(pipeline);
-    Preconditions.checkArgument(pipeline.getMachines() != null);
-    Preconditions.checkArgument(!pipeline.getMachines().isEmpty());
+    Preconditions.checkArgument(pipeline.getNodes() != null);
+    Preconditions.checkArgument(!pipeline.getNodes().isEmpty());
 
     synchronized (clientCache) {
       XceiverClientSpi info = getClient(pipeline);

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -19,7 +19,6 @@
 package org.apache.hadoop.hdds.scm;
 
 import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
 import org.apache.hadoop.io.MultipleIOException;
 import org.apache.ratis.proto.RaftProtos;
 import org.apache.ratis.retry.RetryPolicy;
@@ -27,7 +26,7 @@ import org.apache.ratis.thirdparty.com.google.protobuf
     .InvalidProtocolBufferException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandRequestProto;
@@ -63,19 +62,6 @@ import java.util.concurrent.atomic.AtomicReference;
 public final class XceiverClientRatis extends XceiverClientSpi {
   static final Logger LOG = LoggerFactory.getLogger(XceiverClientRatis.class);
 
-  public static XceiverClientRatis newXceiverClientRatis(
-      Pipeline pipeline, Configuration ozoneConf) {
-    final String rpcType = ozoneConf.get(
-        ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
-        ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
-    final int maxOutstandingRequests =
-        HddsClientUtils.getMaxOutstandingRequests(ozoneConf);
-    final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
-    return new XceiverClientRatis(pipeline,
-        SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests,
-        retryPolicy);
-  }
-
   public static XceiverClientRatis newXceiverClientRatis(
       org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
       Configuration ozoneConf) {
@@ -85,11 +71,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
     final int maxOutstandingRequests =
         HddsClientUtils.getMaxOutstandingRequests(ozoneConf);
     final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
-    Pipeline pipeline1 =
-        new Pipeline(pipeline.getNodes().get(0).getUuidString(),
-            HddsProtos.LifeCycleState.OPEN, pipeline.getType(),
-            pipeline.getFactor(), PipelineID.valueOf(pipeline.getID().getId()));
-    return new XceiverClientRatis(pipeline1,
+    return new XceiverClientRatis(pipeline,
         SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests,
         retryPolicy);
   }
@@ -118,7 +100,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
   public void createPipeline() throws IOException {
     final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
     LOG.debug("creating pipeline:{} with {}", pipeline.getId(), group);
-    callRatisRpc(pipeline.getMachines(),
+    callRatisRpc(pipeline.getNodes(),
         (raftClient, peer) -> raftClient.groupAdd(group, peer.getId()));
   }
 
@@ -128,7 +110,7 @@ public final class XceiverClientRatis extends XceiverClientSpi {
   public void destroyPipeline() throws IOException {
     final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
     LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
-    callRatisRpc(pipeline.getMachines(), (raftClient, peer) -> raftClient
+    callRatisRpc(pipeline.getNodes(), (raftClient, peer) -> raftClient
         .groupRemove(group.getGroupId(), true, peer.getId()));
   }
 
@@ -174,9 +156,8 @@ public final class XceiverClientRatis extends XceiverClientSpi {
 
   @Override
   public void connect() throws Exception {
-    LOG.debug("Connecting to pipeline:{} leader:{}",
-        getPipeline().getId(),
-        RatisHelper.toRaftPeerId(pipeline.getLeader()));
+    LOG.debug("Connecting to pipeline:{} datanode:{}", getPipeline().getId(),
+        RatisHelper.toRaftPeerId(pipeline.getFirstNode()));
     // TODO : XceiverClient ratis should pass the config value of
     // maxOutstandingRequests so as to set the upper bound on max no of async
     // requests to be handled by raft client

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java

@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
@@ -40,11 +40,6 @@ import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
 
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState
-    .ALLOCATED;
-import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState
-    .OPEN;
-
 /**
  * This class provides the client-facing APIs of container operations.
  */
@@ -98,14 +93,10 @@ public class ContainerOperationClient implements ScmClient {
       Pipeline pipeline = containerWithPipeline.getPipeline();
       client = xceiverClientManager.acquireClient(pipeline);
 
-      // Allocated State means that SCM has allocated this pipeline in its
-      // namespace. The client needs to create the pipeline on the machines
-      // which was choosen by the SCM.
-      Preconditions.checkState(pipeline.getLifeCycleState() == ALLOCATED ||
-          pipeline.getLifeCycleState() == OPEN, "Unexpected pipeline state");
-      if (pipeline.getLifeCycleState() == ALLOCATED) {
-        createPipeline(client, pipeline);
-      }
+      Preconditions.checkState(pipeline.isOpen(), String
+          .format("Unexpected state=%s for pipeline=%s, expected state=%s",
+              pipeline.getPipelineState(), pipeline.getId(),
+              Pipeline.PipelineState.OPEN));
       createContainer(client,
           containerWithPipeline.getContainerInfo().getContainerID());
       return containerWithPipeline;
@@ -142,8 +133,7 @@ public class ContainerOperationClient implements ScmClient {
     // creation state.
     if (LOG.isDebugEnabled()) {
       LOG.debug("Created container " + containerId
-          + " leader:" + client.getPipeline().getLeader()
-          + " machines:" + client.getPipeline().getMachines());
+          + " machines:" + client.getPipeline().getNodes());
     }
   }
 
@@ -208,12 +198,6 @@ public class ContainerOperationClient implements ScmClient {
       Pipeline pipeline = containerWithPipeline.getPipeline();
       client = xceiverClientManager.acquireClient(pipeline);
 
-      // Allocated State means that SCM has allocated this pipeline in its
-      // namespace. The client needs to create the pipeline on the machines
-      // which was choosen by the SCM.
-      if (pipeline.getLifeCycleState() == ALLOCATED) {
-        createPipeline(client, pipeline);
-      }
       // connect to pipeline leader and allocate container on leader datanode.
       client = xceiverClientManager.acquireClient(pipeline);
       createContainer(client,
@@ -283,10 +267,8 @@ public class ContainerOperationClient implements ScmClient {
       storageContainerLocationClient
           .deleteContainer(containerId);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Deleted container {}, leader: {}, machines: {} ",
-            containerId,
-            pipeline.getLeader(),
-            pipeline.getMachines());
+        LOG.debug("Deleted container {}, machines: {} ", containerId,
+            pipeline.getNodes());
       }
     } finally {
       if (client != null) {
@@ -336,10 +318,8 @@ public class ContainerOperationClient implements ScmClient {
       ReadContainerResponseProto response =
           ContainerProtocolCalls.readContainer(client, containerID, traceID);
       if (LOG.isDebugEnabled()) {
-        LOG.debug("Read container {}, leader: {}, machines: {} ",
-            containerID,
-            pipeline.getLeader(),
-            pipeline.getMachines());
+        LOG.debug("Read container {}, machines: {} ", containerID,
+            pipeline.getNodes());
       }
       return response.getContainerData();
     } finally {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java

@@ -19,7 +19,7 @@
 package org.apache.hadoop.hdds.scm;
 
 import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandRequestProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.hdds.scm.client;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerData;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java

@@ -36,7 +36,7 @@ import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.util.Time;
 
 /**

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdds.scm.container.common.helpers;
 
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.client.BlockID;
 
 /**

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerWithPipeline.java

@@ -23,6 +23,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder;
 import org.apache.commons.lang3.builder.HashCodeBuilder;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 
 /**
  * Class wraps ozone container info.
@@ -50,7 +51,7 @@ public class ContainerWithPipeline implements Comparator<ContainerWithPipeline>,
       HddsProtos.ContainerWithPipeline allocatedContainer) {
     return new ContainerWithPipeline(
         ContainerInfo.fromProtobuf(allocatedContainer.getContainerInfo()),
-        Pipeline.getFromProtoBuf(allocatedContainer.getPipeline()));
+        Pipeline.getFromProtobuf(allocatedContainer.getPipeline()));
   }
 
   public HddsProtos.ContainerWithPipeline getProtobuf() {

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/Pipeline.java

@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.List;
-
-/**
- * A pipeline represents the group of machines over which a container lives.
- */
-public class Pipeline {
-  static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER";
-  private static final ObjectWriter WRITER;
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    String[] ignorableFieldNames = {"leaderID", "datanodes"};
-    FilterProvider filters = new SimpleFilterProvider()
-        .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter
-            .serializeAllExcept(ignorableFieldNames));
-    mapper.setVisibility(PropertyAccessor.FIELD,
-        JsonAutoDetect.Visibility.ANY);
-    mapper.addMixIn(Object.class, MixIn.class);
-
-    WRITER = mapper.writer(filters);
-  }
-
-  @JsonIgnore
-  private String leaderID;
-  @JsonIgnore
-  private Map<String, DatanodeDetails> datanodes;
-  private HddsProtos.LifeCycleState lifeCycleState;
-  private HddsProtos.ReplicationType type;
-  private HddsProtos.ReplicationFactor factor;
-  private PipelineID id;
-
-  /**
-   * Constructs a new pipeline data structure.
-   *
-   * @param leaderID       -  Leader datanode id
-   * @param lifeCycleState  - Pipeline State
-   * @param replicationType - Replication protocol
-   * @param replicationFactor - replication count on datanodes
-   * @param id  - pipeline ID
-   */
-  public Pipeline(String leaderID, HddsProtos.LifeCycleState lifeCycleState,
-      HddsProtos.ReplicationType replicationType,
-      HddsProtos.ReplicationFactor replicationFactor, PipelineID id) {
-    this.leaderID = leaderID;
-    this.lifeCycleState = lifeCycleState;
-    this.type = replicationType;
-    this.factor = replicationFactor;
-    this.id = id;
-    datanodes = new ConcurrentHashMap<>();
-  }
-
-  @Override
-  public int hashCode() {
-    return id.hashCode();
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    Pipeline that = (Pipeline) o;
-
-    return id.equals(that.id)
-            && factor.equals(that.factor)
-            && type.equals(that.type)
-            && lifeCycleState.equals(that.lifeCycleState)
-            && leaderID.equals(that.leaderID);
-
-  }
-
-  /**
-   * Gets pipeline object from protobuf.
-   *
-   * @param pipelineProto - ProtoBuf definition for the pipeline.
-   * @return Pipeline Object
-   */
-  public static Pipeline getFromProtoBuf(
-      HddsProtos.Pipeline pipelineProto) {
-    Preconditions.checkNotNull(pipelineProto);
-    Pipeline pipeline =
-        new Pipeline(pipelineProto.getLeaderID(),
-            pipelineProto.getState(),
-            pipelineProto.getType(),
-            pipelineProto.getFactor(),
-            PipelineID.getFromProtobuf(pipelineProto.getId()));
-
-    for (HddsProtos.DatanodeDetailsProto dataID :
-        pipelineProto.getMembersList()) {
-      pipeline.addMember(DatanodeDetails.getFromProtoBuf(dataID));
-    }
-    return pipeline;
-  }
-
-  /**
-   * returns the replication count.
-   * @return Replication Factor
-   */
-  public HddsProtos.ReplicationFactor getFactor() {
-    return factor;
-  }
-
-  /**
-   * Returns the first machine in the set of datanodes.
-   *
-   * @return First Machine.
-   */
-  @JsonIgnore
-  public DatanodeDetails getLeader() {
-    return getDatanodes().get(leaderID);
-  }
-
-  /**
-   * Adds a datanode to pipeline
-   * @param datanodeDetails datanode to be added.
-   * @return true if the dn was not earlier present, false otherwise
-   */
-  public boolean addMember(DatanodeDetails datanodeDetails) {
-    return datanodes.put(datanodeDetails.getUuid().toString(),
-        datanodeDetails) == null;
-
-  }
-
-  public void resetPipeline() {
-    // reset datanodes in pipeline and learn about them through
-    // pipeline reports on SCM restart
-    datanodes.clear();
-  }
-
-  public Map<String, DatanodeDetails> getDatanodes() {
-    return datanodes;
-  }
-
-  public boolean isEmpty() {
-    return datanodes.isEmpty();
-  }
-  /**
-   * Returns the leader host.
-   *
-   * @return First Machine.
-   */
-  public String getLeaderHost() {
-    return getDatanodes()
-        .get(leaderID).getHostName();
-  }
-
-  /**
-   *
-   * @return lead
-   */
-  public String getLeaderID() {
-    return leaderID;
-  }
-  /**
-   * Returns all machines that make up this pipeline.
-   *
-   * @return List of Machines.
-   */
-  @JsonIgnore
-  public List<DatanodeDetails> getMachines() {
-    return new ArrayList<>(getDatanodes().values());
-  }
-
-  /**
-   * Returns all machines that make up this pipeline.
-   *
-   * @return List of Machines.
-   */
-  public List<String> getDatanodeHosts() {
-    List<String> dataHosts = new ArrayList<>();
-    for (DatanodeDetails datanode : getDatanodes().values()) {
-      dataHosts.add(datanode.getHostName());
-    }
-    return dataHosts;
-  }
-
-  /**
-   * Return a Protobuf Pipeline message from pipeline.
-   *
-   * @return Protobuf message
-   */
-  @JsonIgnore
-  public HddsProtos.Pipeline getProtobufMessage() {
-    HddsProtos.Pipeline.Builder builder =
-        HddsProtos.Pipeline.newBuilder();
-    for (DatanodeDetails datanode : datanodes.values()) {
-      builder.addMembers(datanode.getProtoBufMessage());
-    }
-    builder.setLeaderID(leaderID);
-
-    if (lifeCycleState != null) {
-      builder.setState(lifeCycleState);
-    }
-    if (type != null) {
-      builder.setType(type);
-    }
-
-    if (factor != null) {
-      builder.setFactor(factor);
-    }
-
-    if (id != null) {
-      builder.setId(id.getProtobuf());
-    }
-    return builder.build();
-  }
-
-  /**
-   * Gets the State of the pipeline.
-   *
-   * @return - LifeCycleStates.
-   */
-  public HddsProtos.LifeCycleState getLifeCycleState() {
-    return lifeCycleState;
-  }
-
-  /**
-   * Update the State of the pipeline.
-   */
-  public void setLifeCycleState(HddsProtos.LifeCycleState nextState) {
-    lifeCycleState = nextState;
-  }
-
-  /**
-   * Gets the pipeline id.
-   *
-   * @return - Id of the pipeline
-   */
-  public PipelineID getId() {
-    return id;
-  }
-
-  /**
-   * Returns the type.
-   *
-   * @return type - Standalone, Ratis, Chained.
-   */
-  public HddsProtos.ReplicationType getType() {
-    return type;
-  }
-
-  @Override
-  public String toString() {
-    final StringBuilder b = new StringBuilder(getClass().getSimpleName())
-        .append("[");
-    getDatanodes().keySet().forEach(
-        node -> b.append(node.endsWith(getLeaderID()) ? "*" + id : id));
-    b.append(" id:").append(id);
-    if (getType() != null) {
-      b.append(" type:").append(getType().toString());
-    }
-    if (getFactor() != null) {
-      b.append(" factor:").append(getFactor().toString());
-    }
-    if (getLifeCycleState() != null) {
-      b.append(" State:").append(getLifeCycleState().toString());
-    }
-    return b.toString();
-  }
-
-  public void setType(HddsProtos.ReplicationType type) {
-    this.type = type;
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   *
-   * @return String - json string
-   * @throws IOException
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  @JsonFilter(PIPELINE_INFO)
-  class MixIn {
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/PipelineID.java

@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.ratis.protocol.RaftGroupId;
-
-import java.util.UUID;
-
-/**
- * ID for the pipeline, the ID is based on UUID so that it can be used
- * in Ratis as RaftGroupId, GroupID is used by the datanodes to initialize
- * the ratis group they are part of.
- */
-public final class PipelineID implements Comparable<PipelineID> {
-
-  private UUID id;
-  private RaftGroupId groupId;
-
-  private PipelineID(UUID id) {
-    this.id = id;
-    this.groupId = RaftGroupId.valueOf(id);
-  }
-
-  public static PipelineID randomId() {
-    return new PipelineID(UUID.randomUUID());
-  }
-
-  public static PipelineID valueOf(UUID id) {
-    return new PipelineID(id);
-  }
-
-  public static PipelineID valueOf(RaftGroupId groupId) {
-    return valueOf(groupId.getUuid());
-  }
-
-  public RaftGroupId getRaftGroupID() {
-    return groupId;
-  }
-
-  public UUID getId() {
-    return id;
-  }
-
-  public HddsProtos.PipelineID getProtobuf() {
-    return HddsProtos.PipelineID.newBuilder().setId(id.toString()).build();
-  }
-
-  public static PipelineID getFromProtobuf(HddsProtos.PipelineID protos) {
-    return new PipelineID(UUID.fromString(protos.getId()));
-  }
-
-  @Override
-  public String toString() {
-    return "pipelineId=" + id;
-  }
-
-  @Override
-  public int compareTo(PipelineID o) {
-    return this.id.compareTo(o.id);
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    PipelineID that = (PipelineID) o;
-
-    return id.equals(that.id);
-  }
-
-  @Override
-  public int hashCode() {
-    return id.hashCode();
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java

@@ -60,7 +60,7 @@ public final class Pipeline {
    *
    * @return PipelineID
    */
-  public PipelineID getID() {
+  public PipelineID getId() {
     return id;
   }
 
@@ -87,11 +87,26 @@ public final class Pipeline {
    *
    * @return - LifeCycleStates.
    */
-  PipelineState getPipelineState() {
-    // TODO: See if we need to expose this.
+  public PipelineState getPipelineState() {
     return state;
   }
 
+  /**
+   * Returns the list of nodes which form this pipeline.
+   *
+   * @return List of DatanodeDetails
+   */
+  public List<DatanodeDetails> getNodes() {
+    return new ArrayList<>(nodeStatus.keySet());
+  }
+
+  public DatanodeDetails getFirstNode() throws IOException {
+    if (nodeStatus.isEmpty()) {
+      throw new IOException(String.format("Pipeline=%s is empty", id));
+    }
+    return nodeStatus.keySet().iterator().next();
+  }
+
   public boolean isClosed() {
     return state == PipelineState.CLOSED;
   }
@@ -117,13 +132,8 @@ public final class Pipeline {
     return true;
   }
 
-  /**
-   * Returns the list of nodes which form this pipeline.
-   *
-   * @return List of DatanodeDetails
-   */
-  public List<DatanodeDetails> getNodes() {
-    return new ArrayList<>(nodeStatus.keySet());
+  public boolean isEmpty() {
+    return nodeStatus.isEmpty();
   }
 
   public HddsProtos.Pipeline getProtobufMessage() {
@@ -138,7 +148,7 @@ public final class Pipeline {
     return builder.build();
   }
 
-  public static Pipeline fromProtobuf(HddsProtos.Pipeline pipeline) {
+  public static Pipeline getFromProtobuf(HddsProtos.Pipeline pipeline) {
     return new Builder().setId(PipelineID.getFromProtobuf(pipeline.getId()))
         .setFactor(pipeline.getFactor())
         .setType(pipeline.getType())
@@ -164,8 +174,7 @@ public final class Pipeline {
         .append(id, that.id)
         .append(type, that.type)
         .append(factor, that.factor)
-        .append(state, that.state)
-        .append(nodeStatus, that.nodeStatus)
+        .append(getNodes(), that.getNodes())
         .isEquals();
   }
 
@@ -175,7 +184,6 @@ public final class Pipeline {
         .append(id)
         .append(type)
         .append(factor)
-        .append(state)
         .append(nodeStatus)
         .toHashCode();
   }
@@ -244,7 +252,10 @@ public final class Pipeline {
     }
   }
 
-  enum PipelineState {
+  /**
+   * Possible Pipeline states in SCM.
+   */
+  public enum PipelineState {
     ALLOCATED, OPEN, CLOSED
   }
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineNotFoundException.java

@@ -0,0 +1,28 @@
+package org.apache.hadoop.hdds.scm.pipeline;
+
+import java.io.IOException;
+
+/**
+ * Signals that a pipeline is missing from PipelineManager.
+ */
+public class PipelineNotFoundException extends IOException{
+  /**
+   * Constructs an {@code PipelineNotFoundException} with {@code null}
+   * as its error detail message.
+   */
+  public PipelineNotFoundException() {
+    super();
+  }
+
+  /**
+   * Constructs an {@code PipelineNotFoundException} with the specified
+   * detail message.
+   *
+   * @param message
+   *        The detail message (which is saved for later retrieval
+   *        by the {@link #getMessage()} method)
+   */
+  public PipelineNotFoundException(String message) {
+    super(message);
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.hdds.scm.protocol;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java

@@ -23,7 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.ScmBlockLocationProtocolProtos
@@ -103,7 +103,7 @@ public final class ScmBlockLocationProtocolClientSideTranslatorPB
     }
     AllocatedBlock.Builder builder = new AllocatedBlock.Builder()
         .setBlockID(BlockID.getFromProtobuf(response.getBlockID()))
-        .setPipeline(Pipeline.getFromProtoBuf(response.getPipeline()))
+        .setPipeline(Pipeline.getFromProtobuf(response.getPipeline()))
         .setShouldCreateContainer(response.getCreateContainer());
     return builder.build();
   }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -29,7 +29,7 @@ import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolPro
 import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
@@ -292,7 +292,7 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB
           PipelineResponseProto.Error.success) {
         Preconditions.checkState(response.hasPipeline(), "With success, " +
             "must come a pipeline");
-        return Pipeline.getFromProtoBuf(response.getPipeline());
+        return Pipeline.getFromProtobuf(response.getPipeline());
       } else {
         String errorMessage = String.format("create replication pipeline " +
                 "failed. code : %s Message: %s", response.getErrorCode(),

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/storage/ContainerProtocolCalls.java

@@ -92,7 +92,7 @@ public final class ContainerProtocolCalls  {
         .newBuilder()
         .setBlockID(datanodeBlockID)
         .setBlockCommitSequenceId(blockCommitSequenceId);
-    String id = xceiverClient.getPipeline().getLeader().getUuidString();
+    String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
 
     ContainerCommandRequestProto request = ContainerCommandRequestProto
         .newBuilder()
@@ -125,7 +125,7 @@ public final class ContainerProtocolCalls  {
         getBlockLengthRequestBuilder =
         ContainerProtos.GetCommittedBlockLengthRequestProto.newBuilder().
             setBlockID(blockID.getDatanodeBlockIDProtobuf());
-    String id = xceiverClient.getPipeline().getLeader().getUuidString();
+    String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto request =
         ContainerCommandRequestProto.newBuilder()
             .setCmdType(Type.GetCommittedBlockLength)
@@ -152,7 +152,7 @@ public final class ContainerProtocolCalls  {
       String traceID) throws IOException {
     PutBlockRequestProto.Builder createBlockRequest =
         PutBlockRequestProto.newBuilder().setBlockData(containerBlockData);
-    String id = xceiverClient.getPipeline().getLeader().getUuidString();
+    String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto request =
         ContainerCommandRequestProto.newBuilder().setCmdType(Type.PutBlock)
             .setContainerID(containerBlockData.getBlockID().getContainerID())
@@ -179,7 +179,7 @@ public final class ContainerProtocolCalls  {
         .newBuilder()
         .setBlockID(blockID.getDatanodeBlockIDProtobuf())
         .setChunkData(chunk);
-    String id = xceiverClient.getPipeline().getLeader().getUuidString();
+    String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto request = ContainerCommandRequestProto
         .newBuilder()
         .setCmdType(Type.ReadChunk)
@@ -211,7 +211,7 @@ public final class ContainerProtocolCalls  {
         .setBlockID(blockID.getDatanodeBlockIDProtobuf())
         .setChunkData(chunk)
         .setData(data);
-    String id = xceiverClient.getPipeline().getLeader().getUuidString();
+    String id = xceiverClient.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto request = ContainerCommandRequestProto
         .newBuilder()
         .setCmdType(Type.WriteChunk)
@@ -260,7 +260,7 @@ public final class ContainerProtocolCalls  {
             .setBlock(createBlockRequest).setData(ByteString.copyFrom(data))
             .build();
 
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto request =
         ContainerCommandRequestProto.newBuilder()
             .setCmdType(Type.PutSmallFile)
@@ -288,7 +288,7 @@ public final class ContainerProtocolCalls  {
     createRequest.setContainerType(ContainerProtos.ContainerType
         .KeyValueContainer);
 
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
     ContainerCommandRequestProto.Builder request =
         ContainerCommandRequestProto.newBuilder();
     request.setCmdType(ContainerProtos.Type.CreateContainer);
@@ -314,7 +314,7 @@ public final class ContainerProtocolCalls  {
     ContainerProtos.DeleteContainerRequestProto.Builder deleteRequest =
         ContainerProtos.DeleteContainerRequestProto.newBuilder();
     deleteRequest.setForceDelete(force);
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
 
     ContainerCommandRequestProto.Builder request =
         ContainerCommandRequestProto.newBuilder();
@@ -338,7 +338,7 @@ public final class ContainerProtocolCalls  {
    */
   public static void closeContainer(XceiverClientSpi client,
       long containerID, String traceID) throws IOException {
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
 
     ContainerCommandRequestProto.Builder request =
         ContainerCommandRequestProto.newBuilder();
@@ -362,7 +362,7 @@ public final class ContainerProtocolCalls  {
   public static ReadContainerResponseProto readContainer(
       XceiverClientSpi client, long containerID,
       String traceID) throws IOException {
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
 
     ContainerCommandRequestProto.Builder request =
         ContainerCommandRequestProto.newBuilder();
@@ -396,7 +396,7 @@ public final class ContainerProtocolCalls  {
         GetSmallFileRequestProto
             .newBuilder().setBlock(getBlock)
             .build();
-    String id = client.getPipeline().getLeader().getUuidString();
+    String id = client.getPipeline().getFirstNode().getUuidString();
 
     ContainerCommandRequestProto request = ContainerCommandRequestProto
         .newBuilder()

hadoop-hdds/common/src/main/java/org/apache/ratis/RatisHelper.java

@@ -19,7 +19,7 @@
 package org.apache.ratis;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.ratis.client.RaftClient;
@@ -40,6 +40,7 @@ import org.apache.ratis.util.TimeDuration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -88,7 +89,7 @@ public interface RatisHelper {
   }
 
   static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
-    return toRaftPeers(pipeline.getMachines());
+    return toRaftPeers(pipeline.getNodes());
   }
 
   static <E extends DatanodeDetails> List<RaftPeer> toRaftPeers(
@@ -125,15 +126,15 @@ public interface RatisHelper {
   }
 
   static RaftGroup newRaftGroup(Pipeline pipeline) {
-    return RaftGroup.valueOf(pipeline.getId().getRaftGroupID(),
+    return RaftGroup.valueOf(RaftGroupId.valueOf(pipeline.getId().getId()),
         toRaftPeers(pipeline));
   }
 
   static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,
-      RetryPolicy retryPolicy) {
-    return newRaftClient(rpcType, toRaftPeerId(pipeline.getLeader()),
-        newRaftGroup(pipeline.getId().getRaftGroupID(), pipeline.getMachines()),
-        retryPolicy);
+      RetryPolicy retryPolicy) throws IOException {
+    return newRaftClient(rpcType, toRaftPeerId(pipeline.getFirstNode()),
+        newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
+            pipeline.getNodes()), retryPolicy);
   }
 
   static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java

@@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
         .StorageContainerDatanodeProtocolProtos.PipelineReport;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.common.helpers.
     StorageContainerException;
 import org.apache.hadoop.ozone.OzoneConfigKeys;

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.protocol.proto
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.PipelineAction;
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
 import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -372,7 +372,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
       ContainerCommandRequestProto request, HddsProtos.PipelineID pipelineID,
       RaftClientRequest.Type type) {
     return new RaftClientRequest(clientId, server.getId(),
-        PipelineID.getFromProtobuf(pipelineID).getRaftGroupID(),
+        RaftGroupId.valueOf(PipelineID.getFromProtobuf(pipelineID).getId()),
         nextCallId(), 0, Message.valueOf(request.toByteString()), type);
   }
 
@@ -405,7 +405,7 @@ public final class XceiverServerRatis implements XceiverServerSpi {
           + roleInfoProto.getRole());
     }
 
-    PipelineID pipelineID = PipelineID.valueOf(groupId);
+    PipelineID pipelineID = PipelineID.valueOf(groupId.getUuid());
     ClosePipelineInfo.Builder closePipelineInfo =
         ClosePipelineInfo.newBuilder()
             .setPipelineID(pipelineID.getProtobuf())
@@ -429,8 +429,8 @@ public final class XceiverServerRatis implements XceiverServerSpi {
       List<PipelineReport> reports = new ArrayList<>();
       for (RaftGroupId groupId : gids) {
         reports.add(PipelineReport.newBuilder()
-                .setPipelineID(PipelineID.valueOf(groupId).getProtobuf())
-                .build());
+            .setPipelineID(PipelineID.valueOf(groupId.getUuid()).getProtobuf())
+            .build());
       }
       return reports;
     } catch (Exception e) {

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java

@@ -22,7 +22,7 @@ import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMCommandProto;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.CloseContainerCommandProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 
 /**
  * Asks datanode to close a container.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/BlockManagerImpl.java

@@ -305,7 +305,7 @@ public class BlockManagerImpl implements EventHandler<Boolean>,
   private AllocatedBlock newBlock(ContainerWithPipeline containerWithPipeline,
       HddsProtos.LifeCycleState state) throws IOException {
     ContainerInfo containerInfo = containerWithPipeline.getContainerInfo();
-    if (containerWithPipeline.getPipeline().getDatanodes().size() == 0) {
+    if (containerWithPipeline.getPipeline().getNodes().size() == 0) {
       LOG.error("Pipeline Machine count is zero.");
       return null;
     }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DatanodeDeletedBlockTransactions.java

@@ -31,7 +31,7 @@ import java.util.UUID;
 import java.util.stream.Collectors;
 
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 
 /**
  * A wrapper class to hold info about datanode and all deleted block
@@ -74,7 +74,7 @@ public class DatanodeDeletedBlockTransactions {
     }
 
     boolean success = false;
-    for (DatanodeDetails dd : pipeline.getMachines()) {
+    for (DatanodeDetails dd : pipeline.getNodes()) {
       UUID dnID = dd.getUuid();
       if (dnsWithTransactionCommitted == null ||
           !dnsWithTransactionCommitted.contains(dnID)) {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImpl.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.scm.command
     .CommandStatusReportHandler.DeleteBlockStatus;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdfs.DFSUtil;
@@ -261,8 +261,7 @@ public class DeletedBlockLogImpl
           Pipeline pipeline =
               containerManager.getContainerWithPipeline(
                   ContainerID.valueof(containerId)).getPipeline();
-          Collection<DatanodeDetails> containerDnsDetails =
-              pipeline.getDatanodes().values();
+          Collection<DatanodeDetails> containerDnsDetails = pipeline.getNodes();
           // The delete entry can be safely removed from the log if all the
           // corresponding nodes commit the txn. It is required to check that
           // the nodes returned in the pipeline match the replication factor.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/CloseContainerEventHandler.java

@@ -19,7 +19,7 @@ package org.apache.hadoop.hdds.scm.container;
 import java.io.IOException;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.server.events.IdentifiableEventPayload;
@@ -123,7 +123,7 @@ public class CloseContainerEventHandler implements EventHandler<ContainerID> {
             info.getReplicationType(), info.getPipelineID());
 
     Pipeline pipeline = containerWithPipeline.getPipeline();
-    pipeline.getMachines().stream()
+    pipeline.getNodes().stream()
         .map(node ->
             new CommandForDatanode<>(node.getUuid(), closeContainerCommand))
         .forEach(command -> publisher.fireEvent(DATANODE_COMMAND, command));

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java

@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -73,7 +73,7 @@ public interface ContainerManager extends Closeable {
    * @throws IOException
    */
   ContainerWithPipeline getContainerWithPipeline(ContainerID containerID)
-      throws ContainerNotFoundException;
+      throws ContainerNotFoundException, PipelineNotFoundException;
 
   /**
    * Returns containers under certain conditions.
@@ -175,6 +175,4 @@ public interface ContainerManager extends Closeable {
   ContainerWithPipeline getMatchingContainerWithPipeline(long size,
       String owner, ReplicationType type, ReplicationFactor factor,
       LifeCycleState state) throws IOException;
-
-  PipelineSelector getPipelineSelector();
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java

@@ -22,11 +22,11 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.StorageUnit;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.container.states.ContainerState;
 import org.apache.hadoop.hdds.scm.container.states.ContainerStateMap;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
@@ -42,6 +42,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Set;
@@ -232,19 +233,28 @@ public class ContainerStateManager {
   /**
    * Allocates a new container based on the type, replication etc.
    *
-   * @param selector -- Pipeline selector class.
+   * @param pipelineManager -- Pipeline Manager class.
    * @param type -- Replication type.
    * @param replicationFactor - Replication replicationFactor.
    * @return ContainerWithPipeline
    * @throws IOException  on Failure.
    */
-  ContainerInfo allocateContainer(final PipelineSelector selector,
+  ContainerInfo allocateContainer(final PipelineManager pipelineManager,
       final HddsProtos.ReplicationType type,
       final HddsProtos.ReplicationFactor replicationFactor, final String owner)
       throws IOException {
 
-    final Pipeline pipeline = selector.getReplicationPipeline(type,
-        replicationFactor);
+    Pipeline pipeline;
+    try {
+      pipeline = pipelineManager.createPipeline(type, replicationFactor);
+    } catch (IOException e) {
+      final List<Pipeline> pipelines =
+          pipelineManager.getPipelines(type, replicationFactor);
+      if (pipelines.isEmpty()) {
+        throw new IOException("Could not allocate container");
+      }
+      pipeline = pipelines.get((int) containerCount.get() % pipelines.size());
+    }
 
     Preconditions.checkNotNull(pipeline, "Pipeline type=%s/"
         + "replication=%s couldn't be found for the new container. "
@@ -263,7 +273,8 @@ public class ContainerStateManager {
         .setReplicationFactor(replicationFactor)
         .setReplicationType(pipeline.getType())
         .build();
-    selector.addContainerToPipeline(pipeline.getId(), containerID);
+    pipelineManager.addContainerToPipeline(pipeline.getId(),
+        ContainerID.valueof(containerID));
     Preconditions.checkNotNull(containerInfo);
     containers.addContainer(containerInfo);
     LOG.trace("New container allocated: {}", containerInfo);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/SCMContainerManager.java

@@ -26,13 +26,13 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
 import org.apache.hadoop.hdds.scm.block.PendingDeleteStatusList;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineNotFoundException;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
@@ -86,7 +86,7 @@ public class SCMContainerManager implements ContainerManager {
 
   private final Lock lock;
   private final MetadataStore containerStore;
-  private final PipelineSelector pipelineSelector;
+  private final PipelineManager pipelineManager;
   private final ContainerStateManager containerStateManager;
   private final LeaseManager<ContainerInfo> containerLeaseManager;
   private final EventPublisher eventPublisher;
@@ -102,12 +102,13 @@ public class SCMContainerManager implements ContainerManager {
    * passed to LevelDB and this memory is allocated in Native code space.
    * CacheSize is specified
    * in MB.
+   * @param pipelineManager - PipelineManager
    * @throws IOException on Failure.
    */
   @SuppressWarnings("unchecked")
   public SCMContainerManager(final Configuration conf,
-      final NodeManager nodeManager, final EventPublisher eventPublisher)
-      throws IOException {
+      final NodeManager nodeManager, PipelineManager pipelineManager,
+      final EventPublisher eventPublisher) throws IOException {
 
     final File metaDir = getOzoneMetaDirPath(conf);
     final File containerDBPath = new File(metaDir, SCM_CONTAINER_DB);
@@ -123,8 +124,7 @@ public class SCMContainerManager implements ContainerManager {
     this.lock = new ReentrantLock();
     this.size = (long) conf.getStorageSize(OZONE_SCM_CONTAINER_SIZE,
         OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
-    this.pipelineSelector = new PipelineSelector(nodeManager,
-            conf, eventPublisher, cacheSize);
+    this.pipelineManager = pipelineManager;
     this.containerStateManager = new ContainerStateManager(conf);
     this.eventPublisher = eventPublisher;
 
@@ -147,8 +147,10 @@ public class SCMContainerManager implements ContainerManager {
           HddsProtos.SCMContainerInfo.PARSER.parseFrom(entry.getValue()));
       Preconditions.checkNotNull(container);
       containerStateManager.loadContainer(container);
-      pipelineSelector.addContainerToPipeline(
-          container.getPipelineID(), container.getContainerID());
+      if (container.isOpen()) {
+        pipelineManager.addContainerToPipeline(container.getPipelineID(),
+            ContainerID.valueof(container.getContainerID()));
+      }
     }
   }
 
@@ -214,28 +216,23 @@ public class SCMContainerManager implements ContainerManager {
    */
   @Override
   public ContainerWithPipeline getContainerWithPipeline(ContainerID containerID)
-      throws ContainerNotFoundException {
+      throws ContainerNotFoundException, PipelineNotFoundException {
     lock.lock();
     try {
       final ContainerInfo contInfo = getContainer(containerID);
       Pipeline pipeline;
-      String leaderId = "";
       if (contInfo.isOpen()) {
         // If pipeline with given pipeline Id already exist return it
-        pipeline = pipelineSelector.getPipeline(contInfo.getPipelineID());
+        pipeline = pipelineManager.getPipeline(contInfo.getPipelineID());
       } else {
         // For close containers create pipeline from datanodes with replicas
         Set<ContainerReplica> dnWithReplicas = containerStateManager
             .getContainerReplicas(contInfo.containerID());
-        if (!dnWithReplicas.isEmpty()) {
-          leaderId = dnWithReplicas.iterator().next()
-              .getDatanodeDetails().getUuidString();
-        }
-        pipeline = new Pipeline(leaderId, contInfo.getState(),
-            ReplicationType.STAND_ALONE, contInfo.getReplicationFactor(),
-            PipelineID.randomId());
-        dnWithReplicas.stream().map(ContainerReplica::getDatanodeDetails).
-            forEach(pipeline::addMember);
+        List<DatanodeDetails> dns =
+            dnWithReplicas.stream().map(ContainerReplica::getDatanodeDetails)
+                .collect(Collectors.toList());
+        pipeline = pipelineManager.createPipeline(ReplicationType.STAND_ALONE,
+            contInfo.getReplicationFactor(), dns);
       }
       return new ContainerWithPipeline(contInfo, pipeline);
     } finally {
@@ -290,8 +287,8 @@ public class SCMContainerManager implements ContainerManager {
     lock.lock();
     try {
       final ContainerInfo containerInfo; containerInfo = containerStateManager
-          .allocateContainer(pipelineSelector, type, replicationFactor, owner);
-      final Pipeline pipeline = pipelineSelector.getPipeline(
+          .allocateContainer(pipelineManager, type, replicationFactor, owner);
+      final Pipeline pipeline = pipelineManager.getPipeline(
           containerInfo.getPipelineID());
 
       try {
@@ -360,8 +357,8 @@ public class SCMContainerManager implements ContainerManager {
       ContainerInfo updatedContainer =
           updateContainerStateInternal(containerID, event);
       if (!updatedContainer.isOpen()) {
-        pipelineSelector.removeContainerFromPipeline(
-            updatedContainer.getPipelineID(), containerID.getId());
+        pipelineManager.removeContainerFromPipeline(
+            updatedContainer.getPipelineID(), containerID);
       }
       final byte[] dbKey = Longs.toByteArray(containerID.getId());
       containerStore.put(dbKey, updatedContainer.getProtobuf().toByteArray());
@@ -485,7 +482,7 @@ public class SCMContainerManager implements ContainerManager {
     if (containerInfo == null) {
       return null;
     }
-    Pipeline pipeline = pipelineSelector
+    Pipeline pipeline = pipelineManager
         .getPipeline(containerInfo.getPipelineID());
     return new ContainerWithPipeline(containerInfo, pipeline);
   }
@@ -647,13 +644,5 @@ public class SCMContainerManager implements ContainerManager {
     if (containerStore != null) {
       containerStore.close();
     }
-
-    if (pipelineSelector != null) {
-      pipelineSelector.shutdown();
-    }
-  }
-
-  public PipelineSelector getPipelineSelector() {
-    return pipelineSelector;
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/events/SCMEvents.java

@@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler
 import org.apache.hadoop.hdds.scm.container.CloseContainerEventHandler
         .CloseContainerRetryableReq;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
         .PipelineReportFromDatanode;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
@@ -102,14 +101,6 @@ public final class SCMEvents {
       PIPELINE_ACTIONS = new TypedEvent<>(PipelineActionsFromDatanode.class,
       "Pipeline_Actions");
 
-  /**
-   * Pipeline close event are triggered to close pipeline because of failure,
-   * stale node, decommissioning etc.
-   */
-  public static final TypedEvent<PipelineID>
-      PIPELINE_CLOSE = new TypedEvent<>(PipelineID.class,
-      "Pipeline_Close");
-
   /**
    * A Command status report will be sent by datanodes. This repoort is received
    * by SCMDatanodeHeartbeatDispatcher and CommandReport event is generated.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java

@@ -19,8 +19,8 @@ package org.apache.hadoop.hdds.scm.node;
 
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.NodeReportProto;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java

@@ -25,8 +25,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
 import org.apache.hadoop.hdds.scm.HddsServerUtil;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

@@ -22,8 +22,8 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.protocol.proto
         .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException;
 import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/StaleNodeHandler.java

@@ -19,24 +19,44 @@
 package org.apache.hadoop.hdds.scm.node;
 
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
 import org.apache.hadoop.hdds.server.events.EventHandler;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Set;
 
 /**
  * Handles Stale node event.
  */
 public class StaleNodeHandler implements EventHandler<DatanodeDetails> {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(StaleNodeHandler.class);
 
-  private final PipelineSelector pipelineSelector;
+  private final NodeManager nodeManager;
+  private final PipelineManager pipelineManager;
 
-  public StaleNodeHandler(PipelineSelector pipelineSelector) {
-    this.pipelineSelector = pipelineSelector;
+  public StaleNodeHandler(NodeManager nodeManager,
+      PipelineManager pipelineManager) {
+    this.nodeManager = nodeManager;
+    this.pipelineManager = pipelineManager;
   }
 
   @Override
   public void onMessage(DatanodeDetails datanodeDetails,
                         EventPublisher publisher) {
-    pipelineSelector.handleStaleNode(datanodeDetails);
+    Set<PipelineID> pipelineIds =
+        nodeManager.getPipelineByDnID(datanodeDetails.getUuid());
+    for (PipelineID pipelineID : pipelineIds) {
+      try {
+        pipelineManager.finalizePipeline(pipelineID);
+      } catch (IOException e) {
+        LOG.info("Could not finalize pipeline={} for dn={}", pipelineID,
+            datanodeDetails);
+      }
+    }
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/states/Node2PipelineMap.java

@@ -19,8 +19,8 @@
 package org.apache.hadoop.hdds.scm.node.states;
 
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 
 import java.util.HashSet;
 import java.util.Set;
@@ -55,7 +55,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
    * @param pipeline Pipeline to be added
    */
   public synchronized void addPipeline(Pipeline pipeline) {
-    for (DatanodeDetails details : pipeline.getDatanodes().values()) {
+    for (DatanodeDetails details : pipeline.getNodes()) {
       UUID dnId = details.getUuid();
       dn2ObjectMap.computeIfAbsent(dnId, k -> new HashSet<>())
           .add(pipeline.getId());
@@ -63,7 +63,7 @@ public class Node2PipelineMap extends Node2ObjectsMap<PipelineID> {
   }
 
   public synchronized void removePipeline(Pipeline pipeline) {
-    for (DatanodeDetails details : pipeline.getDatanodes().values()) {
+    for (DatanodeDetails details : pipeline.getNodes()) {
       UUID dnId = details.getUuid();
       dn2ObjectMap.computeIfPresent(dnId,
           (k, v) -> {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineFactory.java

@@ -50,8 +50,8 @@ public final class PipelineFactory {
     return providers.get(type).create(factor);
   }
 
-  public Pipeline create(ReplicationType type, List<DatanodeDetails> nodes)
-      throws IOException {
-    return providers.get(type).create(nodes);
+  public Pipeline create(ReplicationType type, ReplicationFactor factor,
+      List<DatanodeDetails> nodes) {
+    return providers.get(type).create(factor, nodes);
   }
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java

@@ -36,14 +36,14 @@ public interface PipelineManager extends Closeable {
   Pipeline createPipeline(ReplicationType type, ReplicationFactor factor)
       throws IOException;
 
-  Pipeline createPipeline(ReplicationType type, List<DatanodeDetails> nodes)
-      throws IOException;
+  Pipeline createPipeline(ReplicationType type, ReplicationFactor factor,
+      List<DatanodeDetails> nodes);
 
-  Pipeline getPipeline(PipelineID pipelineID) throws IOException;
+  Pipeline getPipeline(PipelineID pipelineID) throws PipelineNotFoundException;
 
-  List<Pipeline> getPipelinesByType(ReplicationType type);
+  List<Pipeline> getPipelines(ReplicationType type);
 
-  List<Pipeline> getPipelinesByTypeAndFactor(ReplicationType type,
+  List<Pipeline> getPipelines(ReplicationType type,
       ReplicationFactor factor);
 
   void addContainerToPipeline(PipelineID pipelineID, ContainerID containerID)

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineProvider.java

@@ -31,5 +31,5 @@ public interface PipelineProvider {
 
   Pipeline create(ReplicationFactor factor) throws IOException;
 
-  Pipeline create(List<DatanodeDetails> nodes) throws IOException;
+  Pipeline create(ReplicationFactor factor, List<DatanodeDetails> nodes);
 }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java

@@ -76,7 +76,13 @@ public class PipelineReportHandler implements
   private void processPipelineReport(PipelineReport report, DatanodeDetails dn)
       throws IOException {
     PipelineID pipelineID = PipelineID.getFromProtobuf(report.getPipelineID());
-    Pipeline pipeline = pipelineManager.getPipeline(pipelineID);
+    Pipeline pipeline = null;
+    try {
+      pipeline = pipelineManager.getPipeline(pipelineID);
+    } catch (PipelineNotFoundException e) {
+      //TODO: introduce per datanode command for pipeline destroy
+      return;
+    }
 
     if (pipeline.getPipelineState() == Pipeline.PipelineState.ALLOCATED) {
       pipeline.reportDatanode(dn);
@@ -87,14 +93,14 @@ public class PipelineReportHandler implements
     } else if (pipeline.isClosed()) {
       int numContainers = pipelineManager.getNumberOfContainers(pipelineID);
       if (numContainers == 0) {
-        // if all the containers have been closed the pipeline can be destroyed
+        // remove the pipeline from the pipeline manager
+        pipelineManager.removePipeline(pipelineID);
+        // since all the containers have been closed the pipeline can be
+        // destroyed
         try (XceiverClientRatis client =
             XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
           client.destroyPipeline();
         }
-        // after successfully destroying the pipeline, the pipeline can be
-        // removed from the pipeline manager
-        pipelineManager.removePipeline(pipelineID);
       }
     } else {
       // In OPEN state case just report the datanode

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateManager.java

@@ -23,8 +23,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
@@ -39,9 +37,6 @@ import java.util.Set;
  */
 class PipelineStateManager {
 
-  private static final Logger LOG = LoggerFactory.getLogger(
-      org.apache.hadoop.hdds.scm.pipelines.PipelineStateManager.class);
-
   private final PipelineStateMap pipelineStateMap;
 
   PipelineStateManager(Configuration conf) {
@@ -57,17 +52,20 @@ class PipelineStateManager {
     pipelineStateMap.addContainerToPipeline(pipelineId, containerID);
   }
 
-  Pipeline getPipeline(PipelineID pipelineID) throws IOException {
+  Pipeline getPipeline(PipelineID pipelineID) throws PipelineNotFoundException {
     return pipelineStateMap.getPipeline(pipelineID);
   }
 
-  List<Pipeline> getPipelinesByType(ReplicationType type) {
-    return pipelineStateMap.getPipelinesByType(type);
+  List<Pipeline> getPipelines(ReplicationType type) {
+    return pipelineStateMap.getPipelines(type);
+  }
+
+  List<Pipeline> getPipelines(ReplicationType type, ReplicationFactor factor) {
+    return pipelineStateMap.getPipelines(type, factor);
   }
 
-  List<Pipeline> getPipelinesByTypeAndFactor(ReplicationType type,
-      ReplicationFactor factor) {
-    return pipelineStateMap.getPipelinesByTypeAndFactor(type, factor);
+  List<Pipeline> getPipelines(ReplicationType type, PipelineState... states) {
+    return pipelineStateMap.getPipelines(type, states);
   }
 
   Set<ContainerID> getContainers(PipelineID pipelineID) throws IOException {
@@ -78,8 +76,8 @@ class PipelineStateManager {
     return pipelineStateMap.getNumberOfContainers(pipelineID);
   }
 
-  void removePipeline(PipelineID pipelineID) throws IOException {
-    pipelineStateMap.removePipeline(pipelineID);
+  Pipeline removePipeline(PipelineID pipelineID) throws IOException {
+    return pipelineStateMap.removePipeline(pipelineID);
   }
 
   void removeContainerFromPipeline(PipelineID pipelineID,
@@ -87,7 +85,8 @@ class PipelineStateManager {
     pipelineStateMap.removeContainerFromPipeline(pipelineID, containerID);
   }
 
-  Pipeline finalizePipeline(PipelineID pipelineId) throws IOException {
+  Pipeline finalizePipeline(PipelineID pipelineId)
+      throws PipelineNotFoundException {
     Pipeline pipeline = pipelineStateMap.getPipeline(pipelineId);
     if (!pipeline.isClosed()) {
       pipeline = pipelineStateMap

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineStateMap.java

@@ -31,8 +31,7 @@ import java.util.stream.Collectors;
 
 /**
  * Holds the data structures which maintain the information about pipeline and
- * its state. All the read write operations in this class are protected by a
- * lock.
+ * its state.
  * Invariant: If a pipeline exists in PipelineStateMap, both pipelineMap and
  * pipeline2container would have a non-null mapping for it.
  */
@@ -65,12 +64,12 @@ class PipelineStateMap {
         String.format("Nodes size=%d, replication factor=%d do not match ",
                 pipeline.getNodes().size(), pipeline.getFactor().getNumber()));
 
-    if (pipelineMap.putIfAbsent(pipeline.getID(), pipeline) != null) {
-      LOG.warn("Duplicate pipeline ID detected. {}", pipeline.getID());
+    if (pipelineMap.putIfAbsent(pipeline.getId(), pipeline) != null) {
+      LOG.warn("Duplicate pipeline ID detected. {}", pipeline.getId());
       throw new IOException(String
-          .format("Duplicate pipeline ID %s detected.", pipeline.getID()));
+          .format("Duplicate pipeline ID %s detected.", pipeline.getId()));
     }
-    pipeline2container.put(pipeline.getID(), new TreeSet<>());
+    pipeline2container.put(pipeline.getId(), new TreeSet<>());
   }
 
   /**
@@ -85,12 +84,13 @@ class PipelineStateMap {
     Preconditions.checkNotNull(pipelineID,
         "Pipeline Id cannot be null");
     Preconditions.checkNotNull(containerID,
-        "container Id cannot be null");
+        "Container Id cannot be null");
 
     Pipeline pipeline = getPipeline(pipelineID);
-    if (!pipeline.isOpen()) {
-      throw new IOException(
-          String.format("%s is not in open state", pipelineID));
+    if (pipeline.isClosed()) {
+      throw new IOException(String
+          .format("Cannot add container to pipeline=%s in closed state",
+              pipelineID));
     }
     pipeline2container.get(pipelineID).add(containerID);
   }
@@ -102,10 +102,14 @@ class PipelineStateMap {
    * @return Pipeline
    * @throws IOException if pipeline is not found
    */
-  Pipeline getPipeline(PipelineID pipelineID) throws IOException {
+  Pipeline getPipeline(PipelineID pipelineID) throws PipelineNotFoundException {
+    Preconditions.checkNotNull(pipelineID,
+        "Pipeline Id cannot be null");
+
     Pipeline pipeline = pipelineMap.get(pipelineID);
     if (pipeline == null) {
-      throw new IOException(String.format("%s not found", pipelineID));
+      throw new PipelineNotFoundException(
+          String.format("%s not found", pipelineID));
     }
     return pipeline;
   }
@@ -116,28 +120,51 @@ class PipelineStateMap {
    * @param type - ReplicationType
    * @return List of pipelines which have the specified replication type
    */
-  List<Pipeline> getPipelinesByType(ReplicationType type) {
+  List<Pipeline> getPipelines(ReplicationType type) {
     Preconditions.checkNotNull(type, "Replication type cannot be null");
 
-    return pipelineMap.values().stream().filter(p -> p.getType().equals(type))
+    return pipelineMap.values().stream()
+        .filter(p -> p.getType().equals(type))
         .collect(Collectors.toList());
   }
 
   /**
-   * Get open pipeline corresponding to specified replication type and factor.
+   * Get pipeline corresponding to specified replication type and factor.
    *
    * @param type - ReplicationType
    * @param factor - ReplicationFactor
-   * @return List of open pipelines with specified replication type and factor
+   * @return List of pipelines with specified replication type and factor
    */
-  List<Pipeline> getPipelinesByTypeAndFactor(ReplicationType type,
-      ReplicationFactor factor) {
+  List<Pipeline> getPipelines(ReplicationType type, ReplicationFactor factor) {
+    Preconditions.checkNotNull(type, "Replication type cannot be null");
+    Preconditions.checkNotNull(factor, "Replication factor cannot be null");
+
     return pipelineMap.values().stream()
-        .filter(pipeline -> pipeline.isOpen() && pipeline.getType() == type
+        .filter(pipeline -> pipeline.getType() == type
             && pipeline.getFactor() == factor)
         .collect(Collectors.toList());
   }
 
+  /**
+   * Get list of pipeline corresponding to specified replication type and
+   * pipeline states.
+   *
+   * @param type - ReplicationType
+   * @param states - Array of required PipelineState
+   * @return List of pipelines with specified replication type and states
+   */
+  List<Pipeline> getPipelines(ReplicationType type, PipelineState... states) {
+    Preconditions.checkNotNull(type, "Replication type cannot be null");
+    Preconditions.checkNotNull(states, "Pipeline state cannot be null");
+
+    Set<PipelineState> pipelineStates = new HashSet<>();
+    pipelineStates.addAll(Arrays.asList(states));
+    return pipelineMap.values().stream().filter(
+        pipeline -> pipeline.getType() == type && pipelineStates
+            .contains(pipeline.getPipelineState()))
+        .collect(Collectors.toList());
+  }
+
   /**
    * Get set of containerIDs corresponding to a pipeline.
    *
@@ -146,10 +173,14 @@ class PipelineStateMap {
    * @throws IOException if pipeline is not found
    */
   Set<ContainerID> getContainers(PipelineID pipelineID)
-      throws IOException {
+      throws PipelineNotFoundException {
+    Preconditions.checkNotNull(pipelineID,
+        "Pipeline Id cannot be null");
+
     Set<ContainerID> containerIDs = pipeline2container.get(pipelineID);
     if (containerIDs == null) {
-      throw new IOException(String.format("%s not found", pipelineID));
+      throw new PipelineNotFoundException(
+          String.format("%s not found", pipelineID));
     }
     return new HashSet<>(containerIDs);
   }
@@ -161,10 +192,15 @@ class PipelineStateMap {
    * @return Number of containers belonging to the pipeline
    * @throws IOException if pipeline is not found
    */
-  int getNumberOfContainers(PipelineID pipelineID) throws IOException {
+  int getNumberOfContainers(PipelineID pipelineID)
+      throws PipelineNotFoundException {
+    Preconditions.checkNotNull(pipelineID,
+        "Pipeline Id cannot be null");
+
     Set<ContainerID> containerIDs = pipeline2container.get(pipelineID);
     if (containerIDs == null) {
-      throw new IOException(String.format("%s not found", pipelineID));
+      throw new PipelineNotFoundException(
+          String.format("%s not found", pipelineID));
     }
     return containerIDs.size();
   }
@@ -175,7 +211,7 @@ class PipelineStateMap {
    * @param pipelineID - PipelineID of the pipeline to be removed
    * @throws IOException if the pipeline is not empty or does not exist
    */
-  void removePipeline(PipelineID pipelineID) throws IOException {
+  Pipeline removePipeline(PipelineID pipelineID) throws IOException {
     Preconditions.checkNotNull(pipelineID, "Pipeline Id cannot be null");
 
     Pipeline pipeline = getPipeline(pipelineID);
@@ -192,6 +228,7 @@ class PipelineStateMap {
 
     pipelineMap.remove(pipelineID);
     pipeline2container.remove(pipelineID);
+    return pipeline;
   }
 
   /**
@@ -210,6 +247,10 @@ class PipelineStateMap {
         "container Id cannot be null");
 
     Set<ContainerID> containerIDs = pipeline2container.get(pipelineID);
+    if (containerIDs == null) {
+      throw new PipelineNotFoundException(
+          String.format("%s not found", pipelineID));
+    }
     containerIDs.remove(containerID);
   }
 
@@ -223,7 +264,7 @@ class PipelineStateMap {
    * @throws IOException if pipeline does not exist
    */
   Pipeline updatePipelineState(PipelineID pipelineID, PipelineState state)
-      throws IOException {
+      throws PipelineNotFoundException {
     Preconditions.checkNotNull(pipelineID, "Pipeline Id cannot be null");
     Preconditions.checkNotNull(state, "Pipeline LifeCycleState cannot be null");
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/RatisPipelineProvider.java

@@ -93,7 +93,7 @@ public class RatisPipelineProvider implements PipelineProvider {
   public Pipeline create(ReplicationFactor factor) throws IOException {
     // Get set of datanodes already used for ratis pipeline
     Set<DatanodeDetails> dnsUsed = new HashSet<>();
-    stateManager.getPipelinesByType(ReplicationType.RATIS)
+    stateManager.getPipelines(ReplicationType.RATIS)
         .forEach(p -> dnsUsed.addAll(p.getNodes()));
 
     // Get list of healthy nodes
@@ -112,7 +112,7 @@ public class RatisPipelineProvider implements PipelineProvider {
 
     Pipeline pipeline = Pipeline.newBuilder()
         .setId(PipelineID.randomId())
-        .setState(PipelineState.ALLOCATED)
+        .setState(PipelineState.OPEN)
         .setType(ReplicationType.RATIS)
         .setFactor(factor)
         .setNodes(dns)
@@ -122,16 +122,11 @@ public class RatisPipelineProvider implements PipelineProvider {
   }
 
   @Override
-  public Pipeline create(List<DatanodeDetails> nodes) throws IOException {
-    ReplicationFactor factor = ReplicationFactor.valueOf(nodes.size());
-    if (factor == null) {
-      throw new IOException(String
-          .format("Nodes size=%d does not match any replication factor",
-              nodes.size()));
-    }
+  public Pipeline create(ReplicationFactor factor,
+      List<DatanodeDetails> nodes) {
     return Pipeline.newBuilder()
         .setId(PipelineID.randomId())
-        .setState(PipelineState.ALLOCATED)
+        .setState(PipelineState.OPEN)
         .setType(ReplicationType.RATIS)
         .setFactor(factor)
         .setNodes(nodes)

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java

@@ -78,14 +78,14 @@ public class SCMPipelineManager implements PipelineManager {
     File pipelineDBPath = new File(metaDir, SCM_PIPELINE_DB);
     this.pipelineStore =
         MetadataStoreBuilder.newBuilder()
+            .setCreateIfMissing(true)
             .setConf(conf)
             .setDbFile(pipelineDBPath)
             .setCacheSize(cacheSize * OzoneConsts.MB)
             .build();
-    initializePipelineState();
-
     this.eventPublisher = eventPublisher;
     this.nodeManager = nodeManager;
+    initializePipelineState();
   }
 
   private void initializePipelineState() throws IOException {
@@ -97,12 +97,11 @@ public class SCMPipelineManager implements PipelineManager {
         pipelineStore.getSequentialRangeKVs(null, Integer.MAX_VALUE, null);
 
     for (Map.Entry<byte[], byte[]> entry : pipelines) {
-      Pipeline pipeline = Pipeline
-          .fromProtobuf(HddsProtos.Pipeline.PARSER.parseFrom(entry.getValue()));
+      Pipeline pipeline = Pipeline.getFromProtobuf(
+          HddsProtos.Pipeline.PARSER.parseFrom(entry.getValue()));
       Preconditions.checkNotNull(pipeline);
       stateManager.addPipeline(pipeline);
-      // TODO: add pipeline to node manager
-      // nodeManager.addPipeline(pipeline);
+      nodeManager.addPipeline(pipeline);
     }
   }
 
@@ -112,10 +111,10 @@ public class SCMPipelineManager implements PipelineManager {
     lock.writeLock().lock();
     try {
       Pipeline pipeline =  pipelineFactory.create(type, factor);
-      pipelineStore.put(pipeline.getID().getProtobuf().toByteArray(),
+      pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(),
           pipeline.getProtobufMessage().toByteArray());
       stateManager.addPipeline(pipeline);
-      // TODO: add pipeline to node manager
+      nodeManager.addPipeline(pipeline);
       return pipeline;
     } finally {
       lock.writeLock().unlock();
@@ -123,20 +122,20 @@ public class SCMPipelineManager implements PipelineManager {
   }
 
   @Override
-  public Pipeline createPipeline(ReplicationType type,
-                                 List<DatanodeDetails> nodes)
-      throws IOException {
+  public Pipeline createPipeline(ReplicationType type, ReplicationFactor factor,
+                                 List<DatanodeDetails> nodes) {
     // This will mostly be used to create dummy pipeline for SimplePipelines.
     lock.writeLock().lock();
     try {
-      return pipelineFactory.create(type, nodes);
+      return pipelineFactory.create(type, factor, nodes);
     } finally {
       lock.writeLock().unlock();
     }
   }
 
   @Override
-  public Pipeline getPipeline(PipelineID pipelineID) throws IOException {
+  public Pipeline getPipeline(PipelineID pipelineID)
+      throws PipelineNotFoundException {
     lock.readLock().lock();
     try {
       return stateManager.getPipeline(pipelineID);
@@ -146,21 +145,21 @@ public class SCMPipelineManager implements PipelineManager {
   }
 
   @Override
-  public List<Pipeline> getPipelinesByType(ReplicationType type) {
+  public List<Pipeline> getPipelines(ReplicationType type) {
     lock.readLock().lock();
     try {
-      return stateManager.getPipelinesByType(type);
+      return stateManager.getPipelines(type);
     } finally {
       lock.readLock().unlock();
     }
   }
 
   @Override
-  public List<Pipeline> getPipelinesByTypeAndFactor(ReplicationType type,
+  public List<Pipeline> getPipelines(ReplicationType type,
       ReplicationFactor factor) {
     lock.readLock().lock();
     try {
-      return stateManager.getPipelinesByTypeAndFactor(type, factor);
+      return stateManager.getPipelines(type, factor);
     } finally {
       lock.readLock().unlock();
     }
@@ -232,9 +231,9 @@ public class SCMPipelineManager implements PipelineManager {
   public void removePipeline(PipelineID pipelineID) throws IOException {
     lock.writeLock().lock();
     try {
-      stateManager.removePipeline(pipelineID);
       pipelineStore.delete(pipelineID.getProtobuf().toByteArray());
-      // TODO: remove pipeline from node manager
+      Pipeline pipeline = stateManager.removePipeline(pipelineID);
+      nodeManager.removePipeline(pipeline);
     } finally {
       lock.writeLock().unlock();
     }

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SimplePipelineProvider.java

@@ -54,7 +54,7 @@ public class SimplePipelineProvider implements PipelineProvider {
     Collections.shuffle(dns);
     return Pipeline.newBuilder()
         .setId(PipelineID.randomId())
-        .setState(PipelineState.ALLOCATED)
+        .setState(PipelineState.OPEN)
         .setType(ReplicationType.STAND_ALONE)
         .setFactor(factor)
         .setNodes(dns.subList(0, factor.getNumber()))
@@ -62,16 +62,11 @@ public class SimplePipelineProvider implements PipelineProvider {
   }
 
   @Override
-  public Pipeline create(List<DatanodeDetails> nodes) throws IOException {
-    ReplicationFactor factor = ReplicationFactor.valueOf(nodes.size());
-    if (factor == null) {
-      throw new IOException(String
-          .format("Nodes size=%d does not match any replication factor",
-              nodes.size()));
-    }
+  public Pipeline create(ReplicationFactor factor,
+      List<DatanodeDetails> nodes) {
     return Pipeline.newBuilder()
         .setId(PipelineID.randomId())
-        .setState(PipelineState.ALLOCATED)
+        .setState(PipelineState.OPEN)
         .setType(ReplicationType.STAND_ALONE)
         .setFactor(factor)
         .setNodes(nodes)

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineActionEventHandler.java

@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerDatanodeProtocolProtos.PipelineAction;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
-    .PipelineActionsFromDatanode;
-
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handles pipeline actions from datanode.
- */
-public class PipelineActionEventHandler implements
-    EventHandler<PipelineActionsFromDatanode> {
-
-  public static final Logger LOG = LoggerFactory.getLogger(
-      PipelineActionEventHandler.class);
-
-  public PipelineActionEventHandler() {
-
-  }
-
-  @Override
-  public void onMessage(PipelineActionsFromDatanode report,
-      EventPublisher publisher) {
-    for (PipelineAction action : report.getReport().getPipelineActionsList()) {
-      switch (action.getAction()) {
-      case CLOSE:
-        PipelineID pipelineID = PipelineID.
-            getFromProtobuf(action.getClosePipeline().getPipelineID());
-        LOG.info("Closing pipeline " + pipelineID + " for reason:" + action
-            .getClosePipeline().getDetailedReason());
-        publisher.fireEvent(SCMEvents.PIPELINE_CLOSE, pipelineID);
-        break;
-      default:
-        LOG.error("unknown pipeline action:{}" + action.getAction());
-      }
-    }
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineCloseHandler.java

@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handles pipeline close event.
- */
-public class PipelineCloseHandler implements EventHandler<PipelineID> {
-  private static final Logger LOG = LoggerFactory
-          .getLogger(PipelineCloseHandler.class);
-
-  private final PipelineSelector pipelineSelector;
-  public PipelineCloseHandler(PipelineSelector pipelineSelector) {
-    this.pipelineSelector = pipelineSelector;
-  }
-
-  @Override
-  public void onMessage(PipelineID pipelineID, EventPublisher publisher) {
-    Pipeline pipeline = pipelineSelector.getPipeline(pipelineID);
-    try {
-      if (pipeline != null) {
-        pipelineSelector.finalizePipeline(pipeline);
-      } else {
-        LOG.debug("pipeline:{} not found", pipelineID);
-      }
-    } catch (Exception e) {
-      LOG.info("failed to close pipeline:{}", pipelineID, e);
-    }
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineManager.java

@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import java.util.ArrayList;
-import java.util.LinkedList;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Manage Ozone pipelines.
- */
-public abstract class PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PipelineManager.class);
-  protected final ArrayList<ActivePipelines> activePipelines;
-
-  public PipelineManager() {
-    activePipelines = new ArrayList<>();
-    for (ReplicationFactor factor : ReplicationFactor.values()) {
-      activePipelines.add(factor.ordinal(), new ActivePipelines());
-    }
-  }
-
-  /**
-   * List of active pipelines.
-   */
-  public static class ActivePipelines {
-    private final List<PipelineID> activePipelines;
-    private final AtomicInteger pipelineIndex;
-
-    ActivePipelines() {
-      activePipelines = new LinkedList<>();
-      pipelineIndex = new AtomicInteger(0);
-    }
-
-    void addPipeline(PipelineID pipelineID) {
-      if (!activePipelines.contains(pipelineID)) {
-        activePipelines.add(pipelineID);
-      }
-    }
-
-    public void removePipeline(PipelineID pipelineID) {
-      activePipelines.remove(pipelineID);
-    }
-
-    /**
-     * Find a Pipeline that is operational.
-     *
-     * @return - Pipeline or null
-     */
-    PipelineID findOpenPipeline() {
-      if (activePipelines.size() == 0) {
-        LOG.error("No Operational pipelines found. Returning null.");
-        return null;
-      }
-      return activePipelines.get(getNextIndex());
-    }
-
-    /**
-     * gets the next index of the Pipeline to get.
-     *
-     * @return index in the link list to get.
-     */
-    private int getNextIndex() {
-      return pipelineIndex.incrementAndGet() % activePipelines.size();
-    }
-  }
-
-  /**
-   * This function is called by the Container Manager while allocating a new
-   * container. The client specifies what kind of replication pipeline is
-   * needed and based on the replication type in the request appropriate
-   * Interface is invoked.
-   *
-   * @param replicationFactor - Replication Factor
-   * @return a Pipeline.
-   */
-  public synchronized final PipelineID getPipeline(
-      ReplicationFactor replicationFactor, ReplicationType replicationType) {
-    PipelineID id =
-        activePipelines.get(replicationFactor.ordinal()).findOpenPipeline();
-    if (id != null) {
-      LOG.debug("re-used pipeline:{} for container with " +
-              "replicationType:{} replicationFactor:{}",
-          id, replicationType, replicationFactor);
-    }
-    if (id == null) {
-      LOG.error("Get pipeline call failed. We are not able to find" +
-              " operational pipeline.");
-      return null;
-    } else {
-      return id;
-    }
-  }
-
-  void addOpenPipeline(Pipeline pipeline) {
-    activePipelines.get(pipeline.getFactor().ordinal())
-            .addPipeline(pipeline.getId());
-  }
-
-  public abstract Pipeline allocatePipeline(
-      ReplicationFactor replicationFactor);
-
-  /**
-   * Initialize the pipeline.
-   * TODO: move the initialization to Ozone Client later
-   */
-  public abstract void initializePipeline(Pipeline pipeline) throws IOException;
-
-  public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
-    if (pipeline.addMember(dn)
-        &&(pipeline.getDatanodes().size() == pipeline.getFactor().getNumber())
-        && pipeline.getLifeCycleState() == HddsProtos.LifeCycleState.OPEN) {
-      addOpenPipeline(pipeline);
-    }
-  }
-
-  /**
-   * Creates a pipeline with a specified replication factor and type.
-   * @param replicationFactor - Replication Factor.
-   * @param replicationType - Replication Type.
-   */
-  public Pipeline createPipeline(ReplicationFactor replicationFactor,
-      ReplicationType replicationType) throws IOException {
-    Pipeline pipeline = allocatePipeline(replicationFactor);
-    if (pipeline != null) {
-      LOG.debug("created new pipeline:{} for container with "
-              + "replicationType:{} replicationFactor:{}",
-          pipeline.getId(), replicationType, replicationFactor);
-    }
-    return pipeline;
-  }
-
-  /**
-   * Remove the pipeline from active allocation.
-   * @param pipeline pipeline to be finalized
-   */
-  public abstract boolean finalizePipeline(Pipeline pipeline);
-
-  /**
-   *
-   * @param pipeline
-   */
-  public abstract void closePipeline(Pipeline pipeline) throws IOException;
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineReportHandler.java

@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto
-        .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
-import org.apache.hadoop.hdds.scm.server
-        .SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
-import org.apache.hadoop.hdds.server.events.EventHandler;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handles Node Reports from datanode.
- */
-public class PipelineReportHandler implements
-        EventHandler<PipelineReportFromDatanode> {
-
-  private static final Logger LOGGER = LoggerFactory
-      .getLogger(PipelineReportHandler.class);
-  private final PipelineSelector pipelineSelector;
-
-  public PipelineReportHandler(PipelineSelector pipelineSelector) {
-    Preconditions.checkNotNull(pipelineSelector);
-    this.pipelineSelector = pipelineSelector;
-  }
-
-  @Override
-  public void onMessage(PipelineReportFromDatanode pipelineReportFromDatanode,
-      EventPublisher publisher) {
-    Preconditions.checkNotNull(pipelineReportFromDatanode);
-    DatanodeDetails dn = pipelineReportFromDatanode.getDatanodeDetails();
-    PipelineReportsProto pipelineReport =
-            pipelineReportFromDatanode.getReport();
-    Preconditions.checkNotNull(dn, "Pipeline Report is "
-        + "missing DatanodeDetails.");
-    LOGGER.trace("Processing pipeline report for dn: {}", dn);
-    pipelineSelector.processPipelineReport(dn, pipelineReport);
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineSelector.java

@@ -1,481 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.StorageUnit;
-import org.apache.hadoop.hdds.protocol.proto
-        .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .SCMContainerPlacementRandom;
-import org.apache.hadoop.hdds.scm.events.SCMEvents;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipelines.ratis.RatisManagerImpl;
-import org.apache.hadoop.hdds.scm.pipelines.standalone.StandaloneManagerImpl;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.server.events.EventPublisher;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.lease.Lease;
-import org.apache.hadoop.ozone.lease.LeaseException;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.HashMap;
-import java.util.Set;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_PIPELINE_STATE;
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes.FAILED_TO_FIND_ACTIVE_PIPELINE;
-import static org.apache.hadoop.hdds.server
-        .ServerUtils.getOzoneMetaDirPath;
-import static org.apache.hadoop.ozone
-        .OzoneConsts.SCM_PIPELINE_DB;
-
-/**
- * Sends the request to the right pipeline manager.
- */
-public class PipelineSelector {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PipelineSelector.class);
-  private final ContainerPlacementPolicy placementPolicy;
-  private final Map<ReplicationType, PipelineManager> pipelineManagerMap;
-  private final Configuration conf;
-  private final EventPublisher eventPublisher;
-  private final long containerSize;
-  private final MetadataStore pipelineStore;
-  private final PipelineStateManager stateManager;
-  private final NodeManager nodeManager;
-  private final Map<PipelineID, HashSet<ContainerID>> pipeline2ContainerMap;
-  private final Map<PipelineID, Pipeline> pipelineMap;
-  private final LeaseManager<Pipeline> pipelineLeaseManager;
-
-  /**
-   * Constructs a pipeline Selector.
-   *
-   * @param nodeManager - node manager
-   * @param conf - Ozone Config
-   */
-  public PipelineSelector(NodeManager nodeManager, Configuration conf,
-      EventPublisher eventPublisher, int cacheSizeMB) throws IOException {
-    this.conf = conf;
-    this.eventPublisher = eventPublisher;
-    this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf);
-    this.containerSize = (long)this.conf.getStorageSize(
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
-        ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT,
-        StorageUnit.BYTES);
-    pipelineMap = new ConcurrentHashMap<>();
-    pipelineManagerMap = new HashMap<>();
-
-    pipelineManagerMap.put(ReplicationType.STAND_ALONE,
-            new StandaloneManagerImpl(nodeManager, placementPolicy,
-            containerSize));
-    pipelineManagerMap.put(ReplicationType.RATIS,
-            new RatisManagerImpl(nodeManager, placementPolicy,
-                    containerSize, conf));
-    long pipelineCreationLeaseTimeout = conf.getTimeDuration(
-        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT,
-        ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_LEASE_TIMEOUT_DEFAULT,
-        TimeUnit.MILLISECONDS);
-    pipelineLeaseManager = new LeaseManager<>("PipelineCreation",
-        pipelineCreationLeaseTimeout);
-    pipelineLeaseManager.start();
-
-    stateManager = new PipelineStateManager();
-    this.nodeManager = nodeManager;
-    pipeline2ContainerMap = new HashMap<>();
-
-    // Write the container name to pipeline mapping.
-    File metaDir = getOzoneMetaDirPath(conf);
-    File containerDBPath = new File(metaDir, SCM_PIPELINE_DB);
-    pipelineStore = MetadataStoreBuilder.newBuilder()
-            .setConf(conf)
-            .setDbFile(containerDBPath)
-            .setCacheSize(cacheSizeMB * OzoneConsts.MB)
-            .build();
-
-    reloadExistingPipelines();
-  }
-
-  private void reloadExistingPipelines() throws IOException {
-    if (pipelineStore.isEmpty()) {
-      // Nothing to do just return
-      return;
-    }
-
-    List<Map.Entry<byte[], byte[]>> range =
-            pipelineStore.getSequentialRangeKVs(null, Integer.MAX_VALUE, null);
-
-    // Transform the values into the pipelines.
-    // TODO: filter by pipeline state
-    for (Map.Entry<byte[], byte[]> entry : range) {
-      Pipeline pipeline = Pipeline.getFromProtoBuf(
-                HddsProtos.Pipeline.PARSER.parseFrom(entry.getValue()));
-      Preconditions.checkNotNull(pipeline);
-      addExistingPipeline(pipeline);
-    }
-  }
-
-  @VisibleForTesting
-  public Set<ContainerID> getOpenContainerIDsByPipeline(PipelineID pipelineID) {
-    return pipeline2ContainerMap.get(pipelineID);
-  }
-
-  public void addContainerToPipeline(PipelineID pipelineID, long containerID) {
-    pipeline2ContainerMap.get(pipelineID)
-            .add(ContainerID.valueof(containerID));
-  }
-
-  public void removeContainerFromPipeline(PipelineID pipelineID,
-                                          long containerID) throws IOException {
-    pipeline2ContainerMap.get(pipelineID)
-            .remove(ContainerID.valueof(containerID));
-    closePipelineIfNoOpenContainers(pipelineMap.get(pipelineID));
-  }
-
-  /**
-   * Translates a list of nodes, ordered such that the first is the leader, into
-   * a corresponding {@link Pipeline} object.
-   *
-   * @param nodes - list of datanodes on which we will allocate the container.
-   * The first of the list will be the leader node.
-   * @return pipeline corresponding to nodes
-   */
-  public static Pipeline newPipelineFromNodes(
-      List<DatanodeDetails> nodes, ReplicationType replicationType,
-      ReplicationFactor replicationFactor, PipelineID id) {
-    Preconditions.checkNotNull(nodes);
-    Preconditions.checkArgument(nodes.size() > 0);
-    String leaderId = nodes.get(0).getUuidString();
-    // A new pipeline always starts in allocated state
-    Pipeline pipeline = new Pipeline(leaderId, LifeCycleState.ALLOCATED,
-        replicationType, replicationFactor, id);
-    for (DatanodeDetails node : nodes) {
-      pipeline.addMember(node);
-    }
-    return pipeline;
-  }
-
-  /**
-   * Create pluggable container placement policy implementation instance.
-   *
-   * @param nodeManager - SCM node manager.
-   * @param conf - configuration.
-   * @return SCM container placement policy implementation instance.
-   */
-  @SuppressWarnings("unchecked")
-  private static ContainerPlacementPolicy createContainerPlacementPolicy(
-      final NodeManager nodeManager, final Configuration conf) {
-    Class<? extends ContainerPlacementPolicy> implClass =
-        (Class<? extends ContainerPlacementPolicy>) conf.getClass(
-            ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
-            SCMContainerPlacementRandom.class);
-
-    try {
-      Constructor<? extends ContainerPlacementPolicy> ctor =
-          implClass.getDeclaredConstructor(NodeManager.class,
-              Configuration.class);
-      return ctor.newInstance(nodeManager, conf);
-    } catch (RuntimeException e) {
-      throw e;
-    } catch (InvocationTargetException e) {
-      throw new RuntimeException(implClass.getName()
-          + " could not be constructed.", e.getCause());
-    } catch (Exception e) {
-      LOG.error("Unhandled exception occurred, Placement policy will not be " +
-          "functional.");
-      throw new IllegalArgumentException("Unable to load " +
-          "ContainerPlacementPolicy", e);
-    }
-  }
-
-  /**
-   * This function is called by the Container Manager while allocating a new
-   * container. The client specifies what kind of replication pipeline is needed
-   * and based on the replication type in the request appropriate Interface is
-   * invoked.
-   */
-
-  public Pipeline getReplicationPipeline(ReplicationType replicationType,
-      HddsProtos.ReplicationFactor replicationFactor)
-      throws IOException {
-    PipelineManager manager = pipelineManagerMap.get(replicationType);
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Getting replication pipeline forReplicationType {} :" +
-            " ReplicationFactor {}", replicationType.toString(),
-        replicationFactor.toString());
-
-    /**
-     * In the Ozone world, we have a very simple policy.
-     *
-     * 1. Try to create a pipeline if there are enough free nodes.
-     *
-     * 2. This allows all nodes to part of a pipeline quickly.
-     *
-     * 3. if there are not enough free nodes, return already allocated pipeline
-     * in a round-robin fashion.
-     *
-     * TODO: Might have to come up with a better algorithm than this.
-     * Create a new placement policy that returns pipelines in round robin
-     * fashion.
-     */
-    Pipeline pipeline =
-        manager.createPipeline(replicationFactor, replicationType);
-    if (pipeline == null) {
-      // try to return a pipeline from already allocated pipelines
-      PipelineID pipelineId =
-              manager.getPipeline(replicationFactor, replicationType);
-      if (pipelineId == null) {
-        throw new SCMException(FAILED_TO_FIND_ACTIVE_PIPELINE);
-      }
-      pipeline = pipelineMap.get(pipelineId);
-      Preconditions.checkArgument(pipeline.getLifeCycleState() ==
-              LifeCycleState.OPEN);
-    } else {
-      pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(),
-              pipeline.getProtobufMessage().toByteArray());
-      // if a new pipeline is created, initialize its state machine
-      updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.CREATE);
-
-      //TODO: move the initialization of pipeline to Ozone Client
-      manager.initializePipeline(pipeline);
-      updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.CREATED);
-    }
-    return pipeline;
-  }
-
-  /**
-   * This function to return pipeline for given pipeline id.
-   */
-  public Pipeline getPipeline(PipelineID pipelineID) {
-    return pipelineMap.get(pipelineID);
-  }
-
-  /**
-   * Finalize a given pipeline.
-   */
-  public void finalizePipeline(Pipeline pipeline) throws IOException {
-    PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    if (pipeline.getLifeCycleState() == LifeCycleState.CLOSING ||
-        pipeline.getLifeCycleState() == LifeCycleState.CLOSED) {
-      LOG.debug("pipeline:{} already in closing state, skipping",
-          pipeline.getId());
-      // already in closing/closed state
-      return;
-    }
-
-    // Remove the pipeline from active allocation
-    if (manager.finalizePipeline(pipeline)) {
-      LOG.info("Finalizing pipeline. pipelineID: {}", pipeline.getId());
-      updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.FINALIZE);
-      closePipelineIfNoOpenContainers(pipeline);
-    }
-  }
-
-  /**
-   * Close a given pipeline.
-   */
-  private void closePipelineIfNoOpenContainers(Pipeline pipeline)
-      throws IOException {
-    if (pipeline.getLifeCycleState() != LifeCycleState.CLOSING) {
-      return;
-    }
-    HashSet<ContainerID> containerIDS =
-            pipeline2ContainerMap.get(pipeline.getId());
-    if (containerIDS.size() == 0) {
-      updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.CLOSE);
-      LOG.info("Closing pipeline. pipelineID: {}", pipeline.getId());
-    }
-  }
-
-  /**
-   * Close a given pipeline.
-   */
-  private void closePipeline(Pipeline pipeline) throws IOException {
-    PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Closing pipeline. pipelineID: {}", pipeline.getId());
-    HashSet<ContainerID> containers =
-            pipeline2ContainerMap.get(pipeline.getId());
-    Preconditions.checkArgument(containers.size() == 0);
-    manager.closePipeline(pipeline);
-  }
-
-  /**
-   * Add to a given pipeline.
-   */
-  private void addOpenPipeline(Pipeline pipeline) {
-    PipelineManager manager = pipelineManagerMap.get(pipeline.getType());
-    Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
-    LOG.debug("Adding Open pipeline. pipelineID: {}", pipeline.getId());
-    manager.addOpenPipeline(pipeline);
-  }
-
-  private void closeContainersByPipeline(Pipeline pipeline) {
-    HashSet<ContainerID> containers =
-            pipeline2ContainerMap.get(pipeline.getId());
-    for (ContainerID id : containers) {
-      eventPublisher.fireEvent(SCMEvents.CLOSE_CONTAINER, id);
-    }
-  }
-
-  private void addExistingPipeline(Pipeline pipeline) throws IOException {
-    LifeCycleState state = pipeline.getLifeCycleState();
-    switch (state) {
-    case ALLOCATED:
-      // a pipeline in allocated state is only present in SCM and does not exist
-      // on datanode, on SCM restart, this pipeline can be ignored.
-      break;
-    case CREATING:
-    case OPEN:
-    case CLOSING:
-      //TODO: process pipeline report and move pipeline to active queue
-      // when all the nodes have reported.
-      pipelineMap.put(pipeline.getId(), pipeline);
-      pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
-      nodeManager.addPipeline(pipeline);
-      // reset the datanodes in the pipeline
-      // they will be reset on
-      pipeline.resetPipeline();
-      break;
-    case CLOSED:
-      // if the pipeline is in closed state, nothing to do.
-      break;
-    default:
-      throw new IOException("invalid pipeline state:" + state);
-    }
-  }
-
-  public void handleStaleNode(DatanodeDetails dn) {
-    Set<PipelineID> pipelineIDs = nodeManager.getPipelineByDnID(dn.getUuid());
-    for (PipelineID id : pipelineIDs) {
-      LOG.info("closing pipeline {}.", id);
-      eventPublisher.fireEvent(SCMEvents.PIPELINE_CLOSE, id);
-    }
-  }
-
-  void processPipelineReport(DatanodeDetails dn,
-                                    PipelineReportsProto pipelineReport) {
-    Set<PipelineID> reportedPipelines = new HashSet<>();
-    pipelineReport.getPipelineReportList().
-            forEach(p ->
-                    reportedPipelines.add(
-                            processPipelineReport(p.getPipelineID(), dn)));
-
-    //TODO: handle missing pipelines and new pipelines later
-  }
-
-  private PipelineID processPipelineReport(
-          HddsProtos.PipelineID id, DatanodeDetails dn) {
-    PipelineID pipelineID = PipelineID.getFromProtobuf(id);
-    Pipeline pipeline = pipelineMap.get(pipelineID);
-    if (pipeline != null) {
-      pipelineManagerMap.get(pipeline.getType())
-              .processPipelineReport(pipeline, dn);
-    }
-    return pipelineID;
-  }
-
-  /**
-   * Update the Pipeline State to the next state.
-   *
-   * @param pipeline - Pipeline
-   * @param event - LifeCycle Event
-   * @throws SCMException  on Failure.
-   */
-  public void updatePipelineState(Pipeline pipeline,
-      HddsProtos.LifeCycleEvent event) throws IOException {
-    try {
-      switch (event) {
-      case CREATE:
-        pipelineMap.put(pipeline.getId(), pipeline);
-        pipeline2ContainerMap.put(pipeline.getId(), new HashSet<>());
-        nodeManager.addPipeline(pipeline);
-        // Acquire lease on pipeline
-        Lease<Pipeline> pipelineLease = pipelineLeaseManager.acquire(pipeline);
-        // Register callback to be executed in case of timeout
-        pipelineLease.registerCallBack(() -> {
-          updatePipelineState(pipeline, HddsProtos.LifeCycleEvent.TIMEOUT);
-          return null;
-        });
-        break;
-      case CREATED:
-        // Release the lease on pipeline
-        pipelineLeaseManager.release(pipeline);
-        addOpenPipeline(pipeline);
-        break;
-
-      case FINALIZE:
-        closeContainersByPipeline(pipeline);
-        break;
-
-      case CLOSE:
-      case TIMEOUT:
-        closePipeline(pipeline);
-        pipeline2ContainerMap.remove(pipeline.getId());
-        nodeManager.removePipeline(pipeline);
-        pipelineMap.remove(pipeline.getId());
-        break;
-      default:
-        throw new SCMException("Unsupported pipeline LifeCycleEvent.",
-            FAILED_TO_CHANGE_PIPELINE_STATE);
-      }
-
-      stateManager.updatePipelineState(pipeline, event);
-      pipelineStore.put(pipeline.getId().getProtobuf().toByteArray(),
-              pipeline.getProtobufMessage().toByteArray());
-    } catch (LeaseException e) {
-      throw new IOException("Lease Exception.", e);
-    }
-  }
-
-  public void shutdown() throws IOException {
-    if (pipelineLeaseManager != null) {
-      pipelineLeaseManager.shutdown();
-    }
-
-    if (pipelineStore != null) {
-      pipelineStore.close();
-    }
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/PipelineStateManager.java

@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.common.statemachine
-    .InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
-    .FAILED_TO_CHANGE_PIPELINE_STATE;
-
-/**
- * Manages Pipeline states.
- */
-public class PipelineStateManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(PipelineStateManager.class);
-
-  private final StateMachine<HddsProtos.LifeCycleState,
-      HddsProtos.LifeCycleEvent> stateMachine;
-
-  PipelineStateManager() {
-    // Initialize the container state machine.
-    Set<HddsProtos.LifeCycleState> finalStates = new HashSet<>();
-    // These are the steady states of a container.
-    finalStates.add(HddsProtos.LifeCycleState.OPEN);
-    finalStates.add(HddsProtos.LifeCycleState.CLOSED);
-
-    this.stateMachine = new StateMachine<>(HddsProtos.LifeCycleState.ALLOCATED,
-        finalStates);
-    initializeStateMachine();
-  }
-
-  /**
-   * Event and State Transition Mapping.
-   *
-   * State: ALLOCATED ---------------> CREATING
-   * Event:                CREATE
-   *
-   * State: CREATING  ---------------> OPEN
-   * Event:               CREATED
-   *
-   * State: OPEN      ---------------> CLOSING
-   * Event:               FINALIZE
-   *
-   * State: CLOSING   ---------------> CLOSED
-   * Event:                CLOSE
-   *
-   * State: CREATING  ---------------> CLOSED
-   * Event:               TIMEOUT
-   *
-   *
-   * Container State Flow:
-   *
-   * [ALLOCATED]---->[CREATING]------>[OPEN]-------->[CLOSING]
-   *            (CREATE)     | (CREATED)     (FINALIZE)   |
-   *                         |                            |
-   *                         |                            |
-   *                         |(TIMEOUT)                   |(CLOSE)
-   *                         |                            |
-   *                         +--------> [CLOSED] <--------+
-   */
-  private void initializeStateMachine() {
-    stateMachine.addTransition(HddsProtos.LifeCycleState.ALLOCATED,
-        HddsProtos.LifeCycleState.CREATING,
-        HddsProtos.LifeCycleEvent.CREATE);
-
-    stateMachine.addTransition(HddsProtos.LifeCycleState.CREATING,
-        HddsProtos.LifeCycleState.OPEN,
-        HddsProtos.LifeCycleEvent.CREATED);
-
-    stateMachine.addTransition(HddsProtos.LifeCycleState.OPEN,
-        HddsProtos.LifeCycleState.CLOSING,
-        HddsProtos.LifeCycleEvent.FINALIZE);
-
-    stateMachine.addTransition(HddsProtos.LifeCycleState.CLOSING,
-        HddsProtos.LifeCycleState.CLOSED,
-        HddsProtos.LifeCycleEvent.CLOSE);
-
-    stateMachine.addTransition(HddsProtos.LifeCycleState.CREATING,
-        HddsProtos.LifeCycleState.CLOSED,
-        HddsProtos.LifeCycleEvent.TIMEOUT);
-  }
-
-
-  /**
-   * Update the Pipeline State to the next state.
-   *
-   * @param pipeline - Pipeline
-   * @param event - LifeCycle Event
-   * @throws SCMException  on Failure.
-   */
-  public void updatePipelineState(Pipeline pipeline,
-      HddsProtos.LifeCycleEvent event) throws IOException {
-    HddsProtos.LifeCycleState newState;
-    try {
-      newState = stateMachine.getNextState(pipeline.getLifeCycleState(), event);
-    } catch (InvalidStateTransitionException ex) {
-      String error = String.format("Failed to update pipeline state %s, " +
-              "reason: invalid state transition from state: %s upon " +
-              "event: %s.",
-          pipeline.getId(), pipeline.getLifeCycleState(), event);
-      LOG.error(error);
-      throw new SCMException(error, FAILED_TO_CHANGE_PIPELINE_STATE);
-    }
-
-    // This is a post condition after executing getNextState.
-    Preconditions.checkNotNull(newState);
-    Preconditions.checkNotNull(pipeline);
-    pipeline.setLifeCycleState(newState);
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/package-info.java

@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines;
-/**
- Ozone supports the notion of different kind of pipelines.
- That means that we can have a replication pipeline build on
- Ratis, Standalone or some other protocol. All Pipeline managers
- the entities in charge of pipelines reside in the package.
-
- Here is the high level Arch.
-
- 1. A pipeline selector class is instantiated in the Container manager class.
-
- 2. A client when creating a container -- will specify what kind of
- replication type it wants to use. We support 2 types now, Ratis and StandAlone.
-
- 3. Based on the replication type, the pipeline selector class asks the
- corresponding pipeline manager for a pipeline.
-
- 4. We have supported the ability for clients to specify a set of nodes in
- the pipeline or rely in the pipeline manager to select the datanodes if they
- are not specified.
- */

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/RatisManagerImpl.java

@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines.ratis;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.XceiverClientRatis;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Implementation of {@link PipelineManager}.
- *
- * TODO : Introduce a state machine.
- */
-public class RatisManagerImpl extends PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(RatisManagerImpl.class);
-  private final Configuration conf;
-  private final NodeManager nodeManager;
-  private final Set<DatanodeDetails> ratisMembers;
-
-  /**
-   * Constructs a Ratis Pipeline Manager.
-   *
-   * @param nodeManager
-   */
-  public RatisManagerImpl(NodeManager nodeManager,
-      ContainerPlacementPolicy placementPolicy, long size, Configuration conf) {
-    super();
-    this.conf = conf;
-    this.nodeManager = nodeManager;
-    ratisMembers = new HashSet<>();
-  }
-
-  /**
-   * Allocates a new ratis Pipeline from the free nodes.
-   *
-   * @param factor - One or Three
-   * @return Pipeline.
-   */
-  public Pipeline allocatePipeline(ReplicationFactor factor) {
-    List<DatanodeDetails> newNodesList = new LinkedList<>();
-    List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
-    //TODO: Add Raft State to the Nodes, so we can query and skip nodes from
-    // data from datanode instead of maintaining a set.
-    for (DatanodeDetails datanode : datanodes) {
-      Preconditions.checkNotNull(datanode);
-      if (!ratisMembers.contains(datanode)) {
-        newNodesList.add(datanode);
-        if (newNodesList.size() == factor.getNumber()) {
-          // once a datanode has been added to a pipeline, exclude it from
-          // further allocations
-          ratisMembers.addAll(newNodesList);
-          PipelineID pipelineID = PipelineID.randomId();
-          LOG.info("Allocating a new ratis pipeline of size: {} id: {}",
-                  factor.getNumber(), pipelineID);
-          return PipelineSelector.newPipelineFromNodes(newNodesList,
-              ReplicationType.RATIS, factor, pipelineID);
-        }
-      }
-    }
-    return null;
-  }
-
-  public void initializePipeline(Pipeline pipeline) throws IOException {
-    //TODO:move the initialization from SCM to client
-    try (XceiverClientRatis client =
-        XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
-      client.createPipeline();
-    }
-  }
-
-  public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
-    super.processPipelineReport(pipeline, dn);
-    ratisMembers.add(dn);
-  }
-
-  public synchronized boolean finalizePipeline(Pipeline pipeline) {
-    activePipelines.get(pipeline.getFactor().ordinal())
-            .removePipeline(pipeline.getId());
-    return true;
-  }
-
-  /**
-   * Close the pipeline.
-   */
-  public void closePipeline(Pipeline pipeline) throws IOException {
-    try (XceiverClientRatis client =
-        XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
-      client.destroyPipeline();
-    }
-    for (DatanodeDetails node : pipeline.getMachines()) {
-      // A node should always be the in ratis members list.
-      Preconditions.checkArgument(ratisMembers.remove(node));
-    }
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/ratis/package-info.java

@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines.ratis;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/StandaloneManagerImpl.java

@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines.standalone;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.container.placement.algorithms
-    .ContainerPlacementPolicy;
-import org.apache.hadoop.hdds.scm.node.NodeManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineManager;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Standalone Manager Impl to prove that pluggable interface
- * works with current tests.
- */
-public class StandaloneManagerImpl extends PipelineManager {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(StandaloneManagerImpl.class);
-  private final NodeManager nodeManager;
-  private final ContainerPlacementPolicy placementPolicy;
-  private final long containerSize;
-  private final Set<DatanodeDetails> standAloneMembers;
-
-  /**
-   * Constructor for Standalone Node Manager Impl.
-   * @param nodeManager - Node Manager.
-   * @param placementPolicy - Placement Policy
-   * @param containerSize - Container Size.
-   */
-  public StandaloneManagerImpl(NodeManager nodeManager,
-      ContainerPlacementPolicy placementPolicy, long containerSize) {
-    super();
-    this.nodeManager = nodeManager;
-    this.placementPolicy = placementPolicy;
-    this.containerSize =  containerSize;
-    this.standAloneMembers = new HashSet<>();
-  }
-
-
-  /**
-   * Allocates a new standalone Pipeline from the free nodes.
-   *
-   * @param factor - One
-   * @return Pipeline.
-   */
-  public Pipeline allocatePipeline(ReplicationFactor factor) {
-    List<DatanodeDetails> newNodesList = new LinkedList<>();
-    List<DatanodeDetails> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
-    for (DatanodeDetails datanode : datanodes) {
-      Preconditions.checkNotNull(datanode);
-      if (!standAloneMembers.contains(datanode)) {
-        newNodesList.add(datanode);
-        if (newNodesList.size() == factor.getNumber()) {
-          // once a datanode has been added to a pipeline, exclude it from
-          // further allocations
-          standAloneMembers.addAll(newNodesList);
-          // Standalone pipeline use node id as pipeline
-          PipelineID pipelineID =
-                  PipelineID.valueOf(newNodesList.get(0).getUuid());
-          LOG.info("Allocating a new standalone pipeline of size: {} id: {}",
-              factor.getNumber(), pipelineID);
-          return PipelineSelector.newPipelineFromNodes(newNodesList,
-              ReplicationType.STAND_ALONE, ReplicationFactor.ONE, pipelineID);
-        }
-      }
-    }
-    return null;
-  }
-
-  public void initializePipeline(Pipeline pipeline) {
-    // Nothing to be done for standalone pipeline
-  }
-
-  public void processPipelineReport(Pipeline pipeline, DatanodeDetails dn) {
-    super.processPipelineReport(pipeline, dn);
-    standAloneMembers.add(dn);
-  }
-
-  public synchronized boolean finalizePipeline(Pipeline pipeline) {
-    activePipelines.get(pipeline.getFactor().ordinal())
-            .removePipeline(pipeline.getId());
-    return false;
-  }
-
-  /**
-   * Close the pipeline.
-   */
-  public void closePipeline(Pipeline pipeline) throws IOException {
-    for (DatanodeDetails node : pipeline.getMachines()) {
-      // A node should always be the in standalone members list.
-      Preconditions.checkArgument(standAloneMembers.remove(node));
-    }
-  }
-}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipelines/standalone/package-info.java

@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.pipelines.standalone;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -37,7 +37,7 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java

@@ -63,9 +63,10 @@ import org.apache.hadoop.hdds.scm.node.NodeManager;
 import org.apache.hadoop.hdds.scm.node.NodeReportHandler;
 import org.apache.hadoop.hdds.scm.node.SCMNodeManager;
 import org.apache.hadoop.hdds.scm.node.StaleNodeHandler;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineCloseHandler;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineActionEventHandler;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineReportHandler;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler;
 import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 import org.apache.hadoop.hdds.server.events.EventQueue;
@@ -149,6 +150,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
    * State Managers of SCM.
    */
   private final NodeManager scmNodeManager;
+  private final PipelineManager pipelineManager;
   private final ContainerManager containerManager;
   private final BlockManager scmBlockManager;
   private final SCMStorage scmStorage;
@@ -201,8 +203,9 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
 
     scmNodeManager = new SCMNodeManager(
         conf, scmStorage.getClusterID(), this, eventQueue);
+    pipelineManager = new SCMPipelineManager(conf, scmNodeManager, eventQueue);
     containerManager = new SCMContainerManager(
-        conf, scmNodeManager, eventQueue);
+        conf, scmNodeManager, pipelineManager, eventQueue);
     scmBlockManager = new BlockManagerImpl(
         conf, scmNodeManager, containerManager, eventQueue);
 
@@ -213,14 +216,13 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     NodeReportHandler nodeReportHandler =
         new NodeReportHandler(scmNodeManager);
     PipelineReportHandler pipelineReportHandler =
-            new PipelineReportHandler(
-                    containerManager.getPipelineSelector());
+            new PipelineReportHandler(pipelineManager, conf);
     CommandStatusReportHandler cmdStatusReportHandler =
         new CommandStatusReportHandler();
 
     NewNodeHandler newNodeHandler = new NewNodeHandler(scmNodeManager);
     StaleNodeHandler staleNodeHandler =
-        new StaleNodeHandler(containerManager.getPipelineSelector());
+        new StaleNodeHandler(scmNodeManager, pipelineManager);
     DeadNodeHandler deadNodeHandler = new DeadNodeHandler(scmNodeManager,
         containerManager);
     ContainerActionsHandler actionsHandler = new ContainerActionsHandler();
@@ -231,11 +233,8 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
         new ContainerReportHandler(containerManager, scmNodeManager,
             replicationStatus);
 
-    PipelineActionEventHandler pipelineActionEventHandler =
-        new PipelineActionEventHandler();
-
-    PipelineCloseHandler pipelineCloseHandler =
-        new PipelineCloseHandler(containerManager.getPipelineSelector());
+    PipelineActionHandler pipelineActionHandler =
+        new PipelineActionHandler(pipelineManager);
 
     long watcherTimeout =
         conf.getTimeDuration(ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT,
@@ -294,10 +293,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
         .addHandler(SCMEvents.PENDING_DELETE_STATUS, pendingDeleteHandler);
     eventQueue.addHandler(SCMEvents.DELETE_BLOCK_STATUS,
         (DeletedBlockLogImpl) scmBlockManager.getDeletedBlockLog());
-    eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS,
-        pipelineActionEventHandler);
-    eventQueue.addHandler(SCMEvents.PIPELINE_CLOSE, pipelineCloseHandler);
-
+    eventQueue.addHandler(SCMEvents.PIPELINE_ACTIONS, pipelineActionHandler);
     eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, clientProtocolServer);
     eventQueue.addHandler(SCMEvents.PIPELINE_REPORT, pipelineReportHandler);
 
@@ -771,6 +767,7 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
       LOG.error("SCM Event Queue stop failed", ex);
     }
     IOUtils.cleanupWithLogger(LOG, containerManager);
+    IOUtils.cleanupWithLogger(LOG, pipelineManager);
   }
 
   /**
@@ -815,6 +812,16 @@ public final class StorageContainerManager extends ServiceRuntimeInfoImpl
     return scmNodeManager;
   }
 
+  /**
+   * Returns pipeline manager.
+   *
+   * @return - Pipeline Manager
+   */
+  @VisibleForTesting
+  public PipelineManager getPipelineManager() {
+    return pipelineManager;
+  }
+
   @VisibleForTesting
   public BlockManager getScmBlockManager() {
     return scmBlockManager;

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestBlockManager.java

@@ -27,6 +27,8 @@ import org.apache.hadoop.hdds.scm.container.MockNodeManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.scm.server.SCMStorage;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.server.events.EventHandler;
@@ -58,6 +60,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.MB;
 public class TestBlockManager implements EventHandler<Boolean> {
   private static SCMContainerManager mapping;
   private static MockNodeManager nodeManager;
+  private static PipelineManager pipelineManager;
   private static BlockManagerImpl blockManager;
   private static File testDir;
   private final static long DEFAULT_BLOCK_SIZE = 128 * MB;
@@ -83,7 +86,10 @@ public class TestBlockManager implements EventHandler<Boolean> {
       throw new IOException("Unable to create test directory path");
     }
     nodeManager = new MockNodeManager(true, 10);
-    mapping = new SCMContainerManager(conf, nodeManager, eventQueue);
+    pipelineManager =
+        new SCMPipelineManager(conf, nodeManager, eventQueue);
+    mapping = new SCMContainerManager(conf, nodeManager, pipelineManager,
+        eventQueue);
     blockManager = new BlockManagerImpl(conf,
         nodeManager, mapping, eventQueue);
     eventQueue.addHandler(SCMEvents.CHILL_MODE_STATUS, blockManager);
@@ -101,6 +107,7 @@ public class TestBlockManager implements EventHandler<Boolean> {
   @After
   public void cleanup() throws IOException {
     blockManager.close();
+    pipelineManager.close();
     mapping.close();
     FileUtil.fullyDelete(testDir);
   }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/block/TestDeletedBlockLog.java

@@ -24,12 +24,11 @@ import org.apache.hadoop.hdds.scm.container.SCMContainerManager;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.protocol.proto
@@ -102,12 +101,13 @@ public class TestDeletedBlockLog {
 
     ContainerInfo containerInfo =
         new ContainerInfo.Builder().setContainerID(1).build();
-    Pipeline pipeline =
-        new Pipeline(null, LifeCycleState.CLOSED,
-            ReplicationType.RATIS, ReplicationFactor.THREE, null);
-    pipeline.addMember(dnList.get(0));
-    pipeline.addMember(dnList.get(1));
-    pipeline.addMember(dnList.get(2));
+    Pipeline pipeline = Pipeline.newBuilder()
+        .setType(ReplicationType.RATIS)
+        .setFactor(ReplicationFactor.THREE)
+        .setState(Pipeline.PipelineState.CLOSED)
+        .setId(PipelineID.randomId())
+        .setNodes(dnList)
+        .build();
     ContainerWithPipeline containerWithPipeline =
         new ContainerWithPipeline(containerInfo, pipeline);
     when(containerManager.getContainerWithPipeline(anyObject()))
@@ -383,11 +383,15 @@ public class TestDeletedBlockLog {
 
   private void mockContainerInfo(long containerID, DatanodeDetails dd)
       throws IOException {
-    Pipeline pipeline =
-        new Pipeline("fake", LifeCycleState.OPEN,
-            ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
-            PipelineID.randomId());
-    pipeline.addMember(dd);
+    List<DatanodeDetails> dns = new ArrayList<>();
+    dns.add(dd);
+    Pipeline pipeline = Pipeline.newBuilder()
+            .setType(ReplicationType.STAND_ALONE)
+            .setFactor(ReplicationFactor.ONE)
+            .setState(Pipeline.PipelineState.OPEN)
+            .setId(PipelineID.randomId())
+            .setNodes(dns)
+            .build();
 
     ContainerInfo.Builder builder = new ContainerInfo.Builder();
     builder.setPipelineID(pipeline.getId())

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/MockNodeManager.java

@@ -19,8 +19,8 @@ package org.apache.hadoop.hdds.scm.container;
 import org.apache.hadoop.hdds.protocol.proto
         .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
 import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestCloseContainerEventHandler.java

@@ -25,6 +25,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.common.helpers
     .ContainerWithPipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
@@ -65,8 +67,11 @@ public class TestCloseContainerEventHandler {
     configuration
         .set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
     nodeManager = new MockNodeManager(true, 10);
-    containerManager = new SCMContainerManager(configuration, nodeManager,
-        new EventQueue());
+    PipelineManager pipelineManager =
+        new SCMPipelineManager(configuration, nodeManager, eventQueue);
+    containerManager = new
+        SCMContainerManager(configuration, nodeManager,
+        pipelineManager, new EventQueue());
     eventQueue = new EventQueue();
     eventQueue.addHandler(CLOSE_CONTAINER,
         new CloseContainerEventHandler(containerManager));
@@ -110,11 +115,12 @@ public class TestCloseContainerEventHandler {
     GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
         .captureLogs(CloseContainerEventHandler.LOG);
     ContainerWithPipeline containerWithPipeline = containerManager
-        .allocateContainer(HddsProtos.ReplicationType.STAND_ALONE,
+        .allocateContainer(HddsProtos.ReplicationType.RATIS,
             HddsProtos.ReplicationFactor.ONE, "ozone");
     ContainerID id = new ContainerID(
         containerWithPipeline.getContainerInfo().getContainerID());
-    DatanodeDetails datanode = containerWithPipeline.getPipeline().getLeader();
+    DatanodeDetails datanode =
+        containerWithPipeline.getPipeline().getFirstNode();
     int closeCount = nodeManager.getCommandCount(datanode);
     eventQueue.fireEvent(CLOSE_CONTAINER, id);
     eventQueue.processAll(1000);
@@ -149,13 +155,13 @@ public class TestCloseContainerEventHandler {
     eventQueue.processAll(1000);
     int i = 0;
     for (DatanodeDetails details : containerWithPipeline.getPipeline()
-        .getMachines()) {
+        .getNodes()) {
       closeCount[i] = nodeManager.getCommandCount(details);
       i++;
     }
     i = 0;
     for (DatanodeDetails details : containerWithPipeline.getPipeline()
-        .getMachines()) {
+        .getNodes()) {
       Assert.assertEquals(closeCount[i], nodeManager.getCommandCount(details));
       i++;
     }
@@ -166,7 +172,7 @@ public class TestCloseContainerEventHandler {
     i = 0;
     // Make sure close is queued for each datanode on the pipeline
     for (DatanodeDetails details : containerWithPipeline.getPipeline()
-        .getMachines()) {
+        .getNodes()) {
       Assert.assertEquals(closeCount[i] + 1,
           nodeManager.getCommandCount(details));
       Assert.assertEquals(HddsProtos.LifeCycleState.CLOSING,

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerReportHandler.java

@@ -32,6 +32,8 @@ import org.apache.hadoop.hdds.scm.container.replication
     .ReplicationActivityStatus;
 import org.apache.hadoop.hdds.scm.container.replication.ReplicationRequest;
 import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher
     .ContainerReportFromDatanode;
 import org.apache.hadoop.hdds.server.events.Event;
@@ -73,8 +75,11 @@ public class TestContainerReportHandler implements EventPublisher {
     //GIVEN
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir);
+    EventQueue eventQueue = new EventQueue();
+    PipelineManager pipelineManager =
+        new SCMPipelineManager(conf, nodeManager, eventQueue);
     SCMContainerManager containerManager = new SCMContainerManager(
-        conf, nodeManager, new EventQueue());
+        conf, nodeManager, pipelineManager, eventQueue);
 
     ReplicationActivityStatus replicationActivityStatus =
         new ReplicationActivityStatus();

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerStateManager.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdds.scm.container;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Set;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -25,9 +26,10 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.TestUtils;
 
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -102,19 +104,20 @@ public class TestContainerStateManager {
 
   private ContainerInfo allocateContainer() throws IOException {
 
-    PipelineSelector pipelineSelector = Mockito.mock(PipelineSelector.class);
+    PipelineManager pipelineManager = Mockito.mock(SCMPipelineManager.class);
 
-    Pipeline pipeline = new Pipeline("leader", HddsProtos.LifeCycleState.CLOSED,
-        HddsProtos.ReplicationType.STAND_ALONE,
-        HddsProtos.ReplicationFactor.THREE,
-        PipelineID.randomId());
+    Pipeline pipeline =
+        Pipeline.newBuilder().setState(Pipeline.PipelineState.CLOSED)
+            .setId(PipelineID.randomId())
+            .setType(HddsProtos.ReplicationType.STAND_ALONE)
+            .setFactor(HddsProtos.ReplicationFactor.THREE)
+            .setNodes(new ArrayList<>()).build();
 
-    when(pipelineSelector
-        .getReplicationPipeline(HddsProtos.ReplicationType.STAND_ALONE,
-            HddsProtos.ReplicationFactor.THREE)).thenReturn(pipeline);
+    when(pipelineManager.createPipeline(HddsProtos.ReplicationType.STAND_ALONE,
+        HddsProtos.ReplicationFactor.THREE)).thenReturn(pipeline);
 
-    return containerStateManager.allocateContainer(
-        pipelineSelector, HddsProtos.ReplicationType.STAND_ALONE,
+    return containerStateManager.allocateContainer(pipelineManager,
+        HddsProtos.ReplicationType.STAND_ALONE,
         HddsProtos.ReplicationFactor.THREE, "root");
 
   }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestSCMContainerManager.java

@@ -24,13 +24,15 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
@@ -59,6 +61,7 @@ import java.util.concurrent.TimeUnit;
 public class TestSCMContainerManager {
   private static SCMContainerManager containerManager;
   private static MockNodeManager nodeManager;
+  private static PipelineManager pipelineManager;
   private static File testDir;
   private static XceiverClientManager xceiverClientManager;
   private static String containerOwner = "OZONE";
@@ -85,8 +88,10 @@ public class TestSCMContainerManager {
       throw new IOException("Unable to create test directory path");
     }
     nodeManager = new MockNodeManager(true, 10);
+    pipelineManager =
+        new SCMPipelineManager(conf, nodeManager, new EventQueue());
     containerManager = new SCMContainerManager(conf, nodeManager,
-        new EventQueue());
+        pipelineManager, new EventQueue());
     xceiverClientManager = new XceiverClientManager(conf);
     random = new Random();
   }
@@ -96,6 +101,9 @@ public class TestSCMContainerManager {
     if(containerManager != null) {
       containerManager.close();
     }
+    if (pipelineManager != null) {
+      pipelineManager.close();
+    }
     FileUtil.fullyDelete(testDir);
   }
 
@@ -130,7 +138,7 @@ public class TestSCMContainerManager {
 
       Assert.assertNotNull(containerInfo);
       Assert.assertNotNull(containerInfo.getPipeline());
-      pipelineList.add(containerInfo.getPipeline().getLeader()
+      pipelineList.add(containerInfo.getPipeline().getFirstNode()
           .getUuid());
     }
     Assert.assertTrue(pipelineList.size() > 5);
@@ -145,8 +153,8 @@ public class TestSCMContainerManager {
     Pipeline pipeline  = containerInfo.getPipeline();
     Assert.assertNotNull(pipeline);
     Pipeline newPipeline = containerInfo.getPipeline();
-    Assert.assertEquals(pipeline.getLeader().getUuid(),
-        newPipeline.getLeader().getUuid());
+    Assert.assertEquals(pipeline.getFirstNode().getUuid(),
+        newPipeline.getFirstNode().getUuid());
   }
 
   @Test
@@ -191,15 +199,15 @@ public class TestSCMContainerManager {
     contInfo = containerManager.getContainer(contInfo.containerID());
     Assert.assertEquals(contInfo.getState(), LifeCycleState.CLOSED);
     Pipeline pipeline = containerWithPipeline.getPipeline();
-    containerManager.getPipelineSelector().finalizePipeline(pipeline);
+    pipelineManager.finalizePipeline(pipeline.getId());
 
     ContainerWithPipeline containerWithPipeline2 = containerManager
         .getContainerWithPipeline(contInfo.containerID());
     pipeline = containerWithPipeline2.getPipeline();
     Assert.assertNotEquals(containerWithPipeline, containerWithPipeline2);
     Assert.assertNotNull("Pipeline should not be null", pipeline);
-    Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn1.getHostName()));
-    Assert.assertTrue(pipeline.getDatanodeHosts().contains(dn2.getHostName()));
+    Assert.assertTrue(pipeline.getNodes().contains(dn1));
+    Assert.assertTrue(pipeline.getNodes().contains(dn2));
   }
 
   @Test

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/replication/TestReplicationManager.java

@@ -20,23 +20,22 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.IntStream;
 
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ReplicateContainerCommandProto;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.ContainerReplica;
 import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacementPolicy;
 import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationRequestToRepeat;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
@@ -227,18 +226,16 @@ public class TestReplicationManager {
   public static Pipeline createPipeline(Iterable<DatanodeDetails> ids)
       throws IOException {
     Objects.requireNonNull(ids, "ids == null");
-    final Iterator<DatanodeDetails> i = ids.iterator();
-    Preconditions.checkArgument(i.hasNext());
-    final DatanodeDetails leader = i.next();
-    final Pipeline pipeline =
-        new Pipeline(leader.getUuidString(), LifeCycleState.OPEN,
-            ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
-            PipelineID.randomId());
-    pipeline.addMember(leader);
-    while (i.hasNext()) {
-      pipeline.addMember(i.next());
-    }
-    return pipeline;
+    Preconditions.checkArgument(ids.iterator().hasNext());
+    List<DatanodeDetails> dns = new ArrayList<>();
+    ids.forEach(dns::add);
+    return Pipeline.newBuilder()
+        .setState(Pipeline.PipelineState.OPEN)
+        .setId(PipelineID.randomId())
+        .setType(HddsProtos.ReplicationType.STAND_ALONE)
+        .setFactor(ReplicationFactor.ONE)
+        .setNodes(dns)
+        .build();
   }
 
 }

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestContainerPlacement.java

@@ -33,6 +33,8 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -102,7 +104,9 @@ public class TestContainerPlacement {
     EventQueue eventQueue = new EventQueue();
     final int cacheSize = config.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
         OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-    return new SCMContainerManager(config, scmNodeManager,
+    PipelineManager pipelineManager =
+        new SCMPipelineManager(config, scmNodeManager, eventQueue);
+    return new SCMContainerManager(config, scmNodeManager, pipelineManager,
         eventQueue);
 
   }
@@ -156,7 +160,7 @@ public class TestContainerPlacement {
           xceiverClientManager.getType(),
           xceiverClientManager.getFactor(), "OZONE");
       assertEquals(xceiverClientManager.getFactor().getNumber(),
-          containerWithPipeline.getPipeline().getMachines().size());
+          containerWithPipeline.getPipeline().getNodes().size());
     } finally {
       IOUtils.closeQuietly(containerManager);
       IOUtils.closeQuietly(nodeManager);

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java

@@ -42,6 +42,8 @@ import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.events.SCMEvents;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
+import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.NodeReportFromDatanode;
 import org.apache.hadoop.hdds.server.events.EventPublisher;
 
@@ -75,7 +77,10 @@ public class TestDeadNodeHandler {
     conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, storageDir);
     eventQueue = new EventQueue();
     nodeManager = new SCMNodeManager(conf, "cluster1", null, eventQueue);
-    containerManager = new SCMContainerManager(conf, nodeManager, eventQueue);
+    PipelineManager pipelineManager =
+        new SCMPipelineManager(conf, nodeManager, eventQueue);
+    containerManager = new SCMContainerManager(conf, nodeManager,
+        pipelineManager, eventQueue);
     deadNodeHandler = new DeadNodeHandler(nodeManager, containerManager);
     eventQueue.addHandler(SCMEvents.DEAD_NODE, deadNodeHandler);
     publisher = Mockito.mock(EventPublisher.class);

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java

@@ -21,7 +21,7 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java

@@ -51,7 +51,7 @@ import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.StorageReportProto;
 import org.apache.hadoop.hdds.protocol.proto
     .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.container.common.statemachine

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/testutils/ReplicationNodeManagerMock.java

@@ -20,8 +20,8 @@ import com.google.common.base.Preconditions;
 import org.apache.hadoop.hdds.protocol.proto
         .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeMetric;
 import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMNodeStat;
 import org.apache.hadoop.hdds.scm.exceptions.SCMException;

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/InfoSubcommand.java

@@ -82,10 +82,7 @@ public class InfoSubcommand implements Callable<Void> {
       LOG.info("Container Metadata: {}", metadataStr);
 
       // Print pipeline of an existing container.
-      LOG.info("LeaderID: {}", container.getPipeline()
-          .getLeader().getHostName());
-      String machinesStr = container.getPipeline()
-          .getMachines().stream().map(
+      String machinesStr = container.getPipeline().getNodes().stream().map(
               DatanodeDetails::getHostName).collect(Collectors.joining(","));
       LOG.info("Datanodes: [{}]", machinesStr);
       return null;

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java

@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
@@ -282,7 +282,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
 
       // irrespective of the container state, we will always read via Standalone
       // protocol.
-      pipeline.setType(HddsProtos.ReplicationType.STAND_ALONE);
+      if (pipeline.getType() != HddsProtos.ReplicationType.STAND_ALONE) {
+        pipeline = Pipeline.newBuilder(pipeline)
+            .setType(HddsProtos.ReplicationType.STAND_ALONE).build();
+      }
       XceiverClientSpi xceiverClient = xceiverClientManager
           .acquireClient(pipeline);
       boolean success = false;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNode2PipelineMap.java

@@ -24,9 +24,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers
     .ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
-import org.apache.hadoop.hdds.scm.exceptions.SCMException;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.junit.After;
@@ -50,7 +47,7 @@ public class TestNode2PipelineMap {
   private static StorageContainerManager scm;
   private static ContainerWithPipeline ratisContainer;
   private static ContainerManager containerManager;
-  private static PipelineSelector pipelineSelector;
+  private static PipelineManager pipelineManager;
 
   /**
    * Create a MiniDFSCluster for testing.
@@ -66,7 +63,7 @@ public class TestNode2PipelineMap {
     containerManager = scm.getContainerManager();
     ratisContainer = containerManager.allocateContainer(
         RATIS, THREE, "testOwner");
-    pipelineSelector = containerManager.getPipelineSelector();
+    pipelineManager = scm.getPipelineManager();
   }
 
   /**
@@ -83,15 +80,15 @@ public class TestNode2PipelineMap {
   @Test
   public void testPipelineMap() throws IOException {
 
-    Set<ContainerID> set = pipelineSelector.getOpenContainerIDsByPipeline(
-        ratisContainer.getPipeline().getId());
+    Set<ContainerID> set = pipelineManager
+        .getContainersInPipeline(ratisContainer.getPipeline().getId());
 
     ContainerID cId = ratisContainer.getContainerInfo().containerID();
     Assert.assertEquals(1, set.size());
     set.forEach(containerID ->
             Assert.assertEquals(containerID, cId));
 
-    List<DatanodeDetails> dns = ratisContainer.getPipeline().getMachines();
+    List<DatanodeDetails> dns = ratisContainer.getPipeline().getNodes();
     Assert.assertEquals(3, dns.size());
 
     // get pipeline details by dnid
@@ -112,18 +109,14 @@ public class TestNode2PipelineMap {
         .updateContainerState(cId, HddsProtos.LifeCycleEvent.FINALIZE);
     containerManager
         .updateContainerState(cId, HddsProtos.LifeCycleEvent.CLOSE);
-    Set<ContainerID> set2 = pipelineSelector.getOpenContainerIDsByPipeline(
+    Set<ContainerID> set2 = pipelineManager.getContainersInPipeline(
         ratisContainer.getPipeline().getId());
     Assert.assertEquals(0, set2.size());
 
-    try {
-      pipelineSelector.updatePipelineState(ratisContainer.getPipeline(),
-          HddsProtos.LifeCycleEvent.CLOSE);
-      Assert.fail("closing of pipeline without finalize should fail");
-    } catch (Exception e) {
-      Assert.assertTrue(e instanceof SCMException);
-      Assert.assertEquals(((SCMException)e).getResult(),
-          SCMException.ResultCodes.FAILED_TO_CHANGE_PIPELINE_STATE);
-    }
+    pipelineManager.finalizePipeline(ratisContainer.getPipeline().getId());
+    pipelineManager.removePipeline(ratisContainer.getPipeline().getId());
+    pipelines = scm.getScmNodeManager()
+        .getPipelineByDnID(dns.get(0).getUuid());
+    Assert.assertEquals(0, pipelines.size());
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestNodeFailure.java

@@ -20,12 +20,10 @@ package org.apache.hadoop.hdds.scm.pipeline;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers
     .ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -53,6 +51,7 @@ public class TestNodeFailure {
   private static ContainerWithPipeline ratisContainer1;
   private static ContainerWithPipeline ratisContainer2;
   private static ContainerManager containerManager;
+  private static PipelineManager pipelineManager;
   private static long timeForFailure;
 
   /**
@@ -76,6 +75,7 @@ public class TestNodeFailure {
     cluster.waitForClusterToBeReady();
     StorageContainerManager scm = cluster.getStorageContainerManager();
     containerManager = scm.getContainerManager();
+    pipelineManager = scm.getPipelineManager();
     ratisContainer1 = containerManager.allocateContainer(
         RATIS, THREE, "testOwner");
     ratisContainer2 = containerManager.allocateContainer(
@@ -102,21 +102,21 @@ public class TestNodeFailure {
   @Test
   public void testPipelineFail() throws InterruptedException, IOException,
       TimeoutException {
-    Assert.assertEquals(ratisContainer1.getPipeline().getLifeCycleState(),
-        HddsProtos.LifeCycleState.OPEN);
+    Assert.assertEquals(ratisContainer1.getPipeline().getPipelineState(),
+        Pipeline.PipelineState.OPEN);
     Pipeline pipelineToFail = ratisContainer1.getPipeline();
-    DatanodeDetails dnToFail = pipelineToFail.getMachines().get(0);
+    DatanodeDetails dnToFail = pipelineToFail.getFirstNode();
     cluster.shutdownHddsDatanode(dnToFail);
 
     // wait for sufficient time for the callback to be triggered
     Thread.sleep(3 * timeForFailure);
 
-    Assert.assertEquals(HddsProtos.LifeCycleState.CLOSED,
-        ratisContainer1.getPipeline().getLifeCycleState());
-    Assert.assertEquals(HddsProtos.LifeCycleState.OPEN,
-        ratisContainer2.getPipeline().getLifeCycleState());
-    Assert.assertNull(containerManager.getPipelineSelector()
-        .getPipeline(pipelineToFail.getId()));
+    Assert.assertEquals(Pipeline.PipelineState.CLOSED,
+        pipelineManager.getPipeline(ratisContainer1.getPipeline().getId())
+            .getPipelineState());
+    Assert.assertEquals(Pipeline.PipelineState.OPEN,
+        pipelineManager.getPipeline(ratisContainer2.getPipeline().getId())
+            .getPipelineState());
     // Now restart the datanode and make sure that a new pipeline is created.
     cluster.restartHddsDatanode(dnToFail);
     ContainerWithPipeline ratisContainer3 =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineClose.java

@@ -24,8 +24,6 @@ import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
 import org.apache.hadoop.hdds.scm.container.common.helpers
     .ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.pipelines.PipelineSelector;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.junit.AfterClass;
@@ -50,7 +48,7 @@ public class TestPipelineClose {
   private static ContainerWithPipeline ratisContainer1;
   private static ContainerWithPipeline ratisContainer2;
   private static ContainerManager containerManager;
-  private static PipelineSelector pipelineSelector;
+  private static PipelineManager pipelineManager;
 
   /**
    * Create a MiniDFSCluster for testing.
@@ -68,7 +66,7 @@ public class TestPipelineClose {
         .allocateContainer(RATIS, THREE, "testOwner");
     ratisContainer2 = containerManager
         .allocateContainer(RATIS, THREE, "testOwner");
-    pipelineSelector = containerManager.getPipelineSelector();
+    pipelineManager = scm.getPipelineManager();
     // At this stage, there should be 2 pipeline one with 1 open container each.
     // Try closing the both the pipelines, one with a closed container and
     // the other with an open container.
@@ -87,8 +85,8 @@ public class TestPipelineClose {
 
   @Test
   public void testPipelineCloseWithClosedContainer() throws IOException {
-    Set<ContainerID> set = pipelineSelector.getOpenContainerIDsByPipeline(
-        ratisContainer1.getPipeline().getId());
+    Set<ContainerID> set = pipelineManager
+        .getContainersInPipeline(ratisContainer1.getPipeline().getId());
 
     ContainerID cId = ratisContainer1.getContainerInfo().containerID();
     Assert.assertEquals(1, set.size());
@@ -105,17 +103,17 @@ public class TestPipelineClose {
     containerManager
         .updateContainerState(cId, HddsProtos.LifeCycleEvent.CLOSE);
 
-    Set<ContainerID> setClosed = pipelineSelector.getOpenContainerIDsByPipeline(
-        ratisContainer1.getPipeline().getId());
+    Set<ContainerID> setClosed = pipelineManager
+        .getContainersInPipeline(ratisContainer1.getPipeline().getId());
     Assert.assertEquals(0, setClosed.size());
 
-    pipelineSelector.finalizePipeline(ratisContainer1.getPipeline());
-    Pipeline pipeline1 = pipelineSelector
+    pipelineManager.finalizePipeline(ratisContainer1.getPipeline().getId());
+    Pipeline pipeline1 = pipelineManager
         .getPipeline(ratisContainer1.getPipeline().getId());
-    Assert.assertNull(pipeline1);
-    Assert.assertEquals(ratisContainer1.getPipeline().getLifeCycleState(),
-        HddsProtos.LifeCycleState.CLOSED);
-    for (DatanodeDetails dn : ratisContainer1.getPipeline().getMachines()) {
+    Assert.assertEquals(pipeline1.getPipelineState(),
+        Pipeline.PipelineState.CLOSED);
+    pipelineManager.removePipeline(pipeline1.getId());
+    for (DatanodeDetails dn : ratisContainer1.getPipeline().getNodes()) {
       // Assert that the pipeline has been removed from Node2PipelineMap as well
       Assert.assertEquals(scm.getScmNodeManager().getPipelineByDnID(
           dn.getUuid()).size(), 0);
@@ -125,7 +123,7 @@ public class TestPipelineClose {
   @Test
   public void testPipelineCloseWithOpenContainer() throws IOException,
       TimeoutException, InterruptedException {
-    Set<ContainerID> setOpen = pipelineSelector.getOpenContainerIDsByPipeline(
+    Set<ContainerID> setOpen = pipelineManager.getContainersInPipeline(
         ratisContainer2.getPipeline().getId());
     Assert.assertEquals(1, setOpen.size());
 
@@ -134,12 +132,13 @@ public class TestPipelineClose {
         .updateContainerState(cId2, HddsProtos.LifeCycleEvent.CREATE);
     containerManager
         .updateContainerState(cId2, HddsProtos.LifeCycleEvent.CREATED);
-    pipelineSelector.finalizePipeline(ratisContainer2.getPipeline());
-    Assert.assertEquals(ratisContainer2.getPipeline().getLifeCycleState(),
-        HddsProtos.LifeCycleState.CLOSING);
-    Pipeline pipeline2 = pipelineSelector
+    pipelineManager.finalizePipeline(ratisContainer2.getPipeline().getId());
+    Assert.assertEquals(
+        pipelineManager.getPipeline(ratisContainer2.getPipeline().getId())
+            .getPipelineState(), Pipeline.PipelineState.CLOSED);
+    Pipeline pipeline2 = pipelineManager
         .getPipeline(ratisContainer2.getPipeline().getId());
-    Assert.assertEquals(pipeline2.getLifeCycleState(),
-        HddsProtos.LifeCycleState.CLOSING);
+    Assert.assertEquals(pipeline2.getPipelineState(),
+        Pipeline.PipelineState.CLOSED);
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestPipelineStateManager.java

@@ -91,7 +91,7 @@ public class TestPipelineStateManager {
     }
 
     // verify pipeline returned is same
-    Pipeline pipeline1 = stateManager.getPipeline(pipeline.getID());
+    Pipeline pipeline1 = stateManager.getPipeline(pipeline.getId());
     Assert.assertTrue(pipeline == pipeline1);
 
     // clean up
@@ -102,15 +102,17 @@ public class TestPipelineStateManager {
   public void testGetPipelines() throws IOException {
     Set<Pipeline> pipelines = new HashSet<>();
     Pipeline pipeline = createDummyPipeline(1);
-    pipelines.add(pipeline);
     stateManager.addPipeline(pipeline);
-    pipeline = createDummyPipeline(1);
+    stateManager.openPipeline(pipeline.getId());
     pipelines.add(pipeline);
+    pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
+    stateManager.openPipeline(pipeline.getId());
+    pipelines.add(pipeline);
 
-    Set<Pipeline> pipelines1 = new HashSet<>(stateManager.getPipelinesByType(
+    Set<Pipeline> pipelines1 = new HashSet<>(stateManager.getPipelines(
         HddsProtos.ReplicationType.RATIS));
-    Assert.assertEquals(pipelines, pipelines1);
+    Assert.assertEquals(pipelines1.size(), pipelines.size());
     // clean up
     for (Pipeline pipeline1 : pipelines) {
       removePipeline(pipeline1);
@@ -131,16 +133,16 @@ public class TestPipelineStateManager {
           stateManager.addPipeline(pipeline);
           pipelines.add(pipeline);
 
-          // 5 pipelines in allocated state for each type and factor
+          // 5 pipelines in open state for each type and factor
           pipeline = createDummyPipeline(type, factor, factor.getNumber());
           stateManager.addPipeline(pipeline);
-          stateManager.openPipeline(pipeline.getID());
+          stateManager.openPipeline(pipeline.getId());
           pipelines.add(pipeline);
 
-          // 5 pipelines in allocated state for each type and factor
+          // 5 pipelines in closed state for each type and factor
           pipeline = createDummyPipeline(type, factor, factor.getNumber());
           stateManager.addPipeline(pipeline);
-          stateManager.finalizePipeline(pipeline.getID());
+          stateManager.finalizePipeline(pipeline.getId());
           pipelines.add(pipeline);
         }
       }
@@ -152,8 +154,8 @@ public class TestPipelineStateManager {
           .values()) {
         // verify pipelines received
         List<Pipeline> pipelines1 =
-            stateManager.getPipelinesByTypeAndFactor(type, factor);
-        Assert.assertEquals(5, pipelines1.size());
+            stateManager.getPipelines(type, factor);
+        Assert.assertEquals(15, pipelines1.size());
         pipelines1.stream().forEach(p -> {
           Assert.assertEquals(p.getType(), type);
           Assert.assertEquals(p.getFactor(), factor);
@@ -167,41 +169,80 @@ public class TestPipelineStateManager {
     }
   }
 
+  @Test
+  public void testGetPipelinesByTypeAndState() throws IOException {
+    Set<Pipeline> pipelines = new HashSet<>();
+    for (HddsProtos.ReplicationType type : HddsProtos.ReplicationType
+        .values()) {
+      HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
+      for (int i = 0; i < 5; i++) {
+        // 5 pipelines in allocated state for each type and factor
+        Pipeline pipeline =
+            createDummyPipeline(type, factor, factor.getNumber());
+        stateManager.addPipeline(pipeline);
+        pipelines.add(pipeline);
+
+        // 5 pipelines in open state for each type and factor
+        pipeline = createDummyPipeline(type, factor, factor.getNumber());
+        stateManager.addPipeline(pipeline);
+        stateManager.openPipeline(pipeline.getId());
+        pipelines.add(pipeline);
+
+        // 5 pipelines in closed state for each type and factor
+        pipeline = createDummyPipeline(type, factor, factor.getNumber());
+        stateManager.addPipeline(pipeline);
+        stateManager.finalizePipeline(pipeline.getId());
+        pipelines.add(pipeline);
+      }
+    }
+
+    for (HddsProtos.ReplicationType type : HddsProtos.ReplicationType
+        .values()) {
+      // verify pipelines received
+      List<Pipeline> pipelines1 = stateManager
+          .getPipelines(type, Pipeline.PipelineState.OPEN);
+      Assert.assertEquals(5, pipelines1.size());
+      pipelines1.forEach(p -> {
+        Assert.assertEquals(p.getType(), type);
+        Assert.assertEquals(p.getPipelineState(), Pipeline.PipelineState.OPEN);
+      });
+
+      pipelines1 = stateManager
+          .getPipelines(type, Pipeline.PipelineState.OPEN,
+              Pipeline.PipelineState.CLOSED, Pipeline.PipelineState.ALLOCATED);
+      Assert.assertEquals(15, pipelines1.size());
+    }
+
+    //clean up
+    for (Pipeline pipeline : pipelines) {
+      removePipeline(pipeline);
+    }
+  }
+
   @Test
   public void testAddAndGetContainer() throws IOException {
     long containerID = 0;
     Pipeline pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
-    pipeline = stateManager.getPipeline(pipeline.getID());
-
-    try {
-      stateManager.addContainerToPipeline(pipeline.getID(),
-          ContainerID.valueof(++containerID));
-      Assert.fail("Container should not have been added");
-    } catch (IOException e) {
-      // add container possible only in container with open state
-      Assert.assertTrue(e.getMessage().contains("is not in open state"));
-    }
+    pipeline = stateManager.getPipeline(pipeline.getId());
+    stateManager.addContainerToPipeline(pipeline.getId(),
+        ContainerID.valueof(++containerID));
 
     // move pipeline to open state
-    stateManager.openPipeline(pipeline.getID());
-
-    // add three containers
-    stateManager.addContainerToPipeline(pipeline.getID(),
-        ContainerID.valueof(containerID));
-    stateManager.addContainerToPipeline(pipeline.getID(),
+    stateManager.openPipeline(pipeline.getId());
+    stateManager.addContainerToPipeline(pipeline.getId(),
         ContainerID.valueof(++containerID));
-    stateManager.addContainerToPipeline(pipeline.getID(),
+    stateManager.addContainerToPipeline(pipeline.getId(),
         ContainerID.valueof(++containerID));
 
     //verify the number of containers returned
     Set<ContainerID> containerIDs =
-        stateManager.getContainers(pipeline.getID());
+        stateManager.getContainers(pipeline.getId());
     Assert.assertEquals(containerIDs.size(), containerID);
 
     removePipeline(pipeline);
     try {
-      stateManager.addContainerToPipeline(pipeline.getID(),
+      stateManager.addContainerToPipeline(pipeline.getId(),
           ContainerID.valueof(++containerID));
       Assert.fail("Container should not have been added");
     } catch (IOException e) {
@@ -215,12 +256,12 @@ public class TestPipelineStateManager {
     Pipeline pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
     // close the pipeline
-    stateManager.openPipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
     stateManager
-        .addContainerToPipeline(pipeline.getID(), ContainerID.valueof(1));
+        .addContainerToPipeline(pipeline.getId(), ContainerID.valueof(1));
 
     try {
-      stateManager.removePipeline(pipeline.getID());
+      stateManager.removePipeline(pipeline.getId());
       Assert.fail("Pipeline should not have been removed");
     } catch (IOException e) {
       // can not remove a pipeline which already has containers
@@ -228,10 +269,10 @@ public class TestPipelineStateManager {
     }
 
     // close the pipeline
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
 
     try {
-      stateManager.removePipeline(pipeline.getID());
+      stateManager.removePipeline(pipeline.getId());
       Assert.fail("Pipeline should not have been removed");
     } catch (IOException e) {
       // can not remove a pipeline which already has containers
@@ -248,33 +289,33 @@ public class TestPipelineStateManager {
     Pipeline pipeline = createDummyPipeline(1);
     // create an open pipeline in stateMap
     stateManager.addPipeline(pipeline);
-    stateManager.openPipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
 
-    stateManager.addContainerToPipeline(pipeline.getID(),
+    stateManager.addContainerToPipeline(pipeline.getId(),
         ContainerID.valueof(containerID));
-    Assert.assertEquals(1, stateManager.getContainers(pipeline.getID()).size());
-    stateManager.removeContainerFromPipeline(pipeline.getID(),
+    Assert.assertEquals(1, stateManager.getContainers(pipeline.getId()).size());
+    stateManager.removeContainerFromPipeline(pipeline.getId(),
         ContainerID.valueof(containerID));
-    Assert.assertEquals(0, stateManager.getContainers(pipeline.getID()).size());
+    Assert.assertEquals(0, stateManager.getContainers(pipeline.getId()).size());
 
     // add two containers in the pipeline
-    stateManager.addContainerToPipeline(pipeline.getID(),
+    stateManager.addContainerToPipeline(pipeline.getId(),
         ContainerID.valueof(++containerID));
-    stateManager.addContainerToPipeline(pipeline.getID(),
+    stateManager.addContainerToPipeline(pipeline.getId(),
         ContainerID.valueof(++containerID));
-    Assert.assertEquals(2, stateManager.getContainers(pipeline.getID()).size());
+    Assert.assertEquals(2, stateManager.getContainers(pipeline.getId()).size());
 
     // move pipeline to closing state
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
 
-    stateManager.removeContainerFromPipeline(pipeline.getID(),
+    stateManager.removeContainerFromPipeline(pipeline.getId(),
         ContainerID.valueof(containerID));
-    stateManager.removeContainerFromPipeline(pipeline.getID(),
+    stateManager.removeContainerFromPipeline(pipeline.getId(),
         ContainerID.valueof(--containerID));
-    Assert.assertEquals(0, stateManager.getContainers(pipeline.getID()).size());
+    Assert.assertEquals(0, stateManager.getContainers(pipeline.getId()).size());
 
     // clean up
-    stateManager.removePipeline(pipeline.getID());
+    stateManager.removePipeline(pipeline.getId());
   }
 
   @Test
@@ -282,30 +323,30 @@ public class TestPipelineStateManager {
     Pipeline pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
     // finalize on ALLOCATED pipeline
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
     Assert.assertEquals(Pipeline.PipelineState.CLOSED,
-        stateManager.getPipeline(pipeline.getID()).getPipelineState());
+        stateManager.getPipeline(pipeline.getId()).getPipelineState());
     // clean up
     removePipeline(pipeline);
 
     pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
-    stateManager.openPipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
     // finalize on OPEN pipeline
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
     Assert.assertEquals(Pipeline.PipelineState.CLOSED,
-        stateManager.getPipeline(pipeline.getID()).getPipelineState());
+        stateManager.getPipeline(pipeline.getId()).getPipelineState());
     // clean up
     removePipeline(pipeline);
 
     pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
-    stateManager.openPipeline(pipeline.getID());
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
+    stateManager.finalizePipeline(pipeline.getId());
     // finalize should work on already closed pipeline
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
     Assert.assertEquals(Pipeline.PipelineState.CLOSED,
-        stateManager.getPipeline(pipeline.getID()).getPipelineState());
+        stateManager.getPipeline(pipeline.getId()).getPipelineState());
     // clean up
     removePipeline(pipeline);
   }
@@ -315,25 +356,25 @@ public class TestPipelineStateManager {
     Pipeline pipeline = createDummyPipeline(1);
     stateManager.addPipeline(pipeline);
     // open on ALLOCATED pipeline
-    stateManager.openPipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
     Assert.assertEquals(Pipeline.PipelineState.OPEN,
-        stateManager.getPipeline(pipeline.getID()).getPipelineState());
+        stateManager.getPipeline(pipeline.getId()).getPipelineState());
 
-    stateManager.openPipeline(pipeline.getID());
+    stateManager.openPipeline(pipeline.getId());
     // open should work on already open pipeline
     Assert.assertEquals(Pipeline.PipelineState.OPEN,
-        stateManager.getPipeline(pipeline.getID()).getPipelineState());
+        stateManager.getPipeline(pipeline.getId()).getPipelineState());
     // clean up
     removePipeline(pipeline);
   }
 
   private void removePipeline(Pipeline pipeline) throws IOException {
-    stateManager.finalizePipeline(pipeline.getID());
+    stateManager.finalizePipeline(pipeline.getId());
     Set<ContainerID> containerIDs =
-        stateManager.getContainers(pipeline.getID());
+        stateManager.getContainers(pipeline.getId());
     for (ContainerID containerID : containerIDs) {
-      stateManager.removeContainerFromPipeline(pipeline.getID(), containerID);
+      stateManager.removeContainerFromPipeline(pipeline.getId(), containerID);
     }
-    stateManager.removePipeline(pipeline.getID());
+    stateManager.removePipeline(pipeline.getId());
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java

@@ -58,7 +58,7 @@ public class TestRatisPipelineProvider {
     Assert.assertEquals(pipeline.getType(), HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(pipeline.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
 
     factor = HddsProtos.ReplicationFactor.ONE;
@@ -71,7 +71,7 @@ public class TestRatisPipelineProvider {
     Assert.assertEquals(pipeline1.getType(), HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipeline1.getFactor(), factor);
     Assert.assertEquals(pipeline1.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline1.getNodes().size(), factor.getNumber());
   }
 
@@ -86,19 +86,20 @@ public class TestRatisPipelineProvider {
   @Test
   public void testCreatePipelineWithNodes() throws IOException {
     HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
-    Pipeline pipeline = provider.create(createListOfNodes(factor.getNumber()));
+    Pipeline pipeline =
+        provider.create(factor, createListOfNodes(factor.getNumber()));
     Assert.assertEquals(pipeline.getType(), HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(
-        pipeline.getPipelineState(), Pipeline.PipelineState.ALLOCATED);
+        pipeline.getPipelineState(), Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
 
     factor = HddsProtos.ReplicationFactor.ONE;
-    pipeline = provider.create(createListOfNodes(factor.getNumber()));
+    pipeline = provider.create(factor, createListOfNodes(factor.getNumber()));
     Assert.assertEquals(pipeline.getType(), HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(pipeline.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java

@@ -26,7 +26,6 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.MockNodeManager;
-import org.apache.hadoop.hdds.scm.container.TestSCMContainerManager;
 import org.apache.hadoop.hdds.scm.server.SCMDatanodeHeartbeatDispatcher.PipelineReportFromDatanode;
 import org.apache.hadoop.hdds.server.events.EventQueue;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
@@ -54,7 +53,7 @@ public class TestSCMPipelineManager {
   public static void setUp() throws Exception {
     conf = new OzoneConfiguration();
     testDir = GenericTestUtils
-        .getTestDir(TestSCMContainerManager.class.getSimpleName());
+        .getTestDir(TestSCMPipelineManager.class.getSimpleName());
     conf.set(OzoneConfigKeys.OZONE_METADATA_DIRS, testDir.getAbsolutePath());
     boolean folderExisted = testDir.exists() || testDir.mkdirs();
     if (!folderExisted) {
@@ -83,16 +82,18 @@ public class TestSCMPipelineManager {
 
     // new pipeline manager should be able to load the pipelines from the db
     pipelineManager =
-        new SCMPipelineManager(conf, nodeManager,
-            new EventQueue());
+        new SCMPipelineManager(conf, nodeManager, new EventQueue());
+    for (Pipeline p : pipelines) {
+      pipelineManager.openPipeline(p.getId());
+    }
     List<Pipeline> pipelineList =
-        pipelineManager.getPipelinesByType(HddsProtos.ReplicationType.RATIS);
+        pipelineManager.getPipelines(HddsProtos.ReplicationType.RATIS);
     Assert.assertEquals(pipelines, new HashSet<>(pipelineList));
 
     // clean up
     for (Pipeline pipeline : pipelines) {
-      pipelineManager.finalizePipeline(pipeline.getID());
-      pipelineManager.removePipeline(pipeline.getID());
+      pipelineManager.finalizePipeline(pipeline.getId());
+      pipelineManager.removePipeline(pipeline.getId());
     }
     pipelineManager.close();
   }
@@ -104,13 +105,13 @@ public class TestSCMPipelineManager {
     Pipeline pipeline = pipelineManager
         .createPipeline(HddsProtos.ReplicationType.RATIS,
             HddsProtos.ReplicationFactor.THREE);
-    pipelineManager.openPipeline(pipeline.getID());
+    pipelineManager.openPipeline(pipeline.getId());
     pipelineManager
-        .addContainerToPipeline(pipeline.getID(), ContainerID.valueof(1));
-    pipelineManager.finalizePipeline(pipeline.getID());
+        .addContainerToPipeline(pipeline.getId(), ContainerID.valueof(1));
+    pipelineManager.finalizePipeline(pipeline.getId());
     pipelineManager
-        .removeContainerFromPipeline(pipeline.getID(), ContainerID.valueof(1));
-    pipelineManager.removePipeline(pipeline.getID());
+        .removeContainerFromPipeline(pipeline.getId(), ContainerID.valueof(1));
+    pipelineManager.removePipeline(pipeline.getId());
     pipelineManager.close();
 
     // new pipeline manager should not be able to load removed pipelines
@@ -118,7 +119,7 @@ public class TestSCMPipelineManager {
         new SCMPipelineManager(conf, nodeManager,
             new EventQueue());
     try {
-      pipelineManager.getPipeline(pipeline.getID());
+      pipelineManager.getPipeline(pipeline.getId());
       Assert.fail("Pipeline should not have been retrieved");
     } catch (IOException e) {
       Assert.assertTrue(e.getMessage().contains("not found"));
@@ -138,36 +139,36 @@ public class TestSCMPipelineManager {
         .createPipeline(HddsProtos.ReplicationType.RATIS,
             HddsProtos.ReplicationFactor.THREE);
     Assert
-        .assertFalse(pipelineManager.getPipeline(pipeline.getID()).isHealthy());
+        .assertFalse(pipelineManager.getPipeline(pipeline.getId()).isHealthy());
     Assert
-        .assertFalse(pipelineManager.getPipeline(pipeline.getID()).isOpen());
+        .assertTrue(pipelineManager.getPipeline(pipeline.getId()).isOpen());
 
     // get pipeline report from each dn in the pipeline
     PipelineReportHandler pipelineReportHandler =
         new PipelineReportHandler(pipelineManager, conf);
     for (DatanodeDetails dn: pipeline.getNodes()) {
       PipelineReportFromDatanode pipelineReportFromDatanode =
-          TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getID());
+          TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getId());
       // pipeline is not healthy until all dns report
       Assert.assertFalse(
-          pipelineManager.getPipeline(pipeline.getID()).isHealthy());
+          pipelineManager.getPipeline(pipeline.getId()).isHealthy());
       pipelineReportHandler
           .onMessage(pipelineReportFromDatanode, new EventQueue());
     }
 
     // pipeline is healthy when all dns report
     Assert
-        .assertTrue(pipelineManager.getPipeline(pipeline.getID()).isHealthy());
+        .assertTrue(pipelineManager.getPipeline(pipeline.getId()).isHealthy());
     // pipeline should now move to open state
     Assert
-        .assertTrue(pipelineManager.getPipeline(pipeline.getID()).isOpen());
+        .assertTrue(pipelineManager.getPipeline(pipeline.getId()).isOpen());
 
     // close the pipeline
-    pipelineManager.finalizePipeline(pipeline.getID());
+    pipelineManager.finalizePipeline(pipeline.getId());
 
     for (DatanodeDetails dn: pipeline.getNodes()) {
       PipelineReportFromDatanode pipelineReportFromDatanode =
-          TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getID());
+          TestUtils.getRandomPipelineReportFromDatanode(dn, pipeline.getId());
       // pipeline report for a closed pipeline should destroy the pipeline
       // and remove it from the pipeline manager
       pipelineReportHandler
@@ -175,7 +176,7 @@ public class TestSCMPipelineManager {
     }
 
     try {
-      pipelineManager.getPipeline(pipeline.getID());
+      pipelineManager.getPipeline(pipeline.getId());
       Assert.fail("Pipeline should not have been retrieved");
     } catch (IOException e) {
       Assert.assertTrue(e.getMessage().contains("not found"));

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMRestart.java

@@ -19,9 +19,7 @@
 package org.apache.hadoop.hdds.scm.pipeline;
 
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.container.ContainerManager;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.junit.AfterClass;
@@ -48,6 +46,7 @@ public class TestSCMRestart {
   private static Pipeline ratisPipeline2;
   private static ContainerManager containerManager;
   private static ContainerManager newContainerManager;
+  private static PipelineManager pipelineManager;
 
   /**
    * Create a MiniDFSCluster for testing.
@@ -65,6 +64,7 @@ public class TestSCMRestart {
     cluster.waitForClusterToBeReady();
     StorageContainerManager scm = cluster.getStorageContainerManager();
     containerManager = scm.getContainerManager();
+    pipelineManager = scm.getPipelineManager();
     ratisPipeline1 = containerManager.allocateContainer(
         RATIS, THREE, "Owner1").getPipeline();
     ratisPipeline2 = containerManager.allocateContainer(
@@ -75,6 +75,7 @@ public class TestSCMRestart {
     cluster.restartStorageContainerManager();
     newContainerManager = cluster.getStorageContainerManager()
         .getContainerManager();
+    pipelineManager = cluster.getStorageContainerManager().getPipelineManager();
   }
 
   /**
@@ -90,25 +91,15 @@ public class TestSCMRestart {
   @Test
   public void testPipelineWithScmRestart() throws IOException {
     // After restart make sure that the pipeline are still present
-    Pipeline ratisPipeline1AfterRestart = newContainerManager
-        .getPipelineSelector().getPipeline(ratisPipeline1.getId());
-    Pipeline ratisPipeline2AfterRestart = newContainerManager
-        .getPipelineSelector().getPipeline(ratisPipeline2.getId());
+    Pipeline ratisPipeline1AfterRestart =
+        pipelineManager.getPipeline(ratisPipeline1.getId());
+    Pipeline ratisPipeline2AfterRestart =
+        pipelineManager.getPipeline(ratisPipeline2.getId());
     Assert.assertNotSame(ratisPipeline1AfterRestart, ratisPipeline1);
     Assert.assertNotSame(ratisPipeline2AfterRestart, ratisPipeline2);
     Assert.assertEquals(ratisPipeline1AfterRestart, ratisPipeline1);
     Assert.assertEquals(ratisPipeline2AfterRestart, ratisPipeline2);
 
-    for (DatanodeDetails dn : ratisPipeline1.getMachines()) {
-      Assert.assertEquals(dn, ratisPipeline1AfterRestart.getDatanodes()
-              .get(dn.getUuidString()));
-    }
-
-    for (DatanodeDetails dn : ratisPipeline2.getMachines()) {
-      Assert.assertEquals(dn, ratisPipeline2AfterRestart.getDatanodes()
-              .get(dn.getUuidString()));
-    }
-
     // Try creating a new ratis pipeline, it should be from the same pipeline
     // as was before restart
     Pipeline newRatisPipeline = newContainerManager

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSimplePipelineProvider.java

@@ -57,7 +57,7 @@ public class TestSimplePipelineProvider {
         HddsProtos.ReplicationType.STAND_ALONE);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(pipeline.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
 
     factor = HddsProtos.ReplicationFactor.ONE;
@@ -67,7 +67,7 @@ public class TestSimplePipelineProvider {
         HddsProtos.ReplicationType.STAND_ALONE);
     Assert.assertEquals(pipeline1.getFactor(), factor);
     Assert.assertEquals(pipeline1.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline1.getNodes().size(), factor.getNumber());
   }
 
@@ -82,21 +82,22 @@ public class TestSimplePipelineProvider {
   @Test
   public void testCreatePipelineWithNodes() throws IOException {
     HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE;
-    Pipeline pipeline = provider.create(createListOfNodes(factor.getNumber()));
+    Pipeline pipeline =
+        provider.create(factor, createListOfNodes(factor.getNumber()));
     Assert.assertEquals(pipeline.getType(),
         HddsProtos.ReplicationType.STAND_ALONE);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(pipeline.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
 
     factor = HddsProtos.ReplicationFactor.ONE;
-    pipeline = provider.create(createListOfNodes(factor.getNumber()));
+    pipeline = provider.create(factor, createListOfNodes(factor.getNumber()));
     Assert.assertEquals(pipeline.getType(),
         HddsProtos.ReplicationType.STAND_ALONE);
     Assert.assertEquals(pipeline.getFactor(), factor);
     Assert.assertEquals(pipeline.getPipelineState(),
-        Pipeline.PipelineState.ALLOCATED);
+        Pipeline.PipelineState.OPEN);
     Assert.assertEquals(pipeline.getNodes().size(), factor.getNumber());
   }
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/RatisTestHelper.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.ozone;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
 import org.apache.hadoop.ozone.client.rpc.RpcClient;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestContainerStateMachineIdempotency.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.scm.XceiverClientManager;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.container.placement.algorithms.
     ContainerPlacementPolicy;
 import org.apache.hadoop.hdds.scm.container.placement.algorithms.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java

@@ -21,7 +21,7 @@ package org.apache.hadoop.ozone;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
 import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
 import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
@@ -29,7 +29,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.test.PathUtils;
 import org.apache.hadoop.test.TestGenericTestUtils;
 import org.junit.AfterClass;
@@ -40,6 +40,7 @@ import org.junit.Test;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 
@@ -91,18 +92,20 @@ public class TestMiniOzoneCluster {
     assertEquals(numberOfNodes, datanodes.size());
     for(HddsDatanodeService dn : datanodes) {
       // Create a single member pipe line
-      DatanodeDetails datanodeDetails = dn.getDatanodeDetails();
-      final Pipeline pipeline =
-          new Pipeline(datanodeDetails.getUuidString(),
-              HddsProtos.LifeCycleState.OPEN,
-              HddsProtos.ReplicationType.STAND_ALONE,
-              HddsProtos.ReplicationFactor.ONE, PipelineID.randomId());
-      pipeline.addMember(datanodeDetails);
+      List<DatanodeDetails> dns = new ArrayList<>();
+      dns.add(dn.getDatanodeDetails());
+      Pipeline pipeline = Pipeline.newBuilder()
+          .setState(Pipeline.PipelineState.OPEN)
+          .setId(PipelineID.randomId())
+          .setType(HddsProtos.ReplicationType.STAND_ALONE)
+          .setFactor(HddsProtos.ReplicationFactor.ONE)
+          .setNodes(dns)
+          .build();
 
       // Verify client is able to connect to the container
       try (XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf)){
         client.connect();
-        assertTrue(client.isConnected(pipeline.getLeader()));
+        assertTrue(client.isConnected(pipeline.getFirstNode()));
       }
     }
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java

@@ -140,7 +140,7 @@ public class TestStorageContainerManager {
         if (expectPermissionDenied) {
           fail("Operation should fail, expecting an IOException here.");
         } else {
-          Assert.assertEquals(1, container2.getPipeline().getMachines().size());
+          Assert.assertEquals(1, container2.getPipeline().getNodes().size());
         }
       } catch (Exception e) {
         verifyPermissionDeniedException(e, fakeRemoteUsername);
@@ -153,7 +153,7 @@ public class TestStorageContainerManager {
         if (expectPermissionDenied) {
           fail("Operation should fail, expecting an IOException here.");
         } else {
-          Assert.assertEquals(1, container3.getPipeline().getMachines().size());
+          Assert.assertEquals(1, container3.getPipeline().getNodes().size());
         }
       } catch (Exception e) {
         verifyPermissionDeniedException(e, fakeRemoteUsername);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java

@@ -162,9 +162,10 @@ public class TestStorageContainerManagerHelper {
         .getStorageContainerManager().getClientProtocolServer()
         .getContainerWithPipeline(containerID);
 
-    DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader();
+    DatanodeDetails dn =
+        containerWithPipeline.getPipeline().getFirstNode();
     OzoneContainer containerServer =
-        getContainerServerByDatanodeUuid(leadDN.getUuidString());
+        getContainerServerByDatanodeUuid(dn.getUuidString());
     KeyValueContainerData containerData =
         (KeyValueContainerData) containerServer.getContainerSet()
         .getContainer(containerID).getContainerData();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java

@@ -24,7 +24,7 @@ import org.apache.hadoop.hdds.protocol.StorageType;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.*;
 import org.apache.hadoop.ozone.client.*;
 import org.apache.hadoop.hdds.client.OzoneQuota;
@@ -449,7 +449,7 @@ public class TestOzoneRestClient {
         .getContainerManager().getContainerWithPipeline(
             ContainerID.valueof(containerID))
         .getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
     Assert.assertEquals(datanodes.size(), 1);
 
     DatanodeDetails datanodeDetails = datanodes.get(0);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.
     StorageContainerException;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConsts;
@@ -306,7 +306,7 @@ public class TestCloseContainerHandlingByClient {
               .getContainerWithPipeline(ContainerID.valueof(containerID))
               .getPipeline();
       pipelineList.add(pipeline);
-      List<DatanodeDetails> datanodes = pipeline.getMachines();
+      List<DatanodeDetails> datanodes = pipeline.getNodes();
       for (DatanodeDetails details : datanodes) {
         Assert.assertFalse(ContainerTestHelper
             .isContainerClosed(cluster, containerID, details));
@@ -319,7 +319,7 @@ public class TestCloseContainerHandlingByClient {
     int index = 0;
     for (long containerID : containerIdList) {
       Pipeline pipeline = pipelineList.get(index);
-      List<DatanodeDetails> datanodes = pipeline.getMachines();
+      List<DatanodeDetails> datanodes = pipeline.getNodes();
       for (DatanodeDetails datanodeDetails : datanodes) {
         GenericTestUtils.waitFor(() -> ContainerTestHelper
                 .isContainerClosed(cluster, containerID, datanodeDetails), 500,
@@ -352,7 +352,7 @@ public class TestCloseContainerHandlingByClient {
     List<DatanodeDetails> datanodes =
         cluster.getStorageContainerManager().getContainerManager()
             .getContainerWithPipeline(ContainerID.valueof(containerID))
-            .getPipeline().getMachines();
+            .getPipeline().getNodes();
     Assert.assertEquals(1, datanodes.size());
     waitForContainerClose(keyName, key, HddsProtos.ReplicationType.STAND_ALONE);
     dataString = fixedLengthString(keyString, (1 * blockSize));
@@ -455,7 +455,7 @@ public class TestCloseContainerHandlingByClient {
     List<DatanodeDetails> datanodes =
         cluster.getStorageContainerManager().getContainerManager()
             .getContainerWithPipeline(ContainerID.valueof(containerID))
-            .getPipeline().getMachines();
+            .getPipeline().getNodes();
     Assert.assertEquals(1, datanodes.size());
     // move the container on the datanode to Closing state, this will ensure
     // closing the key will hit BLOCK_NOT_COMMITTED_EXCEPTION while trying

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java

@@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientRatis;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.*;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.client.*;
@@ -649,7 +649,7 @@ public class TestOzoneRpcClient {
         cluster.getStorageContainerManager().getContainerManager()
             .getContainerWithPipeline(new ContainerID(containerID));
     Pipeline pipeline = container.getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
 
     DatanodeDetails datanodeDetails = datanodes.get(0);
     Assert.assertNotNull(datanodeDetails);
@@ -754,7 +754,7 @@ public class TestOzoneRpcClient {
         .getContainerManager().getContainerWithPipeline(
             ContainerID.valueof(containerID))
         .getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
     Assert.assertEquals(datanodes.size(), 1);
 
     DatanodeDetails datanodeDetails = datanodes.get(0);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java

@@ -21,7 +21,8 @@ package org.apache.hadoop.ozone.container;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.StorageUnit;
 import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.container.common.impl.ContainerData;
@@ -35,14 +36,12 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandRequestProto;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -128,17 +127,16 @@ public final class ContainerTestHelper {
   public static Pipeline createPipeline(
       Iterable<DatanodeDetails> ids) throws IOException {
     Objects.requireNonNull(ids, "ids == null");
-    final Iterator<DatanodeDetails> i = ids.iterator();
-    Preconditions.checkArgument(i.hasNext());
-    final DatanodeDetails leader = i.next();
-    final Pipeline pipeline =
-        new Pipeline(leader.getUuidString(), LifeCycleState.OPEN,
-            ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
-            PipelineID.randomId());
-    pipeline.addMember(leader);
-    for(; i.hasNext();) {
-      pipeline.addMember(i.next());
-    }
+    Preconditions.checkArgument(ids.iterator().hasNext());
+    List<DatanodeDetails> dns = new ArrayList<>();
+    ids.forEach(dns::add);
+    Pipeline pipeline = Pipeline.newBuilder()
+        .setState(Pipeline.PipelineState.OPEN)
+        .setId(PipelineID.randomId())
+        .setType(HddsProtos.ReplicationType.STAND_ALONE)
+        .setFactor(ReplicationFactor.ONE)
+        .setNodes(dns)
+        .build();
     return pipeline;
   }
 
@@ -218,7 +216,7 @@ public final class ContainerTestHelper {
     request.setContainerID(blockID.getContainerID());
     request.setWriteChunk(writeRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
 
     return request.build();
   }
@@ -260,7 +258,7 @@ public final class ContainerTestHelper {
     request.setContainerID(blockID.getContainerID());
     request.setPutSmallFile(smallFileRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -279,7 +277,7 @@ public final class ContainerTestHelper {
     request.setContainerID(getKey.getGetBlock().getBlockID().getContainerID());
     request.setGetSmallFile(smallFileRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -309,7 +307,7 @@ public final class ContainerTestHelper {
     newRequest.setContainerID(readRequest.getBlockID().getContainerID());
     newRequest.setReadChunk(readRequest);
     newRequest.setTraceID(UUID.randomUUID().toString());
-    newRequest.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    newRequest.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return newRequest.build();
   }
 
@@ -342,7 +340,7 @@ public final class ContainerTestHelper {
     request.setContainerID(writeRequest.getBlockID().getContainerID());
     request.setDeleteChunk(deleteRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -363,7 +361,7 @@ public final class ContainerTestHelper {
     request.setCreateContainer(
         ContainerProtos.CreateContainerRequestProto.getDefaultInstance());
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
 
     return request.build();
   }
@@ -398,7 +396,7 @@ public final class ContainerTestHelper {
     request.setContainerID(containerID);
     request.setUpdateContainer(updateRequestBuilder.build());
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
   /**
@@ -427,7 +425,8 @@ public final class ContainerTestHelper {
    * @return - Request
    */
   public static ContainerCommandRequestProto getPutBlockRequest(
-      Pipeline pipeline, ContainerProtos.WriteChunkRequestProto writeRequest) {
+      Pipeline pipeline, ContainerProtos.WriteChunkRequestProto writeRequest)
+      throws IOException {
     LOG.trace("putBlock: {} to pipeline={}",
         writeRequest.getBlockID());
 
@@ -448,7 +447,7 @@ public final class ContainerTestHelper {
     request.setContainerID(blockData.getContainerID());
     request.setPutBlock(putRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -460,7 +459,8 @@ public final class ContainerTestHelper {
    * immediately.
    */
   public static ContainerCommandRequestProto getBlockRequest(
-      Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) {
+      Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest)
+      throws IOException {
     ContainerProtos.DatanodeBlockID blockID =
         putBlockRequest.getBlockData().getBlockID();
     LOG.trace("getKey: blockID={}", blockID);
@@ -475,7 +475,7 @@ public final class ContainerTestHelper {
     request.setContainerID(blockID.getContainerID());
     request.setGetBlock(getRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -499,7 +499,8 @@ public final class ContainerTestHelper {
    * @return - Request
    */
   public static ContainerCommandRequestProto getDeleteBlockRequest(
-      Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) {
+      Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest)
+      throws IOException {
     ContainerProtos.DatanodeBlockID blockID = putBlockRequest.getBlockData()
         .getBlockID();
     LOG.trace("deleteBlock: name={}", blockID);
@@ -512,7 +513,7 @@ public final class ContainerTestHelper {
     request.setContainerID(blockID.getContainerID());
     request.setDeleteBlock(delRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     return request.build();
   }
 
@@ -523,7 +524,7 @@ public final class ContainerTestHelper {
    * @return ContainerCommandRequestProto.
    */
   public static ContainerCommandRequestProto getCloseContainer(
-      Pipeline pipeline, long containerID) {
+      Pipeline pipeline, long containerID) throws IOException {
     ContainerProtos.ContainerCommandRequestProto cmd =
         ContainerCommandRequestProto.newBuilder()
             .setCmdType(ContainerProtos.Type.CloseContainer)
@@ -531,7 +532,7 @@ public final class ContainerTestHelper {
             .setCloseContainer(
                 ContainerProtos.CloseContainerRequestProto.getDefaultInstance())
             .setTraceID(UUID.randomUUID().toString())
-            .setDatanodeUuid(pipeline.getLeader().getUuidString())
+            .setDatanodeUuid(pipeline.getFirstNode().getUuidString())
             .build();
 
     return cmd;
@@ -544,7 +545,7 @@ public final class ContainerTestHelper {
    * @return ContainerCommandRequestProto without traceId.
    */
   public static ContainerCommandRequestProto getRequestWithoutTraceId(
-      Pipeline pipeline, long containerID) {
+      Pipeline pipeline, long containerID) throws IOException {
     Preconditions.checkNotNull(pipeline);
     ContainerProtos.ContainerCommandRequestProto cmd =
         ContainerCommandRequestProto.newBuilder()
@@ -552,7 +553,7 @@ public final class ContainerTestHelper {
             .setContainerID(containerID)
             .setCloseContainer(
                 ContainerProtos.CloseContainerRequestProto.getDefaultInstance())
-            .setDatanodeUuid(pipeline.getLeader().getUuidString())
+            .setDatanodeUuid(pipeline.getFirstNode().getUuidString())
             .build();
     return cmd;
   }
@@ -563,7 +564,8 @@ public final class ContainerTestHelper {
    * @return ContainerCommandRequestProto.
    */
   public static ContainerCommandRequestProto getDeleteContainer(
-      Pipeline pipeline, long containerID, boolean forceDelete) {
+      Pipeline pipeline, long containerID, boolean forceDelete)
+      throws IOException {
     Preconditions.checkNotNull(pipeline);
     ContainerProtos.DeleteContainerRequestProto deleteRequest =
         ContainerProtos.DeleteContainerRequestProto.newBuilder().
@@ -575,7 +577,7 @@ public final class ContainerTestHelper {
             ContainerProtos.DeleteContainerRequestProto.getDefaultInstance())
         .setDeleteContainer(deleteRequest)
         .setTraceID(UUID.randomUUID().toString())
-        .setDatanodeUuid(pipeline.getLeader().getUuidString())
+        .setDatanodeUuid(pipeline.getFirstNode().getUuidString())
         .build();
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java

@@ -36,7 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .DatanodeBlockID;
 import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -123,7 +123,7 @@ public class TestContainerReplication {
     cluster.getStorageContainerManager().getScmNodeManager()
         .addDatanodeCommand(destinationDatanode.getDatanodeDetails().getUuid(),
             new ReplicateContainerCommand(containerId,
-                sourcePipelines.getMachines()));
+                sourcePipelines.getNodes()));
 
     Thread.sleep(3000);
 
@@ -163,7 +163,7 @@ public class TestContainerReplication {
   private HddsDatanodeService chooseDatanodeWithoutContainer(Pipeline pipeline,
       List<HddsDatanodeService> dataNodes) {
     for (HddsDatanodeService datanode : dataNodes) {
-      if (!pipeline.getMachines().contains(datanode.getDatanodeDetails())) {
+      if (!pipeline.getNodes().contains(datanode.getDatanodeDetails())) {
         return datanode;
       }
     }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestCloseContainerHandler.java

@@ -31,7 +31,7 @@ import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
 import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -144,7 +144,7 @@ public class TestCloseContainerHandler {
       request.setContainerID(blockID.getContainerID());
       request.setWriteChunk(writeRequest);
       request.setTraceID(UUID.randomUUID().toString());
-      request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+      request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
       dispatcher.dispatch(request.build());
       chunkList.add(info);
     }
@@ -179,7 +179,7 @@ public class TestCloseContainerHandler {
     request.setContainerID(blockID.getContainerID());
     request.setPutBlock(putBlockRequestProto);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     dispatcher.dispatch(request.build());
 
     //the open block should be removed from Map
@@ -217,7 +217,7 @@ public class TestCloseContainerHandler {
     request.setDeleteChunk(deleteChunkProto);
     request.setWriteChunk(writeRequest);
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     dispatcher.dispatch(request.build());
     Assert.assertTrue(
         openContainerBlockMap.getBlockDataMap(testContainerID)
@@ -250,7 +250,7 @@ public class TestCloseContainerHandler {
     request.setCloseContainer(
         ContainerProtos.CloseContainerRequestProto.getDefaultInstance());
     request.setTraceID(UUID.randomUUID().toString());
-    request.setDatanodeUuid(pipeline.getLeader().getUuidString());
+    request.setDatanodeUuid(pipeline.getFirstNode().getUuidString());
     dispatcher.dispatch(request.build());
     Assert.assertNull(
         openContainerBlockMap.getBlockDataMap(testContainerID));

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java

@@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.HddsDatanodeService;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.client.ObjectStore;
@@ -106,7 +106,7 @@ public class TestCloseContainerByPipeline {
         .getContainerManager().getContainerWithPipeline(
             ContainerID.valueof(containerID))
         .getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
     Assert.assertEquals(datanodes.size(), 1);
 
     DatanodeDetails datanodeDetails = datanodes.get(0);
@@ -162,7 +162,7 @@ public class TestCloseContainerByPipeline {
         .getContainerManager().getContainerWithPipeline(
             ContainerID.valueof(containerID))
         .getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
     Assert.assertEquals(datanodes.size(), 1);
 
     DatanodeDetails datanodeDetails = datanodes.get(0);
@@ -220,7 +220,7 @@ public class TestCloseContainerByPipeline {
         .getContainerManager().getContainerWithPipeline(
             ContainerID.valueof(containerID))
         .getPipeline();
-    List<DatanodeDetails> datanodes = pipeline.getMachines();
+    List<DatanodeDetails> datanodes = pipeline.getNodes();
     Assert.assertEquals(3, datanodes.size());
 
     GenericTestUtils.LogCapturer logCapturer =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneClient;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java

@@ -33,7 +33,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
 import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
     .ContainerCommandResponseProto;
 import org.apache.hadoop.hdds.scm.*;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.metrics2.MetricsRecordBuilder;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.RatisTestHelper;
@@ -93,7 +93,7 @@ public class TestCSMMetrics {
       final OzoneConfiguration conf = new OzoneConfiguration();
       initConf.accept(pipeline, conf);
 
-      for (DatanodeDetails dn : pipeline.getMachines()) {
+      for (DatanodeDetails dn : pipeline.getNodes()) {
         final XceiverServerSpi s = createServer.apply(dn, conf);
         servers.add(s);
         s.start();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java

@@ -41,7 +41,7 @@ import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
 import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerGrpc;
 import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
 import org.apache.hadoop.hdds.scm.TestUtils;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
 import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
 import org.apache.hadoop.test.GenericTestUtils;
@@ -75,7 +75,7 @@ public class TestContainerMetrics {
           .createSingleNodePipeline();
       OzoneConfiguration conf = new OzoneConfiguration();
       conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-          pipeline.getLeader()
+          pipeline.getFirstNode()
               .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
       conf.setInt(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY,
           interval);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -66,7 +66,8 @@ public class TestOzoneContainer {
       // independently in our test path.
       Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
       conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
-      conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader()
+      conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+          pipeline.getFirstNode()
               .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
       conf.setBoolean(
           OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
@@ -108,7 +109,7 @@ public class TestOzoneContainer {
       Pipeline pipeline =
           ContainerTestHelper.createSingleNodePipeline();
       conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-          pipeline.getLeader()
+          pipeline.getFirstNode()
               .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
 
       cluster = MiniOzoneCluster.newBuilder(conf)
@@ -514,7 +515,7 @@ public class TestOzoneContainer {
     Pipeline pipeline =
         ContainerTestHelper.createSingleNodePipeline();
     conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-        pipeline.getLeader()
+        pipeline.getFirstNode()
             .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
 
     // This client talks to ozone container via datanode.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java

@@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.RatisTestHelper;
 import org.apache.hadoop.ozone.container.ContainerTestHelper;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.ratis.rpc.RpcType;
 import org.apache.ratis.rpc.SupportedRpcType;
 import org.apache.ratis.util.CheckedBiConsumer;
@@ -88,12 +88,12 @@ public class TestOzoneContainerRatis {
       // Create Ratis cluster
 //      final String ratisId = "ratis1";
 //      final PipelineManager manager = RatisManagerImpl.newRatisManager(conf);
-//      manager.createPipeline(ratisId, pipeline.getMachines());
+//      manager.createPipeline(ratisId, pipeline.getNodes());
 //      LOG.info("Created RatisCluster " + ratisId);
 //
 //      // check Ratis cluster members
 //      final List<DatanodeDetails> dns = manager.getMembers(ratisId);
-//      Assert.assertEquals(pipeline.getMachines(), dns);
+//      Assert.assertEquals(pipeline.getNodes(), dns);
 //
 //      // run test
 //      final XceiverClientSpi client = XceiverClientRatis

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java

@@ -44,7 +44,7 @@ import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
 import org.apache.hadoop.hdds.scm.XceiverClientRatis;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.ratis.rpc.RpcType;
 import org.apache.ratis.util.CheckedBiConsumer;
@@ -56,7 +56,6 @@ import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.function.BiConsumer;
 
 import static org.apache.ratis.rpc.SupportedRpcType.GRPC;
 import static org.apache.ratis.rpc.SupportedRpcType.NETTY;
@@ -80,15 +79,15 @@ public class TestContainerServer {
   public void testClientServer() throws Exception {
     DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
     ContainerSet containerSet = new ContainerSet();
-    runTestClientServer(1,
-        (pipeline, conf) -> conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-            pipeline.getLeader()
-                .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()),
+    runTestClientServer(1, (pipeline, conf) -> conf
+            .setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+                pipeline.getFirstNode()
+                    .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue()),
         XceiverClientGrpc::new,
         (dn, conf) -> new XceiverServerGrpc(datanodeDetails, conf,
             new TestContainerDispatcher(),
-            createReplicationService(containerSet)),
-        (dn, p) -> {});
+            createReplicationService(containerSet)), (dn, p) -> {
+        });
   }
 
   @FunctionalInterface
@@ -131,7 +130,7 @@ public class TestContainerServer {
 
   static void runTestClientServer(
       int numDatanodes,
-      BiConsumer<Pipeline, OzoneConfiguration> initConf,
+      CheckedBiConsumer<Pipeline, OzoneConfiguration, IOException> initConf,
       CheckedBiFunction<Pipeline, OzoneConfiguration, XceiverClientSpi,
           IOException> createClient,
       CheckedBiFunction<DatanodeDetails, OzoneConfiguration, XceiverServerSpi,
@@ -146,7 +145,7 @@ public class TestContainerServer {
       final OzoneConfiguration conf = new OzoneConfiguration();
       initConf.accept(pipeline, conf);
 
-      for(DatanodeDetails dn : pipeline.getMachines()) {
+      for (DatanodeDetails dn : pipeline.getNodes()) {
         final XceiverServerSpi s = createServer.apply(dn, conf);
         servers.add(s);
         s.start();
@@ -181,7 +180,7 @@ public class TestContainerServer {
       Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
       OzoneConfiguration conf = new OzoneConfiguration();
       conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-          pipeline.getLeader()
+          pipeline.getFirstNode()
               .getPort(DatanodeDetails.Port.Name.STANDALONE).getValue());
 
       ContainerSet containerSet = new ContainerSet();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java

@@ -69,7 +69,7 @@ public class TestAllocateContainer {
         xceiverClientManager.getFactor(),
         containerOwner);
     Assert.assertNotNull(container);
-    Assert.assertNotNull(container.getPipeline().getLeader());
+    Assert.assertNotNull(container.getPipeline().getFirstNode());
 
   }
 

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestGetCommittedBlockLengthAndPutKey.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager;
 import org.apache.hadoop.hdds.scm.XceiverClientSpi;
 import org.apache.hadoop.hdds.scm.container.common.helpers.
     ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.container.common.helpers.
     StorageContainerException;
 import org.apache.hadoop.hdds.scm.container.placement.algorithms.

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ScmBlockLocationTestIngClient.java

@@ -27,8 +27,8 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
 import org.apache.hadoop.hdds.scm.TestUtils;
 import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.hdds.scm.container.common.helpers.DeleteBlockResult;
-import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineID;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
 import org.apache.hadoop.ozone.common.BlockGroup;
 import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
@@ -127,12 +127,15 @@ public class ScmBlockLocationTestIngClient implements ScmBlockLocationProtocol {
   }
 
   private Pipeline createPipeline(DatanodeDetails datanode) {
-    final Pipeline pipeline =
-        new Pipeline(datanode.getUuidString(), HddsProtos.LifeCycleState.OPEN,
-            HddsProtos.ReplicationType.STAND_ALONE,
-            HddsProtos.ReplicationFactor.ONE,
-            PipelineID.randomId());
-    pipeline.addMember(datanode);
+    List<DatanodeDetails> dns = new ArrayList<>();
+    dns.add(datanode);
+    Pipeline pipeline = Pipeline.newBuilder()
+        .setState(Pipeline.PipelineState.OPEN)
+        .setId(PipelineID.randomId())
+        .setType(HddsProtos.ReplicationType.STAND_ALONE)
+        .setFactor(HddsProtos.ReplicationFactor.ONE)
+        .setNodes(dns)
+        .build();
     return pipeline;
   }