HADOOP-16654:Delete hadoop-ozone and hadoop-hdds subprojects from apache trunk

Signed-off-by: Dinesh Chitlangia <dineshchitlangia@gmail.com>

BUILDING.txt

@@ -104,8 +104,6 @@ Maven main modules:
          - hadoop-hdfs-project          (Hadoop HDFS)
          - hadoop-yarn-project          (Hadoop YARN)
          - hadoop-mapreduce-project     (Hadoop MapReduce)
-         - hadoop-ozone                 (Hadoop Ozone)
-         - hadoop-hdds                  (Hadoop Distributed Data Store)
          - hadoop-tools                 (Hadoop tools like Streaming, Distcp, etc.)
          - hadoop-dist                  (Hadoop distribution assembler)
          - hadoop-client-modules        (Hadoop client modules)

hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml

@@ -56,8 +56,6 @@
         <exclude>**/build/**</exclude>
         <exclude>**/file:/**</exclude>
         <exclude>**/SecurityAuth.audit*</exclude>
-        <exclude>hadoop-ozone/**</exclude>
-        <exclude>hadoop-hdds/**</exclude>
         <exclude>hadoop-submarine/**</exclude>
       </excludes>
     </fileSet>

hadoop-hdds/client/pom.xml

@@ -1,44 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-hdds</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
-  </parent>
-
-  <artifactId>hadoop-hdds-client</artifactId>
-  <version>0.5.0-SNAPSHOT</version>
-  <description>Apache Hadoop Distributed Data Store Client Library</description>
-  <name>Apache Hadoop HDDS Client</name>
-  <packaging>jar</packaging>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-common</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>io.netty</groupId>
-      <artifactId>netty-all</artifactId>
-    </dependency>
-
-  </dependencies>
-</project>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/ClientCredentialInterceptor.java

@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import org.apache.ratis.thirdparty.io.grpc.CallOptions;
-import org.apache.ratis.thirdparty.io.grpc.Channel;
-import org.apache.ratis.thirdparty.io.grpc.ClientCall;
-import org.apache.ratis.thirdparty.io.grpc.ClientInterceptor;
-import org.apache.ratis.thirdparty.io.grpc.ForwardingClientCall;
-import org.apache.ratis.thirdparty.io.grpc.Metadata;
-import org.apache.ratis.thirdparty.io.grpc.MethodDescriptor;
-
-import static org.apache.hadoop.ozone.OzoneConsts.OBT_METADATA_KEY;
-import static org.apache.hadoop.ozone.OzoneConsts.USER_METADATA_KEY;
-
-/**
- * GRPC client interceptor for ozone block token.
- */
-public class ClientCredentialInterceptor implements ClientInterceptor {
-
-  private final String user;
-  private final String token;
-
-  public ClientCredentialInterceptor(String user, String token) {
-    this.user = user;
-    this.token = token;
-  }
-
-  @Override
-  public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
-      MethodDescriptor<ReqT, RespT> method,
-      CallOptions callOptions,
-      Channel next) {
-
-    return new ForwardingClientCall.SimpleForwardingClientCall<ReqT, RespT>(
-        next.newCall(method, callOptions)) {
-      @Override
-      public void start(Listener<RespT> responseListener, Metadata headers) {
-        if (token != null) {
-          headers.put(OBT_METADATA_KEY, token);
-        }
-        if (user != null) {
-          headers.put(USER_METADATA_KEY, user);
-        }
-        super.start(responseListener, headers);
-      }
-    };
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -1,466 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc;
-import org.apache.hadoop.hdds.protocol.datanode.proto.XceiverClientProtocolServiceGrpc.XceiverClientProtocolServiceStub;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.tracing.GrpcClientInterceptor;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
-import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
-import org.apache.ratis.thirdparty.io.grpc.Status;
-import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
-import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
-import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
-import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * A Client for the storageContainer protocol for read object data.
- */
-public class XceiverClientGrpc extends XceiverClientSpi {
-  static final Logger LOG = LoggerFactory.getLogger(XceiverClientGrpc.class);
-  private static final String COMPONENT = "dn";
-  private final Pipeline pipeline;
-  private final Configuration config;
-  private Map<UUID, XceiverClientProtocolServiceStub> asyncStubs;
-  private XceiverClientMetrics metrics;
-  private Map<UUID, ManagedChannel> channels;
-  private final Semaphore semaphore;
-  private boolean closed = false;
-  private SecurityConfig secConfig;
-  private final boolean topologyAwareRead;
-  private X509Certificate caCert;
-
-  /**
-   * Constructs a client that can communicate with the Container framework on
-   * data nodes.
-   *
-   * @param pipeline - Pipeline that defines the machines.
-   * @param config   -- Ozone Config
-   * @param caCert   - SCM ca certificate.
-   */
-  public XceiverClientGrpc(Pipeline pipeline, Configuration config,
-      X509Certificate caCert) {
-    super();
-    Preconditions.checkNotNull(pipeline);
-    Preconditions.checkNotNull(config);
-    this.pipeline = pipeline;
-    this.config = config;
-    this.secConfig = new SecurityConfig(config);
-    this.semaphore =
-        new Semaphore(HddsClientUtils.getMaxOutstandingRequests(config));
-    this.metrics = XceiverClientManager.getXceiverClientMetrics();
-    this.channels = new HashMap<>();
-    this.asyncStubs = new HashMap<>();
-    this.topologyAwareRead = config.getBoolean(
-        OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
-        OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT);
-    this.caCert = caCert;
-  }
-
-  /**
-   * Constructs a client that can communicate with the Container framework on
-   * data nodes.
-   *
-   * @param pipeline - Pipeline that defines the machines.
-   * @param config   -- Ozone Config
-   */
-  public XceiverClientGrpc(Pipeline pipeline, Configuration config) {
-    this(pipeline, config, null);
-  }
-
-  /**
-   * To be used when grpc token is not enabled.
-   */
-  @Override
-  public void connect() throws Exception {
-    // connect to the closest node, if closest node doesn't exist, delegate to
-    // first node, which is usually the leader in the pipeline.
-    DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
-        this.pipeline.getFirstNode();
-    // just make a connection to the picked datanode at the beginning
-    connectToDatanode(dn, null);
-  }
-
-  /**
-   * Passed encoded token to GRPC header when security is enabled.
-   */
-  @Override
-  public void connect(String encodedToken) throws Exception {
-    // connect to the closest node, if closest node doesn't exist, delegate to
-    // first node, which is usually the leader in the pipeline.
-    DatanodeDetails dn = topologyAwareRead ? this.pipeline.getClosestNode() :
-        this.pipeline.getFirstNode();
-    // just make a connection to the picked datanode at the beginning
-    connectToDatanode(dn, encodedToken);
-  }
-
-  private void connectToDatanode(DatanodeDetails dn, String encodedToken)
-      throws IOException {
-    // read port from the data node, on failure use default configured
-    // port.
-    int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
-    if (port == 0) {
-      port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
-          OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
-    }
-
-    // Add credential context to the client call
-    String userName = UserGroupInformation.getCurrentUser().getShortUserName();
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Nodes in pipeline : {}", pipeline.getNodes().toString());
-      LOG.debug("Connecting to server : {}", dn.getIpAddress());
-    }
-    NettyChannelBuilder channelBuilder =
-        NettyChannelBuilder.forAddress(dn.getIpAddress(), port).usePlaintext()
-            .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
-            .intercept(new ClientCredentialInterceptor(userName, encodedToken),
-                new GrpcClientInterceptor());
-    if (secConfig.isGrpcTlsEnabled()) {
-      SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
-      if (caCert != null) {
-        sslContextBuilder.trustManager(caCert);
-      }
-      if (secConfig.useTestCert()) {
-        channelBuilder.overrideAuthority("localhost");
-      }
-      channelBuilder.useTransportSecurity().
-          sslContext(sslContextBuilder.build());
-    } else {
-      channelBuilder.usePlaintext();
-    }
-    ManagedChannel channel = channelBuilder.build();
-    XceiverClientProtocolServiceStub asyncStub =
-        XceiverClientProtocolServiceGrpc.newStub(channel);
-    asyncStubs.put(dn.getUuid(), asyncStub);
-    channels.put(dn.getUuid(), channel);
-  }
-
-  /**
-   * Returns if the xceiver client connects to all servers in the pipeline.
-   *
-   * @return True if the connection is alive, false otherwise.
-   */
-  @VisibleForTesting
-  public boolean isConnected(DatanodeDetails details) {
-    return isConnected(channels.get(details.getUuid()));
-  }
-
-  private boolean isConnected(ManagedChannel channel) {
-    return channel != null && !channel.isTerminated() && !channel.isShutdown();
-  }
-
-  @Override
-  public void close() {
-    closed = true;
-    for (ManagedChannel channel : channels.values()) {
-      channel.shutdownNow();
-      try {
-        channel.awaitTermination(60, TimeUnit.MINUTES);
-      } catch (Exception e) {
-        LOG.error("Unexpected exception while waiting for channel termination",
-            e);
-      }
-    }
-  }
-
-  @Override
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  @Override
-  public ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request) throws IOException {
-    try {
-      XceiverClientReply reply;
-      reply = sendCommandWithTraceIDAndRetry(request, null);
-      ContainerCommandResponseProto responseProto = reply.getResponse().get();
-      return responseProto;
-    } catch (ExecutionException | InterruptedException e) {
-      throw new IOException("Failed to execute command " + request, e);
-    }
-  }
-
-  @Override
-  public ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
-      throws IOException {
-    try {
-      XceiverClientReply reply;
-      reply = sendCommandWithTraceIDAndRetry(request, validators);
-      ContainerCommandResponseProto responseProto = reply.getResponse().get();
-      return responseProto;
-    } catch (ExecutionException | InterruptedException e) {
-      throw new IOException("Failed to execute command " + request, e);
-    }
-  }
-
-  private XceiverClientReply sendCommandWithTraceIDAndRetry(
-      ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
-      throws IOException {
-    try (Scope scope = GlobalTracer.get()
-        .buildSpan("XceiverClientGrpc." + request.getCmdType().name())
-        .startActive(true)) {
-      ContainerCommandRequestProto finalPayload =
-          ContainerCommandRequestProto.newBuilder(request)
-              .setTraceID(TracingUtil.exportCurrentSpan()).build();
-      return sendCommandWithRetry(finalPayload, validators);
-    }
-  }
-
-  private XceiverClientReply sendCommandWithRetry(
-      ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
-      throws IOException {
-    ContainerCommandResponseProto responseProto = null;
-    IOException ioException = null;
-
-    // In case of an exception or an error, we will try to read from the
-    // datanodes in the pipeline in a round robin fashion.
-
-    // TODO: cache the correct leader info in here, so that any subsequent calls
-    // should first go to leader
-    XceiverClientReply reply = new XceiverClientReply(null);
-    List<DatanodeDetails> datanodeList;
-    if ((request.getCmdType() == ContainerProtos.Type.ReadChunk ||
-        request.getCmdType() == ContainerProtos.Type.GetSmallFile) &&
-        topologyAwareRead) {
-      datanodeList = pipeline.getNodesInOrder();
-    } else {
-      datanodeList = pipeline.getNodes();
-      // Shuffle datanode list so that clients do not read in the same order
-      // every time.
-      Collections.shuffle(datanodeList);
-    }
-    for (DatanodeDetails dn : datanodeList) {
-      try {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Executing command " + request + " on datanode " + dn);
-        }
-        // In case the command gets retried on a 2nd datanode,
-        // sendCommandAsyncCall will create a new channel and async stub
-        // in case these don't exist for the specific datanode.
-        reply.addDatanode(dn);
-        responseProto = sendCommandAsync(request, dn).getResponse().get();
-        if (validators != null && !validators.isEmpty()) {
-          for (CheckedBiFunction validator : validators) {
-            validator.apply(request, responseProto);
-          }
-        }
-        break;
-      } catch (ExecutionException | InterruptedException | IOException e) {
-        LOG.error("Failed to execute command " + request + " on datanode " + dn
-            .getUuidString(), e);
-        if (!(e instanceof IOException)) {
-          if (Status.fromThrowable(e.getCause()).getCode()
-              == Status.UNAUTHENTICATED.getCode()) {
-            throw new SCMSecurityException("Failed to authenticate with "
-                + "GRPC XceiverServer with Ozone block token.");
-          }
-          ioException = new IOException(e);
-        } else {
-          ioException = (IOException) e;
-        }
-        responseProto = null;
-      }
-    }
-
-    if (responseProto != null) {
-      reply.setResponse(CompletableFuture.completedFuture(responseProto));
-      return reply;
-    } else {
-      Preconditions.checkNotNull(ioException);
-      LOG.error("Failed to execute command {} on the pipeline {}.", request,
-          pipeline);
-      throw ioException;
-    }
-  }
-
-  // TODO: for a true async API, once the waitable future while executing
-  // the command on one channel fails, it should be retried asynchronously
-  // on the future Task for all the remaining datanodes.
-
-  // Note: this Async api is not used currently used in any active I/O path.
-  // In case it gets used, the asynchronous retry logic needs to be plugged
-  // in here.
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  @Override
-  public XceiverClientReply sendCommandAsync(
-      ContainerCommandRequestProto request)
-      throws IOException, ExecutionException, InterruptedException {
-    try (Scope scope = GlobalTracer.get()
-        .buildSpan("XceiverClientGrpc." + request.getCmdType().name())
-        .startActive(true)) {
-
-      ContainerCommandRequestProto finalPayload =
-          ContainerCommandRequestProto.newBuilder(request)
-              .setTraceID(TracingUtil.exportCurrentSpan())
-              .build();
-      XceiverClientReply asyncReply =
-          sendCommandAsync(finalPayload, pipeline.getFirstNode());
-      // TODO : for now make this API sync in nature as async requests are
-      // served out of order over XceiverClientGrpc. This needs to be fixed
-      // if this API is to be used for I/O path. Currently, this is not
-      // used for Read/Write Operation but for tests.
-      if (!HddsUtils.isReadOnly(request)) {
-        asyncReply.getResponse().get();
-      }
-      return asyncReply;
-    }
-  }
-
-  private XceiverClientReply sendCommandAsync(
-      ContainerCommandRequestProto request, DatanodeDetails dn)
-      throws IOException, ExecutionException, InterruptedException {
-    if (closed) {
-      throw new IOException("This channel is not connected.");
-    }
-
-    UUID dnId = dn.getUuid();
-    ManagedChannel channel = channels.get(dnId);
-    // If the channel doesn't exist for this specific datanode or the channel
-    // is closed, just reconnect
-    String token = request.getEncodedToken();
-    if (!isConnected(channel)) {
-      reconnect(dn, token);
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Send command {} to datanode {}",
-          request.getCmdType().toString(), dn.getNetworkFullPath());
-    }
-    final CompletableFuture<ContainerCommandResponseProto> replyFuture =
-        new CompletableFuture<>();
-    semaphore.acquire();
-    long requestTime = Time.monotonicNowNanos();
-    metrics.incrPendingContainerOpsMetrics(request.getCmdType());
-    // create a new grpc stream for each non-async call.
-
-    // TODO: for async calls, we should reuse StreamObserver resources.
-    final StreamObserver<ContainerCommandRequestProto> requestObserver =
-        asyncStubs.get(dnId)
-            .send(new StreamObserver<ContainerCommandResponseProto>() {
-              @Override
-              public void onNext(ContainerCommandResponseProto value) {
-                replyFuture.complete(value);
-                metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-                metrics.addContainerOpsLatency(request.getCmdType(),
-                    Time.monotonicNowNanos() - requestTime);
-                semaphore.release();
-              }
-
-              @Override
-              public void onError(Throwable t) {
-                replyFuture.completeExceptionally(t);
-                metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-                metrics.addContainerOpsLatency(request.getCmdType(),
-                    Time.monotonicNowNanos() - requestTime);
-                semaphore.release();
-              }
-
-              @Override
-              public void onCompleted() {
-                if (!replyFuture.isDone()) {
-                  replyFuture.completeExceptionally(new IOException(
-                      "Stream completed but no reply for request " + request));
-                }
-              }
-            });
-    requestObserver.onNext(request);
-    requestObserver.onCompleted();
-    return new XceiverClientReply(replyFuture);
-  }
-
-  private void reconnect(DatanodeDetails dn, String encodedToken)
-      throws IOException {
-    ManagedChannel channel;
-    try {
-      connectToDatanode(dn, encodedToken);
-      channel = channels.get(dn.getUuid());
-    } catch (Exception e) {
-      LOG.error("Error while connecting: ", e);
-      throw new IOException(e);
-    }
-
-    if (channel == null || !isConnected(channel)) {
-      throw new IOException("This channel is not connected.");
-    }
-  }
-
-  @Override
-  public XceiverClientReply watchForCommit(long index, long timeout)
-      throws InterruptedException, ExecutionException, TimeoutException,
-      IOException {
-    // there is no notion of watch for commit index in standalone pipeline
-    return null;
-  };
-
-  public long getReplicatedMinCommitIndex() {
-    return 0;
-  }
-  /**
-   * Returns pipeline Type.
-   *
-   * @return - Stand Alone as the type.
-   */
-  @Override
-  public HddsProtos.ReplicationType getPipelineType() {
-    return HddsProtos.ReplicationType.STAND_ALONE;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientManager.java

@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.conf.Config;
-import org.apache.hadoop.hdds.conf.ConfigGroup;
-import org.apache.hadoop.hdds.conf.ConfigType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneSecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-
-import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.apache.hadoop.hdds.conf.ConfigTag.OZONE;
-import static org.apache.hadoop.hdds.conf.ConfigTag.PERFORMANCE;
-
-/**
- * XceiverClientManager is responsible for the lifecycle of XceiverClient
- * instances.  Callers use this class to acquire an XceiverClient instance
- * connected to the desired container pipeline.  When done, the caller also uses
- * this class to release the previously acquired XceiverClient instance.
- *
- *
- * This class caches connection to container for reuse purpose, such that
- * accessing same container frequently will be through the same connection
- * without reestablishing connection. But the connection will be closed if
- * not being used for a period of time.
- */
-public class XceiverClientManager implements Closeable {
-  private static final Logger LOG =
-      LoggerFactory.getLogger(XceiverClientManager.class);
-  //TODO : change this to SCM configuration class
-  private final Configuration conf;
-  private final Cache<String, XceiverClientSpi> clientCache;
-  private final boolean useRatis;
-  private X509Certificate caCert;
-
-  private static XceiverClientMetrics metrics;
-  private boolean isSecurityEnabled;
-  private final boolean topologyAwareRead;
-  /**
-   * Creates a new XceiverClientManager for non secured ozone cluster.
-   * For security enabled ozone cluster, client should use the other constructor
-   * with a valid ca certificate in pem string format.
-   *
-   * @param conf configuration
-   */
-  public XceiverClientManager(Configuration conf) throws IOException {
-    this(conf, OzoneConfiguration.of(conf).getObject(ScmClientConfig.class),
-        null);
-  }
-
-  public XceiverClientManager(Configuration conf, ScmClientConfig clientConf,
-      String caCertPem) throws IOException {
-    Preconditions.checkNotNull(clientConf);
-    Preconditions.checkNotNull(conf);
-    long staleThresholdMs = clientConf.getStaleThreshold(MILLISECONDS);
-    this.useRatis = conf.getBoolean(
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
-        ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
-    this.conf = conf;
-    this.isSecurityEnabled = OzoneSecurityUtil.isSecurityEnabled(conf);
-    if (isSecurityEnabled) {
-      Preconditions.checkNotNull(caCertPem);
-      try {
-        this.caCert = CertificateCodec.getX509Cert(caCertPem);
-      } catch (CertificateException ex) {
-        throw new SCMSecurityException("Error: Fail to get SCM CA certificate",
-            ex);
-      }
-    }
-
-    this.clientCache = CacheBuilder.newBuilder()
-        .expireAfterAccess(staleThresholdMs, MILLISECONDS)
-        .maximumSize(clientConf.getMaxSize())
-        .removalListener(
-            new RemovalListener<String, XceiverClientSpi>() {
-            @Override
-            public void onRemoval(
-                RemovalNotification<String, XceiverClientSpi>
-                  removalNotification) {
-              synchronized (clientCache) {
-                // Mark the entry as evicted
-                XceiverClientSpi info = removalNotification.getValue();
-                info.setEvicted();
-              }
-            }
-          }).build();
-    topologyAwareRead = conf.getBoolean(
-        OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_KEY,
-        OzoneConfigKeys.OZONE_NETWORK_TOPOLOGY_AWARE_READ_DEFAULT);
-  }
-
-  @VisibleForTesting
-  public Cache<String, XceiverClientSpi> getClientCache() {
-    return clientCache;
-  }
-
-  /**
-   * Acquires a XceiverClientSpi connected to a container capable of
-   * storing the specified key.
-   *
-   * If there is already a cached XceiverClientSpi, simply return
-   * the cached otherwise create a new one.
-   *
-   * @param pipeline the container pipeline for the client connection
-   * @return XceiverClientSpi connected to a container
-   * @throws IOException if a XceiverClientSpi cannot be acquired
-   */
-  public XceiverClientSpi acquireClient(Pipeline pipeline)
-      throws IOException {
-    return acquireClient(pipeline, false);
-  }
-
-  /**
-   * Acquires a XceiverClientSpi connected to a container for read.
-   *
-   * If there is already a cached XceiverClientSpi, simply return
-   * the cached otherwise create a new one.
-   *
-   * @param pipeline the container pipeline for the client connection
-   * @return XceiverClientSpi connected to a container
-   * @throws IOException if a XceiverClientSpi cannot be acquired
-   */
-  public XceiverClientSpi acquireClientForReadData(Pipeline pipeline)
-      throws IOException {
-    return acquireClient(pipeline, true);
-  }
-
-  private XceiverClientSpi acquireClient(Pipeline pipeline, boolean read)
-      throws IOException {
-    Preconditions.checkNotNull(pipeline);
-    Preconditions.checkArgument(pipeline.getNodes() != null);
-    Preconditions.checkArgument(!pipeline.getNodes().isEmpty());
-
-    synchronized (clientCache) {
-      XceiverClientSpi info = getClient(pipeline, read);
-      info.incrementReference();
-      return info;
-    }
-  }
-
-  /**
-   * Releases a XceiverClientSpi after use.
-   *
-   * @param client client to release
-   * @param invalidateClient if true, invalidates the client in cache
-   */
-  public void releaseClient(XceiverClientSpi client, boolean invalidateClient) {
-    releaseClient(client, invalidateClient, false);
-  }
-
-  /**
-   * Releases a read XceiverClientSpi after use.
-   *
-   * @param client client to release
-   * @param invalidateClient if true, invalidates the client in cache
-   */
-  public void releaseClientForReadData(XceiverClientSpi client,
-      boolean invalidateClient) {
-    releaseClient(client, invalidateClient, true);
-  }
-
-  private void releaseClient(XceiverClientSpi client, boolean invalidateClient,
-      boolean read) {
-    Preconditions.checkNotNull(client);
-    synchronized (clientCache) {
-      client.decrementReference();
-      if (invalidateClient) {
-        Pipeline pipeline = client.getPipeline();
-        String key = getPipelineCacheKey(pipeline, read);
-        XceiverClientSpi cachedClient = clientCache.getIfPresent(key);
-        if (cachedClient == client) {
-          clientCache.invalidate(key);
-        }
-      }
-    }
-  }
-
-  private XceiverClientSpi getClient(Pipeline pipeline, boolean forRead)
-      throws IOException {
-    HddsProtos.ReplicationType type = pipeline.getType();
-    try {
-      // create different client for read different pipeline node based on
-      // network topology
-      String key = getPipelineCacheKey(pipeline, forRead);
-      // Append user short name to key to prevent a different user
-      // from using same instance of xceiverClient.
-      key = isSecurityEnabled ?
-          key + UserGroupInformation.getCurrentUser().getShortUserName() : key;
-      return clientCache.get(key, new Callable<XceiverClientSpi>() {
-        @Override
-          public XceiverClientSpi call() throws Exception {
-            XceiverClientSpi client = null;
-            switch (type) {
-            case RATIS:
-              client = XceiverClientRatis.newXceiverClientRatis(pipeline, conf,
-                  caCert);
-              client.connect();
-              break;
-            case STAND_ALONE:
-              client = new XceiverClientGrpc(pipeline, conf, caCert);
-              break;
-            case CHAINED:
-            default:
-              throw new IOException("not implemented" + pipeline.getType());
-            }
-            return client;
-          }
-        });
-    } catch (Exception e) {
-      throw new IOException(
-          "Exception getting XceiverClient: " + e.toString(), e);
-    }
-  }
-
-  private String getPipelineCacheKey(Pipeline pipeline, boolean forRead) {
-    String key = pipeline.getId().getId().toString() + pipeline.getType();
-    if (topologyAwareRead && forRead) {
-      try {
-        key += pipeline.getClosestNode().getHostName();
-      } catch (IOException e) {
-        LOG.error("Failed to get closest node to create pipeline cache key:" +
-            e.getMessage());
-      }
-    }
-    return key;
-  }
-
-  /**
-   * Close and remove all the cached clients.
-   */
-  @Override
-  public void close() {
-    //closing is done through RemovalListener
-    clientCache.invalidateAll();
-    clientCache.cleanUp();
-
-    if (metrics != null) {
-      metrics.unRegister();
-    }
-  }
-
-  /**
-   * Tells us if Ratis is enabled for this cluster.
-   * @return True if Ratis is enabled.
-   */
-  public boolean isUseRatis() {
-    return useRatis;
-  }
-
-  /**
-   * Returns hard coded 3 as replication factor.
-   * @return 3
-   */
-  public  HddsProtos.ReplicationFactor getFactor() {
-    if(isUseRatis()) {
-      return HddsProtos.ReplicationFactor.THREE;
-    }
-    return HddsProtos.ReplicationFactor.ONE;
-  }
-
-  /**
-   * Returns the default replication type.
-   * @return Ratis or Standalone
-   */
-  public HddsProtos.ReplicationType getType() {
-    // TODO : Fix me and make Ratis default before release.
-    // TODO: Remove this as replication factor and type are pipeline properties
-    if(isUseRatis()) {
-      return HddsProtos.ReplicationType.RATIS;
-    }
-    return HddsProtos.ReplicationType.STAND_ALONE;
-  }
-
-  public Function<ByteBuffer, ByteString> byteBufferToByteStringConversion(){
-    return ByteStringConversion.createByteBufferConversion(conf);
-  }
-
-  /**
-   * Get xceiver client metric.
-   */
-  public synchronized static XceiverClientMetrics getXceiverClientMetrics() {
-    if (metrics == null) {
-      metrics = XceiverClientMetrics.create();
-    }
-
-    return metrics;
-  }
-
-  /**
-   * Configuration for HDDS client.
-   */
-  @ConfigGroup(prefix = "scm.container.client")
-  public static class ScmClientConfig {
-
-    private int maxSize;
-    private long staleThreshold;
-    private int maxOutstandingRequests;
-
-    public long getStaleThreshold(TimeUnit unit) {
-      return unit.convert(staleThreshold, MILLISECONDS);
-    }
-
-    @Config(key = "idle.threshold",
-        type = ConfigType.TIME, timeUnit = MILLISECONDS,
-        defaultValue = "10s",
-        tags = { OZONE, PERFORMANCE },
-        description =
-            "In the standalone pipelines, the SCM clients use netty to "
-            + " communicate with the container. It also uses connection pooling"
-            + " to reduce client side overheads. This allows a connection to"
-            + " stay idle for a while before the connection is closed."
-    )
-    public void setStaleThreshold(long staleThreshold) {
-      this.staleThreshold = staleThreshold;
-    }
-
-    public int getMaxSize() {
-      return maxSize;
-    }
-
-    @Config(key = "max.size",
-        defaultValue = "256",
-        tags = { OZONE, PERFORMANCE },
-        description =
-            "Controls the maximum number of connections that are cached via"
-            + " client connection pooling. If the number of connections"
-            + " exceed this count, then the oldest idle connection is evicted."
-    )
-    public void setMaxSize(int maxSize) {
-      this.maxSize = maxSize;
-    }
-
-    public int getMaxOutstandingRequests() {
-      return maxOutstandingRequests;
-    }
-
-    @Config(key = "max.outstanding.requests",
-        defaultValue = "100",
-        tags = { OZONE, PERFORMANCE },
-        description =
-            "Controls the maximum number of outstanding async requests that can"
-            + " be handled by the Standalone as well as Ratis client."
-    )
-    public void setMaxOutstandingRequests(int maxOutstandingRequests) {
-      this.maxOutstandingRequests = maxOutstandingRequests;
-    }
-  }
-
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientMetrics.java

@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * The client metrics for the Storage Container protocol.
- */
-@InterfaceAudience.Private
-@Metrics(about = "Storage Container Client Metrics", context = "dfs")
-public class XceiverClientMetrics {
-  public static final String SOURCE_NAME = XceiverClientMetrics.class
-      .getSimpleName();
-
-  private @Metric MutableCounterLong pendingOps;
-  private @Metric MutableCounterLong totalOps;
-  private MutableCounterLong[] pendingOpsArray;
-  private MutableCounterLong[] opsArray;
-  private MutableRate[] containerOpsLatency;
-  private MetricsRegistry registry;
-
-  public XceiverClientMetrics() {
-    int numEnumEntries = ContainerProtos.Type.values().length;
-    this.registry = new MetricsRegistry(SOURCE_NAME);
-
-    this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
-    this.opsArray = new MutableCounterLong[numEnumEntries];
-    this.containerOpsLatency = new MutableRate[numEnumEntries];
-    for (int i = 0; i < numEnumEntries; i++) {
-      pendingOpsArray[i] = registry.newCounter(
-          "numPending" + ContainerProtos.Type.forNumber(i + 1),
-          "number of pending" + ContainerProtos.Type.forNumber(i + 1) + " ops",
-          (long) 0);
-      opsArray[i] = registry
-          .newCounter("opCount" + ContainerProtos.Type.forNumber(i + 1),
-              "number of" + ContainerProtos.Type.forNumber(i + 1) + " ops",
-              (long) 0);
-
-      containerOpsLatency[i] = registry.newRate(
-          ContainerProtos.Type.forNumber(i + 1) + "Latency",
-          "latency of " + ContainerProtos.Type.forNumber(i + 1)
-          + " ops");
-    }
-  }
-
-  public static XceiverClientMetrics create() {
-    DefaultMetricsSystem.initialize(SOURCE_NAME);
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    return ms.register(SOURCE_NAME, "Storage Container Client Metrics",
-        new XceiverClientMetrics());
-  }
-
-  public void incrPendingContainerOpsMetrics(ContainerProtos.Type type) {
-    pendingOps.incr();
-    totalOps.incr();
-    opsArray[type.ordinal()].incr();
-    pendingOpsArray[type.ordinal()].incr();
-  }
-
-  public void decrPendingContainerOpsMetrics(ContainerProtos.Type type) {
-    pendingOps.incr(-1);
-    pendingOpsArray[type.ordinal()].incr(-1);
-  }
-
-  public void addContainerOpsLatency(ContainerProtos.Type type,
-      long latencyNanos) {
-    containerOpsLatency[type.ordinal()].add(latencyNanos);
-  }
-
-  public long getContainerOpsMetrics(ContainerProtos.Type type) {
-    return pendingOpsArray[type.ordinal()].value();
-  }
-
-  @VisibleForTesting
-  public long getTotalOpCount() {
-    return totalOps.value();
-  }
-
-  @VisibleForTesting
-  public long getContainerOpCountMetrics(ContainerProtos.Type type) {
-    return opsArray[type.ordinal()].value();
-  }
-
-  public void unRegister() {
-    MetricsSystem ms = DefaultMetricsSystem.instance();
-    ms.unregisterSource(SOURCE_NAME);
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -1,367 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import java.io.IOException;
-import java.security.cert.X509Certificate;
-import java.util.Collection;
-import java.util.List;
-import java.util.Objects;
-import java.util.OptionalLong;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage;
-import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.hdds.ratis.RatisHelper;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.grpc.GrpcTlsConfig;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.protocol.GroupMismatchException;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftException;
-import org.apache.ratis.retry.RetryPolicy;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import io.opentracing.Scope;
-import io.opentracing.util.GlobalTracer;
-
-/**
- * An abstract implementation of {@link XceiverClientSpi} using Ratis.
- * The underlying RPC mechanism can be chosen via the constructor.
- */
-public final class XceiverClientRatis extends XceiverClientSpi {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(XceiverClientRatis.class);
-
-  public static XceiverClientRatis newXceiverClientRatis(
-      org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
-      Configuration ozoneConf) {
-    return newXceiverClientRatis(pipeline, ozoneConf, null);
-  }
-
-  public static XceiverClientRatis newXceiverClientRatis(
-      org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
-      Configuration ozoneConf, X509Certificate caCert) {
-    final String rpcType = ozoneConf
-        .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
-            ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
-    final TimeDuration clientRequestTimeout =
-        RatisHelper.getClientRequestTimeout(ozoneConf);
-    final int maxOutstandingRequests =
-        HddsClientUtils.getMaxOutstandingRequests(ozoneConf);
-    final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
-    final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new
-        SecurityConfig(ozoneConf), caCert);
-    return new XceiverClientRatis(pipeline,
-        SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests,
-        retryPolicy, tlsConfig, clientRequestTimeout);
-  }
-
-  private final Pipeline pipeline;
-  private final RpcType rpcType;
-  private final AtomicReference<RaftClient> client = new AtomicReference<>();
-  private final int maxOutstandingRequests;
-  private final RetryPolicy retryPolicy;
-  private final GrpcTlsConfig tlsConfig;
-  private final TimeDuration clientRequestTimeout;
-
-  // Map to track commit index at every server
-  private final ConcurrentHashMap<UUID, Long> commitInfoMap;
-
-  private XceiverClientMetrics metrics;
-
-  /**
-   * Constructs a client.
-   */
-  private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
-      int maxOutStandingChunks, RetryPolicy retryPolicy,
-      GrpcTlsConfig tlsConfig, TimeDuration timeout) {
-    super();
-    this.pipeline = pipeline;
-    this.rpcType = rpcType;
-    this.maxOutstandingRequests = maxOutStandingChunks;
-    this.retryPolicy = retryPolicy;
-    commitInfoMap = new ConcurrentHashMap<>();
-    this.tlsConfig = tlsConfig;
-    this.clientRequestTimeout = timeout;
-    metrics = XceiverClientManager.getXceiverClientMetrics();
-  }
-
-  private void updateCommitInfosMap(
-      Collection<RaftProtos.CommitInfoProto> commitInfoProtos) {
-    // if the commitInfo map is empty, just update the commit indexes for each
-    // of the servers
-    if (commitInfoMap.isEmpty()) {
-      commitInfoProtos.forEach(proto -> commitInfoMap
-          .put(RatisHelper.toDatanodeId(proto.getServer()),
-              proto.getCommitIndex()));
-      // In case the commit is happening 2 way, just update the commitIndex
-      // for the servers which have been successfully updating the commit
-      // indexes. This is important because getReplicatedMinCommitIndex()
-      // should always return the min commit index out of the nodes which have
-      // been replicating data successfully.
-    } else {
-      commitInfoProtos.forEach(proto -> commitInfoMap
-          .computeIfPresent(RatisHelper.toDatanodeId(proto.getServer()),
-              (address, index) -> {
-                index = proto.getCommitIndex();
-                return index;
-              }));
-    }
-  }
-
-  /**
-   * Returns Ratis as pipeline Type.
-   *
-   * @return - Ratis
-   */
-  @Override
-  public HddsProtos.ReplicationType getPipelineType() {
-    return HddsProtos.ReplicationType.RATIS;
-  }
-
-  @Override
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  @Override
-  public void connect() throws Exception {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Connecting to pipeline:{} datanode:{}", getPipeline().getId(),
-          RatisHelper.toRaftPeerId(pipeline.getFirstNode()));
-    }
-    // TODO : XceiverClient ratis should pass the config value of
-    // maxOutstandingRequests so as to set the upper bound on max no of async
-    // requests to be handled by raft client
-    if (!client.compareAndSet(null,
-        RatisHelper.newRaftClient(rpcType, getPipeline(), retryPolicy,
-            maxOutstandingRequests, tlsConfig, clientRequestTimeout))) {
-      throw new IllegalStateException("Client is already connected.");
-    }
-  }
-
-  @Override
-  public void connect(String encodedToken) throws Exception {
-    throw new UnsupportedOperationException("Block tokens are not " +
-        "implemented for Ratis clients.");
-  }
-
-  @Override
-  public void close() {
-    final RaftClient c = client.getAndSet(null);
-    if (c != null) {
-      closeRaftClient(c);
-    }
-  }
-
-  private void closeRaftClient(RaftClient raftClient) {
-    try {
-      raftClient.close();
-    } catch (IOException e) {
-      throw new IllegalStateException(e);
-    }
-  }
-
-  private RaftClient getClient() {
-    return Objects.requireNonNull(client.get(), "client is null");
-  }
-
-
-  @VisibleForTesting
-  public ConcurrentHashMap<UUID, Long> getCommitInfoMap() {
-    return commitInfoMap;
-  }
-
-  private CompletableFuture<RaftClientReply> sendRequestAsync(
-      ContainerCommandRequestProto request) {
-    try (Scope scope = GlobalTracer.get()
-        .buildSpan("XceiverClientRatis." + request.getCmdType().name())
-        .startActive(true)) {
-      final ContainerCommandRequestMessage message
-          = ContainerCommandRequestMessage.toMessage(
-              request, TracingUtil.exportCurrentSpan());
-      if (HddsUtils.isReadOnly(request)) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("sendCommandAsync ReadOnly {}", message);
-        }
-        return getClient().sendReadOnlyAsync(message);
-      } else {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("sendCommandAsync {}", message);
-        }
-        return getClient().sendAsync(message);
-      }
-    }
-  }
-
-  // gets the minimum log index replicated to all servers
-  @Override
-  public long getReplicatedMinCommitIndex() {
-    OptionalLong minIndex =
-        commitInfoMap.values().parallelStream().mapToLong(v -> v).min();
-    return minIndex.isPresent() ? minIndex.getAsLong() : 0;
-  }
-
-  private void addDatanodetoReply(UUID address, XceiverClientReply reply) {
-    DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
-    builder.setUuid(address.toString());
-    reply.addDatanode(builder.build());
-  }
-
-  @Override
-  public XceiverClientReply watchForCommit(long index, long timeout)
-      throws InterruptedException, ExecutionException, TimeoutException,
-      IOException {
-    long commitIndex = getReplicatedMinCommitIndex();
-    XceiverClientReply clientReply = new XceiverClientReply(null);
-    if (commitIndex >= index) {
-      // return the min commit index till which the log has been replicated to
-      // all servers
-      clientReply.setLogIndex(commitIndex);
-      return clientReply;
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("commit index : {} watch timeout : {}", index, timeout);
-    }
-    RaftClientReply reply;
-    try {
-      CompletableFuture<RaftClientReply> replyFuture = getClient()
-          .sendWatchAsync(index, RaftProtos.ReplicationLevel.ALL_COMMITTED);
-      replyFuture.get(timeout, TimeUnit.MILLISECONDS);
-    } catch (Exception e) {
-      Throwable t = HddsClientUtils.checkForException(e);
-      LOG.warn("3 way commit failed on pipeline {}", pipeline, e);
-      if (t instanceof GroupMismatchException) {
-        throw e;
-      }
-      reply = getClient()
-          .sendWatchAsync(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
-          .get(timeout, TimeUnit.MILLISECONDS);
-      List<RaftProtos.CommitInfoProto> commitInfoProtoList =
-          reply.getCommitInfos().stream()
-              .filter(i -> i.getCommitIndex() < index)
-              .collect(Collectors.toList());
-      commitInfoProtoList.parallelStream().forEach(proto -> {
-        UUID address = RatisHelper.toDatanodeId(proto.getServer());
-        addDatanodetoReply(address, clientReply);
-        // since 3 way commit has failed, the updated map from now on  will
-        // only store entries for those datanodes which have had successful
-        // replication.
-        commitInfoMap.remove(address);
-        LOG.info(
-            "Could not commit index {} on pipeline {} to all the nodes. " +
-            "Server {} has failed. Committed by majority.",
-            index, pipeline, address);
-      });
-    }
-    clientReply.setLogIndex(index);
-    return clientReply;
-  }
-
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   */
-  @Override
-  public XceiverClientReply sendCommandAsync(
-      ContainerCommandRequestProto request) {
-    XceiverClientReply asyncReply = new XceiverClientReply(null);
-    long requestTime = Time.monotonicNowNanos();
-    CompletableFuture<RaftClientReply> raftClientReply =
-        sendRequestAsync(request);
-    metrics.incrPendingContainerOpsMetrics(request.getCmdType());
-    CompletableFuture<ContainerCommandResponseProto> containerCommandResponse =
-        raftClientReply.whenComplete((reply, e) -> {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("received reply {} for request: cmdType={} containerID={}"
-                    + " pipelineID={} traceID={} exception: {}", reply,
-                request.getCmdType(), request.getContainerID(),
-                request.getPipelineID(), request.getTraceID(), e);
-          }
-          metrics.decrPendingContainerOpsMetrics(request.getCmdType());
-          metrics.addContainerOpsLatency(request.getCmdType(),
-              Time.monotonicNowNanos() - requestTime);
-        }).thenApply(reply -> {
-          try {
-            if (!reply.isSuccess()) {
-              // in case of raft retry failure, the raft client is
-              // not able to connect to the leader hence the pipeline
-              // can not be used but this instance of RaftClient will close
-              // and refreshed again. In case the client cannot connect to
-              // leader, getClient call will fail.
-
-              // No need to set the failed Server ID here. Ozone client
-              // will directly exclude this pipeline in next allocate block
-              // to SCM as in this case, it is the raft client which is not
-              // able to connect to leader in the pipeline, though the
-              // pipeline can still be functional.
-              RaftException exception = reply.getException();
-              Preconditions.checkNotNull(exception, "Raft reply failure but " +
-                  "no exception propagated.");
-              throw new CompletionException(exception);
-            }
-            ContainerCommandResponseProto response =
-                ContainerCommandResponseProto
-                    .parseFrom(reply.getMessage().getContent());
-            UUID serverId = RatisHelper.toDatanodeId(reply.getReplierId());
-            if (response.getResult() == ContainerProtos.Result.SUCCESS) {
-              updateCommitInfosMap(reply.getCommitInfos());
-            }
-            asyncReply.setLogIndex(reply.getLogIndex());
-            addDatanodetoReply(serverId, asyncReply);
-            return response;
-          } catch (InvalidProtocolBufferException e) {
-            throw new CompletionException(e);
-          }
-        });
-    asyncReply.setResponse(containerCommandResponse);
-    return asyncReply;
-  }
-
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/ContainerOperationClient.java

@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerDataProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ReadContainerResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto
-    .StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This class provides the client-facing APIs of container operations.
- */
-public class ContainerOperationClient implements ScmClient {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(ContainerOperationClient.class);
-  private static long containerSizeB = -1;
-  private final StorageContainerLocationProtocol
-      storageContainerLocationClient;
-  private final XceiverClientManager xceiverClientManager;
-
-  public ContainerOperationClient(
-      StorageContainerLocationProtocol
-          storageContainerLocationClient,
-      XceiverClientManager xceiverClientManager) {
-    this.storageContainerLocationClient = storageContainerLocationClient;
-    this.xceiverClientManager = xceiverClientManager;
-  }
-
-  /**
-   * Return the capacity of containers. The current assumption is that all
-   * containers have the same capacity. Therefore one static is sufficient for
-   * any container.
-   * @return The capacity of one container in number of bytes.
-   */
-  public static long getContainerSizeB() {
-    return containerSizeB;
-  }
-
-  /**
-   * Set the capacity of container. Should be exactly once on system start.
-   * @param size Capacity of one container in number of bytes.
-   */
-  public static void setContainerSizeB(long size) {
-    containerSizeB = size;
-  }
-
-
-  @Override
-  public ContainerWithPipeline createContainer(String owner)
-      throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      ContainerWithPipeline containerWithPipeline =
-          storageContainerLocationClient.allocateContainer(
-              xceiverClientManager.getType(),
-              xceiverClientManager.getFactor(), owner);
-      Pipeline pipeline = containerWithPipeline.getPipeline();
-      client = xceiverClientManager.acquireClient(pipeline);
-
-      Preconditions.checkState(pipeline.isOpen(), String
-          .format("Unexpected state=%s for pipeline=%s, expected state=%s",
-              pipeline.getPipelineState(), pipeline.getId(),
-              Pipeline.PipelineState.OPEN));
-      createContainer(client,
-          containerWithPipeline.getContainerInfo().getContainerID());
-      return containerWithPipeline;
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client, false);
-      }
-    }
-  }
-
-  /**
-   * Create a container over pipeline specified by the SCM.
-   *
-   * @param client - Client to communicate with Datanodes.
-   * @param containerId - Container ID.
-   * @throws IOException
-   */
-  public void createContainer(XceiverClientSpi client,
-      long containerId) throws IOException {
-    ContainerProtocolCalls.createContainer(client, containerId, null);
-
-    // Let us log this info after we let SCM know that we have completed the
-    // creation state.
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Created container " + containerId
-          + " machines:" + client.getPipeline().getNodes());
-    }
-  }
-
-  /**
-   * Creates a pipeline over the machines choosen by the SCM.
-   *
-   * @param client - Client
-   * @param pipeline - pipeline to be createdon Datanodes.
-   * @throws IOException
-   */
-  private void createPipeline(XceiverClientSpi client, Pipeline pipeline)
-      throws IOException {
-
-    Preconditions.checkNotNull(pipeline.getId(), "Pipeline " +
-        "name cannot be null when client create flag is set.");
-
-    // Pipeline creation is a three step process.
-    //
-    // 1. Notify SCM that this client is doing a create pipeline on
-    // datanodes.
-    //
-    // 2. Talk to Datanodes to create the pipeline.
-    //
-    // 3. update SCM that pipeline creation was successful.
-
-    // TODO: this has not been fully implemented on server side
-    // SCMClientProtocolServer#notifyObjectStageChange
-    // TODO: when implement the pipeline state machine, change
-    // the pipeline name (string) to pipeline id (long)
-    //storageContainerLocationClient.notifyObjectStageChange(
-    //    ObjectStageChangeRequestProto.Type.pipeline,
-    //    pipeline.getPipelineName(),
-    //    ObjectStageChangeRequestProto.Op.create,
-    //    ObjectStageChangeRequestProto.Stage.begin);
-
-    // client.createPipeline();
-    // TODO: Use PipelineManager to createPipeline
-
-    //storageContainerLocationClient.notifyObjectStageChange(
-    //    ObjectStageChangeRequestProto.Type.pipeline,
-    //    pipeline.getPipelineName(),
-    //    ObjectStageChangeRequestProto.Op.create,
-    //    ObjectStageChangeRequestProto.Stage.complete);
-
-    // TODO : Should we change the state on the client side ??
-    // That makes sense, but it is not needed for the client to work.
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Pipeline creation successful. Pipeline: {}",
-          pipeline.toString());
-    }
-  }
-
-  @Override
-  public ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
-      HddsProtos.ReplicationFactor factor, String owner) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      // allocate container on SCM.
-      ContainerWithPipeline containerWithPipeline =
-          storageContainerLocationClient.allocateContainer(type, factor,
-              owner);
-      Pipeline pipeline = containerWithPipeline.getPipeline();
-      client = xceiverClientManager.acquireClient(pipeline);
-
-      // connect to pipeline leader and allocate container on leader datanode.
-      client = xceiverClientManager.acquireClient(pipeline);
-      createContainer(client,
-          containerWithPipeline.getContainerInfo().getContainerID());
-      return containerWithPipeline;
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client, false);
-      }
-    }
-  }
-
-  /**
-   * Returns a set of Nodes that meet a query criteria.
-   *
-   * @param nodeStatuses - Criteria that we want the node to have.
-   * @param queryScope - Query scope - Cluster or pool.
-   * @param poolName - if it is pool, a pool name is required.
-   * @return A set of nodes that meet the requested criteria.
-   * @throws IOException
-   */
-  @Override
-  public List<HddsProtos.Node> queryNode(HddsProtos.NodeState
-      nodeStatuses, HddsProtos.QueryScope queryScope, String poolName)
-      throws IOException {
-    return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
-        poolName);
-  }
-
-  /**
-   * Creates a specified replication pipeline.
-   */
-  @Override
-  public Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
-      HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
-      throws IOException {
-    return storageContainerLocationClient.createReplicationPipeline(type,
-        factor, nodePool);
-  }
-
-  @Override
-  public List<Pipeline> listPipelines() throws IOException {
-    return storageContainerLocationClient.listPipelines();
-  }
-
-  @Override
-  public void activatePipeline(HddsProtos.PipelineID pipelineID)
-      throws IOException {
-    storageContainerLocationClient.activatePipeline(pipelineID);
-  }
-
-  @Override
-  public void deactivatePipeline(HddsProtos.PipelineID pipelineID)
-      throws IOException {
-    storageContainerLocationClient.deactivatePipeline(pipelineID);
-  }
-
-  @Override
-  public void closePipeline(HddsProtos.PipelineID pipelineID)
-      throws IOException {
-    storageContainerLocationClient.closePipeline(pipelineID);
-  }
-
-  @Override
-  public void close() {
-    try {
-      xceiverClientManager.close();
-    } catch (Exception ex) {
-      LOG.error("Can't close " + this.getClass().getSimpleName(), ex);
-    }
-  }
-
-  /**
-   * Deletes an existing container.
-   *
-   * @param containerId - ID of the container.
-   * @param pipeline    - Pipeline that represents the container.
-   * @param force       - true to forcibly delete the container.
-   * @throws IOException
-   */
-  @Override
-  public void deleteContainer(long containerId, Pipeline pipeline,
-      boolean force) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      client = xceiverClientManager.acquireClient(pipeline);
-      ContainerProtocolCalls
-          .deleteContainer(client, containerId, force, null);
-      storageContainerLocationClient
-          .deleteContainer(containerId);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Deleted container {}, machines: {} ", containerId,
-            pipeline.getNodes());
-      }
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client, false);
-      }
-    }
-  }
-
-  /**
-   * Delete the container, this will release any resource it uses.
-   * @param containerID - containerID.
-   * @param force - True to forcibly delete the container.
-   * @throws IOException
-   */
-  @Override
-  public void deleteContainer(long containerID, boolean force)
-      throws IOException {
-    ContainerWithPipeline info = getContainerWithPipeline(containerID);
-    deleteContainer(containerID, info.getPipeline(), force);
-  }
-
-  @Override
-  public List<ContainerInfo> listContainer(long startContainerID,
-      int count) throws IOException {
-    return storageContainerLocationClient.listContainer(
-        startContainerID, count);
-  }
-
-  /**
-   * Get meta data from an existing container.
-   *
-   * @param containerID - ID of the container.
-   * @param pipeline    - Pipeline where the container is located.
-   * @return ContainerInfo
-   * @throws IOException
-   */
-  @Override
-  public ContainerDataProto readContainer(long containerID,
-      Pipeline pipeline) throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      client = xceiverClientManager.acquireClient(pipeline);
-      ReadContainerResponseProto response =
-          ContainerProtocolCalls.readContainer(client, containerID, null);
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Read container {}, machines: {} ", containerID,
-            pipeline.getNodes());
-      }
-      return response.getContainerData();
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client, false);
-      }
-    }
-  }
-
-  /**
-   * Get meta data from an existing container.
-   * @param containerID - ID of the container.
-   * @return ContainerInfo - a message of protobuf which has basic info
-   * of a container.
-   * @throws IOException
-   */
-  @Override
-  public ContainerDataProto readContainer(long containerID) throws IOException {
-    ContainerWithPipeline info = getContainerWithPipeline(containerID);
-    return readContainer(containerID, info.getPipeline());
-  }
-
-  /**
-   * Given an id, return the pipeline associated with the container.
-   * @param containerId - String Container ID
-   * @return Pipeline of the existing container, corresponding to the given id.
-   * @throws IOException
-   */
-  @Override
-  public ContainerInfo getContainer(long containerId) throws
-      IOException {
-    return storageContainerLocationClient.getContainer(containerId);
-  }
-
-  /**
-   * Gets a container by Name -- Throws if the container does not exist.
-   *
-   * @param containerId - Container ID
-   * @return ContainerWithPipeline
-   * @throws IOException
-   */
-  @Override
-  public ContainerWithPipeline getContainerWithPipeline(long containerId)
-      throws IOException {
-    return storageContainerLocationClient.getContainerWithPipeline(containerId);
-  }
-
-  /**
-   * Close a container.
-   *
-   * @param pipeline the container to be closed.
-   * @throws IOException
-   */
-  @Override
-  public void closeContainer(long containerId, Pipeline pipeline)
-      throws IOException {
-    XceiverClientSpi client = null;
-    try {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Close container {}", pipeline);
-      }
-      /*
-      TODO: two orders here, revisit this later:
-      1. close on SCM first, then on data node
-      2. close on data node first, then on SCM
-
-      with 1: if client failed after closing on SCM, then there is a
-      container SCM thinks as closed, but is actually open. Then SCM will no
-      longer allocate block to it, which is fine. But SCM may later try to
-      replicate this "closed" container, which I'm not sure is safe.
-
-      with 2: if client failed after close on datanode, then there is a
-      container SCM thinks as open, but is actually closed. Then SCM will still
-      try to allocate block to it. Which will fail when actually doing the
-      write. No more data can be written, but at least the correctness and
-      consistency of existing data will maintain.
-
-      For now, take the #2 way.
-       */
-      // Actually close the container on Datanode
-      client = xceiverClientManager.acquireClient(pipeline);
-
-      storageContainerLocationClient.notifyObjectStageChange(
-          ObjectStageChangeRequestProto.Type.container,
-          containerId,
-          ObjectStageChangeRequestProto.Op.close,
-          ObjectStageChangeRequestProto.Stage.begin);
-
-      ContainerProtocolCalls.closeContainer(client, containerId,
-          null);
-      // Notify SCM to close the container
-      storageContainerLocationClient.notifyObjectStageChange(
-          ObjectStageChangeRequestProto.Type.container,
-          containerId,
-          ObjectStageChangeRequestProto.Op.close,
-          ObjectStageChangeRequestProto.Stage.complete);
-    } finally {
-      if (client != null) {
-        xceiverClientManager.releaseClient(client, false);
-      }
-    }
-  }
-
-  /**
-   * Close a container.
-   *
-   * @throws IOException
-   */
-  @Override
-  public void closeContainer(long containerId)
-      throws IOException {
-    ContainerWithPipeline info = getContainerWithPipeline(containerId);
-    Pipeline pipeline = info.getPipeline();
-    closeContainer(containerId, pipeline);
-  }
-
-  /**
-   * Get the the current usage information.
-   * @param containerID - ID of the container.
-   * @return the size of the given container.
-   * @throws IOException
-   */
-  @Override
-  public long getContainerSize(long containerID) throws IOException {
-    // TODO : Fix this, it currently returns the capacity
-    // but not the current usage.
-    long size = getContainerSizeB();
-    if (size == -1) {
-      throw new IOException("Container size unknown!");
-    }
-    return size;
-  }
-
-  /**
-   * Check if SCM is in safe mode.
-   *
-   * @return Returns true if SCM is in safe mode else returns false.
-   * @throws IOException
-   */
-  public boolean inSafeMode() throws IOException {
-    return storageContainerLocationClient.inSafeMode();
-  }
-
-  /**
-   * Force SCM out of safe mode.
-   *
-   * @return returns true if operation is successful.
-   * @throws IOException
-   */
-  public boolean forceExitSafeMode() throws IOException {
-    return storageContainerLocationClient.forceExitSafeMode();
-  }
-
-  @Override
-  public void startReplicationManager() throws IOException {
-    storageContainerLocationClient.startReplicationManager();
-  }
-
-  @Override
-  public void stopReplicationManager() throws IOException {
-    storageContainerLocationClient.stopReplicationManager();
-  }
-
-  @Override
-  public boolean getReplicationManagerStatus() throws IOException {
-    return storageContainerLocationClient.getReplicationManagerStatus();
-  }
-
-
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java

@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
-import org.apache.hadoop.hdds.scm.XceiverClientManager.ScmClientConfig;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.ratis.protocol.AlreadyClosedException;
-import org.apache.ratis.protocol.GroupMismatchException;
-import org.apache.ratis.protocol.NotReplicatedException;
-import org.apache.ratis.protocol.RaftRetryFailureException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.text.ParseException;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Utility methods for Ozone and Container Clients.
- *
- * The methods to retrieve SCM service endpoints assume there is a single
- * SCM service instance. This will change when we switch to replicated service
- * instances for redundancy.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class HddsClientUtils {
-
-  private static final Logger LOG = LoggerFactory.getLogger(
-      HddsClientUtils.class);
-
-  private static final int NO_PORT = -1;
-
-  private HddsClientUtils() {
-  }
-
-  private static final List<Class<? extends Exception>> EXCEPTION_LIST =
-      new ArrayList<Class<? extends Exception>>() {{
-        add(TimeoutException.class);
-        add(StorageContainerException.class);
-        add(RaftRetryFailureException.class);
-        add(AlreadyClosedException.class);
-        add(GroupMismatchException.class);
-        // Not Replicated Exception will be thrown if watch For commit
-        // does not succeed
-        add(NotReplicatedException.class);
-      }};
-
-  /**
-   * Date format that used in ozone. Here the format is thread safe to use.
-   */
-  private static final ThreadLocal<DateTimeFormatter> DATE_FORMAT =
-      ThreadLocal.withInitial(() -> {
-        DateTimeFormatter format =
-            DateTimeFormatter.ofPattern(OzoneConsts.OZONE_DATE_FORMAT);
-        return format.withZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE));
-      });
-
-
-  /**
-   * Convert time in millisecond to a human readable format required in ozone.
-   * @return a human readable string for the input time
-   */
-  public static String formatDateTime(long millis) {
-    ZonedDateTime dateTime = ZonedDateTime.ofInstant(
-        Instant.ofEpochMilli(millis), DATE_FORMAT.get().getZone());
-    return DATE_FORMAT.get().format(dateTime);
-  }
-
-  /**
-   * Convert time in ozone date format to millisecond.
-   * @return time in milliseconds
-   */
-  public static long formatDateTime(String date) throws ParseException {
-    Preconditions.checkNotNull(date, "Date string should not be null.");
-    return ZonedDateTime.parse(date, DATE_FORMAT.get())
-        .toInstant().toEpochMilli();
-  }
-
-  /**
-   * verifies that bucket name / volume name is a valid DNS name.
-   *
-   * @param resName Bucket or volume Name to be validated
-   *
-   * @throws IllegalArgumentException
-   */
-  public static void verifyResourceName(String resName)
-      throws IllegalArgumentException {
-    if (resName == null) {
-      throw new IllegalArgumentException("Bucket or Volume name is null");
-    }
-
-    if (resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH ||
-        resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume length is illegal, "
-              + "valid length is 3-63 characters");
-    }
-
-    if (resName.charAt(0) == '.' || resName.charAt(0) == '-') {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot start with a period or dash");
-    }
-
-    if (resName.charAt(resName.length() - 1) == '.' ||
-        resName.charAt(resName.length() - 1) == '-') {
-      throw new IllegalArgumentException("Bucket or Volume name "
-          + "cannot end with a period or dash");
-    }
-
-    boolean isIPv4 = true;
-    char prev = (char) 0;
-
-    for (int index = 0; index < resName.length(); index++) {
-      char currChar = resName.charAt(index);
-      if (currChar != '.') {
-        isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4;
-      }
-      if (currChar > 'A' && currChar < 'Z') {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name does not support uppercase characters");
-      }
-      if (currChar != '.' && currChar != '-') {
-        if (currChar < '0' || (currChar > '9' && currChar < 'a') ||
-            currChar > 'z') {
-          throw new IllegalArgumentException("Bucket or Volume name has an " +
-              "unsupported character : " +
-              currChar);
-        }
-      }
-      if (prev == '.' && currChar == '.') {
-        throw new IllegalArgumentException("Bucket or Volume name should not " +
-            "have two contiguous periods");
-      }
-      if (prev == '-' && currChar == '.') {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name should not have period after dash");
-      }
-      if (prev == '.' && currChar == '-') {
-        throw new IllegalArgumentException(
-            "Bucket or Volume name should not have dash after period");
-      }
-      prev = currChar;
-    }
-
-    if (isIPv4) {
-      throw new IllegalArgumentException(
-          "Bucket or Volume name cannot be an IPv4 address or all numeric");
-    }
-  }
-
-  /**
-   * verifies that bucket / volume name is a valid DNS name.
-   *
-   * @param resourceNames Array of bucket / volume names to be verified.
-   */
-  public static void verifyResourceName(String... resourceNames) {
-    for (String resourceName : resourceNames) {
-      HddsClientUtils.verifyResourceName(resourceName);
-    }
-  }
-
-  /**
-   * Checks that object parameters passed as reference is not null.
-   *
-   * @param references Array of object references to be checked.
-   * @param <T>
-   */
-  public static <T> void checkNotNull(T... references) {
-    for (T ref: references) {
-      Preconditions.checkNotNull(ref);
-    }
-  }
-
-  /**
-   * Returns the cache value to be used for list calls.
-   * @param conf Configuration object
-   * @return list cache size
-   */
-  public static int getListCacheSize(Configuration conf) {
-    return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
-        OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
-  }
-
-  /**
-   * @return a default instance of {@link CloseableHttpClient}.
-   */
-  public static CloseableHttpClient newHttpClient() {
-    return HddsClientUtils.newHttpClient(new Configuration());
-  }
-
-  /**
-   * Returns a {@link CloseableHttpClient} configured by given configuration.
-   * If conf is null, returns a default instance.
-   *
-   * @param conf configuration
-   * @return a {@link CloseableHttpClient} instance.
-   */
-  public static CloseableHttpClient newHttpClient(Configuration conf) {
-    long socketTimeout = OzoneConfigKeys
-        .OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT;
-    long connectionTimeout = OzoneConfigKeys
-        .OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT;
-    if (conf != null) {
-      socketTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-      connectionTimeout = conf.getTimeDuration(
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
-          OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
-          TimeUnit.MILLISECONDS);
-    }
-
-    CloseableHttpClient client = HttpClients.custom()
-        .setDefaultRequestConfig(
-            RequestConfig.custom()
-                .setSocketTimeout(Math.toIntExact(socketTimeout))
-                .setConnectTimeout(Math.toIntExact(connectionTimeout))
-                .build())
-        .build();
-    return client;
-  }
-
-  /**
-   * Returns the maximum no of outstanding async requests to be handled by
-   * Standalone and Ratis client.
-   */
-  public static int getMaxOutstandingRequests(Configuration config) {
-    return OzoneConfiguration.of(config)
-        .getObject(ScmClientConfig.class)
-        .getMaxOutstandingRequests();
-  }
-
-  /**
-   * Create a scm block client, used by putKey() and getKey().
-   *
-   * @return {@link ScmBlockLocationProtocol}
-   * @throws IOException
-   */
-  public static SCMSecurityProtocol getScmSecurityClient(
-      OzoneConfiguration conf, UserGroupInformation ugi) throws IOException {
-    RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
-        ProtobufRpcEngine.class);
-    long scmVersion =
-        RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
-    InetSocketAddress scmSecurityProtoAdd =
-        HddsUtils.getScmAddressForSecurityProtocol(conf);
-    SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
-        new SCMSecurityProtocolClientSideTranslatorPB(
-            RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion,
-                scmSecurityProtoAdd, ugi, conf,
-                NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf)));
-    return scmSecurityClient;
-  }
-
-  public static Throwable checkForException(Exception e) {
-    Throwable t = e;
-    while (t != null) {
-      for (Class<? extends Exception> cls : getExceptionList()) {
-        if (cls.isInstance(t)) {
-          return t;
-        }
-      }
-      t = t.getCause();
-    }
-    return t;
-  }
-
-  public static RetryPolicy createRetryPolicy(int maxRetryCount,
-      long retryInterval) {
-    // retry with fixed sleep between retries
-    return RetryPolicies.retryUpToMaximumCountWithFixedSleep(
-        maxRetryCount, retryInterval, TimeUnit.MILLISECONDS);
-  }
-
-  public static Map<Class<? extends Throwable>,
-      RetryPolicy> getRetryPolicyByException(int maxRetryCount,
-      long retryInterval) {
-    Map<Class<? extends Throwable>, RetryPolicy> policyMap = new HashMap<>();
-    for (Class<? extends Exception> ex : EXCEPTION_LIST) {
-      if (ex == TimeoutException.class
-          || ex == RaftRetryFailureException.class) {
-        // retry without sleep
-        policyMap.put(ex, createRetryPolicy(maxRetryCount, 0));
-      } else {
-        // retry with fixed sleep between retries
-        policyMap.put(ex, createRetryPolicy(maxRetryCount, retryInterval));
-      }
-    }
-    // Default retry policy
-    policyMap
-        .put(Exception.class, createRetryPolicy(maxRetryCount, retryInterval));
-    return policyMap;
-  }
-
-  public static List<Class<? extends Exception>> getExceptionList() {
-    return EXCEPTION_LIST;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/package-info.java

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.client;
-
-/**
- * Client facing classes for the container operations.
- */

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/package-info.java

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-/**
- * Classes for different type of container service client.
- */

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockInputStream.java

@@ -1,388 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.DatanodeBlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockResponseProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * An {@link InputStream} called from KeyInputStream to read a block from the
- * container.
- * This class encapsulates all state management for iterating
- * through the sequence of chunks through {@link ChunkInputStream}.
- */
-public class BlockInputStream extends InputStream implements Seekable {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(BlockInputStream.class);
-
-  private static final int EOF = -1;
-
-  private final BlockID blockID;
-  private final long length;
-  private Pipeline pipeline;
-  private final Token<OzoneBlockTokenIdentifier> token;
-  private final boolean verifyChecksum;
-  private XceiverClientManager xceiverClientManager;
-  private XceiverClientSpi xceiverClient;
-  private boolean initialized = false;
-
-  // List of ChunkInputStreams, one for each chunk in the block
-  private List<ChunkInputStream> chunkStreams;
-
-  // chunkOffsets[i] stores the index of the first data byte in
-  // chunkStream i w.r.t the block data.
-  // Let’s say we have chunk size as 40 bytes. And let's say the parent
-  // block stores data from index 200 and has length 400.
-  // The first 40 bytes of this block will be stored in chunk[0], next 40 in
-  // chunk[1] and so on. But since the chunkOffsets are w.r.t the block only
-  // and not the key, the values in chunkOffsets will be [0, 40, 80,....].
-  private long[] chunkOffsets = null;
-
-  // Index of the chunkStream corresponding to the current position of the
-  // BlockInputStream i.e offset of the data to be read next from this block
-  private int chunkIndex;
-
-  // Position of the BlockInputStream is maintainted by this variable till
-  // the stream is initialized. This position is w.r.t to the block only and
-  // not the key.
-  // For the above example, if we seek to position 240 before the stream is
-  // initialized, then value of blockPosition will be set to 40.
-  // Once, the stream is initialized, the position of the stream
-  // will be determined by the current chunkStream and its position.
-  private long blockPosition = 0;
-
-  // Tracks the chunkIndex corresponding to the last blockPosition so that it
-  // can be reset if a new position is seeked.
-  private int chunkIndexOfPrevPosition;
-
-  public BlockInputStream(BlockID blockId, long blockLen, Pipeline pipeline,
-      Token<OzoneBlockTokenIdentifier> token, boolean verifyChecksum,
-      XceiverClientManager xceiverClientManager) {
-    this.blockID = blockId;
-    this.length = blockLen;
-    this.pipeline = pipeline;
-    this.token = token;
-    this.verifyChecksum = verifyChecksum;
-    this.xceiverClientManager = xceiverClientManager;
-  }
-
-  /**
-   * Initialize the BlockInputStream. Get the BlockData (list of chunks) from
-   * the Container and create the ChunkInputStreams for each Chunk in the Block.
-   */
-  public synchronized void initialize() throws IOException {
-
-    // Pre-check that the stream has not been intialized already
-    if (initialized) {
-      return;
-    }
-
-    List<ChunkInfo> chunks = getChunkInfos();
-    if (chunks != null && !chunks.isEmpty()) {
-      // For each chunk in the block, create a ChunkInputStream and compute
-      // its chunkOffset
-      this.chunkOffsets = new long[chunks.size()];
-      long tempOffset = 0;
-
-      this.chunkStreams = new ArrayList<>(chunks.size());
-      for (int i = 0; i < chunks.size(); i++) {
-        addStream(chunks.get(i));
-        chunkOffsets[i] = tempOffset;
-        tempOffset += chunks.get(i).getLen();
-      }
-
-      initialized = true;
-      this.chunkIndex = 0;
-
-      if (blockPosition > 0) {
-        // Stream was seeked to blockPosition before initialization. Seek to the
-        // blockPosition now.
-        seek(blockPosition);
-      }
-    }
-  }
-
-  /**
-   * Send RPC call to get the block info from the container.
-   * @return List of chunks in this block.
-   */
-  protected List<ChunkInfo> getChunkInfos() throws IOException {
-    // irrespective of the container state, we will always read via Standalone
-    // protocol.
-    if (pipeline.getType() != HddsProtos.ReplicationType.STAND_ALONE) {
-      pipeline = Pipeline.newBuilder(pipeline)
-          .setType(HddsProtos.ReplicationType.STAND_ALONE).build();
-    }
-    xceiverClient = xceiverClientManager.acquireClientForReadData(pipeline);
-    boolean success = false;
-    List<ChunkInfo> chunks;
-    try {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Initializing BlockInputStream for get key to access {}",
-            blockID.getContainerID());
-      }
-
-      if (token != null) {
-        UserGroupInformation.getCurrentUser().addToken(token);
-      }
-      DatanodeBlockID datanodeBlockID = blockID
-          .getDatanodeBlockIDProtobuf();
-      GetBlockResponseProto response = ContainerProtocolCalls
-          .getBlock(xceiverClient, datanodeBlockID);
-
-      chunks = response.getBlockData().getChunksList();
-      success = true;
-    } finally {
-      if (!success) {
-        xceiverClientManager.releaseClientForReadData(xceiverClient, false);
-      }
-    }
-
-    return chunks;
-  }
-
-  /**
-   * Append another ChunkInputStream to the end of the list. Note that the
-   * ChunkInputStream is only created here. The chunk will be read from the
-   * Datanode only when a read operation is performed on for that chunk.
-   */
-  protected synchronized void addStream(ChunkInfo chunkInfo) {
-    chunkStreams.add(new ChunkInputStream(chunkInfo, blockID,
-        xceiverClient, verifyChecksum));
-  }
-
-  public synchronized long getRemaining() throws IOException {
-    return length - getPos();
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public synchronized int read() throws IOException {
-    byte[] buf = new byte[1];
-    if (read(buf, 0, 1) == EOF) {
-      return EOF;
-    }
-    return Byte.toUnsignedInt(buf[0]);
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public synchronized int read(byte[] b, int off, int len) throws IOException {
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if (off < 0 || len < 0 || len > b.length - off) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return 0;
-    }
-
-    if (!initialized) {
-      initialize();
-    }
-
-    checkOpen();
-    int totalReadLen = 0;
-    while (len > 0) {
-      // if we are at the last chunk and have read the entire chunk, return
-      if (chunkStreams.size() == 0 ||
-          (chunkStreams.size() - 1 <= chunkIndex &&
-              chunkStreams.get(chunkIndex)
-                  .getRemaining() == 0)) {
-        return totalReadLen == 0 ? EOF : totalReadLen;
-      }
-
-      // Get the current chunkStream and read data from it
-      ChunkInputStream current = chunkStreams.get(chunkIndex);
-      int numBytesToRead = Math.min(len, (int)current.getRemaining());
-      int numBytesRead = current.read(b, off, numBytesToRead);
-      if (numBytesRead != numBytesToRead) {
-        // This implies that there is either data loss or corruption in the
-        // chunk entries. Even EOF in the current stream would be covered in
-        // this case.
-        throw new IOException(String.format(
-            "Inconsistent read for chunkName=%s length=%d numBytesRead=%d",
-            current.getChunkName(), current.getLength(), numBytesRead));
-      }
-      totalReadLen += numBytesRead;
-      off += numBytesRead;
-      len -= numBytesRead;
-      if (current.getRemaining() <= 0 &&
-          ((chunkIndex + 1) < chunkStreams.size())) {
-        chunkIndex += 1;
-      }
-    }
-    return totalReadLen;
-  }
-
-  /**
-   * Seeks the BlockInputStream to the specified position. If the stream is
-   * not initialized, save the seeked position via blockPosition. Otherwise,
-   * update the position in 2 steps:
-   *    1. Updating the chunkIndex to the chunkStream corresponding to the
-   *    seeked position.
-   *    2. Seek the corresponding chunkStream to the adjusted position.
-   *
-   * Let’s say we have chunk size as 40 bytes. And let's say the parent block
-   * stores data from index 200 and has length 400. If the key was seeked to
-   * position 90, then this block will be seeked to position 90.
-   * When seek(90) is called on this blockStream, then
-   *    1. chunkIndex will be set to 2 (as indices 80 - 120 reside in chunk[2]).
-   *    2. chunkStream[2] will be seeked to position 10
-   *       (= 90 - chunkOffset[2] (= 80)).
-   */
-  @Override
-  public synchronized void seek(long pos) throws IOException {
-    if (!initialized) {
-      // Stream has not been initialized yet. Save the position so that it
-      // can be seeked when the stream is initialized.
-      blockPosition = pos;
-      return;
-    }
-
-    checkOpen();
-    if (pos < 0 || pos >= length) {
-      if (pos == 0) {
-        // It is possible for length and pos to be zero in which case
-        // seek should return instead of throwing exception
-        return;
-      }
-      throw new EOFException(
-          "EOF encountered at pos: " + pos + " for block: " + blockID);
-    }
-
-    if (chunkIndex >= chunkStreams.size()) {
-      chunkIndex = Arrays.binarySearch(chunkOffsets, pos);
-    } else if (pos < chunkOffsets[chunkIndex]) {
-      chunkIndex =
-          Arrays.binarySearch(chunkOffsets, 0, chunkIndex, pos);
-    } else if (pos >= chunkOffsets[chunkIndex] + chunkStreams
-        .get(chunkIndex).getLength()) {
-      chunkIndex = Arrays.binarySearch(chunkOffsets,
-          chunkIndex + 1, chunkStreams.size(), pos);
-    }
-    if (chunkIndex < 0) {
-      // Binary search returns -insertionPoint - 1  if element is not present
-      // in the array. insertionPoint is the point at which element would be
-      // inserted in the sorted array. We need to adjust the chunkIndex
-      // accordingly so that chunkIndex = insertionPoint - 1
-      chunkIndex = -chunkIndex - 2;
-    }
-
-    // Reset the previous chunkStream's position
-    chunkStreams.get(chunkIndexOfPrevPosition).resetPosition();
-
-    // seek to the proper offset in the ChunkInputStream
-    chunkStreams.get(chunkIndex).seek(pos - chunkOffsets[chunkIndex]);
-    chunkIndexOfPrevPosition = chunkIndex;
-  }
-
-  @Override
-  public synchronized long getPos() throws IOException {
-    if (length == 0) {
-      return 0;
-    }
-
-    if (!initialized) {
-      // The stream is not initialized yet. Return the blockPosition
-      return blockPosition;
-    } else {
-      return chunkOffsets[chunkIndex] + chunkStreams.get(chunkIndex).getPos();
-    }
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-
-  @Override
-  public synchronized void close() {
-    if (xceiverClientManager != null && xceiverClient != null) {
-      xceiverClientManager.releaseClient(xceiverClient, false);
-      xceiverClientManager = null;
-      xceiverClient = null;
-    }
-  }
-
-  public synchronized void resetPosition() {
-    this.blockPosition = 0;
-  }
-
-  /**
-   * Checks if the stream is open.  If not, throw an exception.
-   *
-   * @throws IOException if stream is closed
-   */
-  protected synchronized void checkOpen() throws IOException {
-    if (xceiverClient == null) {
-      throw new IOException("BlockInputStream has been closed.");
-    }
-  }
-
-  public BlockID getBlockID() {
-    return blockID;
-  }
-
-  public long getLength() {
-    return length;
-  }
-
-  @VisibleForTesting
-  synchronized int getChunkIndex() {
-    return chunkIndex;
-  }
-
-  @VisibleForTesting
-  synchronized long getBlockPosition() {
-    return blockPosition;
-  }
-
-  @VisibleForTesting
-  synchronized List<ChunkInputStream> getChunkStreams() {
-    return chunkStreams;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java

@@ -1,640 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.XceiverClientReply;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.ozone.common.Checksum;
-import org.apache.hadoop.ozone.common.ChecksumData;
-import org.apache.hadoop.ozone.common.OzoneChecksumException;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
-    .putBlockAsync;
-import static org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls
-    .writeChunkAsync;
-
-/**
- * An {@link OutputStream} used by the REST service in combination with the
- * SCMClient to write the value of a key to a sequence
- * of container chunks.  Writes are buffered locally and periodically written to
- * the container as a new chunk.  In order to preserve the semantics that
- * replacement of a pre-existing key is atomic, each instance of the stream has
- * an internal unique identifier.  This unique identifier and a monotonically
- * increasing chunk index form a composite key that is used as the chunk name.
- * After all data is written, a putKey call creates or updates the corresponding
- * container key, and this call includes the full list of chunks that make up
- * the key data.  The list of chunks is updated all at once.  Therefore, a
- * concurrent reader never can see an intermediate state in which different
- * chunks of data from different versions of the key data are interleaved.
- * This class encapsulates all state management for buffering and writing
- * through to the container.
- */
-public class BlockOutputStream extends OutputStream {
-  public static final Logger LOG =
-      LoggerFactory.getLogger(BlockOutputStream.class);
-
-  private volatile BlockID blockID;
-
-  private final BlockData.Builder containerBlockData;
-  private XceiverClientManager xceiverClientManager;
-  private XceiverClientSpi xceiverClient;
-  private final ContainerProtos.ChecksumType checksumType;
-  private final int bytesPerChecksum;
-  private int chunkIndex;
-  private int chunkSize;
-  private final long streamBufferFlushSize;
-  private final long streamBufferMaxSize;
-  private BufferPool bufferPool;
-  // The IOException will be set by response handling thread in case there is an
-  // exception received in the response. If the exception is set, the next
-  // request will fail upfront.
-  private AtomicReference<IOException> ioException;
-  private ExecutorService responseExecutor;
-
-  // the effective length of data flushed so far
-  private long totalDataFlushedLength;
-
-  // effective data write attempted so far for the block
-  private long writtenDataLength;
-
-  // List containing buffers for which the putBlock call will
-  // update the length in the datanodes. This list will just maintain
-  // references to the buffers in the BufferPool which will be cleared
-  // when the watchForCommit acknowledges a putBlock logIndex has been
-  // committed on all datanodes. This list will be a  place holder for buffers
-  // which got written between successive putBlock calls.
-  private List<ByteBuffer> bufferList;
-
-  // This object will maintain the commitIndexes and byteBufferList in order
-  // Also, corresponding to the logIndex, the corresponding list of buffers will
-  // be released from the buffer pool.
-  private final CommitWatcher commitWatcher;
-
-  private List<DatanodeDetails> failedServers;
-
-  /**
-   * Creates a new BlockOutputStream.
-   *
-   * @param blockID              block ID
-   * @param xceiverClientManager client manager that controls client
-   * @param pipeline             pipeline where block will be written
-   * @param chunkSize            chunk size
-   * @param bufferPool           pool of buffers
-   * @param streamBufferFlushSize flush size
-   * @param streamBufferMaxSize   max size of the currentBuffer
-   * @param watchTimeout          watch timeout
-   * @param checksumType          checksum type
-   * @param bytesPerChecksum      Bytes per checksum
-   */
-  @SuppressWarnings("parameternumber")
-  public BlockOutputStream(BlockID blockID,
-      XceiverClientManager xceiverClientManager, Pipeline pipeline,
-      int chunkSize, long streamBufferFlushSize, long streamBufferMaxSize,
-      long watchTimeout, BufferPool bufferPool, ChecksumType checksumType,
-      int bytesPerChecksum)
-      throws IOException {
-    this.blockID = blockID;
-    this.chunkSize = chunkSize;
-    KeyValue keyValue =
-        KeyValue.newBuilder().setKey("TYPE").setValue("KEY").build();
-    this.containerBlockData =
-        BlockData.newBuilder().setBlockID(blockID.getDatanodeBlockIDProtobuf())
-            .addMetadata(keyValue);
-    this.xceiverClientManager = xceiverClientManager;
-    this.xceiverClient = xceiverClientManager.acquireClient(pipeline);
-    this.chunkIndex = 0;
-    this.streamBufferFlushSize = streamBufferFlushSize;
-    this.streamBufferMaxSize = streamBufferMaxSize;
-    this.bufferPool = bufferPool;
-    this.checksumType = checksumType;
-    this.bytesPerChecksum = bytesPerChecksum;
-
-    // A single thread executor handle the responses of async requests
-    responseExecutor = Executors.newSingleThreadExecutor();
-    commitWatcher = new CommitWatcher(bufferPool, xceiverClient, watchTimeout);
-    bufferList = null;
-    totalDataFlushedLength = 0;
-    writtenDataLength = 0;
-    failedServers = new ArrayList<>(0);
-    ioException = new AtomicReference<>(null);
-  }
-
-
-  public BlockID getBlockID() {
-    return blockID;
-  }
-
-  public long getTotalAckDataLength() {
-    return commitWatcher.getTotalAckDataLength();
-  }
-
-  public long getWrittenDataLength() {
-    return writtenDataLength;
-  }
-
-  public List<DatanodeDetails> getFailedServers() {
-    return failedServers;
-  }
-
-  @VisibleForTesting
-  public XceiverClientSpi getXceiverClient() {
-    return xceiverClient;
-  }
-
-  @VisibleForTesting
-  public long getTotalDataFlushedLength() {
-    return totalDataFlushedLength;
-  }
-
-  @VisibleForTesting
-  public BufferPool getBufferPool() {
-    return bufferPool;
-  }
-
-  public IOException getIoException() {
-    return ioException.get();
-  }
-
-  @VisibleForTesting
-  public Map<Long, List<ByteBuffer>> getCommitIndex2flushedDataMap() {
-    return commitWatcher.getCommitIndex2flushedDataMap();
-  }
-
-  @Override
-  public void write(int b) throws IOException {
-    checkOpen();
-    byte[] buf = new byte[1];
-    buf[0] = (byte) b;
-    write(buf, 0, 1);
-  }
-
-  @Override
-  public void write(byte[] b, int off, int len) throws IOException {
-    checkOpen();
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length)
-        || ((off + len) < 0)) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return;
-    }
-
-    while (len > 0) {
-      int writeLen;
-      // Allocate a buffer if needed. The buffer will be allocated only
-      // once as needed and will be reused again for multiple blockOutputStream
-      // entries.
-      ByteBuffer  currentBuffer = bufferPool.allocateBufferIfNeeded();
-      int pos = currentBuffer.position();
-      writeLen =
-          Math.min(chunkSize - pos % chunkSize, len);
-      currentBuffer.put(b, off, writeLen);
-      if (!currentBuffer.hasRemaining()) {
-        writeChunk(currentBuffer);
-      }
-      off += writeLen;
-      len -= writeLen;
-      writtenDataLength += writeLen;
-      if (shouldFlush()) {
-        updateFlushLength();
-        executePutBlock();
-      }
-      // Data in the bufferPool can not exceed streamBufferMaxSize
-      if (isBufferPoolFull()) {
-        handleFullBuffer();
-      }
-    }
-  }
-
-  private boolean shouldFlush() {
-    return bufferPool.computeBufferData() % streamBufferFlushSize == 0;
-  }
-
-  private void updateFlushLength() {
-    totalDataFlushedLength += writtenDataLength - totalDataFlushedLength;
-  }
-
-  private boolean isBufferPoolFull() {
-    return bufferPool.computeBufferData() == streamBufferMaxSize;
-  }
-  /**
-   * Will be called on the retryPath in case closedContainerException/
-   * TimeoutException.
-   * @param len length of data to write
-   * @throws IOException if error occurred
-   */
-
-  // In this case, the data is already cached in the currentBuffer.
-  public void writeOnRetry(long len) throws IOException {
-    if (len == 0) {
-      return;
-    }
-    int count = 0;
-    Preconditions.checkArgument(len <= streamBufferMaxSize);
-    while (len > 0) {
-      long writeLen;
-      writeLen = Math.min(chunkSize, len);
-      if (writeLen == chunkSize) {
-        writeChunk(bufferPool.getBuffer(count));
-      }
-      len -= writeLen;
-      count++;
-      writtenDataLength += writeLen;
-      // we should not call isBufferFull/shouldFlush here.
-      // The buffer might already be full as whole data is already cached in
-      // the buffer. We should just validate
-      // if we wrote data of size streamBufferMaxSize/streamBufferFlushSize to
-      // call for handling full buffer/flush buffer condition.
-      if (writtenDataLength % streamBufferFlushSize == 0) {
-        // reset the position to zero as now we will be reading the
-        // next buffer in the list
-        updateFlushLength();
-        executePutBlock();
-      }
-      if (writtenDataLength == streamBufferMaxSize) {
-        handleFullBuffer();
-      }
-    }
-  }
-
-  /**
-   * This is a blocking call. It will wait for the flush till the commit index
-   * at the head of the commitIndex2flushedDataMap gets replicated to all or
-   * majority.
-   * @throws IOException
-   */
-  private void handleFullBuffer() throws IOException {
-    try {
-      checkOpen();
-      if (!commitWatcher.getFutureMap().isEmpty()) {
-        waitOnFlushFutures();
-      }
-    } catch (InterruptedException | ExecutionException e) {
-      setIoException(e);
-      adjustBuffersOnException();
-      throw getIoException();
-    }
-    watchForCommit(true);
-  }
-
-
-  // It may happen that once the exception is encountered , we still might
-  // have successfully flushed up to a certain index. Make sure the buffers
-  // only contain data which have not been sufficiently replicated
-  private void adjustBuffersOnException() {
-    commitWatcher.releaseBuffersOnException();
-  }
-
-  /**
-   * calls watchForCommit API of the Ratis Client. For Standalone client,
-   * it is a no op.
-   * @param bufferFull flag indicating whether bufferFull condition is hit or
-   *              its called as part flush/close
-   * @return minimum commit index replicated to all nodes
-   * @throws IOException IOException in case watch gets timed out
-   */
-  private void watchForCommit(boolean bufferFull) throws IOException {
-    checkOpen();
-    try {
-      XceiverClientReply reply = bufferFull ?
-          commitWatcher.watchOnFirstIndex() : commitWatcher.watchOnLastIndex();
-      if (reply != null) {
-        List<DatanodeDetails> dnList = reply.getDatanodes();
-        if (!dnList.isEmpty()) {
-          Pipeline pipe = xceiverClient.getPipeline();
-
-          LOG.warn("Failed to commit BlockId {} on {}. Failed nodes: {}",
-              blockID, pipe, dnList);
-          failedServers.addAll(dnList);
-        }
-      }
-    } catch (IOException ioe) {
-      setIoException(ioe);
-      throw getIoException();
-    }
-  }
-
-  private CompletableFuture<ContainerProtos.
-      ContainerCommandResponseProto> executePutBlock()
-      throws IOException {
-    checkOpen();
-    long flushPos = totalDataFlushedLength;
-    Preconditions.checkNotNull(bufferList);
-    List<ByteBuffer> byteBufferList = bufferList;
-    bufferList = null;
-    Preconditions.checkNotNull(byteBufferList);
-
-    CompletableFuture<ContainerProtos.
-        ContainerCommandResponseProto> flushFuture;
-    try {
-      XceiverClientReply asyncReply =
-          putBlockAsync(xceiverClient, containerBlockData.build());
-      CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
-          asyncReply.getResponse();
-      flushFuture = future.thenApplyAsync(e -> {
-        try {
-          validateResponse(e);
-        } catch (IOException sce) {
-          throw new CompletionException(sce);
-        }
-        // if the ioException is not set, putBlock is successful
-        if (getIoException() == null) {
-          BlockID responseBlockID = BlockID.getFromProtobuf(
-              e.getPutBlock().getCommittedBlockLength().getBlockID());
-          Preconditions.checkState(blockID.getContainerBlockID()
-              .equals(responseBlockID.getContainerBlockID()));
-          // updates the bcsId of the block
-          blockID = responseBlockID;
-          if (LOG.isDebugEnabled()) {
-            LOG.debug(
-                "Adding index " + asyncReply.getLogIndex() + " commitMap size "
-                    + commitWatcher.getCommitInfoMapSize() + " flushLength "
-                    + flushPos + " numBuffers " + byteBufferList.size()
-                    + " blockID " + blockID + " bufferPool size" + bufferPool
-                    .getSize() + " currentBufferIndex " + bufferPool
-                    .getCurrentBufferIndex());
-          }
-          // for standalone protocol, logIndex will always be 0.
-          commitWatcher
-              .updateCommitInfoMap(asyncReply.getLogIndex(), byteBufferList);
-        }
-        return e;
-      }, responseExecutor).exceptionally(e -> {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(
-              "putBlock failed for blockID " + blockID + " with exception " + e
-                  .getLocalizedMessage());
-        }
-        CompletionException ce =  new CompletionException(e);
-        setIoException(ce);
-        throw ce;
-      });
-    } catch (IOException | InterruptedException | ExecutionException e) {
-      throw new IOException(
-          "Unexpected Storage Container Exception: " + e.toString(), e);
-    }
-    commitWatcher.getFutureMap().put(flushPos, flushFuture);
-    return flushFuture;
-  }
-
-  @Override
-  public void flush() throws IOException {
-    if (xceiverClientManager != null && xceiverClient != null
-        && bufferPool != null && bufferPool.getSize() > 0) {
-      try {
-        handleFlush();
-      } catch (InterruptedException | ExecutionException e) {
-        // just set the exception here as well in order to maintain sanctity of
-        // ioException field
-        setIoException(e);
-        adjustBuffersOnException();
-        throw getIoException();
-      }
-    }
-  }
-
-
-  private void writeChunk(ByteBuffer buffer)
-      throws IOException {
-    // This data in the buffer will be pushed to datanode and a reference will
-    // be added to the bufferList. Once putBlock gets executed, this list will
-    // be marked null. Hence, during first writeChunk call after every putBlock
-    // call or during the first call to writeChunk here, the list will be null.
-
-    if (bufferList == null) {
-      bufferList = new ArrayList<>();
-    }
-    bufferList.add(buffer);
-    // Please note : We are not flipping the slice when we write since
-    // the slices are pointing the currentBuffer start and end as needed for
-    // the chunk write. Also please note, Duplicate does not create a
-    // copy of data, it only creates metadata that points to the data
-    // stream.
-    ByteBuffer chunk = buffer.duplicate();
-    chunk.position(0);
-    chunk.limit(buffer.position());
-    writeChunkToContainer(chunk);
-  }
-
-  private void handleFlush()
-      throws IOException, InterruptedException, ExecutionException {
-    checkOpen();
-    // flush the last chunk data residing on the currentBuffer
-    if (totalDataFlushedLength < writtenDataLength) {
-      ByteBuffer currentBuffer = bufferPool.getCurrentBuffer();
-      Preconditions.checkArgument(currentBuffer.position() > 0);
-      if (currentBuffer.position() != chunkSize) {
-        writeChunk(currentBuffer);
-      }
-      // This can be a partially filled chunk. Since we are flushing the buffer
-      // here, we just limit this buffer to the current position. So that next
-      // write will happen in new buffer
-      updateFlushLength();
-      executePutBlock();
-    }
-    waitOnFlushFutures();
-    watchForCommit(false);
-    // just check again if the exception is hit while waiting for the
-    // futures to ensure flush has indeed succeeded
-
-    // irrespective of whether the commitIndex2flushedDataMap is empty
-    // or not, ensure there is no exception set
-    checkOpen();
-  }
-
-  @Override
-  public void close() throws IOException {
-    if (xceiverClientManager != null && xceiverClient != null
-        && bufferPool != null && bufferPool.getSize() > 0) {
-      try {
-        handleFlush();
-      } catch (InterruptedException | ExecutionException e) {
-        setIoException(e);
-        adjustBuffersOnException();
-        throw getIoException();
-      } finally {
-        cleanup(false);
-      }
-      // TODO: Turn the below buffer empty check on when Standalone pipeline
-      // is removed in the write path in tests
-      // Preconditions.checkArgument(buffer.position() == 0);
-      // bufferPool.checkBufferPoolEmpty();
-
-    }
-  }
-
-  private void waitOnFlushFutures()
-      throws InterruptedException, ExecutionException {
-    CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(
-        commitWatcher.getFutureMap().values().toArray(
-            new CompletableFuture[commitWatcher.getFutureMap().size()]));
-    // wait for all the transactions to complete
-    combinedFuture.get();
-  }
-
-  private void validateResponse(
-      ContainerProtos.ContainerCommandResponseProto responseProto)
-      throws IOException {
-    try {
-      // if the ioException is already set, it means a prev request has failed
-      // just throw the exception. The current operation will fail with the
-      // original error
-      IOException exception = getIoException();
-      if (exception != null) {
-        throw exception;
-      }
-      ContainerProtocolCalls.validateContainerResponse(responseProto);
-    } catch (StorageContainerException sce) {
-      LOG.error("Unexpected Storage Container Exception: ", sce);
-      setIoException(sce);
-      throw sce;
-    }
-  }
-
-
-  private void setIoException(Exception e) {
-    if (getIoException() == null) {
-      IOException exception =  new IOException(
-          "Unexpected Storage Container Exception: " + e.toString(), e);
-      ioException.compareAndSet(null, exception);
-    }
-  }
-
-  public void cleanup(boolean invalidateClient) {
-    if (xceiverClientManager != null) {
-      xceiverClientManager.releaseClient(xceiverClient, invalidateClient);
-    }
-    xceiverClientManager = null;
-    xceiverClient = null;
-    commitWatcher.cleanup();
-    if (bufferList !=  null) {
-      bufferList.clear();
-    }
-    bufferList = null;
-    responseExecutor.shutdown();
-  }
-
-  /**
-   * Checks if the stream is open or exception has occured.
-   * If not, throws an exception.
-   *
-   * @throws IOException if stream is closed
-   */
-  private void checkOpen() throws IOException {
-    if (isClosed()) {
-      throw new IOException("BlockOutputStream has been closed.");
-    } else if (getIoException() != null) {
-      adjustBuffersOnException();
-      throw getIoException();
-    }
-  }
-
-  public boolean isClosed() {
-    return xceiverClient == null;
-  }
-
-  /**
-   * Writes buffered data as a new chunk to the container and saves chunk
-   * information to be used later in putKey call.
-   *
-   * @throws IOException if there is an I/O error while performing the call
-   * @throws OzoneChecksumException if there is an error while computing
-   * checksum
-   */
-  private void writeChunkToContainer(ByteBuffer chunk) throws IOException {
-    int effectiveChunkSize = chunk.remaining();
-    ByteString data = bufferPool.byteStringConversion().apply(chunk);
-    Checksum checksum = new Checksum(checksumType, bytesPerChecksum);
-    ChecksumData checksumData = checksum.computeChecksum(chunk);
-    ChunkInfo chunkInfo = ChunkInfo.newBuilder()
-        .setChunkName(blockID.getLocalID() + "_chunk_" + ++chunkIndex)
-        .setOffset(0)
-        .setLen(effectiveChunkSize)
-        .setChecksumData(checksumData.getProtoBufMessage())
-        .build();
-
-    try {
-      XceiverClientReply asyncReply =
-          writeChunkAsync(xceiverClient, chunkInfo, blockID, data);
-      CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
-          asyncReply.getResponse();
-      future.thenApplyAsync(e -> {
-        try {
-          validateResponse(e);
-        } catch (IOException sce) {
-          future.completeExceptionally(sce);
-        }
-        return e;
-      }, responseExecutor).exceptionally(e -> {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(
-              "writing chunk failed " + chunkInfo.getChunkName() + " blockID "
-                  + blockID + " with exception " + e.getLocalizedMessage());
-        }
-        CompletionException ce = new CompletionException(e);
-        setIoException(ce);
-        throw ce;
-      });
-    } catch (IOException | InterruptedException | ExecutionException e) {
-      throw new IOException(
-          "Unexpected Storage Container Exception: " + e.toString(), e);
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(
-          "writing chunk " + chunkInfo.getChunkName() + " blockID " + blockID
-              + " length " + effectiveChunkSize);
-    }
-    containerBlockData.addChunks(chunkInfo);
-  }
-
-  @VisibleForTesting
-  public void setXceiverClient(XceiverClientSpi xceiverClient) {
-    this.xceiverClient = xceiverClient;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BufferPool.java

@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.ByteStringConversion;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Function;
-
-/**
- * This class creates and manages pool of n buffers.
- */
-public class BufferPool {
-
-  private List<ByteBuffer> bufferList;
-  private int currentBufferIndex;
-  private final int bufferSize;
-  private final int capacity;
-  private final Function<ByteBuffer, ByteString> byteStringConversion;
-
-  public BufferPool(int bufferSize, int capacity) {
-    this(bufferSize, capacity,
-        ByteStringConversion.createByteBufferConversion(null));
-  }
-
-  public BufferPool(int bufferSize, int capacity,
-      Function<ByteBuffer, ByteString> byteStringConversion){
-    this.capacity = capacity;
-    this.bufferSize = bufferSize;
-    bufferList = new ArrayList<>(capacity);
-    currentBufferIndex = -1;
-    this.byteStringConversion = byteStringConversion;
-  }
-
-  public Function<ByteBuffer, ByteString> byteStringConversion(){
-    return byteStringConversion;
-  }
-
-  public ByteBuffer getCurrentBuffer() {
-    return currentBufferIndex == -1 ? null : bufferList.get(currentBufferIndex);
-  }
-
-  /**
-   * If the currentBufferIndex is less than the buffer size - 1,
-   * it means, the next buffer in the list has been freed up for
-   * rewriting. Reuse the next available buffer in such cases.
-   *
-   * In case, the currentBufferIndex == buffer.size and buffer size is still
-   * less than the capacity to be allocated, just allocate a buffer of size
-   * chunk size.
-   *
-   */
-  public ByteBuffer allocateBufferIfNeeded() {
-    ByteBuffer buffer = getCurrentBuffer();
-    if (buffer != null && buffer.hasRemaining()) {
-      return buffer;
-    }
-    if (currentBufferIndex < bufferList.size() - 1) {
-      buffer = getBuffer(currentBufferIndex + 1);
-    } else {
-      buffer = ByteBuffer.allocate(bufferSize);
-      bufferList.add(buffer);
-    }
-    Preconditions.checkArgument(bufferList.size() <= capacity);
-    currentBufferIndex++;
-    // TODO: Turn the below precondition check on when Standalone pipeline
-    // is removed in the write path in tests
-    // Preconditions.checkArgument(buffer.position() == 0);
-    return buffer;
-  }
-
-  public void releaseBuffer(ByteBuffer byteBuffer) {
-    // always remove from head of the list and append at last
-    ByteBuffer buffer = bufferList.remove(0);
-    // Ensure the buffer to be removed is always at the head of the list.
-    Preconditions.checkArgument(buffer.equals(byteBuffer));
-    buffer.clear();
-    bufferList.add(buffer);
-    Preconditions.checkArgument(currentBufferIndex >= 0);
-    currentBufferIndex--;
-  }
-
-  public void clearBufferPool() {
-    bufferList.clear();
-    currentBufferIndex = -1;
-  }
-
-  public void checkBufferPoolEmpty() {
-    Preconditions.checkArgument(computeBufferData() == 0);
-  }
-
-  public long computeBufferData() {
-    return bufferList.stream().mapToInt(value -> value.position())
-        .sum();
-  }
-
-  public int getSize() {
-    return bufferList.size();
-  }
-
-  public ByteBuffer getBuffer(int index) {
-    return bufferList.get(index);
-  }
-
-  int getCurrentBufferIndex() {
-    return currentBufferIndex;
-  }
-
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/ChunkInputStream.java

@@ -1,544 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkResponseProto;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.common.Checksum;
-import org.apache.hadoop.ozone.common.ChecksumData;
-import org.apache.hadoop.ozone.common.OzoneChecksumException;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.List;
-
-/**
- * An {@link InputStream} called from BlockInputStream to read a chunk from the
- * container. Each chunk may contain multiple underlying {@link ByteBuffer}
- * instances.
- */
-public class ChunkInputStream extends InputStream implements Seekable {
-
-  private ChunkInfo chunkInfo;
-  private final long length;
-  private final BlockID blockID;
-  private XceiverClientSpi xceiverClient;
-  private boolean verifyChecksum;
-  private boolean allocated = false;
-
-  // Buffer to store the chunk data read from the DN container
-  private List<ByteBuffer> buffers;
-
-  // Index of the buffers corresponding to the current position of the buffers
-  private int bufferIndex;
-
-  // The offset of the current data residing in the buffers w.r.t the start
-  // of chunk data
-  private long bufferOffset;
-
-  // The number of bytes of chunk data residing in the buffers currently
-  private long bufferLength;
-
-  // Position of the ChunkInputStream is maintained by this variable (if a
-  // seek is performed. This position is w.r.t to the chunk only and not the
-  // block or key. This variable is set only if either the buffers are not
-  // yet allocated or the if the allocated buffers do not cover the seeked
-  // position. Once the chunk is read, this variable is reset.
-  private long chunkPosition = -1;
-
-  private static final int EOF = -1;
-
-  ChunkInputStream(ChunkInfo chunkInfo, BlockID blockId, 
-          XceiverClientSpi xceiverClient, boolean verifyChecksum) {
-    this.chunkInfo = chunkInfo;
-    this.length = chunkInfo.getLen();
-    this.blockID = blockId;
-    this.xceiverClient = xceiverClient;
-    this.verifyChecksum = verifyChecksum;
-  }
-
-  public synchronized long getRemaining() throws IOException {
-    return length - getPos();
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public synchronized int read() throws IOException {
-    checkOpen();
-    int available = prepareRead(1);
-    int dataout = EOF;
-
-    if (available == EOF) {
-      // There is no more data in the chunk stream. The buffers should have
-      // been released by now
-      Preconditions.checkState(buffers == null);
-    } else {
-      dataout = Byte.toUnsignedInt(buffers.get(bufferIndex).get());
-    }
-
-    if (chunkStreamEOF()) {
-      // consumer might use getPos to determine EOF,
-      // so release buffers when serving the last byte of data
-      releaseBuffers();
-    }
-
-    return dataout;
-  }
-
-  /**
-   * {@inheritDoc}
-   */
-  @Override
-  public synchronized int read(byte[] b, int off, int len) throws IOException {
-    // According to the JavaDocs for InputStream, it is recommended that
-    // subclasses provide an override of bulk read if possible for performance
-    // reasons.  In addition to performance, we need to do it for correctness
-    // reasons.  The Ozone REST service uses PipedInputStream and
-    // PipedOutputStream to relay HTTP response data between a Jersey thread and
-    // a Netty thread.  It turns out that PipedInputStream/PipedOutputStream
-    // have a subtle dependency (bug?) on the wrapped stream providing separate
-    // implementations of single-byte read and bulk read.  Without this, get key
-    // responses might close the connection before writing all of the bytes
-    // advertised in the Content-Length.
-    if (b == null) {
-      throw new NullPointerException();
-    }
-    if (off < 0 || len < 0 || len > b.length - off) {
-      throw new IndexOutOfBoundsException();
-    }
-    if (len == 0) {
-      return 0;
-    }
-    checkOpen();
-    int total = 0;
-    while (len > 0) {
-      int available = prepareRead(len);
-      if (available == EOF) {
-        // There is no more data in the chunk stream. The buffers should have
-        // been released by now
-        Preconditions.checkState(buffers == null);
-        return total != 0 ? total : EOF;
-      }
-      buffers.get(bufferIndex).get(b, off + total, available);
-      len -= available;
-      total += available;
-    }
-
-    if (chunkStreamEOF()) {
-      // smart consumers determine EOF by calling getPos()
-      // so we release buffers when serving the final bytes of data
-      releaseBuffers();
-    }
-
-    return total;
-  }
-
-  /**
-   * Seeks the ChunkInputStream to the specified position. This is done by
-   * updating the chunkPosition to the seeked position in case the buffers
-   * are not allocated or buffers do not contain the data corresponding to
-   * the seeked position (determined by buffersHavePosition()). Otherwise,
-   * the buffers position is updated to the seeked position.
-   */
-  @Override
-  public synchronized void seek(long pos) throws IOException {
-    if (pos < 0 || pos >= length) {
-      if (pos == 0) {
-        // It is possible for length and pos to be zero in which case
-        // seek should return instead of throwing exception
-        return;
-      }
-      throw new EOFException("EOF encountered at pos: " + pos + " for chunk: "
-          + chunkInfo.getChunkName());
-    }
-
-    if (buffersHavePosition(pos)) {
-      // The bufferPosition is w.r.t the current chunk.
-      // Adjust the bufferIndex and position to the seeked position.
-      adjustBufferPosition(pos - bufferOffset);
-    } else {
-      chunkPosition = pos;
-    }
-  }
-
-  @Override
-  public synchronized long getPos() throws IOException {
-    if (chunkPosition >= 0) {
-      return chunkPosition;
-    }
-    if (chunkStreamEOF()) {
-      return length;
-    }
-    if (buffersHaveData()) {
-      return bufferOffset + buffers.get(bufferIndex).position();
-    }
-    if (buffersAllocated()) {
-      return bufferOffset + bufferLength;
-    }
-    return 0;
-  }
-
-  @Override
-  public boolean seekToNewSource(long targetPos) throws IOException {
-    return false;
-  }
-
-  @Override
-  public synchronized void close() {
-    if (xceiverClient != null) {
-      xceiverClient = null;
-    }
-  }
-
-  /**
-   * Checks if the stream is open.  If not, throw an exception.
-   *
-   * @throws IOException if stream is closed
-   */
-  protected synchronized void checkOpen() throws IOException {
-    if (xceiverClient == null) {
-      throw new IOException("BlockInputStream has been closed.");
-    }
-  }
-
-  /**
-   * Prepares to read by advancing through buffers or allocating new buffers,
-   * as needed until it finds data to return, or encounters EOF.
-   * @param len desired lenght of data to read
-   * @return length of data available to read, possibly less than desired length
-   */
-  private synchronized int prepareRead(int len) throws IOException {
-    for (;;) {
-      if (chunkPosition >= 0) {
-        if (buffersHavePosition(chunkPosition)) {
-          // The current buffers have the seeked position. Adjust the buffer
-          // index and position to point to the chunkPosition.
-          adjustBufferPosition(chunkPosition - bufferOffset);
-        } else {
-          // Read a required chunk data to fill the buffers with seeked
-          // position data
-          readChunkFromContainer(len);
-        }
-      }
-      if (buffersHaveData()) {
-        // Data is available from buffers
-        ByteBuffer bb = buffers.get(bufferIndex);
-        return len > bb.remaining() ? bb.remaining() : len;
-      } else  if (dataRemainingInChunk()) {
-        // There is more data in the chunk stream which has not
-        // been read into the buffers yet.
-        readChunkFromContainer(len);
-      } else {
-        // All available input from this chunk stream has been consumed.
-        return EOF;
-      }
-    }
-  }
-
-  /**
-   * Reads full or partial Chunk from DN Container based on the current
-   * position of the ChunkInputStream, the number of bytes of data to read
-   * and the checksum boundaries.
-   * If successful, then the read data in saved in the buffers so that
-   * subsequent read calls can utilize it.
-   * @param len number of bytes of data to be read
-   * @throws IOException if there is an I/O error while performing the call
-   * to Datanode
-   */
-  private synchronized void readChunkFromContainer(int len) throws IOException {
-
-    // index of first byte to be read from the chunk
-    long startByteIndex;
-    if (chunkPosition >= 0) {
-      // If seek operation was called to advance the buffer position, the
-      // chunk should be read from that position onwards.
-      startByteIndex = chunkPosition;
-    } else {
-      // Start reading the chunk from the last chunkPosition onwards.
-      startByteIndex = bufferOffset + bufferLength;
-    }
-
-    if (verifyChecksum) {
-      // Update the bufferOffset and bufferLength as per the checksum
-      // boundary requirement.
-      computeChecksumBoundaries(startByteIndex, len);
-    } else {
-      // Read from the startByteIndex
-      bufferOffset = startByteIndex;
-      bufferLength = len;
-    }
-
-    // Adjust the chunkInfo so that only the required bytes are read from
-    // the chunk.
-    final ChunkInfo adjustedChunkInfo = ChunkInfo.newBuilder(chunkInfo)
-        .setOffset(bufferOffset)
-        .setLen(bufferLength)
-        .build();
-
-    ByteString byteString = readChunk(adjustedChunkInfo);
-
-    buffers = byteString.asReadOnlyByteBufferList();
-    bufferIndex = 0;
-    allocated = true;
-
-    // If the stream was seeked to position before, then the buffer
-    // position should be adjusted as the reads happen at checksum boundaries.
-    // The buffers position might need to be adjusted for the following
-    // scenarios:
-    //    1. Stream was seeked to a position before the chunk was read
-    //    2. Chunk was read from index < the current position to account for
-    //    checksum boundaries.
-    adjustBufferPosition(startByteIndex - bufferOffset);
-  }
-
-  /**
-   * Send RPC call to get the chunk from the container.
-   */
-  @VisibleForTesting
-  protected ByteString readChunk(ChunkInfo readChunkInfo) throws IOException {
-    ReadChunkResponseProto readChunkResponse;
-
-    try {
-      List<CheckedBiFunction> validators =
-          ContainerProtocolCalls.getValidatorList();
-      validators.add(validator);
-
-      readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient,
-          readChunkInfo, blockID, validators);
-
-    } catch (IOException e) {
-      if (e instanceof StorageContainerException) {
-        throw e;
-      }
-      throw new IOException("Unexpected OzoneException: " + e.toString(), e);
-    }
-
-    return readChunkResponse.getData();
-  }
-
-  private CheckedBiFunction<ContainerCommandRequestProto,
-      ContainerCommandResponseProto, IOException> validator =
-          (request, response) -> {
-            final ChunkInfo reqChunkInfo =
-                request.getReadChunk().getChunkData();
-
-            ReadChunkResponseProto readChunkResponse = response.getReadChunk();
-            ByteString byteString = readChunkResponse.getData();
-
-            if (byteString.size() != reqChunkInfo.getLen()) {
-              // Bytes read from chunk should be equal to chunk size.
-              throw new OzoneChecksumException(String
-                  .format("Inconsistent read for chunk=%s len=%d bytesRead=%d",
-                      reqChunkInfo.getChunkName(), reqChunkInfo.getLen(),
-                      byteString.size()));
-            }
-
-            if (verifyChecksum) {
-              ChecksumData checksumData = ChecksumData.getFromProtoBuf(
-                  chunkInfo.getChecksumData());
-
-              // ChecksumData stores checksum for each 'numBytesPerChecksum'
-              // number of bytes in a list. Compute the index of the first
-              // checksum to match with the read data
-
-              int checkumStartIndex = (int) (reqChunkInfo.getOffset() /
-                  checksumData.getBytesPerChecksum());
-              Checksum.verifyChecksum(
-                  byteString, checksumData, checkumStartIndex);
-            }
-          };
-
-  /**
-   * Return the offset and length of bytes that need to be read from the
-   * chunk file to cover the checksum boundaries covering the actual start and
-   * end of the chunk index to be read.
-   * For example, lets say the client is reading from index 120 to 450 in the
-   * chunk. And let's say checksum is stored for every 100 bytes in the chunk
-   * i.e. the first checksum is for bytes from index 0 to 99, the next for
-   * bytes from index 100 to 199 and so on. To verify bytes from 120 to 450,
-   * we would need to read from bytes 100 to 499 so that checksum
-   * verification can be done.
-   *
-   * @param startByteIndex the first byte index to be read by client
-   * @param dataLen number of bytes to be read from the chunk
-   */
-  private void computeChecksumBoundaries(long startByteIndex, int dataLen) {
-
-    int bytesPerChecksum = chunkInfo.getChecksumData().getBytesPerChecksum();
-    // index of the last byte to be read from chunk, inclusively.
-    final long endByteIndex = startByteIndex + dataLen - 1;
-
-    bufferOffset =  (startByteIndex / bytesPerChecksum)
-        * bytesPerChecksum; // inclusive
-    final long endIndex = ((endByteIndex / bytesPerChecksum) + 1)
-        * bytesPerChecksum; // exclusive
-    bufferLength = Math.min(endIndex, length) - bufferOffset;
-  }
-
-  /**
-   * Adjust the buffers position to account for seeked position and/ or checksum
-   * boundary reads.
-   * @param bufferPosition the position to which the buffers must be advanced
-   */
-  private void adjustBufferPosition(long bufferPosition) {
-    // The bufferPosition is w.r.t the current chunk.
-    // Adjust the bufferIndex and position to the seeked chunkPosition.
-    long tempOffest = 0;
-    for (int i = 0; i < buffers.size(); i++) {
-      if (bufferPosition - tempOffest >= buffers.get(i).capacity()) {
-        tempOffest += buffers.get(i).capacity();
-      } else {
-        bufferIndex = i;
-        break;
-      }
-    }
-    buffers.get(bufferIndex).position((int) (bufferPosition - tempOffest));
-
-    // Reset the chunkPosition as chunk stream has been initialized i.e. the
-    // buffers have been allocated.
-    resetPosition();
-  }
-
-  /**
-   * Check if the buffers have been allocated data and false otherwise.
-   */
-  private boolean buffersAllocated() {
-    return buffers != null && !buffers.isEmpty();
-  }
-
-  /**
-   * Check if the buffers have any data remaining between the current
-   * position and the limit.
-   */
-  private boolean buffersHaveData() {
-    boolean hasData = false;
-
-    if (buffersAllocated()) {
-      while (bufferIndex < (buffers.size())) {
-        if (buffers.get(bufferIndex).hasRemaining()) {
-          // current buffer has data
-          hasData = true;
-          break;
-        } else {
-          if (buffersRemaining()) {
-            // move to next available buffer
-            ++bufferIndex;
-            Preconditions.checkState(bufferIndex < buffers.size());
-          } else {
-            // no more buffers remaining
-            break;
-          }
-        }
-      }
-    }
-
-    return hasData;
-  }
-
-  private boolean buffersRemaining() {
-    return (bufferIndex < (buffers.size() - 1));
-  }
-
-  /**
-   * Check if curernt buffers have the data corresponding to the input position.
-   */
-  private boolean buffersHavePosition(long pos) {
-    // Check if buffers have been allocated
-    if (buffersAllocated()) {
-      // Check if the current buffers cover the input position
-      return pos >= bufferOffset &&
-          pos < bufferOffset + bufferLength;
-    }
-    return false;
-  }
-
-  /**
-   * Check if there is more data in the chunk which has not yet been read
-   * into the buffers.
-   */
-  private boolean dataRemainingInChunk() {
-    long bufferPos;
-    if (chunkPosition >= 0) {
-      bufferPos = chunkPosition;
-    } else {
-      bufferPos = bufferOffset + bufferLength;
-    }
-
-    return bufferPos < length;
-  }
-
-  /**
-   * Check if end of chunkStream has been reached.
-   */
-  private boolean chunkStreamEOF() {
-    if (!allocated) {
-      // Chunk data has not been read yet
-      return false;
-    }
-
-    if (buffersHaveData() || dataRemainingInChunk()) {
-      return false;
-    } else {
-      Preconditions.checkState(bufferOffset + bufferLength == length,
-          "EOF detected, but not at the last byte of the chunk");
-      return true;
-    }
-  }
-
-  /**
-   * If EOF is reached, release the buffers.
-   */
-  private void releaseBuffers() {
-    buffers = null;
-    bufferIndex = 0;
-  }
-
-  /**
-   * Reset the chunkPosition once the buffers are allocated.
-   */
-  void resetPosition() {
-    this.chunkPosition = -1;
-  }
-
-  String getChunkName() {
-    return chunkInfo.getChunkName();
-  }
-
-  protected long getLength() {
-    return length;
-  }
-
-  @VisibleForTesting
-  protected long getChunkPosition() {
-    return chunkPosition;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/CommitWatcher.java

@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/**
- * This class maintains the map of the commitIndexes to be watched for
- * successful replication in the datanodes in a given pipeline. It also releases
- * the buffers associated with the user data back to {@Link BufferPool} once
- * minimum replication criteria is achieved during an ozone key write.
- */
-package org.apache.hadoop.hdds.scm.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.scm.XceiverClientReply;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.ExecutionException;
-
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.stream.Collectors;
-
-/**
- * This class executes watchForCommit on ratis pipeline and releases
- * buffers once data successfully gets replicated.
- */
-public class CommitWatcher {
-
-  private static final Logger LOG =
-      LoggerFactory.getLogger(CommitWatcher.class);
-
-  // A reference to the pool of buffers holding the data
-  private BufferPool bufferPool;
-
-  // The map should maintain the keys (logIndexes) in order so that while
-  // removing we always end up updating incremented data flushed length.
-  // Also, corresponding to the logIndex, the corresponding list of buffers will
-  // be released from the buffer pool.
-  private ConcurrentSkipListMap<Long, List<ByteBuffer>>
-      commitIndex2flushedDataMap;
-
-  // future Map to hold up all putBlock futures
-  private ConcurrentHashMap<Long,
-      CompletableFuture<ContainerProtos.ContainerCommandResponseProto>>
-      futureMap;
-
-  private XceiverClientSpi xceiverClient;
-
-  private final long watchTimeout;
-
-  // total data which has been successfully flushed and acknowledged
-  // by all servers
-  private long totalAckDataLength;
-
-  public CommitWatcher(BufferPool bufferPool, XceiverClientSpi xceiverClient,
-      long watchTimeout) {
-    this.bufferPool = bufferPool;
-    this.xceiverClient = xceiverClient;
-    this.watchTimeout = watchTimeout;
-    commitIndex2flushedDataMap = new ConcurrentSkipListMap<>();
-    totalAckDataLength = 0;
-    futureMap = new ConcurrentHashMap<>();
-  }
-
-  /**
-   * just update the totalAckDataLength. In case of failure,
-   * we will read the data starting from totalAckDataLength.
-   */
-  private long releaseBuffers(List<Long> indexes) {
-    Preconditions.checkArgument(!commitIndex2flushedDataMap.isEmpty());
-    for (long index : indexes) {
-      Preconditions.checkState(commitIndex2flushedDataMap.containsKey(index));
-      List<ByteBuffer> buffers = commitIndex2flushedDataMap.remove(index);
-      long length = buffers.stream().mapToLong(value -> {
-        int pos = value.position();
-        return pos;
-      }).sum();
-      totalAckDataLength += length;
-      // clear the future object from the future Map
-      Preconditions.checkNotNull(futureMap.remove(totalAckDataLength));
-      for (ByteBuffer byteBuffer : buffers) {
-        bufferPool.releaseBuffer(byteBuffer);
-      }
-    }
-    return totalAckDataLength;
-  }
-
-  public void updateCommitInfoMap(long index, List<ByteBuffer> byteBufferList) {
-    commitIndex2flushedDataMap
-        .put(index, byteBufferList);
-  }
-
-  int getCommitInfoMapSize() {
-    return commitIndex2flushedDataMap.size();
-  }
-
-  /**
-   * Calls watch for commit for the first index in commitIndex2flushedDataMap to
-   * the Ratis client.
-   * @return reply reply from raft client
-   * @throws IOException in case watchForCommit fails
-   */
-  public XceiverClientReply watchOnFirstIndex() throws IOException {
-    if (!commitIndex2flushedDataMap.isEmpty()) {
-      // wait for the  first commit index in the commitIndex2flushedDataMap
-      // to get committed to all or majority of nodes in case timeout
-      // happens.
-      long index =
-          commitIndex2flushedDataMap.keySet().stream().mapToLong(v -> v).min()
-              .getAsLong();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("waiting for first index " + index + " to catch up");
-      }
-      return watchForCommit(index);
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Calls watch for commit for the first index in commitIndex2flushedDataMap to
-   * the Ratis client.
-   * @return reply reply from raft client
-   * @throws IOException in case watchForCommit fails
-   */
-  public XceiverClientReply watchOnLastIndex()
-      throws IOException {
-    if (!commitIndex2flushedDataMap.isEmpty()) {
-      // wait for the  commit index in the commitIndex2flushedDataMap
-      // to get committed to all or majority of nodes in case timeout
-      // happens.
-      long index =
-          commitIndex2flushedDataMap.keySet().stream().mapToLong(v -> v).max()
-              .getAsLong();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("waiting for last flush Index " + index + " to catch up");
-      }
-      return watchForCommit(index);
-    } else {
-      return null;
-    }
-  }
-
-
-  private void adjustBuffers(long commitIndex) {
-    List<Long> keyList = commitIndex2flushedDataMap.keySet().stream()
-        .filter(p -> p <= commitIndex).collect(Collectors.toList());
-    if (keyList.isEmpty()) {
-      return;
-    } else {
-      releaseBuffers(keyList);
-    }
-  }
-
-  // It may happen that once the exception is encountered , we still might
-  // have successfully flushed up to a certain index. Make sure the buffers
-  // only contain data which have not been sufficiently replicated
-  void releaseBuffersOnException() {
-    adjustBuffers(xceiverClient.getReplicatedMinCommitIndex());
-  }
-
-
-  /**
-   * calls watchForCommit API of the Ratis Client. For Standalone client,
-   * it is a no op.
-   * @param commitIndex log index to watch for
-   * @return minimum commit index replicated to all nodes
-   * @throws IOException IOException in case watch gets timed out
-   */
-  public XceiverClientReply watchForCommit(long commitIndex)
-      throws IOException {
-    long index;
-    try {
-      XceiverClientReply reply =
-          xceiverClient.watchForCommit(commitIndex, watchTimeout);
-      if (reply == null) {
-        index = 0;
-      } else {
-        index = reply.getLogIndex();
-      }
-      adjustBuffers(index);
-      return reply;
-    } catch (TimeoutException | InterruptedException | ExecutionException e) {
-      LOG.warn("watchForCommit failed for index " + commitIndex, e);
-      IOException ioException = new IOException(
-          "Unexpected Storage Container Exception: " + e.toString(), e);
-      releaseBuffersOnException();
-      throw ioException;
-    }
-  }
-
-  @VisibleForTesting
-  public ConcurrentSkipListMap<Long,
-      List<ByteBuffer>> getCommitIndex2flushedDataMap() {
-    return commitIndex2flushedDataMap;
-  }
-
-  public ConcurrentHashMap<Long,
-      CompletableFuture<ContainerProtos.
-          ContainerCommandResponseProto>> getFutureMap() {
-    return futureMap;
-  }
-
-  public long getTotalAckDataLength() {
-    return totalAckDataLength;
-  }
-
-  public void cleanup() {
-    if (commitIndex2flushedDataMap != null) {
-      commitIndex2flushedDataMap.clear();
-    }
-    if (futureMap != null) {
-      futureMap.clear();
-    }
-    commitIndex2flushedDataMap = null;
-  }
-}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/package-info.java

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-/**
- * Low level IO streams to upload/download chunks from container service.
- */

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java

@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-import com.google.common.primitives.Bytes;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.client.ContainerBlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.scm.XceiverClientManager;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.security.token.OzoneBlockTokenIdentifier;
-import org.apache.hadoop.ozone.common.Checksum;
-import org.apache.hadoop.security.token.Token;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import static org.apache.hadoop.hdds.scm.storage.TestChunkInputStream.generateRandomData;
-
-/**
- * Tests for {@link BlockInputStream}'s functionality.
- */
-public class TestBlockInputStream {
-
-  private static final int CHUNK_SIZE = 100;
-  private static Checksum checksum;
-
-  private BlockInputStream blockStream;
-  private byte[] blockData;
-  private int blockSize;
-  private List<ChunkInfo> chunks;
-  private Map<String, byte[]> chunkDataMap;
-
-  @Before
-  public void setup() throws Exception {
-    BlockID blockID = new BlockID(new ContainerBlockID(1, 1));
-    checksum = new Checksum(ChecksumType.NONE, CHUNK_SIZE);
-    createChunkList(5);
-
-    blockStream = new DummyBlockInputStream(blockID, blockSize, null, null,
-        false, null);
-  }
-
-  /**
-   * Create a mock list of chunks. The first n-1 chunks of length CHUNK_SIZE
-   * and the last chunk with length CHUNK_SIZE/2.
-   */
-  private void createChunkList(int numChunks)
-      throws Exception {
-
-    chunks = new ArrayList<>(numChunks);
-    chunkDataMap = new HashMap<>();
-    blockData = new byte[0];
-    int i, chunkLen;
-    byte[] byteData;
-    String chunkName;
-
-    for (i = 0; i < numChunks; i++) {
-      chunkName = "chunk-" + i;
-      chunkLen = CHUNK_SIZE;
-      if (i == numChunks - 1) {
-        chunkLen = CHUNK_SIZE / 2;
-      }
-      byteData = generateRandomData(chunkLen);
-      ChunkInfo chunkInfo = ChunkInfo.newBuilder()
-          .setChunkName(chunkName)
-          .setOffset(0)
-          .setLen(chunkLen)
-          .setChecksumData(checksum.computeChecksum(
-              byteData, 0, chunkLen).getProtoBufMessage())
-          .build();
-
-      chunkDataMap.put(chunkName, byteData);
-      chunks.add(chunkInfo);
-
-      blockSize += chunkLen;
-      blockData = Bytes.concat(blockData, byteData);
-    }
-  }
-
-  /**
-   * A dummy BlockInputStream to mock read block call to DN.
-   */
-  private class DummyBlockInputStream extends BlockInputStream {
-
-    DummyBlockInputStream(BlockID blockId,
-        long blockLen,
-        Pipeline pipeline,
-        Token<OzoneBlockTokenIdentifier> token,
-        boolean verifyChecksum,
-        XceiverClientManager xceiverClientManager) {
-      super(blockId, blockLen, pipeline, token, verifyChecksum,
-          xceiverClientManager);
-    }
-
-    @Override
-    protected List<ChunkInfo> getChunkInfos() {
-      return chunks;
-    }
-
-    @Override
-    protected void addStream(ChunkInfo chunkInfo) {
-      TestChunkInputStream testChunkInputStream = new TestChunkInputStream();
-      getChunkStreams().add(testChunkInputStream.new DummyChunkInputStream(
-          chunkInfo, null, null, false,
-          chunkDataMap.get(chunkInfo.getChunkName()).clone()));
-    }
-
-    @Override
-    protected synchronized void checkOpen() throws IOException {
-      // No action needed
-    }
-  }
-
-  private void seekAndVerify(int pos) throws Exception {
-    blockStream.seek(pos);
-    Assert.assertEquals("Current position of buffer does not match with the " +
-        "seeked position", pos, blockStream.getPos());
-  }
-
-  /**
-   * Match readData with the chunkData byte-wise.
-   * @param readData Data read through ChunkInputStream
-   * @param inputDataStartIndex first index (inclusive) in chunkData to compare
-   *                            with read data
-   * @param length the number of bytes of data to match starting from
-   *               inputDataStartIndex
-   */
-  private void matchWithInputData(byte[] readData, int inputDataStartIndex,
-      int length) {
-    for (int i = inputDataStartIndex; i < inputDataStartIndex + length; i++) {
-      Assert.assertEquals(blockData[i], readData[i - inputDataStartIndex]);
-    }
-  }
-
-  @Test
-  public void testSeek() throws Exception {
-    // Seek to position 0
-    int pos = 0;
-    seekAndVerify(pos);
-    Assert.assertEquals("ChunkIndex is incorrect", 0,
-        blockStream.getChunkIndex());
-
-    // Before BlockInputStream is initialized (initialization happens during
-    // read operation), seek should update the BlockInputStream#blockPosition
-    pos = CHUNK_SIZE;
-    seekAndVerify(pos);
-    Assert.assertEquals("ChunkIndex is incorrect", 0,
-        blockStream.getChunkIndex());
-    Assert.assertEquals(pos, blockStream.getBlockPosition());
-
-    // Initialize the BlockInputStream. After initializtion, the chunkIndex
-    // should be updated to correspond to the seeked position.
-    blockStream.initialize();
-    Assert.assertEquals("ChunkIndex is incorrect", 1,
-        blockStream.getChunkIndex());
-
-    pos = (CHUNK_SIZE * 4) + 5;
-    seekAndVerify(pos);
-    Assert.assertEquals("ChunkIndex is incorrect", 4,
-        blockStream.getChunkIndex());
-
-    try {
-      // Try seeking beyond the blockSize.
-      pos = blockSize + 10;
-      seekAndVerify(pos);
-      Assert.fail("Seek to position beyond block size should fail.");
-    } catch (EOFException e) {
-      System.out.println(e);
-    }
-
-    // Seek to random positions between 0 and the block size.
-    Random random = new Random();
-    for (int i = 0; i < 10; i++) {
-      pos = random.nextInt(blockSize);
-      seekAndVerify(pos);
-    }
-  }
-
-  @Test
-  public void testRead() throws Exception {
-    // read 200 bytes of data starting from position 50. Chunk0 contains
-    // indices 0 to 99, chunk1 from 100 to 199 and chunk3 from 200 to 299. So
-    // the read should result in 3 ChunkInputStream reads
-    seekAndVerify(50);
-    byte[] b = new byte[200];
-    blockStream.read(b, 0, 200);
-    matchWithInputData(b, 50, 200);
-
-    // The new position of the blockInputStream should be the last index read
-    // + 1.
-    Assert.assertEquals(250, blockStream.getPos());
-    Assert.assertEquals(2, blockStream.getChunkIndex());
-  }
-
-  @Test
-  public void testSeekAndRead() throws Exception {
-    // Seek to a position and read data
-    seekAndVerify(50);
-    byte[] b1 = new byte[100];
-    blockStream.read(b1, 0, 100);
-    matchWithInputData(b1, 50, 100);
-
-    // Next read should start from the position of the last read + 1 i.e. 100
-    byte[] b2 = new byte[100];
-    blockStream.read(b2, 0, 100);
-    matchWithInputData(b2, 150, 100);
-  }
-}

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestChunkInputStream.java

@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.storage;
-
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdds.scm.XceiverClientSpi;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.common.Checksum;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.EOFException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-/**
- * Tests for {@link ChunkInputStream}'s functionality.
- */
-public class TestChunkInputStream {
-
-  private static final int CHUNK_SIZE = 100;
-  private static final int BYTES_PER_CHECKSUM = 20;
-  private static final String CHUNK_NAME = "dummyChunk";
-  private static final Random RANDOM = new Random();
-  private static Checksum checksum;
-
-  private DummyChunkInputStream chunkStream;
-  private ChunkInfo chunkInfo;
-  private byte[] chunkData;
-
-  @Before
-  public void setup() throws Exception {
-    checksum = new Checksum(ChecksumType.valueOf(
-        OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT),
-        BYTES_PER_CHECKSUM);
-
-    chunkData = generateRandomData(CHUNK_SIZE);
-
-    chunkInfo = ChunkInfo.newBuilder()
-        .setChunkName(CHUNK_NAME)
-        .setOffset(0)
-        .setLen(CHUNK_SIZE)
-        .setChecksumData(checksum.computeChecksum(
-            chunkData, 0, CHUNK_SIZE).getProtoBufMessage())
-        .build();
-
-    chunkStream = new DummyChunkInputStream(chunkInfo, null, null, true);
-  }
-
-  static byte[] generateRandomData(int length) {
-    byte[] bytes = new byte[length];
-    RANDOM.nextBytes(bytes);
-    return bytes;
-  }
-
-  /**
-   * A dummy ChunkInputStream to mock read chunk calls to DN.
-   */
-  public class DummyChunkInputStream extends ChunkInputStream {
-
-    // Stores the read chunk data in each readChunk call
-    private List<ByteString> readByteBuffers = new ArrayList<>();
-
-    DummyChunkInputStream(ChunkInfo chunkInfo,
-        BlockID blockId,
-        XceiverClientSpi xceiverClient,
-        boolean verifyChecksum) {
-      super(chunkInfo, blockId, xceiverClient, verifyChecksum);
-    }
-
-    public DummyChunkInputStream(ChunkInfo chunkInfo,
-        BlockID blockId,
-        XceiverClientSpi xceiverClient,
-        boolean verifyChecksum,
-        byte[] data) {
-      super(chunkInfo, blockId, xceiverClient, verifyChecksum);
-      chunkData = data;
-    }
-
-    @Override
-    protected ByteString readChunk(ChunkInfo readChunkInfo) {
-      ByteString byteString = ByteString.copyFrom(chunkData,
-          (int) readChunkInfo.getOffset(),
-          (int) readChunkInfo.getLen());
-      readByteBuffers.add(byteString);
-      return byteString;
-    }
-
-    @Override
-    protected void checkOpen() {
-      // No action needed
-    }
-  }
-
-  /**
-   * Match readData with the chunkData byte-wise.
-   * @param readData Data read through ChunkInputStream
-   * @param inputDataStartIndex first index (inclusive) in chunkData to compare
-   *                            with read data
-   * @param length the number of bytes of data to match starting from
-   *               inputDataStartIndex
-   */
-  private void matchWithInputData(byte[] readData, int inputDataStartIndex,
-      int length) {
-    for (int i = inputDataStartIndex; i < inputDataStartIndex + length; i++) {
-      Assert.assertEquals(chunkData[i], readData[i - inputDataStartIndex]);
-    }
-  }
-
-  /**
-   * Seek to a position and verify through getPos().
-   */
-  private void seekAndVerify(int pos) throws Exception {
-    chunkStream.seek(pos);
-    Assert.assertEquals("Current position of buffer does not match with the " +
-        "seeked position", pos, chunkStream.getPos());
-  }
-
-  @Test
-  public void testFullChunkRead() throws Exception {
-    byte[] b = new byte[CHUNK_SIZE];
-    chunkStream.read(b, 0, CHUNK_SIZE);
-
-    matchWithInputData(b, 0, CHUNK_SIZE);
-  }
-
-  @Test
-  public void testPartialChunkRead() throws Exception {
-    int len = CHUNK_SIZE / 2;
-    byte[] b = new byte[len];
-
-    chunkStream.read(b, 0, len);
-
-    matchWithInputData(b, 0, len);
-
-    // To read chunk data from index 0 to 49 (len = 50), we need to read
-    // chunk from offset 0 to 60 as the checksum boundary is at every 20
-    // bytes. Verify that 60 bytes of chunk data are read and stored in the
-    // buffers.
-    matchWithInputData(chunkStream.readByteBuffers.get(0).toByteArray(),
-        0, 60);
-
-  }
-
-  @Test
-  public void testSeek() throws Exception {
-    seekAndVerify(0);
-
-    try {
-      seekAndVerify(CHUNK_SIZE);
-      Assert.fail("Seeking to Chunk Length should fail.");
-    } catch (EOFException e) {
-      GenericTestUtils.assertExceptionContains("EOF encountered at pos: "
-          + CHUNK_SIZE + " for chunk: " + CHUNK_NAME, e);
-    }
-
-    // Seek before read should update the ChunkInputStream#chunkPosition
-    seekAndVerify(25);
-    Assert.assertEquals(25, chunkStream.getChunkPosition());
-
-    // Read from the seeked position.
-    // Reading from index 25 to 54 should result in the ChunkInputStream
-    // copying chunk data from index 20 to 59 into the buffers (checksum
-    // boundaries).
-    byte[] b = new byte[30];
-    chunkStream.read(b, 0, 30);
-    matchWithInputData(b, 25, 30);
-    matchWithInputData(chunkStream.readByteBuffers.get(0).toByteArray(),
-        20, 40);
-
-    // After read, the position of the chunkStream is evaluated from the
-    // buffers and the chunkPosition should be reset to -1.
-    Assert.assertEquals(-1, chunkStream.getChunkPosition());
-
-    // Seek to a position within the current buffers. Current buffers contain
-    // data from index 20 to 59. ChunkPosition should still not be used to
-    // set the position.
-    seekAndVerify(35);
-    Assert.assertEquals(-1, chunkStream.getChunkPosition());
-
-    // Seek to a position outside the current buffers. In this case, the
-    // chunkPosition should be updated to the seeked position.
-    seekAndVerify(75);
-    Assert.assertEquals(75, chunkStream.getChunkPosition());
-  }
-
-  @Test
-  public void testSeekAndRead() throws Exception {
-    // Seek to a position and read data
-    seekAndVerify(50);
-    byte[] b1 = new byte[20];
-    chunkStream.read(b1, 0, 20);
-    matchWithInputData(b1, 50, 20);
-
-    // Next read should start from the position of the last read + 1 i.e. 70
-    byte[] b2 = new byte[20];
-    chunkStream.read(b2, 0, 20);
-    matchWithInputData(b2, 70, 20);
-  }
-}

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/package-info.java

@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * This package contains Ozone InputStream related tests.
- */
-package org.apache.hadoop.hdds.scm.storage;

hadoop-hdds/common/dev-support/findbugsExcludeFile.xml

@@ -1,33 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<FindBugsFilter>
-  <Match>
-    <Package name="org.apache.hadoop.hdds.protocol.proto"/>
-  </Match>
-  <Match>
-    <Package name="org.apache.hadoop.hdds.protocol.datanode.proto"/>
-  </Match>
-  <Match>
-    <Class name="org.apache.hadoop.hdds.cli.GenericCli"></Class>
-    <Bug pattern="DM_EXIT" />
-  </Match>
-  <Match>
-    <Class name="org.apache.hadoop.ozone.common.ChecksumByteBuffer$CrcIntTable" />
-    <Method name="update" />
-    <Bug pattern="SF_SWITCH_FALLTHROUGH,SF_SWITCH_NO_DEFAULT" />
-  </Match>
-</FindBugsFilter>

hadoop-hdds/common/pom.xml

@@ -1,285 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-https://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>org.apache.hadoop</groupId>
-    <artifactId>hadoop-hdds</artifactId>
-    <version>0.5.0-SNAPSHOT</version>
-  </parent>
-  <artifactId>hadoop-hdds-common</artifactId>
-  <version>0.5.0-SNAPSHOT</version>
-  <description>Apache Hadoop Distributed Data Store Common</description>
-  <name>Apache Hadoop HDDS Common</name>
-  <packaging>jar</packaging>
-
-  <properties>
-    <hdds.version>0.5.0-SNAPSHOT</hdds.version>
-    <log4j2.version>2.11.0</log4j2.version>
-    <disruptor.version>3.4.2</disruptor.version>
-    <declared.hdds.version>${hdds.version}</declared.hdds.version>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdds-config</artifactId>
-    </dependency>
-
-    <dependency>
-      <groupId>javax.annotation</groupId>
-      <artifactId>javax.annotation-api</artifactId>
-      <version>1.2</version>
-    </dependency>
-
-    <dependency>
-      <groupId>org.fusesource.leveldbjni</groupId>
-      <artifactId>leveldbjni-all</artifactId>
-    </dependency>
-
-    <dependency>
-      <artifactId>ratis-server</artifactId>
-      <groupId>org.apache.ratis</groupId>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>io.dropwizard.metrics</groupId>
-          <artifactId>metrics-core</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.bouncycastle</groupId>
-          <artifactId>bcprov-jdk15on</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <artifactId>ratis-netty</artifactId>
-      <groupId>org.apache.ratis</groupId>
-    </dependency>
-    <dependency>
-      <artifactId>ratis-grpc</artifactId>
-      <groupId>org.apache.ratis</groupId>
-    </dependency>
-    <dependency>
-      <groupId>com.google.errorprone</groupId>
-      <artifactId>error_prone_annotations</artifactId>
-      <version>2.2.0</version>
-      <optional>true</optional>
-    </dependency>
-
-    <dependency>
-      <groupId>org.rocksdb</groupId>
-      <artifactId>rocksdbjni</artifactId>
-      <version>6.0.1</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>test</scope>
-      <type>test-jar</type>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-api</artifactId>
-      <version>${log4j2.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.logging.log4j</groupId>
-      <artifactId>log4j-core</artifactId>
-      <version>${log4j2.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>com.lmax</groupId>
-      <artifactId>disruptor</artifactId>
-      <version>${disruptor.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.commons</groupId>
-      <artifactId>commons-pool2</artifactId>
-      <version>2.6.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.bouncycastle</groupId>
-      <artifactId>bcpkix-jdk15on</artifactId>
-      <version>${bouncycastle.version}</version>
-    </dependency>
-    <!-- https://mvnrepository.com/artifact/commons-validator/commons-validator -->
-    <dependency>
-      <groupId>commons-validator</groupId>
-      <artifactId>commons-validator</artifactId>
-      <version>1.6</version>
-    </dependency>
-    <dependency>
-      <groupId>org.junit.jupiter</groupId>
-      <artifactId>junit-jupiter-api</artifactId>
-    </dependency>
-    <dependency>
-      <groupId>io.jaegertracing</groupId>
-      <artifactId>jaeger-client</artifactId>
-      <version>${jaeger.version}</version>
-    </dependency>
-    <dependency>
-      <groupId>io.opentracing</groupId>
-      <artifactId>opentracing-util</artifactId>
-      <version>0.31.0</version>
-    </dependency>
-    <dependency>
-      <groupId>org.yaml</groupId>
-      <artifactId>snakeyaml</artifactId>
-      <version>1.16</version>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <resources>
-      <resource>
-        <directory>${basedir}/src/main/resources</directory>
-        <excludes>
-          <exclude>hdds-version-info.properties</exclude>
-        </excludes>
-        <filtering>false</filtering>
-      </resource>
-      <resource>
-        <directory>${basedir}/src/main/resources</directory>
-        <includes>
-          <include>hdds-version-info.properties</include>
-        </includes>
-        <filtering>true</filtering>
-      </resource>
-    </resources>
-    <extensions>
-      <extension>
-        <groupId>kr.motd.maven</groupId>
-        <artifactId>os-maven-plugin</artifactId>
-        <version>${os-maven-plugin.version}</version>
-      </extension>
-    </extensions>
-    <plugins>
-      <plugin>
-        <groupId>org.xolstice.maven.plugins</groupId>
-        <artifactId>protobuf-maven-plugin</artifactId>
-        <version>${protobuf-maven-plugin.version}</version>
-        <extensions>true</extensions>
-        <configuration>
-          <protocArtifact>
-            com.google.protobuf:protoc:${protobuf-compile.version}:exe:${os.detected.classifier}
-          </protocArtifact>
-          <protoSourceRoot>${basedir}/src/main/proto/</protoSourceRoot>
-          <includes>
-            <include>DatanodeContainerProtocol.proto</include>
-          </includes>
-          <outputDirectory>target/generated-sources/java</outputDirectory>
-          <clearOutputDirectory>false</clearOutputDirectory>
-        </configuration>
-        <executions>
-          <execution>
-            <id>compile-protoc</id>
-              <goals>
-                <goal>compile</goal>
-                <goal>test-compile</goal>
-                <goal>compile-custom</goal>
-                <goal>test-compile-custom</goal>
-              </goals>
-              <configuration>
-                <pluginId>grpc-java</pluginId>
-                <pluginArtifact>
-                  io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}
-                </pluginArtifact>
-              </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <executions>
-          <execution>
-            <phase>generate-sources</phase>
-            <configuration>
-              <tasks>
-                <replace token="com.google.protobuf" value="org.apache.ratis.thirdparty.com.google.protobuf"
-                  dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
-                </replace>
-                <replace token="io.grpc" value="org.apache.ratis.thirdparty.io.grpc"
-                  dir="target/generated-sources/java/org/apache/hadoop/hdds/protocol/datanode/proto">
-                </replace>
-              </tasks>
-            </configuration>
-            <goals>
-              <goal>run</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-maven-plugins</artifactId>
-        <executions>
-          <execution>
-            <id>version-info</id>
-            <phase>generate-resources</phase>
-            <goals>
-              <goal>version-info</goal>
-            </goals>
-            <configuration>
-              <source>
-                <directory>${basedir}/../</directory>
-                <includes>
-                  <include>*/src/main/java/**/*.java</include>
-                  <include>*/src/main/proto/*.proto</include>
-                </includes>
-              </source>
-            </configuration>
-          </execution>
-          <execution>
-            <id>compile-protoc</id>
-            <goals>
-              <goal>protoc</goal>
-            </goals>
-            <configuration>
-              <protocVersion>${protobuf.version}</protocVersion>
-              <protocCommand>${protoc.path}</protocCommand>
-              <imports>
-                <param>${basedir}/src/main/proto</param>
-              </imports>
-              <source>
-                <directory>${basedir}/src/main/proto</directory>
-                <includes>
-                  <include>StorageContainerLocationProtocol.proto</include>
-                  <include>hdds.proto</include>
-                  <include>ScmBlockLocationProtocol.proto</include>
-                  <include>SCMSecurityProtocol.proto</include>
-                </includes>
-              </source>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>com.github.spotbugs</groupId>
-        <artifactId>spotbugs-maven-plugin</artifactId>
-        <configuration>
-          <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-</project>

hadoop-hdds/common/src/main/bin/hadoop-config.cmd

@@ -1,317 +0,0 @@
-@echo off
-@rem Licensed to the Apache Software Foundation (ASF) under one or more
-@rem contributor license agreements.  See the NOTICE file distributed with
-@rem this work for additional information regarding copyright ownership.
-@rem The ASF licenses this file to You under the Apache License, Version 2.0
-@rem (the "License"); you may not use this file except in compliance with
-@rem the License.  You may obtain a copy of the License at
-@rem
-@rem     http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-
-@rem included in all the hadoop scripts with source command
-@rem should not be executable directly
-@rem also should not be passed any arguments, since we need original %*
-
-if not defined HADOOP_COMMON_DIR (
-  set HADOOP_COMMON_DIR=share\hadoop\common
-)
-if not defined HADOOP_COMMON_LIB_JARS_DIR (
-  set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib
-)
-if not defined HADOOP_COMMON_LIB_NATIVE_DIR (
-  set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native
-)
-if not defined HDFS_DIR (
-  set HDFS_DIR=share\hadoop\hdfs
-)
-if not defined HDFS_LIB_JARS_DIR (
-  set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib
-)
-if not defined YARN_DIR (
-  set YARN_DIR=share\hadoop\yarn
-)
-if not defined YARN_LIB_JARS_DIR (
-  set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib
-)
-if not defined MAPRED_DIR (
-  set MAPRED_DIR=share\hadoop\mapreduce
-)
-if not defined MAPRED_LIB_JARS_DIR (
-  set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib
-)
-
-@rem the root of the Hadoop installation
-set HADOOP_HOME=%~dp0
-for %%i in (%HADOOP_HOME%.) do (
-  set HADOOP_HOME=%%~dpi
-)
-if "%HADOOP_HOME:~-1%" == "\" (
-  set HADOOP_HOME=%HADOOP_HOME:~0,-1%
-)
-
-if not exist %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar (
-    @echo +================================================================+
-    @echo ^|      Error: HADOOP_HOME is not set correctly                   ^|
-    @echo +----------------------------------------------------------------+
-    @echo ^| Please set your HADOOP_HOME variable to the absolute path of   ^|
-    @echo ^| the directory that contains the hadoop distribution            ^|
-    @echo +================================================================+
-    exit /b 1
-)
-
-if not defined HADOOP_CONF_DIR (
-  set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop
-)
-
-@rem
-@rem Allow alternate conf dir location.
-@rem
-
-if "%1" == "--config" (
-  set HADOOP_CONF_DIR=%2
-  shift
-  shift
-)
-
-@rem
-@rem check to see it is specified whether to use the workers or the
-@rem masters file
-@rem
-
-if "%1" == "--hosts" (
-  set HADOOP_WORKERS=%HADOOP_CONF_DIR%\%2
-  shift
-  shift
-)
-
-@rem
-@rem Set log level. Default to INFO.
-@rem
-
-if "%1" == "--loglevel" (
-  set HADOOP_LOGLEVEL=%2
-  shift
-  shift
-)
-
-if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
-  call %HADOOP_CONF_DIR%\hadoop-env.cmd
-)
-
-@rem
-@rem setup java environment variables
-@rem
-
-if not defined JAVA_HOME (
-  echo Error: JAVA_HOME is not set.
-  goto :eof
-)
-
-if not exist %JAVA_HOME%\bin\java.exe (
-  echo Error: JAVA_HOME is incorrectly set.
-  echo        Please update %HADOOP_CONF_DIR%\hadoop-env.cmd
-  goto :eof
-)
-
-set JAVA=%JAVA_HOME%\bin\java
-@rem some Java parameters
-set JAVA_HEAP_MAX=-Xmx1000m
-
-@rem
-@rem check envvars which might override default args
-@rem
-
-if defined HADOOP_HEAPSIZE (
-  set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m
-)
-
-@rem
-@rem CLASSPATH initially contains %HADOOP_CONF_DIR%
-@rem
-
-set CLASSPATH=%HADOOP_CONF_DIR%
-
-if not defined HADOOP_COMMON_HOME (
-  if exist %HADOOP_HOME%\share\hadoop\common (
-    set HADOOP_COMMON_HOME=%HADOOP_HOME%
-  )
-)
-
-@rem
-@rem for releases, add core hadoop jar & webapps to CLASSPATH
-@rem
-
-if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\webapps (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%
-)
-
-if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR% (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\*
-)
-
-set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\*
-
-@rem
-@rem default log directory % file
-@rem
-
-if not defined HADOOP_LOG_DIR (
-  set HADOOP_LOG_DIR=%HADOOP_HOME%\logs
-)
-
-if not defined HADOOP_LOGFILE (
-  set HADOOP_LOGFILE=hadoop.log
-)
-
-if not defined HADOOP_LOGLEVEL (
-  set HADOOP_LOGLEVEL=INFO
-)
-
-if not defined HADOOP_ROOT_LOGGER (
-  set HADOOP_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
-)
-
-@rem
-@rem default policy file for service-level authorization
-@rem
-
-if not defined HADOOP_POLICYFILE (
-  set HADOOP_POLICYFILE=hadoop-policy.xml
-)
-
-@rem
-@rem Determine the JAVA_PLATFORM
-@rem
-
-for /f "delims=" %%A in ('%JAVA% -Xmx32m %HADOOP_JAVA_PLATFORM_OPTS% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName') do set JAVA_PLATFORM=%%A
-@rem replace space with underscore
-set JAVA_PLATFORM=%JAVA_PLATFORM: =_%
-
-@rem
-@rem setup 'java.library.path' for native hadoop code if necessary
-@rem
-
-@rem Check if we're running hadoop directly from the build
-if exist %HADOOP_COMMON_HOME%\target\bin (
-  if defined JAVA_LIBRARY_PATH (
-    set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_COMMON_HOME%\target\bin
-   ) else (
-    set JAVA_LIBRARY_PATH=%HADOOP_COMMON_HOME%\target\bin
-   )
-)
-
-@rem For the distro case, check the bin folder
-if exist %HADOOP_COMMON_HOME%\bin (
-  if defined JAVA_LIBRARY_PATH (
-    set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_COMMON_HOME%\bin
-  ) else (
-    set JAVA_LIBRARY_PATH=%HADOOP_COMMON_HOME%\bin
-  )
-)
-
-@rem
-@rem setup a default TOOL_PATH
-@rem
-set TOOL_PATH=%HADOOP_HOME%\share\hadoop\tools\lib\*
-
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR%
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE%
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_HOME%
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING%
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER%
-
-if defined JAVA_LIBRARY_PATH (
-  set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH%
-)
-set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE%
-
-@rem
-@rem Disable ipv6 as it can cause issues
-@rem
-
-set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
-
-@rem
-@rem put hdfs in classpath if present
-@rem
-
-if not defined HADOOP_HDFS_HOME (
-  if exist %HADOOP_HOME%\%HDFS_DIR% (
-    set HADOOP_HDFS_HOME=%HADOOP_HOME%
-  )
-)
-
-if exist %HADOOP_HDFS_HOME%\%HDFS_DIR%\webapps (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%
-)
-
-if exist %HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR% (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\*
-)
-
-set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\*
-
-@rem
-@rem put yarn in classpath if present
-@rem
-
-if not defined HADOOP_YARN_HOME (
-  if exist %HADOOP_HOME%\%YARN_DIR% (
-    set HADOOP_YARN_HOME=%HADOOP_HOME%
-  )
-)
-
-if exist %HADOOP_YARN_HOME%\%YARN_DIR%\webapps (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%
-)
-
-if exist %HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR% (
-  set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\*
-)
-
-set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\*
-
-@rem
-@rem put mapred in classpath if present AND different from YARN
-@rem
-
-if not defined HADOOP_MAPRED_HOME (
-  if exist %HADOOP_HOME%\%MAPRED_DIR% (
-    set HADOOP_MAPRED_HOME=%HADOOP_HOME%
-  )
-)
-
-if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" (
-
-  if exist %HADOOP_MAPRED_HOME%\%MAPRED_DIR%\webapps (
-    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%
-  )
-
-  if exist %HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR% (
-    set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\*
-  )
-
-  set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\*
-)
-
-@rem
-@rem add user-specified CLASSPATH last
-@rem
-
-if defined HADOOP_CLASSPATH (
-  if not defined HADOOP_USE_CLIENT_CLASSLOADER (
-    if defined HADOOP_USER_CLASSPATH_FIRST (
-      set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%;
-    ) else (
-      set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%;
-    )
-  )
-)
-
-:eof

hadoop-hdds/common/src/main/bin/hadoop-config.sh

@@ -1,165 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-####
-# IMPORTANT
-####
-
-## The hadoop-config.sh tends to get executed by non-Hadoop scripts.
-## Those parts expect this script to parse/manipulate $@. In order
-## to maintain backward compatibility, this means a surprising
-## lack of functions for bits that would be much better off in
-## a function.
-##
-## In other words, yes, there is some bad things happen here and
-## unless we break the rest of the ecosystem, we can't change it. :(
-
-
-# included in all the hadoop scripts with source command
-# should not be executable directly
-# also should not be passed any arguments, since we need original $*
-#
-# after doing more config, caller should also exec finalize
-# function to finish last minute/default configs for
-# settings that might be different between daemons & interactive
-
-# you must be this high to ride the ride
-if [[ -z "${BASH_VERSINFO[0]}" ]] \
-   || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
-   || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
-  echo "bash v3.2+ is required. Sorry."
-  exit 1
-fi
-
-# In order to get partially bootstrapped, we need to figure out where
-# we are located. Chances are good that our caller has already done
-# this work for us, but just in case...
-
-if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
-  _hadoop_common_this="${BASH_SOURCE-$0}"
-  HADOOP_LIBEXEC_DIR=$(cd -P -- "$(dirname -- "${_hadoop_common_this}")" >/dev/null && pwd -P)
-fi
-
-# get our functions defined for usage later
-if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
-   [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh" ]]; then
-  # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
-  . "${HADOOP_COMMON_HOME}/libexec/hadoop-functions.sh"
-elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh" ]]; then
-  # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
-  . "${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh"
-else
-  echo "ERROR: Unable to exec ${HADOOP_LIBEXEC_DIR}/hadoop-functions.sh." 1>&2
-  exit 1
-fi
-
-hadoop_deprecate_envvar HADOOP_PREFIX HADOOP_HOME
-
-# allow overrides of the above and pre-defines of the below
-if [[ -n "${HADOOP_COMMON_HOME}" ]] &&
-   [[ -e "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh" ]]; then
-  # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
-  . "${HADOOP_COMMON_HOME}/libexec/hadoop-layout.sh"
-elif [[ -e "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh" ]]; then
-  # shellcheck source=./hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example
-  . "${HADOOP_LIBEXEC_DIR}/hadoop-layout.sh"
-fi
-
-#
-# IMPORTANT! We are not executing user provided code yet!
-#
-
-# Let's go!  Base definitions so we can move forward
-hadoop_bootstrap
-
-# let's find our conf.
-#
-# first, check and process params passed to us
-# we process this in-line so that we can directly modify $@
-# if something downstream is processing that directly,
-# we need to make sure our params have been ripped out
-# note that we do many of them here for various utilities.
-# this provides consistency and forces a more consistent
-# user experience
-
-
-# save these off in case our caller needs them
-# shellcheck disable=SC2034
-HADOOP_USER_PARAMS=("$@")
-
-hadoop_parse_args "$@"
-shift "${HADOOP_PARSE_COUNTER}"
-
-#
-# Setup the base-line environment
-#
-hadoop_find_confdir
-hadoop_exec_hadoopenv
-hadoop_import_shellprofiles
-hadoop_exec_userfuncs
-
-#
-# IMPORTANT! User provided code is now available!
-#
-
-hadoop_exec_user_hadoopenv
-hadoop_verify_confdir
-
-hadoop_deprecate_envvar HADOOP_SLAVES HADOOP_WORKERS
-hadoop_deprecate_envvar HADOOP_SLAVE_NAMES HADOOP_WORKER_NAMES
-hadoop_deprecate_envvar HADOOP_SLAVE_SLEEP HADOOP_WORKER_SLEEP
-
-# do all the OS-specific startup bits here
-# this allows us to get a decent JAVA_HOME,
-# call crle for LD_LIBRARY_PATH, etc.
-hadoop_os_tricks
-
-hadoop_java_setup
-
-hadoop_basic_init
-
-# inject any sub-project overrides, defaults, etc.
-if declare -F hadoop_subproject_init >/dev/null ; then
-  hadoop_subproject_init
-fi
-
-hadoop_shellprofiles_init
-
-# get the native libs in there pretty quick
-hadoop_add_javalibpath "${HADOOP_HOME}/build/native"
-hadoop_add_javalibpath "${HADOOP_HOME}/${HADOOP_COMMON_LIB_NATIVE_DIR}"
-
-hadoop_shellprofiles_nativelib
-
-# get the basic java class path for these subprojects
-# in as quickly as possible since other stuff
-# will definitely depend upon it.
-
-hadoop_add_common_to_classpath
-hadoop_shellprofiles_classpath
-
-# user API commands can now be run since the runtime
-# environment has been configured
-hadoop_exec_hadooprc
-
-#
-# backwards compatibility. new stuff should
-# call this when they are ready
-#
-if [[ -z "${HADOOP_NEW_CONFIG}" ]]; then
-  hadoop_finalize
-fi

hadoop-hdds/common/src/main/bin/hadoop-daemons.sh

@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Run a Hadoop command on all slave hosts.
-
-function hadoop_usage
-{
-  echo "Usage: hadoop-daemons.sh [--config confdir] [--hosts hostlistfile] (start|stop|status) <hadoop-command> <args...>"
-}
-
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
-  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
-  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
-  . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
-else
-  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
-  exit 1
-fi
-
-if [[ $# = 0 ]]; then
-  hadoop_exit_with_usage 1
-fi
-
-daemonmode=$1
-shift
-
-if [[ -z "${HADOOP_HDFS_HOME}" ]]; then
-  hdfsscript="${HADOOP_HOME}/bin/hdfs"
-else
-  hdfsscript="${HADOOP_HDFS_HOME}/bin/hdfs"
-fi
-
-hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
-hadoop_error "WARNING: Attempting to execute replacement \"hdfs --workers --daemon ${daemonmode}\" instead."
-
-#
-# Original input was usually:
-#  hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
-# we're going to turn this into
-#  hdfs --workers --daemon (start|stop) (rest of options)
-#
-for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ ))
-do
-  if [[ "${HADOOP_USER_PARAMS[$i]}" =~ ^start$ ]] ||
-     [[ "${HADOOP_USER_PARAMS[$i]}" =~ ^stop$ ]] ||
-     [[ "${HADOOP_USER_PARAMS[$i]}" =~ ^status$ ]]; then
-    unset HADOOP_USER_PARAMS[$i]
-  fi
-done
-
-${hdfsscript} --workers --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}"

hadoop-hdds/common/src/main/bin/hadoop-functions.sh

@@ -1,2732 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# we need to declare this globally as an array, which can only
-# be done outside of a function
-declare -a HADOOP_SUBCMD_USAGE
-declare -a HADOOP_OPTION_USAGE
-declare -a HADOOP_SUBCMD_USAGE_TYPES
-
-## @description  Print a message to stderr
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function hadoop_error
-{
-  echo "$*" 1>&2
-}
-
-## @description  Print a message to stderr if --debug is turned on
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        string
-function hadoop_debug
-{
-  if [[ -n "${HADOOP_SHELL_SCRIPT_DEBUG}" ]]; then
-    echo "DEBUG: $*" 1>&2
-  fi
-}
-
-## @description  Given a filename or dir, return the absolute version of it
-## @description  This works as an alternative to readlink, which isn't
-## @description  portable.
-## @audience     public
-## @stability    stable
-## @param        fsobj
-## @replaceable  no
-## @return       0 success
-## @return       1 failure
-## @return       stdout abspath
-function hadoop_abs
-{
-  declare obj=$1
-  declare dir
-  declare fn
-  declare dirret
-
-  if [[ ! -e ${obj} ]]; then
-    return 1
-  elif [[ -d ${obj} ]]; then
-    dir=${obj}
-  else
-    dir=$(dirname -- "${obj}")
-    fn=$(basename -- "${obj}")
-    fn="/${fn}"
-  fi
-
-  dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
-  dirret=$?
-  if [[ ${dirret} = 0 ]]; then
-    echo "${dir}${fn}"
-    return 0
-  fi
-  return 1
-}
-
-## @description  Given variable $1 delete $2 from it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function hadoop_delete_entry
-{
-  if [[ ${!1} =~ \ ${2}\  ]] ; then
-    hadoop_debug "Removing ${2} from ${1}"
-    eval "${1}"=\""${!1// ${2} }"\"
-  fi
-}
-
-## @description  Given variable $1 add $2 to it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-function hadoop_add_entry
-{
-  if [[ ! ${!1} =~ \ ${2}\  ]] ; then
-    hadoop_debug "Adding ${2} to ${1}"
-    #shellcheck disable=SC2140
-    eval "${1}"=\""${!1} ${2} "\"
-  fi
-}
-
-## @description  Given variable $1 determine if $2 is in it
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @return       0 = yes, 1 = no
-function hadoop_verify_entry
-{
-  # this unfortunately can't really be tested by bats. :(
-  # so if this changes, be aware that unit tests effectively
-  # do this function in them
-  [[ ${!1} =~ \ ${2}\  ]]
-}
-
-## @description  Check if an array has a given value
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        element
-## @param        array
-## @returns      0 = yes
-## @returns      1 = no
-function hadoop_array_contains
-{
-  declare element=$1
-  shift
-  declare val
-
-  if [[ "$#" -eq 0 ]]; then
-    return 1
-  fi
-
-  for val in "${@}"; do
-    if [[ "${val}" == "${element}" ]]; then
-      return 0
-    fi
-  done
-  return 1
-}
-
-## @description  Add the `appendstring` if `checkstring` is not
-## @description  present in the given array
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        envvar
-## @param        appendstring
-function hadoop_add_array_param
-{
-  declare arrname=$1
-  declare add=$2
-
-  declare arrref="${arrname}[@]"
-  declare array=("${!arrref}")
-
-  if ! hadoop_array_contains "${add}" "${array[@]}"; then
-    #shellcheck disable=SC1083,SC2086
-    eval ${arrname}=\(\"\${array[@]}\" \"${add}\" \)
-    hadoop_debug "$1 accepted $2"
-  else
-    hadoop_debug "$1 declined $2"
-  fi
-}
-
-## @description  Sort an array (must not contain regexps)
-## @description  present in the given array
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        arrayvar
-function hadoop_sort_array
-{
-  declare arrname=$1
-  declare arrref="${arrname}[@]"
-  declare array=("${!arrref}")
-  declare oifs
-
-  declare globstatus
-  declare -a sa
-
-  globstatus=$(set -o | grep noglob | awk '{print $NF}')
-
-  set -f
-  oifs=${IFS}
-
-  # shellcheck disable=SC2034
-  IFS=$'\n' sa=($(sort <<<"${array[*]}"))
-
-  # shellcheck disable=SC1083
-  eval "${arrname}"=\(\"\${sa[@]}\"\)
-
-  IFS=${oifs}
-  if [[ "${globstatus}" = off ]]; then
-    set +f
-  fi
-}
-
-## @description  Check if we are running with priv
-## @description  by default, this implementation looks for
-## @description  EUID=0.  For OSes that have true priv
-## @description  separation, this should be something more complex
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @return       1 = no priv
-## @return       0 = priv
-function hadoop_privilege_check
-{
-  [[ "${EUID}" = 0 ]]
-}
-
-## @description  Execute a command via su when running as root
-## @description  if the given user is found or exit with
-## @description  failure if not.
-## @description  otherwise just run it.  (This is intended to
-## @description  be used by the start-*/stop-* scripts.)
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        user
-## @param        commandstring
-## @return       exitstatus
-function hadoop_su
-{
-  declare user=$1
-  shift
-
-  if hadoop_privilege_check; then
-    if hadoop_verify_user_resolves user; then
-       su -l "${user}" -- "$@"
-    else
-      hadoop_error "ERROR: Refusing to run as root: ${user} account is not found. Aborting."
-      return 1
-    fi
-  else
-    "$@"
-  fi
-}
-
-## @description  Execute a command via su when running as root
-## @description  with extra support for commands that might
-## @description  legitimately start as root (e.g., datanode)
-## @description  (This is intended to
-## @description  be used by the start-*/stop-* scripts.)
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        user
-## @param        commandstring
-## @return       exitstatus
-function hadoop_uservar_su
-{
-
-  ## startup matrix:
-  #
-  # if $EUID != 0, then exec
-  # if $EUID =0 then
-  #    if hdfs_subcmd_user is defined, call hadoop_su to exec
-  #    if hdfs_subcmd_user is not defined, error
-  #
-  # For secure daemons, this means both the secure and insecure env vars need to be
-  # defined.  e.g., HDFS_DATANODE_USER=root HDFS_DATANODE_SECURE_USER=hdfs
-  # This function will pick up the "normal" var, switch to that user, then
-  # execute the command which will then pick up the "secure" version.
-  #
-
-  declare program=$1
-  declare command=$2
-  shift 2
-
-  declare uprogram
-  declare ucommand
-  declare uvar
-  declare svar
-
-  if hadoop_privilege_check; then
-    uvar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" USER)
-
-    svar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" SECURE_USER)
-
-    if [[ -n "${!uvar}" ]]; then
-      hadoop_su "${!uvar}" "$@"
-    elif [[ -n "${!svar}" ]]; then
-      ## if we are here, then SECURE_USER with no USER defined
-      ## we are already privileged, so just run the command and hope
-      ## for the best
-      "$@"
-    else
-      hadoop_error "ERROR: Attempting to operate on ${program} ${command} as root"
-      hadoop_error "ERROR: but there is no ${uvar} defined. Aborting operation."
-      return 1
-    fi
-  else
-    "$@"
-  fi
-}
-
-## @description  Add a subcommand to the usage output
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        subcommand
-## @param        subcommandtype
-## @param        subcommanddesc
-function hadoop_add_subcommand
-{
-  declare subcmd=$1
-  declare subtype=$2
-  declare text=$3
-
-  hadoop_debug "${subcmd} as a ${subtype}"
-
-  hadoop_add_array_param HADOOP_SUBCMD_USAGE_TYPES "${subtype}"
-
-  # done in this order so that sort works later
-  HADOOP_SUBCMD_USAGE[${HADOOP_SUBCMD_USAGE_COUNTER}]="${subcmd}@${subtype}@${text}"
-  ((HADOOP_SUBCMD_USAGE_COUNTER=HADOOP_SUBCMD_USAGE_COUNTER+1))
-}
-
-## @description  Add an option to the usage output
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        subcommand
-## @param        subcommanddesc
-function hadoop_add_option
-{
-  local option=$1
-  local text=$2
-
-  HADOOP_OPTION_USAGE[${HADOOP_OPTION_USAGE_COUNTER}]="${option}@${text}"
-  ((HADOOP_OPTION_USAGE_COUNTER=HADOOP_OPTION_USAGE_COUNTER+1))
-}
-
-## @description  Reset the usage information to blank
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_reset_usage
-{
-  HADOOP_SUBCMD_USAGE=()
-  HADOOP_OPTION_USAGE=()
-  HADOOP_SUBCMD_USAGE_TYPES=()
-  HADOOP_SUBCMD_USAGE_COUNTER=0
-  HADOOP_OPTION_USAGE_COUNTER=0
-}
-
-## @description  Print a screen-size aware two-column output
-## @description  if reqtype is not null, only print those requested
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        reqtype
-## @param        array
-function hadoop_generic_columnprinter
-{
-  declare reqtype=$1
-  shift
-  declare -a input=("$@")
-  declare -i i=0
-  declare -i counter=0
-  declare line
-  declare text
-  declare option
-  declare giventext
-  declare -i maxoptsize
-  declare -i foldsize
-  declare -a tmpa
-  declare numcols
-  declare brup
-
-  if [[ -n "${COLUMNS}" ]]; then
-    numcols=${COLUMNS}
-  else
-    numcols=$(tput cols) 2>/dev/null
-    COLUMNS=${numcols}
-  fi
-
-  if [[ -z "${numcols}"
-     || ! "${numcols}" =~ ^[0-9]+$ ]]; then
-    numcols=75
-  else
-    ((numcols=numcols-5))
-  fi
-
-  while read -r line; do
-    tmpa[${counter}]=${line}
-    ((counter=counter+1))
-    IFS='@' read -ra brup <<< "${line}"
-    option="${brup[0]}"
-    if [[ ${#option} -gt ${maxoptsize} ]]; then
-      maxoptsize=${#option}
-    fi
-  done < <(for text in "${input[@]}"; do
-    echo "${text}"
-  done | sort)
-
-  i=0
-  ((foldsize=numcols-maxoptsize))
-
-  until [[ $i -eq ${#tmpa[@]} ]]; do
-    IFS='@' read -ra brup <<< "${tmpa[$i]}"
-
-    option="${brup[0]}"
-    cmdtype="${brup[1]}"
-    giventext="${brup[2]}"
-
-    if [[ -n "${reqtype}" ]]; then
-      if [[ "${cmdtype}" != "${reqtype}" ]]; then
-        ((i=i+1))
-        continue
-      fi
-    fi
-
-    if [[ -z "${giventext}" ]]; then
-      giventext=${cmdtype}
-    fi
-
-    while read -r line; do
-      printf "%-${maxoptsize}s   %-s\n" "${option}" "${line}"
-      option=" "
-    done < <(echo "${giventext}"| fold -s -w ${foldsize})
-    ((i=i+1))
-  done
-}
-
-## @description  generate standard usage output
-## @description  and optionally takes a class
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-## @param        execname
-## @param        true|false
-## @param        [text to use in place of SUBCOMMAND]
-function hadoop_generate_usage
-{
-  declare cmd=$1
-  declare takesclass=$2
-  declare subcmdtext=${3:-"SUBCOMMAND"}
-  declare haveoptions
-  declare optstring
-  declare havesubs
-  declare subcmdstring
-  declare cmdtype
-
-  cmd=${cmd##*/}
-
-  if [[ -n "${HADOOP_OPTION_USAGE_COUNTER}"
-        && "${HADOOP_OPTION_USAGE_COUNTER}" -gt 0 ]]; then
-    haveoptions=true
-    optstring=" [OPTIONS]"
-  fi
-
-  if [[ -n "${HADOOP_SUBCMD_USAGE_COUNTER}"
-        && "${HADOOP_SUBCMD_USAGE_COUNTER}" -gt 0 ]]; then
-    havesubs=true
-    subcmdstring=" ${subcmdtext} [${subcmdtext} OPTIONS]"
-  fi
-
-  echo "Usage: ${cmd}${optstring}${subcmdstring}"
-  if [[ ${takesclass} = true ]]; then
-    echo " or    ${cmd}${optstring} CLASSNAME [CLASSNAME OPTIONS]"
-    echo "  where CLASSNAME is a user-provided Java class"
-  fi
-
-  if [[ "${haveoptions}" = true ]]; then
-    echo ""
-    echo "  OPTIONS is none or any of:"
-    echo ""
-
-    hadoop_generic_columnprinter "" "${HADOOP_OPTION_USAGE[@]}"
-  fi
-
-  if [[ "${havesubs}" = true ]]; then
-    echo ""
-    echo "  ${subcmdtext} is one of:"
-    echo ""
-
-    if [[ "${#HADOOP_SUBCMD_USAGE_TYPES[@]}" -gt 0 ]]; then
-
-      hadoop_sort_array HADOOP_SUBCMD_USAGE_TYPES
-      for subtype in "${HADOOP_SUBCMD_USAGE_TYPES[@]}"; do
-        #shellcheck disable=SC2086
-        cmdtype="$(tr '[:lower:]' '[:upper:]' <<< ${subtype:0:1})${subtype:1}"
-        printf "\n    %s Commands:\n\n" "${cmdtype}"
-        hadoop_generic_columnprinter "${subtype}" "${HADOOP_SUBCMD_USAGE[@]}"
-      done
-    else
-      hadoop_generic_columnprinter "" "${HADOOP_SUBCMD_USAGE[@]}"
-    fi
-    echo ""
-    echo "${subcmdtext} may print help when invoked w/o parameters or with -h."
-  fi
-}
-
-## @description  Replace `oldvar` with `newvar` if `oldvar` exists.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        oldvar
-## @param        newvar
-function hadoop_deprecate_envvar
-{
-  local oldvar=$1
-  local newvar=$2
-  local oldval=${!oldvar}
-  local newval=${!newvar}
-
-  if [[ -n "${oldval}" ]]; then
-    hadoop_error "WARNING: ${oldvar} has been replaced by ${newvar}. Using value of ${oldvar}."
-    # shellcheck disable=SC2086
-    eval ${newvar}=\"${oldval}\"
-
-    # shellcheck disable=SC2086
-    newval=${oldval}
-
-    # shellcheck disable=SC2086
-    eval ${newvar}=\"${newval}\"
-  fi
-}
-
-## @description  Declare `var` being used and print its value.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        var
-function hadoop_using_envvar
-{
-  local var=$1
-  local val=${!var}
-
-  if [[ -n "${val}" ]]; then
-    hadoop_debug "${var} = ${val}"
-  fi
-}
-
-## @description  Create the directory 'dir'.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        dir
-function hadoop_mkdir
-{
-  local dir=$1
-
-  if [[ ! -w "${dir}" ]] && [[ ! -d "${dir}" ]]; then
-    hadoop_error "WARNING: ${dir} does not exist. Creating."
-    if ! mkdir -p "${dir}"; then
-      hadoop_error "ERROR: Unable to create ${dir}. Aborting."
-      exit 1
-    fi
-  fi
-}
-
-## @description  Bootstraps the Hadoop shell environment
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_bootstrap
-{
-  # the root of the Hadoop installation
-  # See HADOOP-6255 for the expected directory structure layout
-
-  if [[ -n "${DEFAULT_LIBEXEC_DIR}" ]]; then
-    hadoop_error "WARNING: DEFAULT_LIBEXEC_DIR ignored. It has been replaced by HADOOP_DEFAULT_LIBEXEC_DIR."
-  fi
-
-  # By now, HADOOP_LIBEXEC_DIR should have been defined upstream
-  # We can piggyback off of that to figure out where the default
-  # HADOOP_FREFIX should be.  This allows us to run without
-  # HADOOP_HOME ever being defined by a human! As a consequence
-  # HADOOP_LIBEXEC_DIR now becomes perhaps the single most powerful
-  # env var within Hadoop.
-  if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
-    hadoop_error "HADOOP_LIBEXEC_DIR is not defined.  Exiting."
-    exit 1
-  fi
-  HADOOP_DEFAULT_PREFIX=$(cd -P -- "${HADOOP_LIBEXEC_DIR}/.." >/dev/null && pwd -P)
-  HADOOP_HOME=${HADOOP_HOME:-$HADOOP_DEFAULT_PREFIX}
-  export HADOOP_HOME
-
-  #
-  # short-cuts. vendors may redefine these as well, preferably
-  # in hadoop-layout.sh
-  #
-  HADOOP_COMMON_DIR=${HADOOP_COMMON_DIR:-"share/hadoop/common"}
-  HADOOP_COMMON_LIB_JARS_DIR=${HADOOP_COMMON_LIB_JARS_DIR:-"share/hadoop/common/lib"}
-  HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_COMMON_LIB_NATIVE_DIR:-"lib/native"}
-  HDFS_DIR=${HDFS_DIR:-"share/hadoop/hdfs"}
-  HDFS_LIB_JARS_DIR=${HDFS_LIB_JARS_DIR:-"share/hadoop/hdfs/lib"}
-  YARN_DIR=${YARN_DIR:-"share/hadoop/yarn"}
-  YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
-  MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
-  MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
-  HDDS_DIR=${HDDS_DIR:-"share/hadoop/hdds"}
-  HDDS_LIB_JARS_DIR=${HDDS_LIB_JARS_DIR:-"share/hadoop/hdds/lib"}
-  OZONE_DIR=${OZONE_DIR:-"share/hadoop/ozone"}
-  OZONE_LIB_JARS_DIR=${OZONE_LIB_JARS_DIR:-"share/hadoop/ozone/lib"}
-  OZONEFS_DIR=${OZONEFS_DIR:-"share/hadoop/ozonefs"}
-
-  HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}}
-  HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
-  HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
-
-  # by default, whatever we are about to run doesn't support
-  # daemonization
-  HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
-
-  # by default, we have not been self-re-execed
-  HADOOP_REEXECED_CMD=false
-
-  HADOOP_SUBCMD_SECURESERVICE=false
-
-  # This is the default we claim in hadoop-env.sh
-  JSVC_HOME=${JSVC_HOME:-"/usr/bin"}
-
-  # usage output set to zero
-  hadoop_reset_usage
-
-  export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
-
-  # defaults
-  export HADOOP_OPTS=${HADOOP_OPTS:-"-Djava.net.preferIPv4Stack=true"}
-  hadoop_debug "Initial HADOOP_OPTS=${HADOOP_OPTS}"
-}
-
-## @description  Locate Hadoop's configuration directory
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_find_confdir
-{
-  local conf_dir
-
-  # An attempt at compatibility with some Hadoop 1.x
-  # installs.
-  if [[ -e "${HADOOP_HOME}/conf/hadoop-env.sh" ]]; then
-    conf_dir="conf"
-  else
-    conf_dir="etc/hadoop"
-  fi
-  export HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-${HADOOP_HOME}/${conf_dir}}"
-
-  hadoop_debug "HADOOP_CONF_DIR=${HADOOP_CONF_DIR}"
-}
-
-## @description  Validate ${HADOOP_CONF_DIR}
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @return       will exit on failure conditions
-function hadoop_verify_confdir
-{
-  # Check only log4j.properties by default.
-  # --loglevel does not work without logger settings in log4j.log4j.properties.
-  if [[ ! -f "${HADOOP_CONF_DIR}/log4j.properties" ]]; then
-    hadoop_error "WARNING: log4j.properties is not found. HADOOP_CONF_DIR may be incomplete."
-  fi
-}
-
-## @description  Import the hadoop-env.sh settings
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_exec_hadoopenv
-{
-  if [[ -z "${HADOOP_ENV_PROCESSED}" ]]; then
-    if [[ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]]; then
-      export HADOOP_ENV_PROCESSED=true
-      # shellcheck source=./hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
-      . "${HADOOP_CONF_DIR}/hadoop-env.sh"
-    fi
-  fi
-}
-
-## @description  Import the replaced functions
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_exec_userfuncs
-{
-  if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then
-    # shellcheck disable=SC1090
-    . "${HADOOP_CONF_DIR}/hadoop-user-functions.sh"
-  fi
-}
-
-## @description  Read the user's settings.  This provides for users to
-## @description  override and/or append hadoop-env.sh. It is not meant
-## @description  as a complete system override.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_exec_user_hadoopenv
-{
-  if [[ -f "${HOME}/.hadoop-env" ]]; then
-    hadoop_debug "Applying the user's .hadoop-env"
-    # shellcheck disable=SC1090
-    . "${HOME}/.hadoop-env"
-  fi
-}
-
-## @description  Read the user's settings.  This provides for users to
-## @description  run Hadoop Shell API after system bootstrap
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_exec_hadooprc
-{
-  if [[ -f "${HOME}/.hadooprc" ]]; then
-    hadoop_debug "Applying the user's .hadooprc"
-    # shellcheck disable=SC1090
-    . "${HOME}/.hadooprc"
-  fi
-}
-
-## @description  Import shellprofile.d content
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_import_shellprofiles
-{
-  local i
-  local files1
-  local files2
-
-  if [[ -d "${HADOOP_LIBEXEC_DIR}/shellprofile.d" ]]; then
-    files1=(${HADOOP_LIBEXEC_DIR}/shellprofile.d/*.sh)
-    hadoop_debug "shellprofiles: ${files1[*]}"
-  else
-    hadoop_error "WARNING: ${HADOOP_LIBEXEC_DIR}/shellprofile.d doesn't exist. Functionality may not work."
-  fi
-
-  if [[ -d "${HADOOP_CONF_DIR}/shellprofile.d" ]]; then
-    files2=(${HADOOP_CONF_DIR}/shellprofile.d/*.sh)
-  fi
-
-  # enable bundled shellprofiles that come
-  # from hadoop-tools.  This converts the user-facing HADOOP_OPTIONAL_TOOLS
-  # to the HADOOP_TOOLS_OPTIONS that the shell profiles expect.
-  # See dist-tools-hooks-maker for how the example HADOOP_OPTIONAL_TOOLS
-  # gets populated into hadoop-env.sh
-
-  for i in ${HADOOP_OPTIONAL_TOOLS//,/ }; do
-    hadoop_add_entry HADOOP_TOOLS_OPTIONS "${i}"
-  done
-
-  for i in "${files1[@]}" "${files2[@]}"
-  do
-    if [[ -n "${i}"
-      && -f "${i}" ]]; then
-      hadoop_debug "Profiles: importing ${i}"
-      # shellcheck disable=SC1090
-      . "${i}"
-    fi
-  done
-}
-
-## @description  Initialize the registered shell profiles
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_shellprofiles_init
-{
-  local i
-
-  for i in ${HADOOP_SHELL_PROFILES}
-  do
-    if declare -F _${i}_hadoop_init >/dev/null ; then
-       hadoop_debug "Profiles: ${i} init"
-       # shellcheck disable=SC2086
-       _${i}_hadoop_init
-    fi
-  done
-}
-
-## @description  Apply the shell profile classpath additions
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_shellprofiles_classpath
-{
-  local i
-
-  for i in ${HADOOP_SHELL_PROFILES}
-  do
-    if declare -F _${i}_hadoop_classpath >/dev/null ; then
-       hadoop_debug "Profiles: ${i} classpath"
-       # shellcheck disable=SC2086
-       _${i}_hadoop_classpath
-    fi
-  done
-}
-
-## @description  Apply the shell profile native library additions
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_shellprofiles_nativelib
-{
-  local i
-
-  for i in ${HADOOP_SHELL_PROFILES}
-  do
-    if declare -F _${i}_hadoop_nativelib >/dev/null ; then
-       hadoop_debug "Profiles: ${i} nativelib"
-       # shellcheck disable=SC2086
-       _${i}_hadoop_nativelib
-    fi
-  done
-}
-
-## @description  Apply the shell profile final configuration
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_shellprofiles_finalize
-{
-  local i
-
-  for i in ${HADOOP_SHELL_PROFILES}
-  do
-    if declare -F _${i}_hadoop_finalize >/dev/null ; then
-       hadoop_debug "Profiles: ${i} finalize"
-       # shellcheck disable=SC2086
-       _${i}_hadoop_finalize
-    fi
-  done
-}
-
-## @description  Initialize the Hadoop shell environment, now that
-## @description  user settings have been imported
-## @audience     private
-## @stability    evolving
-## @replaceable  no
-function hadoop_basic_init
-{
-  # Some of these are also set in hadoop-env.sh.
-  # we still set them here just in case hadoop-env.sh is
-  # broken in some way, set up defaults, etc.
-  #
-  # but it is important to note that if you update these
-  # you also need to update hadoop-env.sh as well!!!
-
-  CLASSPATH=""
-  hadoop_debug "Initialize CLASSPATH"
-
-  if [[ -z "${HADOOP_COMMON_HOME}" ]] &&
-  [[ -d "${HADOOP_HOME}/${HADOOP_COMMON_DIR}" ]]; then
-    export HADOOP_COMMON_HOME="${HADOOP_HOME}"
-  fi
-
-  # default policy file for service-level authorization
-  HADOOP_POLICYFILE=${HADOOP_POLICYFILE:-"hadoop-policy.xml"}
-
-  # define HADOOP_HDFS_HOME
-  if [[ -z "${HADOOP_HDFS_HOME}" ]] &&
-     [[ -d "${HADOOP_HOME}/${HDFS_DIR}" ]]; then
-    export HADOOP_HDFS_HOME="${HADOOP_HOME}"
-  fi
-
-  # define HADOOP_YARN_HOME
-  if [[ -z "${HADOOP_YARN_HOME}" ]] &&
-     [[ -d "${HADOOP_HOME}/${YARN_DIR}" ]]; then
-    export HADOOP_YARN_HOME="${HADOOP_HOME}"
-  fi
-
-  # define HADOOP_MAPRED_HOME
-  if [[ -z "${HADOOP_MAPRED_HOME}" ]] &&
-     [[ -d "${HADOOP_HOME}/${MAPRED_DIR}" ]]; then
-    export HADOOP_MAPRED_HOME="${HADOOP_HOME}"
-  fi
-
-  if [[ ! -d "${HADOOP_COMMON_HOME}" ]]; then
-    hadoop_error "ERROR: Invalid HADOOP_COMMON_HOME"
-    exit 1
-  fi
-
-  if [[ ! -d "${HADOOP_HDFS_HOME}" ]]; then
-    hadoop_error "ERROR: Invalid HADOOP_HDFS_HOME"
-    exit 1
-  fi
-
-  if [[ ! -d "${HADOOP_YARN_HOME}" ]]; then
-    hadoop_error "ERROR: Invalid HADOOP_YARN_HOME"
-    exit 1
-  fi
-
-  if [[ ! -d "${HADOOP_MAPRED_HOME}" ]]; then
-    hadoop_error "ERROR: Invalid HADOOP_MAPRED_HOME"
-    exit 1
-  fi
-
-  # if for some reason the shell doesn't have $USER defined
-  # (e.g., ssh'd in to execute a command)
-  # let's get the effective username and use that
-  USER=${USER:-$(id -nu)}
-  HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER}
-  HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_HOME}/logs"}
-  HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log}
-  HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
-  HADOOP_NICENESS=${HADOOP_NICENESS:-0}
-  HADOOP_STOP_TIMEOUT=${HADOOP_STOP_TIMEOUT:-5}
-  HADOOP_PID_DIR=${HADOOP_PID_DIR:-/tmp}
-  HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}
-  HADOOP_DAEMON_ROOT_LOGGER=${HADOOP_DAEMON_ROOT_LOGGER:-${HADOOP_LOGLEVEL},RFA}
-  HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}
-  HADOOP_SSH_OPTS=${HADOOP_SSH_OPTS-"-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"}
-  HADOOP_SECURE_LOG_DIR=${HADOOP_SECURE_LOG_DIR:-${HADOOP_LOG_DIR}}
-  HADOOP_SECURE_PID_DIR=${HADOOP_SECURE_PID_DIR:-${HADOOP_PID_DIR}}
-  HADOOP_SSH_PARALLEL=${HADOOP_SSH_PARALLEL:-10}
-}
-
-## @description  Set the worker support information to the contents
-## @description  of `filename`
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        filename
-## @return       will exit if file does not exist
-function hadoop_populate_workers_file
-{
-  local workersfile=$1
-  shift
-  if [[ -f "${workersfile}" ]]; then
-    HADOOP_WORKERS="${workersfile}"
-  elif [[ -f "${HADOOP_CONF_DIR}/${workersfile}" ]]; then
-    HADOOP_WORKERS="${HADOOP_CONF_DIR}/${workersfile}"
-  else
-    hadoop_error "ERROR: Cannot find hosts file \"${workersfile}\""
-    hadoop_exit_with_usage 1
-  fi
-}
-
-## @description  Rotates the given `file` until `number` of
-## @description  files exist.
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        filename
-## @param        [number]
-## @return       $? will contain last mv's return value
-function hadoop_rotate_log
-{
-  #
-  # Users are likely to replace this one for something
-  # that gzips or uses dates or who knows what.
-  #
-  # be aware that &1 and &2 might go through here
-  # so don't do anything too crazy...
-  #
-  local log=$1;
-  local num=${2:-5};
-
-  if [[ -f "${log}" ]]; then # rotate logs
-    while [[ ${num} -gt 1 ]]; do
-      #shellcheck disable=SC2086
-      let prev=${num}-1
-      if [[ -f "${log}.${prev}" ]]; then
-        mv "${log}.${prev}" "${log}.${num}"
-      fi
-      num=${prev}
-    done
-    mv "${log}" "${log}.${num}"
-  fi
-}
-
-## @description  Via ssh, log into `hostname` and run `command`
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        hostname
-## @param        command
-## @param        [...]
-function hadoop_actual_ssh
-{
-  # we are passing this function to xargs
-  # should get hostname followed by rest of command line
-  local worker=$1
-  shift
-
-  # shellcheck disable=SC2086
-  ssh ${HADOOP_SSH_OPTS} ${worker} $"${@// /\\ }" 2>&1 | sed "s/^/$worker: /"
-}
-
-## @description  Connect to ${HADOOP_WORKERS} or ${HADOOP_WORKER_NAMES}
-## @description  and execute command.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        [...]
-function hadoop_connect_to_hosts
-{
-  # shellcheck disable=SC2124
-  local params="$@"
-  local worker_file
-  local tmpslvnames
-
-  #
-  # ssh (or whatever) to a host
-  #
-  # User can specify hostnames or a file where the hostnames are (not both)
-  if [[ -n "${HADOOP_WORKERS}" && -n "${HADOOP_WORKER_NAMES}" ]] ; then
-    hadoop_error "ERROR: Both HADOOP_WORKERS and HADOOP_WORKER_NAMES were defined. Aborting."
-    exit 1
-  elif [[ -z "${HADOOP_WORKER_NAMES}" ]]; then
-    if [[ -n "${HADOOP_WORKERS}" ]]; then
-      worker_file=${HADOOP_WORKERS}
-    elif [[ -f "${HADOOP_CONF_DIR}/workers" ]]; then
-      worker_file=${HADOOP_CONF_DIR}/workers
-    elif [[ -f "${HADOOP_CONF_DIR}/slaves" ]]; then
-      hadoop_error "WARNING: 'slaves' file has been deprecated. Please use 'workers' file instead."
-      worker_file=${HADOOP_CONF_DIR}/slaves
-    fi
-  fi
-
-  # if pdsh is available, let's use it.  otherwise default
-  # to a loop around ssh.  (ugh)
-  if [[ -e '/usr/bin/pdsh' ]]; then
-    if [[ -z "${HADOOP_WORKER_NAMES}" ]] ; then
-      # if we were given a file, just let pdsh deal with it.
-      # shellcheck disable=SC2086
-      PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \
-      -f "${HADOOP_SSH_PARALLEL}" -w ^"${worker_file}" $"${@// /\\ }" 2>&1
-    else
-      # no spaces allowed in the pdsh arg host list
-      # shellcheck disable=SC2086
-      tmpslvnames=$(echo ${HADOOP_WORKER_NAMES} | tr -s ' ' ,)
-      PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \
-        -f "${HADOOP_SSH_PARALLEL}" \
-        -w "${tmpslvnames}" $"${@// /\\ }" 2>&1
-    fi
-  else
-    if [[ -z "${HADOOP_WORKER_NAMES}" ]]; then
-      HADOOP_WORKER_NAMES=$(sed 's/#.*$//;/^$/d' "${worker_file}")
-    fi
-    hadoop_connect_to_hosts_without_pdsh "${params}"
-  fi
-}
-
-## @description  Connect to ${HADOOP_WORKER_NAMES} and execute command
-## @description  under the environment which does not support pdsh.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        [...]
-function hadoop_connect_to_hosts_without_pdsh
-{
-  # shellcheck disable=SC2124
-  local params="$@"
-  local workers=(${HADOOP_WORKER_NAMES})
-  for (( i = 0; i < ${#workers[@]}; i++ ))
-  do
-    if (( i != 0 && i % HADOOP_SSH_PARALLEL == 0 )); then
-      wait
-    fi
-    # shellcheck disable=SC2086
-    hadoop_actual_ssh "${workers[$i]}" ${params} &
-  done
-  wait
-}
-
-## @description  Utility routine to handle --workers mode
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        commandarray
-function hadoop_common_worker_mode_execute
-{
-  #
-  # input should be the command line as given by the user
-  # in the form of an array
-  #
-  local argv=("$@")
-
-  # if --workers is still on the command line, remove it
-  # to prevent loops
-  # Also remove --hostnames and --hosts along with arg values
-  local argsSize=${#argv[@]};
-  for (( i = 0; i < argsSize; i++ ))
-  do
-    if [[ "${argv[$i]}" =~ ^--workers$ ]]; then
-      unset argv[$i]
-    elif [[ "${argv[$i]}" =~ ^--hostnames$ ]] ||
-      [[ "${argv[$i]}" =~ ^--hosts$ ]]; then
-      unset argv[$i];
-      let i++;
-      unset argv[$i];
-    fi
-  done
-  if [[ ${QATESTMODE} = true ]]; then
-    echo "${argv[@]}"
-    return
-  fi
-  hadoop_connect_to_hosts -- "${argv[@]}"
-}
-
-## @description  Verify that a shell command was passed a valid
-## @description  class name
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        classname
-## @return       0 = success
-## @return       1 = failure w/user message
-function hadoop_validate_classname
-{
-  local class=$1
-  shift 1
-
-  if [[ ! ${class} =~ \. ]]; then
-    # assuming the arg is typo of command if it does not conatain ".".
-    # class belonging to no package is not allowed as a result.
-    hadoop_error "ERROR: ${class} is not COMMAND nor fully qualified CLASSNAME."
-    return 1
-  fi
-  return 0
-}
-
-## @description  Append the `appendstring` if `checkstring` is not
-## @description  present in the given `envvar`
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        envvar
-## @param        checkstring
-## @param        appendstring
-function hadoop_add_param
-{
-  #
-  # general param dedupe..
-  # $1 is what we are adding to
-  # $2 is the name of what we want to add (key)
-  # $3 is the key+value of what we're adding
-  #
-  # doing it this way allows us to support all sorts of
-  # different syntaxes, just so long as they are space
-  # delimited
-  #
-  if [[ ! ${!1} =~ $2 ]] ; then
-    #shellcheck disable=SC2140
-    eval "$1"="'${!1} $3'"
-    if [[ ${!1:0:1} = ' ' ]]; then
-      #shellcheck disable=SC2140
-      eval "$1"="'${!1# }'"
-    fi
-    hadoop_debug "$1 accepted $3"
-  else
-    hadoop_debug "$1 declined $3"
-  fi
-}
-
-## @description  Register the given `shellprofile` to the Hadoop
-## @description  shell subsystem
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        shellprofile
-function hadoop_add_profile
-{
-  # shellcheck disable=SC2086
-  hadoop_add_param HADOOP_SHELL_PROFILES $1 $1
-}
-
-## @description  Add a file system object (directory, file,
-## @description  wildcard, ...) to the classpath. Optionally provide
-## @description  a hint as to where in the classpath it should go.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        object
-## @param        [before|after]
-## @return       0 = success (added or duplicate)
-## @return       1 = failure (doesn't exist or some other reason)
-function hadoop_add_classpath
-{
-  # However, with classpath (& JLP), we can do dedupe
-  # along with some sanity checking (e.g., missing directories)
-  # since we have a better idea of what is legal
-  #
-  # for wildcard at end, we can
-  # at least check the dir exists
-  if [[ $1 =~ ^.*\*$ ]]; then
-    local mp
-    mp=$(dirname "$1")
-    if [[ ! -d "${mp}" ]]; then
-      hadoop_debug "Rejected CLASSPATH: $1 (not a dir)"
-      return 1
-    fi
-
-    # no wildcard in the middle, so check existence
-    # (doesn't matter *what* it is)
-  elif [[ ! $1 =~ ^.*\*.*$ ]] && [[ ! -e "$1" ]]; then
-    hadoop_debug "Rejected CLASSPATH: $1 (does not exist)"
-    return 1
-  fi
-  if [[ -z "${CLASSPATH}" ]]; then
-    CLASSPATH=$1
-    hadoop_debug "Initial CLASSPATH=$1"
-  elif [[ ":${CLASSPATH}:" != *":$1:"* ]]; then
-    if [[ "$2" = "before" ]]; then
-      CLASSPATH="$1:${CLASSPATH}"
-      hadoop_debug "Prepend CLASSPATH: $1"
-    else
-      CLASSPATH+=:$1
-      hadoop_debug "Append CLASSPATH: $1"
-    fi
-  else
-    hadoop_debug "Dupe CLASSPATH: $1"
-  fi
-  return 0
-}
-
-## @description  Add a file system object (directory, file,
-## @description  wildcard, ...) to the colonpath.  Optionally provide
-## @description  a hint as to where in the colonpath it should go.
-## @description  Prior to adding, objects are checked for duplication
-## @description  and check for existence.  Many other functions use
-## @description  this function as their base implementation
-## @description  including `hadoop_add_javalibpath` and `hadoop_add_ldlibpath`.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        envvar
-## @param        object
-## @param        [before|after]
-## @return       0 = success (added or duplicate)
-## @return       1 = failure (doesn't exist or some other reason)
-function hadoop_add_colonpath
-{
-  # this is CLASSPATH, JLP, etc but with dedupe but no
-  # other checking
-  if [[ -d "${2}" ]] && [[ ":${!1}:" != *":$2:"* ]]; then
-    if [[ -z "${!1}" ]]; then
-      # shellcheck disable=SC2086
-      eval $1="'$2'"
-      hadoop_debug "Initial colonpath($1): $2"
-    elif [[ "$3" = "before" ]]; then
-      # shellcheck disable=SC2086
-      eval $1="'$2:${!1}'"
-      hadoop_debug "Prepend colonpath($1): $2"
-    else
-      # shellcheck disable=SC2086
-      eval $1+=":'$2'"
-      hadoop_debug "Append colonpath($1): $2"
-    fi
-    return 0
-  fi
-  hadoop_debug "Rejected colonpath($1): $2"
-  return 1
-}
-
-## @description  Add a file system object (directory, file,
-## @description  wildcard, ...) to the Java JNI path.  Optionally
-## @description  provide a hint as to where in the Java JNI path
-## @description  it should go.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        object
-## @param        [before|after]
-## @return       0 = success (added or duplicate)
-## @return       1 = failure (doesn't exist or some other reason)
-function hadoop_add_javalibpath
-{
-  # specialized function for a common use case
-  hadoop_add_colonpath JAVA_LIBRARY_PATH "$1" "$2"
-}
-
-## @description  Add a file system object (directory, file,
-## @description  wildcard, ...) to the LD_LIBRARY_PATH.  Optionally
-## @description  provide a hint as to where in the LD_LIBRARY_PATH
-## @description  it should go.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        object
-## @param        [before|after]
-## @return       0 = success (added or duplicate)
-## @return       1 = failure (doesn't exist or some other reason)
-function hadoop_add_ldlibpath
-{
-  local status
-  # specialized function for a common use case
-  hadoop_add_colonpath LD_LIBRARY_PATH "$1" "$2"
-  status=$?
-
-  # note that we export this
-  export LD_LIBRARY_PATH
-  return ${status}
-}
-
-## @description  Add the common/core Hadoop components to the
-## @description  environment
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @returns      1 on failure, may exit
-## @returns      0 on success
-function hadoop_add_common_to_classpath
-{
-  #
-  # get all of the common jars+config in the path
-  #
-
-  if [[ -z "${HADOOP_COMMON_HOME}"
-    || -z "${HADOOP_COMMON_DIR}"
-    || -z "${HADOOP_COMMON_LIB_JARS_DIR}" ]]; then
-    hadoop_debug "COMMON_HOME=${HADOOP_COMMON_HOME}"
-    hadoop_debug "COMMON_DIR=${HADOOP_COMMON_DIR}"
-    hadoop_debug "COMMON_LIB_JARS_DIR=${HADOOP_COMMON_LIB_JARS_DIR}"
-    hadoop_error "ERROR: HADOOP_COMMON_HOME or related vars are not configured."
-    exit 1
-  fi
-
-  # developers
-  if [[ -n "${HADOOP_ENABLE_BUILD_PATHS}" ]]; then
-    hadoop_add_classpath "${HADOOP_COMMON_HOME}/hadoop-common/target/classes"
-  fi
-
-  hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"'/*'
-  hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*'
-}
-
-## @description  Run libexec/tools/module.sh to add to the classpath
-## @description  environment
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        module
-function hadoop_add_to_classpath_tools
-{
-  declare module=$1
-
-  if [[ -f "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh" ]]; then
-    # shellcheck disable=SC1090
-    . "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh"
-  else
-    hadoop_error "ERROR: Tools helper ${HADOOP_LIBEXEC_DIR}/tools/${module}.sh was not found."
-  fi
-
-  if declare -f hadoop_classpath_tools_${module} >/dev/null 2>&1; then
-    "hadoop_classpath_tools_${module}"
-  fi
-}
-
-## @description  Add the user's custom classpath settings to the
-## @description  environment
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_add_to_classpath_userpath
-{
-  # Add the user-specified HADOOP_CLASSPATH to the
-  # official CLASSPATH env var if HADOOP_USE_CLIENT_CLASSLOADER
-  # is not set.
-  # Add it first or last depending on if user has
-  # set env-var HADOOP_USER_CLASSPATH_FIRST
-  # we'll also dedupe it, because we're cool like that.
-  #
-  declare -a array
-  declare -i c=0
-  declare -i j
-  declare -i i
-  declare idx
-
-  if [[ -n "${HADOOP_CLASSPATH}" ]]; then
-    # I wonder if Java runs on VMS.
-    for idx in $(echo "${HADOOP_CLASSPATH}" | tr : '\n'); do
-      array[${c}]=${idx}
-      ((c=c+1))
-    done
-
-    # bats gets confused by j getting set to 0
-    ((j=c-1)) || ${QATESTMODE}
-
-    if [[ -z "${HADOOP_USE_CLIENT_CLASSLOADER}" ]]; then
-      if [[ -z "${HADOOP_USER_CLASSPATH_FIRST}" ]]; then
-        for ((i=0; i<=j; i++)); do
-          hadoop_add_classpath "${array[$i]}" after
-        done
-      else
-        for ((i=j; i>=0; i--)); do
-          hadoop_add_classpath "${array[$i]}" before
-        done
-      fi
-    fi
-  fi
-}
-
-## @description  Routine to configure any OS-specific settings.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @return       may exit on failure conditions
-function hadoop_os_tricks
-{
-  local bindv6only
-
-  HADOOP_IS_CYGWIN=false
-  case ${HADOOP_OS_TYPE} in
-    Darwin)
-      if [[ -z "${JAVA_HOME}" ]]; then
-        if [[ -x /usr/libexec/java_home ]]; then
-          JAVA_HOME="$(/usr/libexec/java_home)"
-          export JAVA_HOME
-        else
-          JAVA_HOME=/Library/Java/Home
-          export JAVA_HOME
-        fi
-      fi
-    ;;
-    Linux)
-
-      # Newer versions of glibc use an arena memory allocator that
-      # causes virtual # memory usage to explode. This interacts badly
-      # with the many threads that we use in Hadoop. Tune the variable
-      # down to prevent vmem explosion.
-      export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
-      # we put this in QA test mode off so that non-Linux can test
-      if [[ "${QATESTMODE}" = true ]]; then
-        return
-      fi
-
-      # NOTE! HADOOP_ALLOW_IPV6 is a developer hook.  We leave it
-      # undocumented in hadoop-env.sh because we don't want users to
-      # shoot themselves in the foot while devs make IPv6 work.
-
-      bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
-
-      if [[ -n "${bindv6only}" ]] &&
-         [[ "${bindv6only}" -eq "1" ]] &&
-         [[ "${HADOOP_ALLOW_IPV6}" != "yes" ]]; then
-        hadoop_error "ERROR: \"net.ipv6.bindv6only\" is set to 1 "
-        hadoop_error "ERROR: Hadoop networking could be broken. Aborting."
-        hadoop_error "ERROR: For more info: http://wiki.apache.org/hadoop/HadoopIPv6"
-        exit 1
-      fi
-    ;;
-    CYGWIN*)
-      # Flag that we're running on Cygwin to trigger path translation later.
-      HADOOP_IS_CYGWIN=true
-    ;;
-  esac
-}
-
-## @description  Configure/verify ${JAVA_HOME}
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @return       may exit on failure conditions
-function hadoop_java_setup
-{
-  # Bail if we did not detect it
-  if [[ -z "${JAVA_HOME}" ]]; then
-    hadoop_error "ERROR: JAVA_HOME is not set and could not be found."
-    exit 1
-  fi
-
-  if [[ ! -d "${JAVA_HOME}" ]]; then
-    hadoop_error "ERROR: JAVA_HOME ${JAVA_HOME} does not exist."
-    exit 1
-  fi
-
-  JAVA="${JAVA_HOME}/bin/java"
-
-  if [[ ! -x "$JAVA" ]]; then
-    hadoop_error "ERROR: $JAVA is not executable."
-    exit 1
-  fi
-}
-
-## @description  Finish Java JNI paths prior to execution
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_finalize_libpaths
-{
-  if [[ -n "${JAVA_LIBRARY_PATH}" ]]; then
-    hadoop_translate_cygwin_path JAVA_LIBRARY_PATH
-    hadoop_add_param HADOOP_OPTS java.library.path \
-      "-Djava.library.path=${JAVA_LIBRARY_PATH}"
-    export LD_LIBRARY_PATH
-  fi
-}
-
-## @description  Finish Java heap parameters prior to execution
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_finalize_hadoop_heap
-{
-  if [[ -n "${HADOOP_HEAPSIZE_MAX}" ]]; then
-    if [[ "${HADOOP_HEAPSIZE_MAX}" =~ ^[0-9]+$ ]]; then
-      HADOOP_HEAPSIZE_MAX="${HADOOP_HEAPSIZE_MAX}m"
-    fi
-    hadoop_add_param HADOOP_OPTS Xmx "-Xmx${HADOOP_HEAPSIZE_MAX}"
-  fi
-
-  # backwards compatibility
-  if [[ -n "${HADOOP_HEAPSIZE}" ]]; then
-    if [[ "${HADOOP_HEAPSIZE}" =~ ^[0-9]+$ ]]; then
-      HADOOP_HEAPSIZE="${HADOOP_HEAPSIZE}m"
-    fi
-    hadoop_add_param HADOOP_OPTS Xmx "-Xmx${HADOOP_HEAPSIZE}"
-  fi
-
-  if [[ -n "${HADOOP_HEAPSIZE_MIN}" ]]; then
-    if [[ "${HADOOP_HEAPSIZE_MIN}" =~ ^[0-9]+$ ]]; then
-      HADOOP_HEAPSIZE_MIN="${HADOOP_HEAPSIZE_MIN}m"
-    fi
-    hadoop_add_param HADOOP_OPTS Xms "-Xms${HADOOP_HEAPSIZE_MIN}"
-  fi
-}
-
-## @description  Converts the contents of the variable name
-## @description  `varnameref` into the equivalent Windows path.
-## @description  If the second parameter is true, then `varnameref`
-## @description  is treated as though it was a path list.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        varnameref
-## @param        [true]
-function hadoop_translate_cygwin_path
-{
-  if [[ "${HADOOP_IS_CYGWIN}" = "true" ]]; then
-    if [[ "$2" = "true" ]]; then
-      #shellcheck disable=SC2016
-      eval "$1"='$(cygpath -p -w "${!1}" 2>/dev/null)'
-    else
-      #shellcheck disable=SC2016
-      eval "$1"='$(cygpath -w "${!1}" 2>/dev/null)'
-    fi
-  fi
-}
-
-## @description  Adds the HADOOP_CLIENT_OPTS variable to
-## @description  HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-function hadoop_add_client_opts
-{
-  if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = false
-     || -z "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" ]]; then
-    hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-  fi
-}
-
-## @description  Finish configuring Hadoop specific system properties
-## @description  prior to executing Java
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_finalize_hadoop_opts
-{
-  hadoop_translate_cygwin_path HADOOP_LOG_DIR
-  hadoop_add_param HADOOP_OPTS hadoop.log.dir "-Dhadoop.log.dir=${HADOOP_LOG_DIR}"
-  hadoop_add_param HADOOP_OPTS hadoop.log.file "-Dhadoop.log.file=${HADOOP_LOGFILE}"
-  hadoop_translate_cygwin_path HADOOP_HOME
-  export HADOOP_HOME
-  hadoop_add_param HADOOP_OPTS hadoop.home.dir "-Dhadoop.home.dir=${HADOOP_HOME}"
-  hadoop_add_param HADOOP_OPTS hadoop.id.str "-Dhadoop.id.str=${HADOOP_IDENT_STRING}"
-  hadoop_add_param HADOOP_OPTS hadoop.root.logger "-Dhadoop.root.logger=${HADOOP_ROOT_LOGGER}"
-  hadoop_add_param HADOOP_OPTS hadoop.policy.file "-Dhadoop.policy.file=${HADOOP_POLICYFILE}"
-  hadoop_add_param HADOOP_OPTS hadoop.security.logger "-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER}"
-}
-
-## @description  Finish Java classpath prior to execution
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_finalize_classpath
-{
-  hadoop_add_classpath "${HADOOP_CONF_DIR}" before
-
-  # user classpath gets added at the last minute. this allows
-  # override of CONF dirs and more
-  hadoop_add_to_classpath_userpath
-  hadoop_translate_cygwin_path CLASSPATH true
-}
-
-## @description  Finish all the remaining environment settings prior
-## @description  to executing Java.  This is a wrapper that calls
-## @description  the other `finalize` routines.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_finalize
-{
-  hadoop_shellprofiles_finalize
-
-  hadoop_finalize_classpath
-  hadoop_finalize_libpaths
-  hadoop_finalize_hadoop_heap
-  hadoop_finalize_hadoop_opts
-
-  hadoop_translate_cygwin_path HADOOP_HOME
-  hadoop_translate_cygwin_path HADOOP_CONF_DIR
-  hadoop_translate_cygwin_path HADOOP_COMMON_HOME
-  hadoop_translate_cygwin_path HADOOP_HDFS_HOME
-  hadoop_translate_cygwin_path HADOOP_YARN_HOME
-  hadoop_translate_cygwin_path HADOOP_MAPRED_HOME
-}
-
-## @description  Print usage information and exit with the passed
-## @description  `exitcode`
-## @audience     public
-## @stability    stable
-## @replaceable  no
-## @param        exitcode
-## @return       This function will always exit.
-function hadoop_exit_with_usage
-{
-  local exitcode=$1
-  if [[ -z $exitcode ]]; then
-    exitcode=1
-  fi
-  # shellcheck disable=SC2034
-  if declare -F hadoop_usage >/dev/null ; then
-    hadoop_usage
-  elif [[ -x /usr/bin/cowsay ]]; then
-    /usr/bin/cowsay -f elephant "Sorry, no help available."
-  else
-    hadoop_error "Sorry, no help available."
-  fi
-  exit $exitcode
-}
-
-## @description  Verify that prerequisites have been met prior to
-## @description  excuting a privileged program.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @return       This routine may exit.
-function hadoop_verify_secure_prereq
-{
-  # if you are on an OS like Illumos that has functional roles
-  # and you are using pfexec, you'll probably want to change
-  # this.
-
-  if ! hadoop_privilege_check && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
-    hadoop_error "ERROR: You must be a privileged user in order to run a secure service."
-    exit 1
-  else
-    return 0
-  fi
-}
-
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_setup_secure_service
-{
-  # need a more complicated setup? replace me!
-
-  HADOOP_PID_DIR=${HADOOP_SECURE_PID_DIR}
-  HADOOP_LOG_DIR=${HADOOP_SECURE_LOG_DIR}
-}
-
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_verify_piddir
-{
-  if [[ -z "${HADOOP_PID_DIR}" ]]; then
-    hadoop_error "No pid directory defined."
-    exit 1
-  fi
-  hadoop_mkdir "${HADOOP_PID_DIR}"
-  touch "${HADOOP_PID_DIR}/$$" >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Unable to write in ${HADOOP_PID_DIR}. Aborting."
-    exit 1
-  fi
-  rm "${HADOOP_PID_DIR}/$$" >/dev/null 2>&1
-}
-
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-function hadoop_verify_logdir
-{
-  if [[ -z "${HADOOP_LOG_DIR}" ]]; then
-    hadoop_error "No log directory defined."
-    exit 1
-  fi
-  hadoop_mkdir "${HADOOP_LOG_DIR}"
-  touch "${HADOOP_LOG_DIR}/$$" >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Unable to write in ${HADOOP_LOG_DIR}. Aborting."
-    exit 1
-  fi
-  rm "${HADOOP_LOG_DIR}/$$" >/dev/null 2>&1
-}
-
-## @description  Determine the status of the daemon referenced
-## @description  by `pidfile`
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        pidfile
-## @return       (mostly) LSB 4.1.0 compatible status
-function hadoop_status_daemon
-{
-  #
-  # LSB 4.1.0 compatible status command (1)
-  #
-  # 0 = program is running
-  # 1 = dead, but still a pid (2)
-  # 2 = (not used by us)
-  # 3 = not running
-  #
-  # 1 - this is not an endorsement of the LSB
-  #
-  # 2 - technically, the specification says /var/run/pid, so
-  #     we should never return this value, but we're giving
-  #     them the benefit of a doubt and returning 1 even if
-  #     our pid is not in in /var/run .
-  #
-
-  local pidfile=$1
-  shift
-
-  local pid
-  local pspid
-
-  if [[ -f "${pidfile}" ]]; then
-    pid=$(cat "${pidfile}")
-    if pspid=$(ps -o args= -p"${pid}" 2>/dev/null); then
-      # this is to check that the running process we found is actually the same
-      # daemon that we're interested in
-      if [[ ${pspid} =~ -Dproc_${daemonname} ]]; then
-        return 0
-      fi
-    fi
-    return 1
-  fi
-  return 3
-}
-
-## @description  Execute the Java `class`, passing along any `options`.
-## @description  Additionally, set the Java property -Dproc_`command`.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        class
-## @param        [options]
-function hadoop_java_exec
-{
-  # run a java command.  this is used for
-  # non-daemons
-
-  local command=$1
-  local class=$2
-  shift 2
-
-  hadoop_debug "Final CLASSPATH: ${CLASSPATH}"
-  hadoop_debug "Final HADOOP_OPTS: ${HADOOP_OPTS}"
-  hadoop_debug "Final JAVA_HOME: ${JAVA_HOME}"
-  hadoop_debug "java: ${JAVA}"
-  hadoop_debug "Class name: ${class}"
-  hadoop_debug "Command line options: $*"
-
-  export CLASSPATH
-  #shellcheck disable=SC2086
-  exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
-}
-
-## @description  Start a non-privileged daemon in the foreground.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        class
-## @param        pidfile
-## @param        [options]
-function hadoop_start_daemon
-{
-  # this is our non-privileged daemon starter
-  # that fires up a daemon in the *foreground*
-  # so complex! so wow! much java!
-  local command=$1
-  local class=$2
-  local pidfile=$3
-  shift 3
-
-  hadoop_debug "Final CLASSPATH: ${CLASSPATH}"
-  hadoop_debug "Final HADOOP_OPTS: ${HADOOP_OPTS}"
-  hadoop_debug "Final JAVA_HOME: ${JAVA_HOME}"
-  hadoop_debug "java: ${JAVA}"
-  hadoop_debug "Class name: ${class}"
-  hadoop_debug "Command line options: $*"
-
-  # this is for the non-daemon pid creation
-  #shellcheck disable=SC2086
-  echo $$ > "${pidfile}" 2>/dev/null
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR:  Cannot write ${command} pid ${pidfile}."
-  fi
-
-  export CLASSPATH
-  #shellcheck disable=SC2086
-  exec "${JAVA}" "-Dproc_${command}" ${HADOOP_OPTS} "${class}" "$@"
-}
-
-## @description  Start a non-privileged daemon in the background.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        class
-## @param        pidfile
-## @param        outfile
-## @param        [options]
-function hadoop_start_daemon_wrapper
-{
-  local daemonname=$1
-  local class=$2
-  local pidfile=$3
-  local outfile=$4
-  shift 4
-
-  local counter
-
-  hadoop_rotate_log "${outfile}"
-
-  hadoop_start_daemon "${daemonname}" \
-    "$class" \
-    "${pidfile}" \
-    "$@" >> "${outfile}" 2>&1 < /dev/null &
-
-  # we need to avoid a race condition here
-  # so let's wait for the fork to finish
-  # before overriding with the daemonized pid
-  (( counter=0 ))
-  while [[ ! -f ${pidfile} && ${counter} -le 5 ]]; do
-    sleep 1
-    (( counter++ ))
-  done
-
-  # this is for daemon pid creation
-  #shellcheck disable=SC2086
-  echo $! > "${pidfile}" 2>/dev/null
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR:  Cannot write ${daemonname} pid ${pidfile}."
-  fi
-
-  # shellcheck disable=SC2086
-  renice "${HADOOP_NICENESS}" $! >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Cannot set priority of ${daemonname} process $!"
-  fi
-
-  # shellcheck disable=SC2086
-  disown %+ >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Cannot disconnect ${daemonname} process $!"
-  fi
-  sleep 1
-
-  # capture the ulimit output
-  ulimit -a >> "${outfile}" 2>&1
-
-  # shellcheck disable=SC2086
-  if ! ps -p $! >/dev/null 2>&1; then
-    return 1
-  fi
-  return 0
-}
-
-## @description  Start a privileged daemon in the foreground.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        class
-## @param        daemonpidfile
-## @param        daemonoutfile
-## @param        daemonerrfile
-## @param        wrapperpidfile
-## @param        [options]
-function hadoop_start_secure_daemon
-{
-  # this is used to launch a secure daemon in the *foreground*
-  #
-  local daemonname=$1
-  local class=$2
-
-  # pid file to create for our daemon
-  local daemonpidfile=$3
-
-  # where to send stdout. jsvc has bad habits so this *may* be &1
-  # which means you send it to stdout!
-  local daemonoutfile=$4
-
-  # where to send stderr.  same thing, except &2 = stderr
-  local daemonerrfile=$5
-  local privpidfile=$6
-  shift 6
-
-  hadoop_rotate_log "${daemonoutfile}"
-  hadoop_rotate_log "${daemonerrfile}"
-
-  # shellcheck disable=SC2153
-  jsvc="${JSVC_HOME}/jsvc"
-  if [[ ! -f "${jsvc}" ]]; then
-    hadoop_error "JSVC_HOME is not set or set incorrectly. jsvc is required to run secure"
-    hadoop_error "or privileged daemons. Please download and install jsvc from "
-    hadoop_error "http://archive.apache.org/dist/commons/daemon/binaries/ "
-    hadoop_error "and set JSVC_HOME to the directory containing the jsvc binary."
-    exit 1
-  fi
-
-  # note that shellcheck will throw a
-  # bogus for-our-use-case 2086 here.
-  # it doesn't properly support multi-line situations
-
-  hadoop_debug "Final CLASSPATH: ${CLASSPATH}"
-  hadoop_debug "Final HADOOP_OPTS: ${HADOOP_OPTS}"
-  hadoop_debug "Final JSVC_HOME: ${JSVC_HOME}"
-  hadoop_debug "jsvc: ${jsvc}"
-  hadoop_debug "Final HADOOP_DAEMON_JSVC_EXTRA_OPTS: ${HADOOP_DAEMON_JSVC_EXTRA_OPTS}"
-  hadoop_debug "Class name: ${class}"
-  hadoop_debug "Command line options: $*"
-
-  #shellcheck disable=SC2086
-  echo $$ > "${privpidfile}" 2>/dev/null
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR:  Cannot write ${daemonname} pid ${privpidfile}."
-  fi
-
-  # shellcheck disable=SC2086
-  exec "${jsvc}" \
-    "-Dproc_${daemonname}" \
-    ${HADOOP_DAEMON_JSVC_EXTRA_OPTS} \
-    -outfile "${daemonoutfile}" \
-    -errfile "${daemonerrfile}" \
-    -pidfile "${daemonpidfile}" \
-    -nodetach \
-    -user "${HADOOP_SECURE_USER}" \
-    -cp "${CLASSPATH}" \
-    ${HADOOP_OPTS} \
-    "${class}" "$@"
-}
-
-## @description  Start a privileged daemon in the background.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        command
-## @param        class
-## @param        daemonpidfile
-## @param        daemonoutfile
-## @param        wrapperpidfile
-## @param        warpperoutfile
-## @param        daemonerrfile
-## @param        [options]
-function hadoop_start_secure_daemon_wrapper
-{
-  # this wraps hadoop_start_secure_daemon to take care
-  # of the dirty work to launch a daemon in the background!
-  local daemonname=$1
-  local class=$2
-
-  # same rules as hadoop_start_secure_daemon except we
-  # have some additional parameters
-
-  local daemonpidfile=$3
-
-  local daemonoutfile=$4
-
-  # the pid file of the subprocess that spawned our
-  # secure launcher
-  local jsvcpidfile=$5
-
-  # the output of the subprocess that spawned our secure
-  # launcher
-  local jsvcoutfile=$6
-
-  local daemonerrfile=$7
-  shift 7
-
-  local counter
-
-  hadoop_rotate_log "${jsvcoutfile}"
-
-  hadoop_start_secure_daemon \
-    "${daemonname}" \
-    "${class}" \
-    "${daemonpidfile}" \
-    "${daemonoutfile}" \
-    "${daemonerrfile}" \
-    "${jsvcpidfile}"  "$@" >> "${jsvcoutfile}" 2>&1 < /dev/null &
-
-  # we need to avoid a race condition here
-  # so let's wait for the fork to finish
-  # before overriding with the daemonized pid
-  (( counter=0 ))
-  while [[ ! -f ${daemonpidfile} && ${counter} -le 5 ]]; do
-    sleep 1
-    (( counter++ ))
-  done
-
-  #shellcheck disable=SC2086
-  if ! echo $! > "${jsvcpidfile}"; then
-    hadoop_error "ERROR:  Cannot write ${daemonname} pid ${jsvcpidfile}."
-  fi
-
-  sleep 1
-  #shellcheck disable=SC2086
-  renice "${HADOOP_NICENESS}" $! >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Cannot set priority of ${daemonname} process $!"
-  fi
-  if [[ -f "${daemonpidfile}" ]]; then
-    #shellcheck disable=SC2046
-    renice "${HADOOP_NICENESS}" $(cat "${daemonpidfile}" 2>/dev/null) >/dev/null 2>&1
-    if [[ $? -gt 0 ]]; then
-      hadoop_error "ERROR: Cannot set priority of ${daemonname} process $(cat "${daemonpidfile}" 2>/dev/null)"
-    fi
-  fi
-  #shellcheck disable=SC2046
-  disown %+ >/dev/null 2>&1
-  if [[ $? -gt 0 ]]; then
-    hadoop_error "ERROR: Cannot disconnect ${daemonname} process $!"
-  fi
-  # capture the ulimit output
-  su "${HADOOP_SECURE_USER}" -c 'bash -c "ulimit -a"' >> "${jsvcoutfile}" 2>&1
-  #shellcheck disable=SC2086
-  if ! ps -p $! >/dev/null 2>&1; then
-    return 1
-  fi
-  return 0
-}
-
-## @description  Wait till process dies or till timeout
-## @audience     private
-## @stability    evolving
-## @param        pid
-## @param        timeout
-function wait_process_to_die_or_timeout
-{
-  local pid=$1
-  local timeout=$2
-
-  # Normalize timeout
-  # Round up or down
-  timeout=$(printf "%.0f\n" "${timeout}")
-  if [[ ${timeout} -lt 1  ]]; then
-    # minimum 1 second
-    timeout=1
-  fi
-
-  # Wait to see if it's still alive
-  for (( i=0; i < "${timeout}"; i++ ))
-  do
-    if kill -0 "${pid}" > /dev/null 2>&1; then
-      sleep 1
-    else
-      break
-    fi
-  done
-}
-
-## @description  Stop the non-privileged `command` daemon with that
-## @description  that is running at `pidfile`.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        pidfile
-function hadoop_stop_daemon
-{
-  local cmd=$1
-  local pidfile=$2
-  shift 2
-
-  local pid
-  local cur_pid
-
-  if [[ -f "${pidfile}" ]]; then
-    pid=$(cat "$pidfile")
-
-    kill "${pid}" >/dev/null 2>&1
-
-    wait_process_to_die_or_timeout "${pid}" "${HADOOP_STOP_TIMEOUT}"
-
-    if kill -0 "${pid}" > /dev/null 2>&1; then
-      hadoop_error "WARNING: ${cmd} did not stop gracefully after ${HADOOP_STOP_TIMEOUT} seconds: Trying to kill with kill -9"
-      kill -9 "${pid}" >/dev/null 2>&1
-    fi
-    wait_process_to_die_or_timeout "${pid}" "${HADOOP_STOP_TIMEOUT}"
-    if ps -p "${pid}" > /dev/null 2>&1; then
-      hadoop_error "ERROR: Unable to kill ${pid}"
-    else
-      cur_pid=$(cat "$pidfile")
-      if [[ "${pid}" = "${cur_pid}" ]]; then
-        rm -f "${pidfile}" >/dev/null 2>&1
-      else
-        hadoop_error "WARNING: pid has changed for ${cmd}, skip deleting pid file"
-      fi
-    fi
-  fi
-}
-
-## @description  Stop the privileged `command` daemon with that
-## @description  that is running at `daemonpidfile` and launched with
-## @description  the wrapper at `wrapperpidfile`.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        daemonpidfile
-## @param        wrapperpidfile
-function hadoop_stop_secure_daemon
-{
-  local command=$1
-  local daemonpidfile=$2
-  local privpidfile=$3
-  shift 3
-  local ret
-
-  local daemon_pid
-  local priv_pid
-  local cur_daemon_pid
-  local cur_priv_pid
-
-  daemon_pid=$(cat "$daemonpidfile")
-  priv_pid=$(cat "$privpidfile")
-
-  hadoop_stop_daemon "${command}" "${daemonpidfile}"
-  ret=$?
-
-  cur_daemon_pid=$(cat "$daemonpidfile")
-  cur_priv_pid=$(cat "$privpidfile")
-
-  if [[ "${daemon_pid}" = "${cur_daemon_pid}" ]]; then
-    rm -f "${daemonpidfile}" >/dev/null 2>&1
-  else
-    hadoop_error "WARNING: daemon pid has changed for ${command}, skip deleting daemon pid file"
-  fi
-
-  if [[ "${priv_pid}" = "${cur_priv_pid}" ]]; then
-    rm -f "${privpidfile}" >/dev/null 2>&1
-  else
-    hadoop_error "WARNING: priv pid has changed for ${command}, skip deleting priv pid file"
-  fi
-  return ${ret}
-}
-
-## @description  Manage a non-privileged daemon.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        [start|stop|status|default]
-## @param        command
-## @param        class
-## @param        daemonpidfile
-## @param        daemonoutfile
-## @param        [options]
-function hadoop_daemon_handler
-{
-  local daemonmode=$1
-  local daemonname=$2
-  local class=$3
-  local daemon_pidfile=$4
-  local daemon_outfile=$5
-  shift 5
-
-  case ${daemonmode} in
-    status)
-      hadoop_status_daemon "${daemon_pidfile}"
-      exit $?
-    ;;
-
-    stop)
-      hadoop_stop_daemon "${daemonname}" "${daemon_pidfile}"
-      exit $?
-    ;;
-
-    ##COMPAT  -- older hadoops would also start daemons by default
-    start|default)
-      hadoop_verify_piddir
-      hadoop_verify_logdir
-      hadoop_status_daemon "${daemon_pidfile}"
-      if [[ $? == 0  ]]; then
-        hadoop_error "${daemonname} is running as process $(cat "${daemon_pidfile}").  Stop it first."
-        exit 1
-      else
-        # stale pid file, so just remove it and continue on
-        rm -f "${daemon_pidfile}" >/dev/null 2>&1
-      fi
-      ##COMPAT  - differenticate between --daemon start and nothing
-      # "nothing" shouldn't detach
-      if [[ "$daemonmode" = "default" ]]; then
-        hadoop_start_daemon "${daemonname}" "${class}" "${daemon_pidfile}" "$@"
-      else
-        hadoop_start_daemon_wrapper "${daemonname}" \
-        "${class}" "${daemon_pidfile}" "${daemon_outfile}" "$@"
-      fi
-    ;;
-  esac
-}
-
-## @description  Manage a privileged daemon.
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        [start|stop|status|default]
-## @param        command
-## @param        class
-## @param        daemonpidfile
-## @param        daemonoutfile
-## @param        wrapperpidfile
-## @param        wrapperoutfile
-## @param        wrappererrfile
-## @param        [options]
-function hadoop_secure_daemon_handler
-{
-  local daemonmode=$1
-  local daemonname=$2
-  local classname=$3
-  local daemon_pidfile=$4
-  local daemon_outfile=$5
-  local priv_pidfile=$6
-  local priv_outfile=$7
-  local priv_errfile=$8
-  shift 8
-
-  case ${daemonmode} in
-    status)
-      hadoop_status_daemon "${daemon_pidfile}"
-      exit $?
-    ;;
-
-    stop)
-      hadoop_stop_secure_daemon "${daemonname}" \
-      "${daemon_pidfile}" "${priv_pidfile}"
-      exit $?
-    ;;
-
-    ##COMPAT  -- older hadoops would also start daemons by default
-    start|default)
-      hadoop_verify_piddir
-      hadoop_verify_logdir
-      hadoop_status_daemon "${daemon_pidfile}"
-      if [[ $? == 0  ]]; then
-        hadoop_error "${daemonname} is running as process $(cat "${daemon_pidfile}").  Stop it first."
-        exit 1
-      else
-        # stale pid file, so just remove it and continue on
-        rm -f "${daemon_pidfile}" >/dev/null 2>&1
-      fi
-
-      ##COMPAT  - differenticate between --daemon start and nothing
-      # "nothing" shouldn't detach
-      if [[ "${daemonmode}" = "default" ]]; then
-        hadoop_start_secure_daemon "${daemonname}" "${classname}" \
-        "${daemon_pidfile}" "${daemon_outfile}" \
-        "${priv_errfile}" "${priv_pidfile}" "$@"
-      else
-        hadoop_start_secure_daemon_wrapper "${daemonname}" "${classname}" \
-        "${daemon_pidfile}" "${daemon_outfile}" \
-        "${priv_pidfile}" "${priv_outfile}" "${priv_errfile}"  "$@"
-      fi
-    ;;
-  esac
-}
-
-## @description autodetect whether this is a priv subcmd
-## @description by whether or not a priv user var exists
-## @description and if HADOOP_SECURE_CLASSNAME is defined
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        subcommand
-## @return       1 = not priv
-## @return       0 = priv
-function hadoop_detect_priv_subcmd
-{
-  declare program=$1
-  declare command=$2
-
-  if [[ -z "${HADOOP_SECURE_CLASSNAME}" ]]; then
-    hadoop_debug "No secure classname defined."
-    return 1
-  fi
-
-  uvar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" SECURE_USER)
-  if [[ -z "${!uvar}" ]]; then
-    hadoop_debug "No secure user defined."
-    return 1
-  fi
-  return 0
-}
-
-## @description  Build custom subcommand var
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        subcommand
-## @param        customid
-## @return       string
-function hadoop_build_custom_subcmd_var
-{
-  declare program=$1
-  declare command=$2
-  declare custom=$3
-  declare uprogram
-  declare ucommand
-
-  if [[ -z "${BASH_VERSINFO[0]}" ]] \
-     || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
-    uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
-    ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
-  else
-    uprogram=${program^^}
-    ucommand=${command^^}
-  fi
-
-  echo "${uprogram}_${ucommand}_${custom}"
-}
-
-## @description  Verify that username in a var converts to user id
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        userstring
-## @return       0 for success
-## @return       1 for failure
-function hadoop_verify_user_resolves
-{
-  declare userstr=$1
-
-  if [[ -z ${userstr} || -z ${!userstr} ]] ; then
-    return 1
-  fi
-
-  id -u "${!userstr}" >/dev/null 2>&1
-}
-
-## @description  Verify that ${USER} is allowed to execute the
-## @description  given subcommand.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        command
-## @param        subcommand
-## @return       return 0 on success
-## @return       exit 1 on failure
-function hadoop_verify_user_perm
-{
-  declare program=$1
-  declare command=$2
-  declare uvar
-
-  if [[ ${command} =~ \. ]]; then
-    return 1
-  fi
-
-  uvar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" USER)
-
-  if [[ -n ${!uvar} ]]; then
-    if [[ ${!uvar} !=  "${USER}" ]]; then
-      hadoop_error "ERROR: ${command} can only be executed by ${!uvar}."
-      exit 1
-    fi
-  fi
-  return 0
-}
-
-## @description  Verify that ${USER} is allowed to execute the
-## @description  given subcommand.
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        subcommand
-## @return       1 on no re-exec needed
-## @return       0 on need to re-exec
-function hadoop_need_reexec
-{
-  declare program=$1
-  declare command=$2
-  declare uvar
-
-  # we've already been re-execed, bail
-
-  if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then
-    return 1
-  fi
-
-  if [[ ${command} =~ \. ]]; then
-    return 1
-  fi
-
-  # if we have privilege, and the _USER is defined, and _USER is
-  # set to someone who isn't us, then yes, we should re-exec.
-  # otherwise no, don't re-exec and let the system deal with it.
-
-  if hadoop_privilege_check; then
-    uvar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" USER)
-    if [[ -n ${!uvar} ]]; then
-      if [[ ${!uvar} !=  "${USER}" ]]; then
-        return 0
-      fi
-    fi
-  fi
-  return 1
-}
-
-## @description  Add custom (program)_(command)_OPTS to HADOOP_OPTS.
-## @description  Also handles the deprecated cases from pre-3.x.
-## @audience     public
-## @stability    evolving
-## @replaceable  yes
-## @param        program
-## @param        subcommand
-## @return       will exit on failure conditions
-function hadoop_subcommand_opts
-{
-  declare program=$1
-  declare command=$2
-  declare uvar
-  declare depvar
-  declare uprogram
-  declare ucommand
-
-  if [[ -z "${program}" || -z "${command}" ]]; then
-    return 1
-  fi
-
-  if [[ ${command} =~ \. ]]; then
-    return 1
-  fi
-
-  # bash 4 and up have built-in ways to upper and lower
-  # case the contents of vars.  This is faster than
-  # calling tr.
-
-  ## We don't call hadoop_build_custom_subcmd_var here
-  ## since we need to construct this for the deprecation
-  ## cases. For Hadoop 4.x, this needs to get cleaned up.
-
-  if [[ -z "${BASH_VERSINFO[0]}" ]] \
-     || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
-    uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
-    ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
-  else
-    uprogram=${program^^}
-    ucommand=${command^^}
-  fi
-
-  uvar="${uprogram}_${ucommand}_OPTS"
-
-  # Let's handle all of the deprecation cases early
-  # HADOOP_NAMENODE_OPTS -> HDFS_NAMENODE_OPTS
-
-  depvar="HADOOP_${ucommand}_OPTS"
-
-  if [[ "${depvar}" != "${uvar}" ]]; then
-    if [[ -n "${!depvar}" ]]; then
-      hadoop_deprecate_envvar "${depvar}" "${uvar}"
-    fi
-  fi
-
-  if [[ -n ${!uvar} ]]; then
-    hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
-    return 0
-  fi
-}
-
-## @description  Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
-## @description  This *does not* handle the pre-3.x deprecated cases
-## @audience     public
-## @stability    stable
-## @replaceable  yes
-## @param        program
-## @param        subcommand
-## @return       will exit on failure conditions
-function hadoop_subcommand_secure_opts
-{
-  declare program=$1
-  declare command=$2
-  declare uvar
-  declare uprogram
-  declare ucommand
-
-  if [[ -z "${program}" || -z "${command}" ]]; then
-    return 1
-  fi
-
-  # HDFS_DATANODE_SECURE_EXTRA_OPTS
-  # HDFS_NFS3_SECURE_EXTRA_OPTS
-  # ...
-  uvar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" SECURE_EXTRA_OPTS)
-
-  if [[ -n ${!uvar} ]]; then
-    hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
-    HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
-    return 0
-  fi
-}
-
-## @description  Perform the 'hadoop classpath', etc subcommand with the given
-## @description  parameters
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        [parameters]
-## @return       will print & exit with no params
-function hadoop_do_classpath_subcommand
-{
-  if [[ "$#" -gt 1 ]]; then
-    eval "$1"=org.apache.hadoop.util.Classpath
-  else
-    hadoop_finalize
-    echo "${CLASSPATH}"
-    exit 0
-  fi
-}
-
-## @description  generic shell script option parser.  sets
-## @description  HADOOP_PARSE_COUNTER to set number the
-## @description  caller should shift
-## @audience     private
-## @stability    evolving
-## @replaceable  yes
-## @param        [parameters, typically "$@"]
-function hadoop_parse_args
-{
-  HADOOP_DAEMON_MODE="default"
-  HADOOP_PARSE_COUNTER=0
-
-  # not all of the options supported here are supported by all commands
-  # however these are:
-  hadoop_add_option "--config dir" "Hadoop config directory"
-  hadoop_add_option "--debug" "turn on shell script debug mode"
-  hadoop_add_option "--help" "usage information"
-
-  while true; do
-    hadoop_debug "hadoop_parse_args: processing $1"
-    case $1 in
-      --buildpaths)
-        HADOOP_ENABLE_BUILD_PATHS=true
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
-      ;;
-      --config)
-        shift
-        confdir=$1
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
-        if [[ -d "${confdir}" ]]; then
-          HADOOP_CONF_DIR="${confdir}"
-        elif [[ -z "${confdir}" ]]; then
-          hadoop_error "ERROR: No parameter provided for --config "
-          hadoop_exit_with_usage 1
-        else
-          hadoop_error "ERROR: Cannot find configuration directory \"${confdir}\""
-          hadoop_exit_with_usage 1
-        fi
-      ;;
-      --daemon)
-        shift
-        HADOOP_DAEMON_MODE=$1
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
-        if [[ -z "${HADOOP_DAEMON_MODE}" || \
-          ! "${HADOOP_DAEMON_MODE}" =~ ^st(art|op|atus)$ ]]; then
-          hadoop_error "ERROR: --daemon must be followed by either \"start\", \"stop\", or \"status\"."
-          hadoop_exit_with_usage 1
-        fi
-      ;;
-      --debug)
-        shift
-        HADOOP_SHELL_SCRIPT_DEBUG=true
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
-      ;;
-      --help|-help|-h|help|--h|--\?|-\?|\?)
-        hadoop_exit_with_usage 0
-      ;;
-      --hostnames)
-        shift
-        HADOOP_WORKER_NAMES="$1"
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
-      ;;
-      --hosts)
-        shift
-        hadoop_populate_workers_file "$1"
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
-      ;;
-      --loglevel)
-        shift
-        # shellcheck disable=SC2034
-        HADOOP_LOGLEVEL="$1"
-        shift
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
-      ;;
-      --reexec)
-        shift
-        if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then
-          hadoop_error "ERROR: re-exec fork bomb prevention: --reexec already called"
-          exit 1
-        fi
-        HADOOP_REEXECED_CMD=true
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
-      ;;
-      --workers)
-        shift
-        # shellcheck disable=SC2034
-        HADOOP_WORKER_MODE=true
-        ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
-      ;;
-      *)
-        break
-      ;;
-    esac
-  done
-
-  hadoop_debug "hadoop_parse: asking caller to skip ${HADOOP_PARSE_COUNTER}"
-}
-
-## @description Handle subcommands from main program entries
-## @audience private
-## @stability evolving
-## @replaceable yes
-function hadoop_generic_java_subcmd_handler
-{
-  declare priv_outfile
-  declare priv_errfile
-  declare priv_pidfile
-  declare daemon_outfile
-  declare daemon_pidfile
-  declare secureuser
-
-  # The default/expected way to determine if a daemon is going to run in secure
-  # mode is defined by hadoop_detect_priv_subcmd.  If this returns true
-  # then setup the secure user var and tell the world we're in secure mode
-
-  if hadoop_detect_priv_subcmd "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"; then
-    HADOOP_SUBCMD_SECURESERVICE=true
-    secureuser=$(hadoop_build_custom_subcmd_var "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" SECURE_USER)
-
-    if ! hadoop_verify_user_resolves "${secureuser}"; then
-      hadoop_error "ERROR: User defined in ${secureuser} (${!secureuser}) does not exist. Aborting."
-      exit 1
-    fi
-
-    HADOOP_SECURE_USER="${!secureuser}"
-  fi
-
-  # check if we're running in secure mode.
-  # breaking this up from the above lets 3rd parties
-  # do things a bit different
-  # secure services require some extra setup
-  # if yes, then we need to define all of the priv and daemon stuff
-  # if not, then we just need to define daemon stuff.
-  # note the daemon vars are purposefully different between the two
-
-  if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
-
-    hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
-
-    hadoop_verify_secure_prereq
-    hadoop_setup_secure_service
-    priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
-    priv_errfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.err"
-    priv_pidfile="${HADOOP_PID_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
-    daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
-    daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
-  else
-    daemon_outfile="${HADOOP_LOG_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
-    daemon_pidfile="${HADOOP_PID_DIR}/hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}.pid"
-  fi
-
-  # are we actually in daemon mode?
-  # if yes, use the daemon logger and the appropriate log file.
-  if [[ "${HADOOP_DAEMON_MODE}" != "default" ]]; then
-    HADOOP_ROOT_LOGGER="${HADOOP_DAEMON_ROOT_LOGGER}"
-    if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
-      HADOOP_LOGFILE="hadoop-${HADOOP_SECURE_USER}-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
-    else
-      HADOOP_LOGFILE="hadoop-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.log"
-    fi
-  fi
-
-  # finish defining the environment: system properties, env vars, class paths, etc.
-  hadoop_finalize
-
-  # do the hard work of launching a daemon or just executing our interactive
-  # java class
-  if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = true ]]; then
-    if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
-      hadoop_secure_daemon_handler \
-        "${HADOOP_DAEMON_MODE}" \
-        "${HADOOP_SUBCMD}" \
-        "${HADOOP_SECURE_CLASSNAME}" \
-        "${daemon_pidfile}" \
-        "${daemon_outfile}" \
-        "${priv_pidfile}" \
-        "${priv_outfile}" \
-        "${priv_errfile}" \
-        "${HADOOP_SUBCMD_ARGS[@]}"
-    else
-      hadoop_daemon_handler \
-        "${HADOOP_DAEMON_MODE}" \
-        "${HADOOP_SUBCMD}" \
-        "${HADOOP_CLASSNAME}" \
-        "${daemon_pidfile}" \
-        "${daemon_outfile}" \
-        "${HADOOP_SUBCMD_ARGS[@]}"
-    fi
-    exit $?
-  else
-    hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "${HADOOP_SUBCMD_ARGS[@]}"
-  fi
-}

hadoop-hdds/common/src/main/bin/workers.sh

@@ -1,59 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-# Run a shell command on all worker hosts.
-#
-# Environment Variables
-#
-#   HADOOP_WORKERS    File naming remote hosts.
-#     Default is ${HADOOP_CONF_DIR}/workers.
-#   HADOOP_CONF_DIR  Alternate conf dir. Default is ${HADOOP_HOME}/conf.
-#   HADOOP_WORKER_SLEEP Seconds to sleep between spawning remote commands.
-#   HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
-##
-
-function hadoop_usage
-{
-  echo "Usage: workers.sh [--config confdir] command..."
-}
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
-  HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
-  this="${BASH_SOURCE-$0}"
-  bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-  HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh" ]]; then
-  . "${HADOOP_LIBEXEC_DIR}/hadoop-config.sh"
-else
-  echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
-  exit 1
-fi
-
-# if no args specified, show usage
-if [[ $# -le 0 ]]; then
-  hadoop_exit_with_usage 1
-fi
-
-hadoop_connect_to_hosts "$@"

hadoop-hdds/common/src/main/conf/core-site.xml

@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-</configuration>

hadoop-hdds/common/src/main/conf/hadoop-env.cmd

@@ -1,90 +0,0 @@
-@echo off
-@rem Licensed to the Apache Software Foundation (ASF) under one or more
-@rem contributor license agreements.  See the NOTICE file distributed with
-@rem this work for additional information regarding copyright ownership.
-@rem The ASF licenses this file to You under the Apache License, Version 2.0
-@rem (the "License"); you may not use this file except in compliance with
-@rem the License.  You may obtain a copy of the License at
-@rem
-@rem     http://www.apache.org/licenses/LICENSE-2.0
-@rem
-@rem Unless required by applicable law or agreed to in writing, software
-@rem distributed under the License is distributed on an "AS IS" BASIS,
-@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-@rem See the License for the specific language governing permissions and
-@rem limitations under the License.
-
-@rem Set Hadoop-specific environment variables here.
-
-@rem The only required environment variable is JAVA_HOME.  All others are
-@rem optional.  When running a distributed configuration it is best to
-@rem set JAVA_HOME in this file, so that it is correctly defined on
-@rem remote nodes.
-
-@rem The java implementation to use.  Required.
-set JAVA_HOME=%JAVA_HOME%
-
-@rem The jsvc implementation to use. Jsvc is required to run secure datanodes.
-@rem set JSVC_HOME=%JSVC_HOME%
-
-@rem set HADOOP_CONF_DIR=
-
-@rem Extra Java CLASSPATH elements.  Automatically insert capacity-scheduler.
-if exist %HADOOP_HOME%\contrib\capacity-scheduler (
-  if not defined HADOOP_CLASSPATH (
-    set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
-  ) else (
-    set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar
-  )
-)
-
-@rem The maximum amount of heap to use, in MB. Default is 1000.
-@rem set HADOOP_HEAPSIZE=
-@rem set HADOOP_NAMENODE_INIT_HEAPSIZE=""
-
-@rem Extra Java runtime options.  Empty by default.
-@rem set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true
-
-@rem Command specific options appended to HADOOP_OPTS when specified
-if not defined HADOOP_SECURITY_LOGGER (
-  set HADOOP_SECURITY_LOGGER=INFO,RFAS
-)
-if not defined HDFS_AUDIT_LOGGER (
-  set HDFS_AUDIT_LOGGER=INFO,NullAppender
-)
-
-set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS%
-set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS%
-set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS%
-
-@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc)
-set HADOOP_CLIENT_OPTS=-Xmx512m %HADOOP_CLIENT_OPTS%
-@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%"
-
-@rem On secure datanodes, user to run the datanode as after dropping privileges
-set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER%
-
-@rem Where log files are stored.  %HADOOP_HOME%/logs by default.
-@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME%
-
-@rem Where log files are stored in the secure data environment.
-set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER%
-
-@rem
-@rem Router-based HDFS Federation specific parameters
-@rem Specify the JVM options to be used when starting the RBF Routers.
-@rem These options will be appended to the options specified as HADOOP_OPTS
-@rem and therefore may override any similar flags set in HADOOP_OPTS
-@rem
-@rem set HADOOP_DFSROUTER_OPTS=""
-@rem
-
-@rem The directory where pid files are stored. /tmp by default.
-@rem NOTE: this should be set to a directory that can only be written to by
-@rem       the user that will run the hadoop daemons.  Otherwise there is the
-@rem       potential for a symlink attack.
-set HADOOP_PID_DIR=%HADOOP_PID_DIR%
-set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR%
-
-@rem A string representing this instance of hadoop. %USERNAME% by default.
-set HADOOP_IDENT_STRING=%USERNAME%

hadoop-hdds/common/src/main/conf/hadoop-env.sh

@@ -1,439 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Set Hadoop-specific environment variables here.
-
-##
-## THIS FILE ACTS AS THE MASTER FILE FOR ALL HADOOP PROJECTS.
-## SETTINGS HERE WILL BE READ BY ALL HADOOP COMMANDS.  THEREFORE,
-## ONE CAN USE THIS FILE TO SET YARN, HDFS, AND MAPREDUCE
-## CONFIGURATION OPTIONS INSTEAD OF xxx-env.sh.
-##
-## Precedence rules:
-##
-## {yarn-env.sh|hdfs-env.sh} > hadoop-env.sh > hard-coded defaults
-##
-## {YARN_xyz|HDFS_xyz} > HADOOP_xyz > hard-coded defaults
-##
-
-# Many of the options here are built from the perspective that users
-# may want to provide OVERWRITING values on the command line.
-# For example:
-#
-#  JAVA_HOME=/usr/java/testing hdfs dfs -ls
-#
-# Therefore, the vast majority (BUT NOT ALL!) of these defaults
-# are configured for substitution and not append.  If append
-# is preferable, modify this file accordingly.
-
-###
-# Generic settings for HADOOP
-###
-
-# Technically, the only required environment variable is JAVA_HOME.
-# All others are optional.  However, the defaults are probably not
-# preferred.  Many sites configure these options outside of Hadoop,
-# such as in /etc/profile.d
-
-# The java implementation to use. By default, this environment
-# variable is REQUIRED on ALL platforms except OS X!
-# export JAVA_HOME=
-
-# Location of Hadoop.  By default, Hadoop will attempt to determine
-# this location based upon its execution path.
-# export HADOOP_HOME=
-
-# Location of Hadoop's configuration information.  i.e., where this
-# file is living. If this is not defined, Hadoop will attempt to
-# locate it based upon its execution path.
-#
-# NOTE: It is recommend that this variable not be set here but in
-# /etc/profile.d or equivalent.  Some options (such as
-# --config) may react strangely otherwise.
-#
-# export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop
-
-# The maximum amount of heap to use (Java -Xmx).  If no unit
-# is provided, it will be converted to MB.  Daemons will
-# prefer any Xmx setting in their respective _OPT variable.
-# There is no default; the JVM will autoscale based upon machine
-# memory size.
-# export HADOOP_HEAPSIZE_MAX=
-
-# The minimum amount of heap to use (Java -Xms).  If no unit
-# is provided, it will be converted to MB.  Daemons will
-# prefer any Xms setting in their respective _OPT variable.
-# There is no default; the JVM will autoscale based upon machine
-# memory size.
-# export HADOOP_HEAPSIZE_MIN=
-
-# Enable extra debugging of Hadoop's JAAS binding, used to set up
-# Kerberos security.
-# export HADOOP_JAAS_DEBUG=true
-
-# Extra Java runtime options for all Hadoop commands. We don't support
-# IPv6 yet/still, so by default the preference is set to IPv4.
-# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
-# For Kerberos debugging, an extended option set logs more information
-# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true -Dsun.security.krb5.debug=true -Dsun.security.spnego.debug"
-
-# Some parts of the shell code may do special things dependent upon
-# the operating system.  We have to set this here. See the next
-# section as to why....
-export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
-
-# Extra Java runtime options for some Hadoop commands
-# and clients (i.e., hdfs dfs -blah).  These get appended to HADOOP_OPTS for
-# such commands.  In most cases, # this should be left empty and
-# let users supply it on the command line.
-# export HADOOP_CLIENT_OPTS=""
-
-#
-# A note about classpaths.
-#
-# By default, Apache Hadoop overrides Java's CLASSPATH
-# environment variable.  It is configured such
-# that it starts out blank with new entries added after passing
-# a series of checks (file/dir exists, not already listed aka
-# de-deduplication).  During de-deduplication, wildcards and/or
-# directories are *NOT* expanded to keep it simple. Therefore,
-# if the computed classpath has two specific mentions of
-# awesome-methods-1.0.jar, only the first one added will be seen.
-# If two directories are in the classpath that both contain
-# awesome-methods-1.0.jar, then Java will pick up both versions.
-
-# An additional, custom CLASSPATH. Site-wide configs should be
-# handled via the shellprofile functionality, utilizing the
-# hadoop_add_classpath function for greater control and much
-# harder for apps/end-users to accidentally override.
-# Similarly, end users should utilize ${HOME}/.hadooprc .
-# This variable should ideally only be used as a short-cut,
-# interactive way for temporary additions on the command line.
-# export HADOOP_CLASSPATH="/some/cool/path/on/your/machine"
-
-# Should HADOOP_CLASSPATH be first in the official CLASSPATH?
-# export HADOOP_USER_CLASSPATH_FIRST="yes"
-
-# If HADOOP_USE_CLIENT_CLASSLOADER is set, the classpath along
-# with the main jar are handled by a separate isolated
-# client classloader when 'hadoop jar', 'yarn jar', or 'mapred job'
-# is utilized. If it is set, HADOOP_CLASSPATH and
-# HADOOP_USER_CLASSPATH_FIRST are ignored.
-# export HADOOP_USE_CLIENT_CLASSLOADER=true
-
-# HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES overrides the default definition of
-# system classes for the client classloader when HADOOP_USE_CLIENT_CLASSLOADER
-# is enabled. Names ending in '.' (period) are treated as package names, and
-# names starting with a '-' are treated as negative matches. For example,
-# export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop."
-
-# Enable optional, bundled Hadoop features
-# This is a comma delimited list.  It may NOT be overridden via .hadooprc
-# Entries may be added/removed as needed.
-# export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@"
-
-###
-# Options for remote shell connectivity
-###
-
-# There are some optional components of hadoop that allow for
-# command and control of remote hosts.  For example,
-# start-dfs.sh will attempt to bring up all NNs, DNS, etc.
-
-# Options to pass to SSH when one of the "log into a host and
-# start/stop daemons" scripts is executed
-# export HADOOP_SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10s"
-
-# The built-in ssh handler will limit itself to 10 simultaneous connections.
-# For pdsh users, this sets the fanout size ( -f )
-# Change this to increase/decrease as necessary.
-# export HADOOP_SSH_PARALLEL=10
-
-# Filename which contains all of the hosts for any remote execution
-# helper scripts # such as workers.sh, start-dfs.sh, etc.
-# export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers"
-
-###
-# Options for all daemons
-###
-#
-
-#
-# Many options may also be specified as Java properties.  It is
-# very common, and in many cases, desirable, to hard-set these
-# in daemon _OPTS variables.  Where applicable, the appropriate
-# Java property is also identified.  Note that many are re-used
-# or set differently in certain contexts (e.g., secure vs
-# non-secure)
-#
-
-# Where (primarily) daemon log files are stored.
-# ${HADOOP_HOME}/logs by default.
-# Java property: hadoop.log.dir
-# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
-
-# A string representing this instance of hadoop. $USER by default.
-# This is used in writing log and pid files, so keep that in mind!
-# Java property: hadoop.id.str
-# export HADOOP_IDENT_STRING=$USER
-
-# How many seconds to pause after stopping a daemon
-# export HADOOP_STOP_TIMEOUT=5
-
-# Where pid files are stored.  /tmp by default.
-# export HADOOP_PID_DIR=/tmp
-
-# Default log4j setting for interactive commands
-# Java property: hadoop.root.logger
-# export HADOOP_ROOT_LOGGER=INFO,console
-
-# Default log4j setting for daemons spawned explicitly by
-# --daemon option of hadoop, hdfs, mapred and yarn command.
-# Java property: hadoop.root.logger
-# export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA
-
-# Default log level and output location for security-related messages.
-# You will almost certainly want to change this on a per-daemon basis via
-# the Java property (i.e., -Dhadoop.security.logger=foo). (Note that the
-# defaults for the NN and 2NN override this by default.)
-# Java property: hadoop.security.logger
-# export HADOOP_SECURITY_LOGGER=INFO,NullAppender
-
-# Default process priority level
-# Note that sub-processes will also run at this level!
-# export HADOOP_NICENESS=0
-
-# Default name for the service level authorization file
-# Java property: hadoop.policy.file
-# export HADOOP_POLICYFILE="hadoop-policy.xml"
-
-#
-# NOTE: this is not used by default!  <-----
-# You can define variables right here and then re-use them later on.
-# For example, it is common to use the same garbage collection settings
-# for all the daemons.  So one could define:
-#
-# export HADOOP_GC_SETTINGS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
-#
-# .. and then use it as per the b option under the namenode.
-
-###
-# Secure/privileged execution
-###
-
-#
-# Out of the box, Hadoop uses jsvc from Apache Commons to launch daemons
-# on privileged ports.  This functionality can be replaced by providing
-# custom functions.  See hadoop-functions.sh for more information.
-#
-
-# The jsvc implementation to use. Jsvc is required to run secure datanodes
-# that bind to privileged ports to provide authentication of data transfer
-# protocol.  Jsvc is not required if SASL is configured for authentication of
-# data transfer protocol using non-privileged ports.
-# export JSVC_HOME=/usr/bin
-
-#
-# This directory contains pids for secure and privileged processes.
-#export HADOOP_SECURE_PID_DIR=${HADOOP_PID_DIR}
-
-#
-# This directory contains the logs for secure and privileged processes.
-# Java property: hadoop.log.dir
-# export HADOOP_SECURE_LOG=${HADOOP_LOG_DIR}
-
-#
-# When running a secure daemon, the default value of HADOOP_IDENT_STRING
-# ends up being a bit bogus.  Therefore, by default, the code will
-# replace HADOOP_IDENT_STRING with HADOOP_xx_SECURE_USER.  If one wants
-# to keep HADOOP_IDENT_STRING untouched, then uncomment this line.
-# export HADOOP_SECURE_IDENT_PRESERVE="true"
-
-###
-# NameNode specific parameters
-###
-
-# Default log level and output location for file system related change
-# messages. For non-namenode daemons, the Java property must be set in
-# the appropriate _OPTS if one wants something other than INFO,NullAppender
-# Java property: hdfs.audit.logger
-# export HDFS_AUDIT_LOGGER=INFO,NullAppender
-
-# Specify the JVM options to be used when starting the NameNode.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# a) Set JMX options
-# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
-#
-# b) Set garbage collection logs
-# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
-#
-# c) ... or set them directly
-# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
-
-# this is the default:
-# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
-
-###
-# SecondaryNameNode specific parameters
-###
-# Specify the JVM options to be used when starting the SecondaryNameNode.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# This is the default:
-# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
-
-###
-# DataNode specific parameters
-###
-# Specify the JVM options to be used when starting the DataNode.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# This is the default:
-# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
-
-# On secure datanodes, user to run the datanode as after dropping privileges.
-# This **MUST** be uncommented to enable secure HDFS if using privileged ports
-# to provide authentication of data transfer protocol.  This **MUST NOT** be
-# defined if SASL is configured for authentication of data transfer protocol
-# using non-privileged ports.
-# This will replace the hadoop.id.str Java property in secure mode.
-# export HDFS_DATANODE_SECURE_USER=hdfs
-
-# Supplemental options for secure datanodes
-# By default, Hadoop uses jsvc which needs to know to launch a
-# server jvm.
-# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
-
-###
-# NFS3 Gateway specific parameters
-###
-# Specify the JVM options to be used when starting the NFS3 Gateway.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_NFS3_OPTS=""
-
-# Specify the JVM options to be used when starting the Hadoop portmapper.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_PORTMAP_OPTS="-Xmx512m"
-
-# Supplemental options for priviliged gateways
-# By default, Hadoop uses jsvc which needs to know to launch a
-# server jvm.
-# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"
-
-# On privileged gateways, user to run the gateway as after dropping privileges
-# This will replace the hadoop.id.str Java property in secure mode.
-# export HDFS_NFS3_SECURE_USER=nfsserver
-
-###
-# ZKFailoverController specific parameters
-###
-# Specify the JVM options to be used when starting the ZKFailoverController.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_ZKFC_OPTS=""
-
-###
-# QuorumJournalNode specific parameters
-###
-# Specify the JVM options to be used when starting the QuorumJournalNode.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_JOURNALNODE_OPTS=""
-
-###
-# HDFS Balancer specific parameters
-###
-# Specify the JVM options to be used when starting the HDFS Balancer.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_BALANCER_OPTS=""
-
-###
-# HDFS Mover specific parameters
-###
-# Specify the JVM options to be used when starting the HDFS Mover.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_MOVER_OPTS=""
-
-###
-# Router-based HDFS Federation specific parameters
-# Specify the JVM options to be used when starting the RBF Routers.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_DFSROUTER_OPTS=""
-
-###
-# Ozone Manager specific parameters
-###
-# Specify the JVM options to be used when starting the Ozone Manager.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_OM_OPTS=""
-
-###
-# HDFS StorageContainerManager specific parameters
-###
-# Specify the JVM options to be used when starting the HDFS Storage Container Manager.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_STORAGECONTAINERMANAGER_OPTS=""
-
-###
-# Advanced Users Only!
-###
-
-#
-# When building Hadoop, one can add the class paths to the commands
-# via this special env var:
-# export HADOOP_ENABLE_BUILD_PATHS="true"
-
-#
-# To prevent accidents, shell commands be (superficially) locked
-# to only allow certain users to execute certain subcommands.
-# It uses the format of (command)_(subcommand)_USER.
-#
-# For example, to limit who can execute the namenode command,
-# export HDFS_NAMENODE_USER=hdfs
-
-
-###
-# Registry DNS specific parameters
-###
-# For privileged registry DNS, user to run as after dropping privileges
-# This will replace the hadoop.id.str Java property in secure mode.
-# export HADOOP_REGISTRYDNS_SECURE_USER=yarn
-
-# Supplemental options for privileged registry DNS
-# By default, Hadoop uses jsvc which needs to know to launch a
-# server jvm.
-# export HADOOP_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"

hadoop-hdds/common/src/main/conf/hadoop-metrics2.properties

@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# syntax: [prefix].[source|sink].[instance].[options]
-# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
-
-*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
-# default sampling period, in seconds
-*.period=10
-
-# The namenode-metrics.out will contain metrics from all context
-#namenode.sink.file.filename=namenode-metrics.out
-# Specifying a special sampling period for namenode:
-#namenode.sink.*.period=8
-
-#datanode.sink.file.filename=datanode-metrics.out
-
-#resourcemanager.sink.file.filename=resourcemanager-metrics.out
-
-#nodemanager.sink.file.filename=nodemanager-metrics.out
-
-#mrappmaster.sink.file.filename=mrappmaster-metrics.out
-
-#jobhistoryserver.sink.file.filename=jobhistoryserver-metrics.out
-
-# the following example split metrics of different
-# context to different sinks (in this case files)
-#nodemanager.sink.file_jvm.class=org.apache.hadoop.metrics2.sink.FileSink
-#nodemanager.sink.file_jvm.context=jvm
-#nodemanager.sink.file_jvm.filename=nodemanager-jvm-metrics.out
-#nodemanager.sink.file_mapred.class=org.apache.hadoop.metrics2.sink.FileSink
-#nodemanager.sink.file_mapred.context=mapred
-#nodemanager.sink.file_mapred.filename=nodemanager-mapred-metrics.out
-
-#
-# Below are for sending metrics to Ganglia
-#
-# for Ganglia 3.0 support
-# *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink30
-#
-# for Ganglia 3.1 support
-# *.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31
-
-# *.sink.ganglia.period=10
-
-# default for supportsparse is false
-# *.sink.ganglia.supportsparse=true
-
-#*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
-#*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
-
-# Tag values to use for the ganglia prefix. If not defined no tags are used.
-# If '*' all tags are used. If specifying multiple tags separate them with
-# commas. Note that the last segment of the property name is the context name.
-#
-# A typical use of tags is separating the metrics by the HDFS rpc port
-# and HDFS service rpc port.
-# For example:
-#   With following HDFS configuration:
-#       dfs.namenode.rpc-address is set as namenodeAddress:9110
-#       dfs.namenode.servicerpc-address is set as namenodeAddress:9111
-#   If no tags are used, following metric would be gathered:
-#       rpc.rpc.NumOpenConnections
-#   If using "*.sink.ganglia.tagsForPrefix.rpc=port",
-#   following metrics would be gathered:
-#       rpc.rpc.port=9110.NumOpenConnections
-#       rpc.rpc.port=9111.NumOpenConnections
-#
-#*.sink.ganglia.tagsForPrefix.jvm=ProcessName
-#*.sink.ganglia.tagsForPrefix.dfs=HAState,IsOutOfSync
-#*.sink.ganglia.tagsForPrefix.rpc=port
-#*.sink.ganglia.tagsForPrefix.rpcdetailed=port
-#*.sink.ganglia.tagsForPrefix.metricssystem=*
-#*.sink.ganglia.tagsForPrefix.ugi=*
-#*.sink.ganglia.tagsForPrefix.mapred=
-
-#namenode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
-
-#datanode.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
-
-#resourcemanager.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
-
-#nodemanager.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
-
-#mrappmaster.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649
-
-#jobhistoryserver.sink.ganglia.servers=yourgangliahost_1:8649,yourgangliahost_2:8649

hadoop-hdds/common/src/main/conf/hadoop-policy.xml

@@ -1,275 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-  <property>
-    <name>security.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientProtocol, which is used by user code
-    via the DistributedFileSystem.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.client.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ClientDatanodeProtocol, the client-to-datanode protocol
-    for block recovery.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for DatanodeProtocol, which is used by datanodes to
-    communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.inter.datanode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterDatanodeProtocol, the inter-datanode protocol
-    for updating generation timestamp.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.namenode.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for NamenodeProtocol, the protocol used by the secondary
-    namenode to communicate with the namenode.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
- <property>
-    <name>security.admin.operations.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for AdminOperationsProtocol. Used for admin commands.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.user.mappings.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for RefreshUserMappingsProtocol. Used to refresh
-    users mappings. The ACL is a comma-separated list of user and
-    group names. The user and group list is separated by a blank. For
-    e.g. "alice,bob users,wheel".  A special value of "*" means all
-    users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.refresh.policy.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
-    dfsadmin and mradmin commands to refresh the security policy in-effect.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.ha.service.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for HAService protocol used by HAAdmin to manage the
-      active and stand-by states of namenode.</description>
-  </property>
-
-  <property>
-    <name>security.router.admin.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for RouterAdmin Protocol. The ACL is a comma-separated
-    list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.
-    </description>
-  </property>
-
-  <property>
-    <name>security.zkfc.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for access to the ZK Failover Controller
-    </description>
-  </property>
-
-  <property>
-    <name>security.qjournal.service.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for QJournalProtocol, used by the NN to communicate with
-    JNs when using the QuorumJournalManager for edit logs.</description>
-  </property>
-
-  <property>
-    <name>security.interqjournal.service.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for InterQJournalProtocol, used by the JN to
-    communicate with other JN
-    </description>
-  </property>
-
-  <property>
-    <name>security.mrhs.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for HSClientProtocol, used by job clients to
-    communciate with the MR History Server job status etc.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <!-- YARN Protocols -->
-
-  <property>
-    <name>security.resourcetracker.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ResourceTrackerProtocol, used by the
-    ResourceManager and NodeManager to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.resourcemanager-administration.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ResourceManagerAdministrationProtocol, for admin commands.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationclient.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ApplicationClientProtocol, used by the ResourceManager
-    and applications submission clients to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationmaster.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ApplicationMasterProtocol, used by the ResourceManager
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.containermanagement.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ContainerManagementProtocol protocol, used by the NodeManager
-    and ApplicationMasters to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.resourcelocalizer.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ResourceLocalizer protocol, used by the NodeManager
-    and ResourceLocalizer to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.task.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for TaskUmbilicalProtocol, used by the map and reduce
-    tasks to communicate with the parent tasktracker.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.job.client.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for MRClientProtocol, used by job clients to
-    communciate with the MR ApplicationMaster to query job status etc.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationhistory.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ApplicationHistoryProtocol, used by the timeline
-    server and the generic history service client to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.collector-nodemanager.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for CollectorNodemanagerProtocol, used by nodemanager
-    if timeline service v2 is enabled, for the timeline collector and nodemanager
-    to communicate with each other.
-    The ACL is a comma-separated list of user and group names. The user and
-    group list is separated by a blank. For e.g. "alice,bob users,wheel".
-    A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.applicationmaster-nodemanager.applicationmaster.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for ApplicationMasterProtocol, used by the Nodemanager
-        and ApplicationMasters to communicate.
-        The ACL is a comma-separated list of user and group names. The user and
-        group list is separated by a blank. For e.g. "alice,bob users,wheel".
-        A special value of "*" means all users are allowed.</description>
-  </property>
-
-  <property>
-    <name>security.distributedscheduling.protocol.acl</name>
-    <value>*</value>
-    <description>ACL for DistributedSchedulingAMProtocol, used by the Nodemanager
-        and Resourcemanager to communicate.
-        The ACL is a comma-separated list of user and group names. The user and
-        group list is separated by a blank. For e.g. "alice,bob users,wheel".
-        A special value of "*" means all users are allowed.</description>
-    </property>
-</configuration>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds;
-
-import org.apache.hadoop.hdds.utils.db.DBProfile;
-
-/**
- * This class contains constants for configuration keys and default values
- * used in hdds.
- */
-public final class HddsConfigKeys {
-
-  public static final String HDDS_HEARTBEAT_INTERVAL =
-      "hdds.heartbeat.interval";
-  public static final String HDDS_HEARTBEAT_INTERVAL_DEFAULT =
-      "30s";
-  public static final String HDDS_NODE_REPORT_INTERVAL =
-      "hdds.node.report.interval";
-  public static final String HDDS_NODE_REPORT_INTERVAL_DEFAULT =
-      "60s";
-  public static final String HDDS_CONTAINER_REPORT_INTERVAL =
-      "hdds.container.report.interval";
-  public static final String HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT =
-      "60s";
-  public static final String HDDS_PIPELINE_REPORT_INTERVAL =
-          "hdds.pipeline.report.interval";
-  public static final String HDDS_PIPELINE_REPORT_INTERVAL_DEFAULT =
-          "60s";
-  public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL =
-      "hdds.command.status.report.interval";
-  public static final String HDDS_COMMAND_STATUS_REPORT_INTERVAL_DEFAULT =
-      "60s";
-  public static final String HDDS_CONTAINER_ACTION_MAX_LIMIT =
-      "hdds.container.action.max.limit";
-  public static final int HDDS_CONTAINER_ACTION_MAX_LIMIT_DEFAULT =
-      20;
-  public static final String HDDS_PIPELINE_ACTION_MAX_LIMIT =
-      "hdds.pipeline.action.max.limit";
-  public static final int HDDS_PIPELINE_ACTION_MAX_LIMIT_DEFAULT =
-      20;
-  // Configuration to allow volume choosing policy.
-  public static final String HDDS_DATANODE_VOLUME_CHOOSING_POLICY =
-      "hdds.datanode.volume.choosing.policy";
-  // DB PKIProfile used by ROCKDB instances.
-  public static final String HDDS_DB_PROFILE = "hdds.db.profile";
-  public static final DBProfile HDDS_DEFAULT_DB_PROFILE = DBProfile.DISK;
-  // Once a container usage crosses this threshold, it is eligible for
-  // closing.
-  public static final String HDDS_CONTAINER_CLOSE_THRESHOLD =
-      "hdds.container.close.threshold";
-  public static final float HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
-  public static final String HDDS_SCM_SAFEMODE_ENABLED =
-      "hdds.scm.safemode.enabled";
-
-  public static final boolean HDDS_SCM_SAFEMODE_ENABLED_DEFAULT = true;
-  public static final String HDDS_SCM_SAFEMODE_MIN_DATANODE =
-      "hdds.scm.safemode.min.datanode";
-  public static final int HDDS_SCM_SAFEMODE_MIN_DATANODE_DEFAULT = 1;
-
-  public static final String
-      HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT =
-      "hdds.scm.wait.time.after.safemode.exit";
-
-  public static final String
-      HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT_DEFAULT = "5m";
-
-  public static final String HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK =
-      "hdds.scm.safemode.pipeline-availability.check";
-  public static final boolean
-      HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = false;
-
-  // % of containers which should have at least one reported replica
-  // before SCM comes out of safe mode.
-  public static final String HDDS_SCM_SAFEMODE_THRESHOLD_PCT =
-      "hdds.scm.safemode.threshold.pct";
-  public static final double HDDS_SCM_SAFEMODE_THRESHOLD_PCT_DEFAULT = 0.99;
-
-
-  // percentage of healthy pipelines, where all 3 datanodes are reported in the
-  // pipeline.
-  public static final String HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT =
-      "hdds.scm.safemode.healthy.pipelie.pct";
-  public static final double
-      HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT_DEFAULT = 0.10;
-
-  public static final String HDDS_SCM_SAFEMODE_ONE_NODE_REPORTED_PIPELINE_PCT =
-      "hdds.scm.safemode.atleast.one.node.reported.pipeline.pct";
-  public static final double
-      HDDS_SCM_SAFEMODE_ONE_NODE_REPORTED_PIPELINE_PCT_DEFAULT = 0.90;
-
-  public static final String HDDS_LOCK_MAX_CONCURRENCY =
-      "hdds.lock.max.concurrency";
-  public static final int HDDS_LOCK_MAX_CONCURRENCY_DEFAULT = 100;
-  // This configuration setting is used as a fallback location by all
-  // Ozone/HDDS services for their metadata. It is useful as a single
-  // config point for test/PoC clusters.
-  //
-  // In any real cluster where performance matters, the SCM, OM and DN
-  // metadata locations must be configured explicitly.
-  public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
-
-  public static final String HDDS_PROMETHEUS_ENABLED =
-      "hdds.prometheus.endpoint.enabled";
-
-  public static final String HDDS_PROFILER_ENABLED =
-      "hdds.profiler.endpoint.enabled";
-
-  public static final String HDDS_KEY_LEN = "hdds.key.len";
-  public static final int HDDS_DEFAULT_KEY_LEN = 2048;
-  public static final String HDDS_KEY_ALGORITHM = "hdds.key.algo";
-  public static final String HDDS_DEFAULT_KEY_ALGORITHM = "RSA";
-  public static final String HDDS_SECURITY_PROVIDER = "hdds.security.provider";
-  public static final String HDDS_DEFAULT_SECURITY_PROVIDER = "BC";
-  public static final String HDDS_KEY_DIR_NAME = "hdds.key.dir.name";
-  public static final String HDDS_KEY_DIR_NAME_DEFAULT = "keys";
-  // TODO : Talk to StorageIO classes and see if they can return a secure
-  // storage location for each node.
-  public static final String HDDS_METADATA_DIR_NAME = "hdds.metadata.dir";
-  public static final String HDDS_PRIVATE_KEY_FILE_NAME =
-      "hdds.priv.key.file.name";
-  public static final String HDDS_PRIVATE_KEY_FILE_NAME_DEFAULT = "private.pem";
-  public static final String HDDS_PUBLIC_KEY_FILE_NAME = "hdds.public.key.file"
-      + ".name";
-  public static final String HDDS_PUBLIC_KEY_FILE_NAME_DEFAULT = "public.pem";
-
-  public static final String HDDS_BLOCK_TOKEN_EXPIRY_TIME =
-      "hdds.block.token.expiry.time";
-  public static final String HDDS_BLOCK_TOKEN_EXPIRY_TIME_DEFAULT = "1d";
-  /**
-   * Maximum duration of certificates issued by SCM including Self-Signed Roots.
-   * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
-   * Default value is 5 years and written as P1865D.
-   */
-  public static final String HDDS_X509_MAX_DURATION = "hdds.x509.max.duration";
-  // Limit Certificate duration to a max value of 5 years.
-  public static final String HDDS_X509_MAX_DURATION_DEFAULT= "P1865D";
-  public static final String HDDS_X509_SIGNATURE_ALGO =
-      "hdds.x509.signature.algorithm";
-  public static final String HDDS_X509_SIGNATURE_ALGO_DEFAULT = "SHA256withRSA";
-  public static final String HDDS_BLOCK_TOKEN_ENABLED =
-      "hdds.block.token.enabled";
-  public static final boolean HDDS_BLOCK_TOKEN_ENABLED_DEFAULT = false;
-
-  public static final String HDDS_X509_DIR_NAME = "hdds.x509.dir.name";
-  public static final String HDDS_X509_DIR_NAME_DEFAULT = "certs";
-  public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
-  public static final String HDDS_X509_FILE_NAME_DEFAULT = "certificate.crt";
-
-  /**
-   * Default duration of certificates issued by SCM CA.
-   * The formats accepted are based on the ISO-8601 duration format PnDTnHnMn.nS
-   * Default value is 5 years and written as P1865D.
-   */
-  public static final String HDDS_X509_DEFAULT_DURATION = "hdds.x509.default" +
-      ".duration";
-  // Default Certificate duration to one year.
-  public static final String HDDS_X509_DEFAULT_DURATION_DEFAULT = "P365D";
-
-  /**
-   * Do not instantiate.
-   */
-  private HddsConfigKeys() {
-  }
-
-  // Enable TLS for GRPC clients/server in ozone.
-  public static final String HDDS_GRPC_TLS_ENABLED = "hdds.grpc.tls.enabled";
-  public static final boolean HDDS_GRPC_TLS_ENABLED_DEFAULT = false;
-
-  // Choose TLS provider the default is set to OPENSSL for better performance.
-  public static final String HDDS_GRPC_TLS_PROVIDER = "hdds.grpc.tls.provider";
-  public static final String HDDS_GRPC_TLS_PROVIDER_DEFAULT = "OPENSSL";
-
-  // Test only settings for using test signed certificate, authority assume to
-  // be localhost.
-  public static final String HDDS_GRPC_TLS_TEST_CERT = "hdds.grpc.tls" +
-      ".test.cert";
-  public static final boolean HDDS_GRPC_TLS_TEST_CERT_DEFAULT = false;
-
-  // Comma separated acls (users, groups) allowing clients accessing
-  // datanode container protocol
-  // when hadoop.security.authorization is true, this needs to be set in
-  // hadoop-policy.xml, "*" allows all users/groups to access.
-  public static final String
-      HDDS_SECURITY_CLIENT_DATANODE_CONTAINER_PROTOCOL_ACL =
-      "hdds.security.client.datanode.container.protocol.acl";
-
-  // Comma separated acls (users, groups) allowing clients accessing
-  // scm container protocol
-  // when hadoop.security.authorization is true, this needs to be set in
-  // hadoop-policy.xml, "*" allows all users/groups to access.
-  public static final String HDDS_SECURITY_CLIENT_SCM_CONTAINER_PROTOCOL_ACL =
-      "hdds.security.client.scm.container.protocol.acl";
-
-  // Comma separated acls (users, groups) allowing clients accessing
-  // scm block protocol
-  // when hadoop.security.authorization is true, this needs to be set in
-  // hadoop-policy.xml, "*" allows all users/groups to access.
-  public static final String HDDS_SECURITY_CLIENT_SCM_BLOCK_PROTOCOL_ACL =
-      "hdds.security.client.scm.block.protocol.acl";
-
-  // Comma separated acls (users, groups) allowing clients accessing
-  // scm certificate protocol
-  // when hadoop.security.authorization is true, this needs to be set in
-  // hadoop-policy.xml, "*" allows all users/groups to access.
-  public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
-      "hdds.security.client.scm.certificate.protocol.acl";
-
-  // Determines if the Container Chunk Manager will write user data to disk
-  // Set to false only for specific performance tests
-  public static final String HDDS_CONTAINER_PERSISTDATA =
-      "hdds.container.chunk.persistdata";
-  public static final boolean HDDS_CONTAINER_PERSISTDATA_DEFAULT = true;
-
-  public static final String HDDS_CONTAINER_SCRUB_ENABLED =
-      "hdds.container.scrub.enabled";
-  public static final boolean HDDS_CONTAINER_SCRUB_ENABLED_DEFAULT = false;
-
-  public static final String HDDS_DATANODE_HTTP_ENABLED_KEY =
-      "hdds.datanode.http.enabled";
-  public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY =
-      "hdds.datanode.http-bind-host";
-  public static final String HDDS_DATANODE_HTTPS_BIND_HOST_KEY =
-      "hdds.datanode.https-bind-host";
-  public static final String HDDS_DATANODE_HTTP_ADDRESS_KEY =
-      "hdds.datanode.http-address";
-  public static final String HDDS_DATANODE_HTTPS_ADDRESS_KEY =
-      "hdds.datanode.https-address";
-
-  public static final String HDDS_DATANODE_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
-  public static final int HDDS_DATANODE_HTTP_BIND_PORT_DEFAULT = 9882;
-  public static final int HDDS_DATANODE_HTTPS_BIND_PORT_DEFAULT = 9883;
-  public static final String
-      HDDS_DATANODE_HTTP_KERBEROS_PRINCIPAL_KEY =
-      "hdds.datanode.http.kerberos.principal";
-  public static final String
-      HDDS_DATANODE_HTTP_KERBEROS_KEYTAB_FILE_KEY =
-      "hdds.datanode.http.kerberos.keytab";
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsIdFactory.java

@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds;
-
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * HDDS Id generator.
- */
-public final class HddsIdFactory {
-  private HddsIdFactory() {
-  }
-
-  private static final AtomicLong LONG_COUNTER = new AtomicLong(
-      System.currentTimeMillis());
-
-  /**
-   * Returns an incrementing long. This class doesn't
-   * persist initial value for long Id's, so incremental id's after restart
-   * may collide with previously generated Id's.
-   *
-   * @return long
-   */
-  public static long getLongId() {
-    return LONG_COUNTER.incrementAndGet();
-  }
-
-  /**
-   * Returns a uuid.
-   *
-   * @return UUID.
-   */
-  public static UUID getUUId() {
-    return UUID.randomUUID();
-  }
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java

@@ -1,505 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
-import java.util.Calendar;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Optional;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
-import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.source.JvmMetrics;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.DNS;
-import org.apache.hadoop.net.NetUtils;
-
-import com.google.common.net.HostAndPort;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_INTERFACE_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DNS_NAMESERVER_KEY;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HOST_NAME_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT;
-
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * HDDS specific stateless utility functions.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public final class HddsUtils {
-
-
-  private static final Logger LOG = LoggerFactory.getLogger(HddsUtils.class);
-
-  /**
-   * The service ID of the solitary Ozone SCM service.
-   */
-  public static final String OZONE_SCM_SERVICE_ID = "OzoneScmService";
-  public static final String OZONE_SCM_SERVICE_INSTANCE_ID =
-      "OzoneScmServiceInstance";
-  private static final TimeZone UTC_ZONE = TimeZone.getTimeZone("UTC");
-
-
-  private static final int NO_PORT = -1;
-
-  private HddsUtils() {
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM client endpoint.
-   */
-  public static InetSocketAddress getScmAddressForClients(Configuration conf) {
-    Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      // Fallback to Ozone SCM names.
-      Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
-      if (scmAddresses.size() > 1) {
-        throw new IllegalArgumentException(
-            ScmConfigKeys.OZONE_SCM_NAMES +
-                " must contain a single hostname. Multiple SCM hosts are " +
-                "currently unsupported");
-      }
-      host = Optional.of(scmAddresses.iterator().next().getHostName());
-    }
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY + " must be defined. See"
-              + " https://wiki.apache.org/hadoop/Ozone#Configuration for "
-              + "details"
-              + " on configuring Ozone.");
-    }
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(host.get() + ":" + port
-        .orElse(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM for block service. If
-   * {@link ScmConfigKeys#OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY} is not defined
-   * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither
-   * is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM block client endpoint.
-   * @throws IllegalArgumentException if configuration is not defined.
-   */
-  public static InetSocketAddress getScmAddressForBlockClients(
-      Configuration conf) {
-    Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      host = getHostNameFromConfigKeys(conf,
-          ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-    }
-
-    if (!host.isPresent()) {
-      // Fallback to Ozone SCM names.
-      Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
-      if (scmAddresses.size() > 1) {
-        throw new IllegalArgumentException(
-            ScmConfigKeys.OZONE_SCM_NAMES +
-                " must contain a single hostname. Multiple SCM hosts are " +
-                "currently unsupported");
-      }
-      host = Optional.of(scmAddresses.iterator().next().getHostName());
-    }
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY
-              + " must be defined. See"
-              + " https://wiki.apache.org/hadoop/Ozone#Configuration"
-              + " for details on configuring Ozone.");
-    }
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
-    return NetUtils.createSocketAddr(host.get() + ":" + port
-        .orElse(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
-  }
-
-  /**
-   * Create a scm security client.
-   * @param conf    - Ozone configuration.
-   *
-   * @return {@link SCMSecurityProtocol}
-   * @throws IOException
-   */
-  public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
-      OzoneConfiguration conf) throws IOException {
-    RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
-        ProtobufRpcEngine.class);
-    long scmVersion =
-        RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
-    InetSocketAddress address =
-        getScmAddressForSecurityProtocol(conf);
-    RetryPolicy retryPolicy =
-        RetryPolicies.retryForeverWithFixedSleep(
-            1000, TimeUnit.MILLISECONDS);
-    SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
-        new SCMSecurityProtocolClientSideTranslatorPB(
-            RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion,
-                address, UserGroupInformation.getCurrentUser(),
-                conf, NetUtils.getDefaultSocketFactory(conf),
-                Client.getRpcTimeout(conf), retryPolicy).getProxy());
-    return scmSecurityClient;
-  }
-
-  /**
-   * Retrieve the hostname, trying the supplied config keys in order.
-   * Each config value may be absent, or if present in the format
-   * host:port (the :port part is optional).
-   *
-   * @param conf  - Conf
-   * @param keys a list of configuration key names.
-   *
-   * @return first hostname component found from the given keys, or absent.
-   * @throws IllegalArgumentException if any values are not in the 'host'
-   *             or host:port format.
-   */
-  public static Optional<String> getHostNameFromConfigKeys(Configuration conf,
-      String... keys) {
-    for (final String key : keys) {
-      final String value = conf.getTrimmed(key);
-      final Optional<String> hostName = getHostName(value);
-      if (hostName.isPresent()) {
-        return hostName;
-      }
-    }
-    return Optional.empty();
-  }
-
-  /**
-   * Gets the hostname or Indicates that it is absent.
-   * @param value host or host:port
-   * @return hostname
-   */
-  public static Optional<String> getHostName(String value) {
-    if ((value == null) || value.isEmpty()) {
-      return Optional.empty();
-    }
-    String hostname = value.replaceAll("\\:[0-9]+$", "");
-    if (hostname.length() == 0) {
-      return Optional.empty();
-    } else {
-      return Optional.of(hostname);
-    }
-  }
-
-  /**
-   * Gets the port if there is one, throws otherwise.
-   * @param value  String in host:port format.
-   * @return Port
-   */
-  public static Optional<Integer> getHostPort(String value) {
-    if ((value == null) || value.isEmpty()) {
-      return Optional.empty();
-    }
-    int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT);
-    if (port == NO_PORT) {
-      return Optional.empty();
-    } else {
-      return Optional.of(port);
-    }
-  }
-
-  /**
-   * Retrieve the port number, trying the supplied config keys in order.
-   * Each config value may be absent, or if present in the format
-   * host:port (the :port part is optional).
-   *
-   * @param conf Conf
-   * @param keys a list of configuration key names.
-   *
-   * @return first port number component found from the given keys, or absent.
-   * @throws IllegalArgumentException if any values are not in the 'host'
-   *             or host:port format.
-   */
-  public static Optional<Integer> getPortNumberFromConfigKeys(
-      Configuration conf, String... keys) {
-    for (final String key : keys) {
-      final String value = conf.getTrimmed(key);
-      final Optional<Integer> hostPort = getHostPort(value);
-      if (hostPort.isPresent()) {
-        return hostPort;
-      }
-    }
-    return Optional.empty();
-  }
-
-  /**
-   * Retrieve the socket addresses of all storage container managers.
-   *
-   * @param conf
-   * @return A collection of SCM addresses
-   * @throws IllegalArgumentException If the configuration is invalid
-   */
-  public static Collection<InetSocketAddress> getSCMAddresses(
-      Configuration conf) throws IllegalArgumentException {
-    Collection<InetSocketAddress> addresses =
-        new HashSet<InetSocketAddress>();
-    Collection<String> names =
-        conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
-    if (names == null || names.isEmpty()) {
-      throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
-          + " need to be a set of valid DNS names or IP addresses."
-          + " Null or empty address list found.");
-    }
-
-    final Optional<Integer> defaultPort = Optional
-        .of(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT);
-    for (String address : names) {
-      Optional<String> hostname = getHostName(address);
-      if (!hostname.isPresent()) {
-        throw new IllegalArgumentException("Invalid hostname for SCM: "
-            + hostname);
-      }
-      Optional<Integer> port = getHostPort(address);
-      InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(),
-          port.orElse(defaultPort.get()));
-      addresses.add(addr);
-    }
-    return addresses;
-  }
-
-  public static boolean isHddsEnabled(Configuration conf) {
-    return conf.getBoolean(OZONE_ENABLED, OZONE_ENABLED_DEFAULT);
-  }
-
-
-  /**
-   * Returns the hostname for this datanode. If the hostname is not
-   * explicitly configured in the given config, then it is determined
-   * via the DNS class.
-   *
-   * @param conf Configuration
-   *
-   * @return the hostname (NB: may not be a FQDN)
-   * @throws UnknownHostException if the dfs.datanode.dns.interface
-   *    option is used and the hostname can not be determined
-   */
-  public static String getHostName(Configuration conf)
-      throws UnknownHostException {
-    String name = conf.get(DFS_DATANODE_HOST_NAME_KEY);
-    if (name == null) {
-      String dnsInterface = conf.get(
-          CommonConfigurationKeys.HADOOP_SECURITY_DNS_INTERFACE_KEY);
-      String nameServer = conf.get(
-          CommonConfigurationKeys.HADOOP_SECURITY_DNS_NAMESERVER_KEY);
-      boolean fallbackToHosts = false;
-
-      if (dnsInterface == null) {
-        // Try the legacy configuration keys.
-        dnsInterface = conf.get(DFS_DATANODE_DNS_INTERFACE_KEY);
-        nameServer = conf.get(DFS_DATANODE_DNS_NAMESERVER_KEY);
-      } else {
-        // If HADOOP_SECURITY_DNS_* is set then also attempt hosts file
-        // resolution if DNS fails. We will not use hosts file resolution
-        // by default to avoid breaking existing clusters.
-        fallbackToHosts = true;
-      }
-
-      name = DNS.getDefaultHost(dnsInterface, nameServer, fallbackToHosts);
-    }
-    return name;
-  }
-
-  /**
-   * Checks if the container command is read only or not.
-   * @param proto ContainerCommand Request proto
-   * @return True if its readOnly , false otherwise.
-   */
-  public static boolean isReadOnly(
-      ContainerProtos.ContainerCommandRequestProto proto) {
-    switch (proto.getCmdType()) {
-    case ReadContainer:
-    case ReadChunk:
-    case ListBlock:
-    case GetBlock:
-    case GetSmallFile:
-    case ListContainer:
-    case ListChunk:
-    case GetCommittedBlockLength:
-      return true;
-    case CloseContainer:
-    case WriteChunk:
-    case UpdateContainer:
-    case CompactChunk:
-    case CreateContainer:
-    case DeleteChunk:
-    case DeleteContainer:
-    case DeleteBlock:
-    case PutBlock:
-    case PutSmallFile:
-    default:
-      return false;
-    }
-  }
-
-  /**
-   * Register the provided MBean with additional JMX ObjectName properties.
-   * If additional properties are not supported then fallback to registering
-   * without properties.
-   *
-   * @param serviceName - see {@link MBeans#register}
-   * @param mBeanName - see {@link MBeans#register}
-   * @param jmxProperties - additional JMX ObjectName properties.
-   * @param mBean - the MBean to register.
-   * @return the named used to register the MBean.
-   */
-  public static ObjectName registerWithJmxProperties(
-      String serviceName, String mBeanName, Map<String, String> jmxProperties,
-      Object mBean) {
-    try {
-
-      // Check support for registering with additional properties.
-      final Method registerMethod = MBeans.class.getMethod(
-          "register", String.class, String.class,
-          Map.class, Object.class);
-
-      return (ObjectName) registerMethod.invoke(
-          null, serviceName, mBeanName, jmxProperties, mBean);
-
-    } catch (NoSuchMethodException | IllegalAccessException |
-        InvocationTargetException e) {
-
-      // Fallback
-      if (LOG.isTraceEnabled()) {
-        LOG.trace("Registering MBean {} without additional properties {}",
-            mBeanName, jmxProperties);
-      }
-      return MBeans.register(serviceName, mBeanName, mBean);
-    }
-  }
-
-  /**
-   * Get the current UTC time in milliseconds.
-   * @return the current UTC time in milliseconds.
-   */
-  public static long getUtcTime() {
-    return Calendar.getInstance(UTC_ZONE).getTimeInMillis();
-  }
-
-  /**
-   * Retrieve the socket address that should be used by clients to connect
-   * to the SCM for
-   * {@link org.apache.hadoop.hdds.protocol.SCMSecurityProtocol}. If
-   * {@link ScmConfigKeys#OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY} is not defined
-   * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used. If neither
-   * is defined then {@link ScmConfigKeys#OZONE_SCM_NAMES} is used.
-   *
-   * @param conf
-   * @return Target InetSocketAddress for the SCM block client endpoint.
-   * @throws IllegalArgumentException if configuration is not defined.
-   */
-  public static InetSocketAddress getScmAddressForSecurityProtocol(
-      Configuration conf) {
-    Optional<String> host = getHostNameFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY);
-
-    if (!host.isPresent()) {
-      host = getHostNameFromConfigKeys(conf,
-          ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-    }
-
-    if (!host.isPresent()) {
-      // Fallback to Ozone SCM names.
-      Collection<InetSocketAddress> scmAddresses = getSCMAddresses(conf);
-      if (scmAddresses.size() > 1) {
-        throw new IllegalArgumentException(
-            ScmConfigKeys.OZONE_SCM_NAMES +
-                " must contain a single hostname. Multiple SCM hosts are " +
-                "currently unsupported");
-      }
-      host = Optional.of(scmAddresses.iterator().next().getHostName());
-    }
-
-    if (!host.isPresent()) {
-      throw new IllegalArgumentException(
-          ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY
-              + " must be defined. See"
-              + " https://wiki.apache.org/hadoop/Ozone#Configuration"
-              + " for details on configuring Ozone.");
-    }
-
-    final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
-        ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY);
-
-    return NetUtils.createSocketAddr(host.get() + ":" + port
-        .orElse(ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT));
-  }
-
-  /**
-   * Initialize hadoop metrics system for Ozone servers.
-   * @param configuration OzoneConfiguration to use.
-   * @param serverName    The logical name of the server components.
-   * @return
-   */
-  public static MetricsSystem initializeMetrics(
-      OzoneConfiguration configuration, String serverName) {
-    MetricsSystem metricsSystem = DefaultMetricsSystem.initialize(serverName);
-    JvmMetrics.create(serverName,
-        configuration.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY),
-        DefaultMetricsSystem.instance());
-    return metricsSystem;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericCli.java

@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.cli;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.Callable;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.Path;
-
-import com.google.common.annotations.VisibleForTesting;
-import picocli.CommandLine;
-import picocli.CommandLine.ExecutionException;
-import picocli.CommandLine.Option;
-import picocli.CommandLine.RunLast;
-
-/**
- * This is a generic parent class for all the ozone related cli tools.
- */
-public class GenericCli implements Callable<Void>, GenericParentCommand {
-
-  @Option(names = {"--verbose"},
-      description = "More verbose output. Show the stack trace of the errors.")
-  private boolean verbose;
-
-  @Option(names = {"-D", "--set"})
-  private Map<String, String> configurationOverrides = new HashMap<>();
-
-  @Option(names = {"-conf"})
-  private String configurationPath;
-
-  private final CommandLine cmd;
-
-  public GenericCli() {
-    cmd = new CommandLine(this);
-  }
-
-  public void run(String[] argv) {
-    try {
-      execute(argv);
-    } catch (ExecutionException ex) {
-      printError(ex.getCause() == null ? ex : ex.getCause());
-      System.exit(-1);
-    }
-  }
-
-  @VisibleForTesting
-  public void execute(String[] argv) {
-    cmd.parseWithHandler(new RunLast(), argv);
-  }
-
-  protected void printError(Throwable error) {
-    //message could be null in case of NPE. This is unexpected so we can
-    //print out the stack trace.
-    if (verbose || error.getMessage() == null
-        || error.getMessage().length() == 0) {
-      error.printStackTrace(System.err);
-    } else {
-      System.err.println(error.getMessage().split("\n")[0]);
-    }
-  }
-
-  @Override
-  public Void call() throws Exception {
-    throw new MissingSubcommandException(cmd);
-  }
-
-  @Override
-  public OzoneConfiguration createOzoneConfiguration() {
-    OzoneConfiguration ozoneConf = new OzoneConfiguration();
-    if (configurationPath != null) {
-      ozoneConf.addResource(new Path(configurationPath));
-    }
-    if (configurationOverrides != null) {
-      for (Entry<String, String> entry : configurationOverrides.entrySet()) {
-        ozoneConf.set(entry.getKey(), entry.getValue());
-      }
-    }
-    return ozoneConf;
-  }
-
-  @VisibleForTesting
-  public picocli.CommandLine getCmd() {
-    return cmd;
-  }
-
-  @Override
-  public boolean isVerbose() {
-    return verbose;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/GenericParentCommand.java

@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.cli;
-
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-
-/**
- * Interface to access the higher level parameters.
- */
-public interface GenericParentCommand {
-
-  boolean isVerbose();
-
-  OzoneConfiguration createOzoneConfiguration();
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/HddsVersionProvider.java

@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.cli;
-
-import org.apache.hadoop.hdds.utils.HddsVersionInfo;
-
-import picocli.CommandLine.IVersionProvider;
-
-/**
- * Version provider for the CLI interface.
- */
-public class HddsVersionProvider implements IVersionProvider {
-  @Override
-  public String[] getVersion() throws Exception {
-    String[] result = new String[] {
-        HddsVersionInfo.HDDS_VERSION_INFO.getBuildVersion()
-    };
-    return result;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/MissingSubcommandException.java

@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.cli;
-
-import picocli.CommandLine;
-
-/**
- * Exception to throw if subcommand is not selected but required.
- */
-public class MissingSubcommandException extends CommandLine.ParameterException {
-
-  public MissingSubcommandException(CommandLine cmd) {
-    super(cmd, "Incomplete command");
-  }
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/cli/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Generic helper class to make instantiate picocli based cli tools.
- */
-package org.apache.hadoop.hdds.cli;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/BlockID.java

@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.client;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-import java.util.Objects;
-
-/**
- * BlockID of Ozone (containerID + localID + blockCommitSequenceId).
- */
-
-public class BlockID {
-
-  private ContainerBlockID containerBlockID;
-  private long blockCommitSequenceId;
-
-  public BlockID(long containerID, long localID) {
-    this(containerID, localID, 0);
-  }
-
-  private BlockID(long containerID, long localID, long bcsID) {
-    containerBlockID = new ContainerBlockID(containerID, localID);
-    blockCommitSequenceId = bcsID;
-  }
-
-  public BlockID(ContainerBlockID containerBlockID) {
-    this(containerBlockID, 0);
-  }
-
-  private BlockID(ContainerBlockID containerBlockID, long bcsId) {
-    this.containerBlockID = containerBlockID;
-    blockCommitSequenceId = bcsId;
-  }
-
-  public long getContainerID() {
-    return containerBlockID.getContainerID();
-  }
-
-  public long getLocalID() {
-    return containerBlockID.getLocalID();
-  }
-
-  public long getBlockCommitSequenceId() {
-    return blockCommitSequenceId;
-  }
-
-  public void setBlockCommitSequenceId(long blockCommitSequenceId) {
-    this.blockCommitSequenceId = blockCommitSequenceId;
-  }
-
-  public ContainerBlockID getContainerBlockID() {
-    return containerBlockID;
-  }
-
-  public void setContainerBlockID(ContainerBlockID containerBlockID) {
-    this.containerBlockID = containerBlockID;
-  }
-
-  @Override
-  public String toString() {
-    return new StringBuilder().append(getContainerBlockID().toString())
-        .append(" bcsId: ")
-        .append(blockCommitSequenceId)
-        .toString();
-  }
-
-  public ContainerProtos.DatanodeBlockID getDatanodeBlockIDProtobuf() {
-    return ContainerProtos.DatanodeBlockID.newBuilder().
-        setContainerID(containerBlockID.getContainerID())
-        .setLocalID(containerBlockID.getLocalID())
-        .setBlockCommitSequenceId(blockCommitSequenceId).build();
-  }
-
-  public static BlockID getFromProtobuf(
-      ContainerProtos.DatanodeBlockID blockID) {
-    return new BlockID(blockID.getContainerID(),
-        blockID.getLocalID(), blockID.getBlockCommitSequenceId());
-  }
-
-  public HddsProtos.BlockID getProtobuf() {
-    return HddsProtos.BlockID.newBuilder()
-        .setContainerBlockID(containerBlockID.getProtobuf())
-        .setBlockCommitSequenceId(blockCommitSequenceId).build();
-  }
-
-  public static BlockID getFromProtobuf(HddsProtos.BlockID blockID) {
-    return new BlockID(
-        ContainerBlockID.getFromProtobuf(blockID.getContainerBlockID()),
-        blockID.getBlockCommitSequenceId());
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    BlockID blockID = (BlockID) o;
-    return containerBlockID.equals(blockID.getContainerBlockID())
-        && blockCommitSequenceId == blockID.getBlockCommitSequenceId();
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects
-        .hash(containerBlockID.getContainerID(), containerBlockID.getLocalID(),
-            blockCommitSequenceId);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ContainerBlockID.java

@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.client;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-import java.util.Objects;
-
-/**
- * BlockID returned by SCM during allocation of block (containerID + localID).
- */
-public class ContainerBlockID {
-  private long containerID;
-  private long localID;
-
-  public ContainerBlockID(long containerID, long localID) {
-    this.containerID = containerID;
-    this.localID = localID;
-  }
-
-  public long getContainerID() {
-    return containerID;
-  }
-
-  public long getLocalID() {
-    return localID;
-  }
-
-  @Override
-  public String toString() {
-    return new StringBuffer()
-        .append("conID: ")
-        .append(containerID)
-        .append(" locID: ")
-        .append(localID).toString();
-  }
-
-  public HddsProtos.ContainerBlockID getProtobuf() {
-    return HddsProtos.ContainerBlockID.newBuilder().
-        setContainerID(containerID).setLocalID(localID).build();
-  }
-
-  public static ContainerBlockID getFromProtobuf(
-      HddsProtos.ContainerBlockID containerBlockID) {
-    return new ContainerBlockID(containerBlockID.getContainerID(),
-        containerBlockID.getLocalID());
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    ContainerBlockID blockID = (ContainerBlockID) o;
-    return containerID == blockID.containerID && localID == blockID.localID;
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(containerID, localID);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/OzoneQuota.java

@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.client;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-
-
-/**
- * represents an OzoneQuota Object that can be applied to
- * a storage volume.
- */
-public class OzoneQuota {
-
-  public static final String OZONE_QUOTA_BYTES = "BYTES";
-  public static final String OZONE_QUOTA_MB = "MB";
-  public static final String OZONE_QUOTA_GB = "GB";
-  public static final String OZONE_QUOTA_TB = "TB";
-
-  private Units unit;
-  private long size;
-
-  /** Quota Units.*/
-  public enum Units {UNDEFINED, BYTES, KB, MB, GB, TB}
-
-  /**
-   * Returns size.
-   *
-   * @return long
-   */
-  public long getSize() {
-    return size;
-  }
-
-  /**
-   * Returns Units.
-   *
-   * @return Unit in MB, GB or TB
-   */
-  public Units getUnit() {
-    return unit;
-  }
-
-  /**
-   * Constructs a default Quota object.
-   */
-  public OzoneQuota() {
-    this.size = 0;
-    this.unit = Units.UNDEFINED;
-  }
-
-  /**
-   * Constructor for Ozone Quota.
-   *
-   * @param size Long Size
-   * @param unit MB, GB  or TB
-   */
-  public OzoneQuota(long size, Units unit) {
-    this.size = size;
-    this.unit = unit;
-  }
-
-  /**
-   * Formats a quota as a string.
-   *
-   * @param quota the quota to format
-   * @return string representation of quota
-   */
-  public static String formatQuota(OzoneQuota quota) {
-    return String.valueOf(quota.size) + quota.unit;
-  }
-
-  /**
-   * Parses a user provided string and returns the
-   * Quota Object.
-   *
-   * @param quotaString Quota String
-   *
-   * @return OzoneQuota object
-   *
-   * @throws IllegalArgumentException
-   */
-  public static OzoneQuota parseQuota(String quotaString)
-      throws IllegalArgumentException {
-
-    if ((quotaString == null) || (quotaString.isEmpty())) {
-      throw new IllegalArgumentException(
-          "Quota string cannot be null or empty.");
-    }
-
-    String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
-    String size = "";
-    int nSize;
-    Units currUnit = Units.MB;
-    Boolean found = false;
-    if (uppercase.endsWith(OZONE_QUOTA_MB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_MB.length());
-      currUnit = Units.MB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_GB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_GB.length());
-      currUnit = Units.GB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_TB)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_TB.length());
-      currUnit = Units.TB;
-      found = true;
-    }
-
-    if (uppercase.endsWith(OZONE_QUOTA_BYTES)) {
-      size = uppercase
-          .substring(0, uppercase.length() - OZONE_QUOTA_BYTES.length());
-      currUnit = Units.BYTES;
-      found = true;
-    }
-
-    if (!found) {
-      throw new IllegalArgumentException(
-          "Quota unit not recognized. Supported values are BYTES, MB, GB and " +
-              "TB.");
-    }
-
-    nSize = Integer.parseInt(size);
-    if (nSize < 0) {
-      throw new IllegalArgumentException("Quota cannot be negative.");
-    }
-
-    return new OzoneQuota(nSize, currUnit);
-  }
-
-
-  /**
-   * Returns size in Bytes or -1 if there is no Quota.
-   */
-  public long sizeInBytes() {
-    switch (this.unit) {
-    case BYTES:
-      return this.getSize();
-    case MB:
-      return this.getSize() * OzoneConsts.MB;
-    case GB:
-      return this.getSize() * OzoneConsts.GB;
-    case TB:
-      return this.getSize() * OzoneConsts.TB;
-    case UNDEFINED:
-    default:
-      return -1;
-    }
-  }
-
-  /**
-   * Returns OzoneQuota corresponding to size in bytes.
-   *
-   * @param sizeInBytes size in bytes to be converted
-   *
-   * @return OzoneQuota object
-   */
-  public static OzoneQuota getOzoneQuota(long sizeInBytes) {
-    long size;
-    Units unit;
-    if (sizeInBytes % OzoneConsts.TB == 0) {
-      size = sizeInBytes / OzoneConsts.TB;
-      unit = Units.TB;
-    } else if (sizeInBytes % OzoneConsts.GB == 0) {
-      size = sizeInBytes / OzoneConsts.GB;
-      unit = Units.GB;
-    } else if (sizeInBytes % OzoneConsts.MB == 0) {
-      size = sizeInBytes / OzoneConsts.MB;
-      unit = Units.MB;
-    } else {
-      size = sizeInBytes;
-      unit = Units.BYTES;
-    }
-    return new OzoneQuota((int)size, unit);
-  }
-
-  @Override
-  public String toString() {
-    return size + " " + unit;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationFactor.java

@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.client;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-/**
- * The replication factor to be used while writing key into ozone.
- */
-public enum ReplicationFactor {
-  ONE(1),
-  THREE(3);
-
-  /**
-   * Integer representation of replication.
-   */
-  private int value;
-
-  /**
-   * Initializes ReplicationFactor with value.
-   * @param value replication value
-   */
-  ReplicationFactor(int value) {
-    this.value = value;
-  }
-
-  /**
-   * Returns enum value corresponding to the int value.
-   * @param value replication value
-   * @return ReplicationFactor
-   */
-  public static ReplicationFactor valueOf(int value) {
-    if(value == 1) {
-      return ONE;
-    }
-    if (value == 3) {
-      return THREE;
-    }
-    throw new IllegalArgumentException("Unsupported value: " + value);
-  }
-
-  public static ReplicationFactor fromProto(
-      HddsProtos.ReplicationFactor replicationFactor) {
-    if (replicationFactor == null) {
-      return null;
-    }
-    switch (replicationFactor) {
-    case ONE:
-      return ReplicationFactor.ONE;
-    case THREE:
-      return ReplicationFactor.THREE;
-    default:
-      throw new IllegalArgumentException(
-          "Unsupported ProtoBuf replication factor: " + replicationFactor);
-    }
-  }
-
-  /**
-   * Returns integer representation of ReplicationFactor.
-   * @return replication value
-   */
-  public int getValue() {
-    return value;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationType.java

@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.client;
-
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-/**
- * The replication type to be used while writing key into ozone.
- */
-public enum ReplicationType {
-    RATIS,
-    STAND_ALONE,
-  CHAINED;
-
-  public static ReplicationType fromProto(
-      HddsProtos.ReplicationType replicationType) {
-    if (replicationType == null) {
-      return null;
-    }
-    switch (replicationType) {
-    case RATIS:
-      return ReplicationType.RATIS;
-    case STAND_ALONE:
-      return ReplicationType.STAND_ALONE;
-    case CHAINED:
-      return ReplicationType.CHAINED;
-    default:
-      throw new IllegalArgumentException(
-          "Unsupported ProtoBuf replication type: " + replicationType);
-    }
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/package-info.java

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.client;
-
-/**
- * Base property types for HDDS containers and replications.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java

@@ -1,190 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.conf;
-
-import com.google.gson.Gson;
-import java.io.IOException;
-import java.io.Writer;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Properties;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.HttpHeaders;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer2;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY;
-
-/**
- * A servlet to print out the running configuration data.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Unstable
-public class HddsConfServlet extends HttpServlet {
-
-  private static final long serialVersionUID = 1L;
-
-  protected static final String FORMAT_JSON = "json";
-  protected static final String FORMAT_XML = "xml";
-  private static final String COMMAND = "cmd";
-  private static final OzoneConfiguration OZONE_CONFIG =
-      new OzoneConfiguration();
-  private static final transient Logger LOG =
-      LoggerFactory.getLogger(HddsConfServlet.class);
-
-
-  /**
-   * Return the Configuration of the daemon hosting this servlet.
-   * This is populated when the HttpServer starts.
-   */
-  private Configuration getConfFromContext() {
-    Configuration conf = (Configuration) getServletContext().getAttribute(
-        HttpServer2.CONF_CONTEXT_ATTRIBUTE);
-    assert conf != null;
-    return conf;
-  }
-
-  @Override
-  public void doGet(HttpServletRequest request, HttpServletResponse response)
-      throws ServletException, IOException {
-
-    if (!HttpServer2.isInstrumentationAccessAllowed(getServletContext(),
-        request, response)) {
-      return;
-    }
-
-    String format = parseAcceptHeader(request);
-    if (FORMAT_XML.equals(format)) {
-      response.setContentType("text/xml; charset=utf-8");
-    } else if (FORMAT_JSON.equals(format)) {
-      response.setContentType("application/json; charset=utf-8");
-    }
-
-    String name = request.getParameter("name");
-    Writer out = response.getWriter();
-    String cmd = request.getParameter(COMMAND);
-
-    processCommand(cmd, format, request, response, out, name);
-    out.close();
-  }
-
-  private void processCommand(String cmd, String format,
-      HttpServletRequest request, HttpServletResponse response, Writer out,
-      String name)
-      throws IOException {
-    try {
-      if (cmd == null) {
-        if (FORMAT_XML.equals(format)) {
-          response.setContentType("text/xml; charset=utf-8");
-        } else if (FORMAT_JSON.equals(format)) {
-          response.setContentType("application/json; charset=utf-8");
-        }
-
-        writeResponse(getConfFromContext(), out, format, name);
-      } else {
-        processConfigTagRequest(request, out);
-      }
-    } catch (BadFormatException bfe) {
-      response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
-    } catch (IllegalArgumentException iae) {
-      response.sendError(HttpServletResponse.SC_NOT_FOUND, iae.getMessage());
-    }
-  }
-
-  @VisibleForTesting
-  static String parseAcceptHeader(HttpServletRequest request) {
-    String format = request.getHeader(HttpHeaders.ACCEPT);
-    return format != null && format.contains(FORMAT_JSON) ?
-        FORMAT_JSON : FORMAT_XML;
-  }
-
-  /**
-   * Guts of the servlet - extracted for easy testing.
-   */
-  static void writeResponse(Configuration conf,
-      Writer out, String format, String propertyName)
-      throws IOException, IllegalArgumentException, BadFormatException {
-    if (FORMAT_JSON.equals(format)) {
-      Configuration.dumpConfiguration(conf, propertyName, out);
-    } else if (FORMAT_XML.equals(format)) {
-      conf.writeXml(propertyName, out);
-    } else {
-      throw new BadFormatException("Bad format: " + format);
-    }
-  }
-
-  /**
-   * Exception for signal bad content type.
-   */
-  public static class BadFormatException extends Exception {
-
-    private static final long serialVersionUID = 1L;
-
-    public BadFormatException(String msg) {
-      super(msg);
-    }
-  }
-
-  private void processConfigTagRequest(HttpServletRequest request,
-      Writer out) throws IOException {
-    String cmd = request.getParameter(COMMAND);
-    Gson gson = new Gson();
-    Configuration config = getOzoneConfig();
-
-    switch (cmd) {
-    case "getOzoneTags":
-      out.write(gson.toJson(config.get(OZONE_TAGS_SYSTEM_KEY)
-          .split(",")));
-      break;
-    case "getPropertyByTag":
-      String tags = request.getParameter("tags");
-      Map<String, Properties> propMap = new HashMap<>();
-
-      for (String tag : tags.split(",")) {
-        if (config.isPropertyTag(tag)) {
-          Properties properties = config.getAllPropertiesByTag(tag);
-          propMap.put(tag, properties);
-        } else {
-          if (LOG.isDebugEnabled()) {
-            LOG.debug("Not a valid tag" + tag);
-          }
-        }
-      }
-      out.write(gson.toJsonTree(propMap).toString());
-      break;
-    default:
-      throw new IllegalArgumentException(cmd + " is not a valid command.");
-    }
-
-  }
-
-  private static Configuration getOzoneConfig() {
-    return OZONE_CONFIG;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java

@@ -1,328 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.conf;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.Properties;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Configuration for ozone.
- */
-@InterfaceAudience.Private
-public class OzoneConfiguration extends Configuration {
-  static {
-    activate();
-  }
-
-  public static OzoneConfiguration of(Configuration conf) {
-    Preconditions.checkNotNull(conf);
-
-    return conf instanceof OzoneConfiguration
-        ? (OzoneConfiguration) conf
-        : new OzoneConfiguration(conf);
-  }
-
-  public OzoneConfiguration() {
-    OzoneConfiguration.activate();
-    loadDefaults();
-  }
-
-  public OzoneConfiguration(Configuration conf) {
-    super(conf);
-    //load the configuration from the classloader of the original conf.
-    setClassLoader(conf.getClassLoader());
-    if (!(conf instanceof OzoneConfiguration)) {
-      loadDefaults();
-    }
-  }
-
-  private void loadDefaults() {
-    try {
-      //there could be multiple ozone-default-generated.xml files on the
-      // classpath, which are generated by the annotation processor.
-      // Here we add all of them to the list of the available configuration.
-      Enumeration<URL> generatedDefaults =
-          OzoneConfiguration.class.getClassLoader().getResources(
-              "ozone-default-generated.xml");
-      while (generatedDefaults.hasMoreElements()) {
-        addResource(generatedDefaults.nextElement());
-      }
-    } catch (IOException e) {
-      e.printStackTrace();
-    }
-    addResource("ozone-site.xml");
-  }
-
-  public List<Property> readPropertyFromXml(URL url) throws JAXBException {
-    JAXBContext context = JAXBContext.newInstance(XMLConfiguration.class);
-    Unmarshaller um = context.createUnmarshaller();
-
-    XMLConfiguration config = (XMLConfiguration) um.unmarshal(url);
-    return config.getProperties();
-  }
-
-  /**
-   * Create a Configuration object and inject the required configuration values.
-   *
-   * @param configurationClass The class where the fields are annotated with
-   *                           the configuration.
-   * @return Initiated java object where the config fields are injected.
-   */
-  public <T> T getObject(Class<T> configurationClass) {
-
-    T configuration;
-
-    try {
-      configuration = configurationClass.newInstance();
-    } catch (InstantiationException | IllegalAccessException e) {
-      throw new ConfigurationException(
-          "Configuration class can't be created: " + configurationClass, e);
-    }
-    ConfigGroup configGroup =
-        configurationClass.getAnnotation(ConfigGroup.class);
-    String prefix = configGroup.prefix();
-
-    for (Method setterMethod : configurationClass.getMethods()) {
-      if (setterMethod.isAnnotationPresent(Config.class)) {
-
-        String methodLocation =
-            configurationClass + "." + setterMethod.getName();
-
-        Config configAnnotation = setterMethod.getAnnotation(Config.class);
-
-        String key = prefix + "." + configAnnotation.key();
-
-        Class<?>[] parameterTypes = setterMethod.getParameterTypes();
-        if (parameterTypes.length != 1) {
-          throw new ConfigurationException(
-              "@Config annotation should be used on simple setter: "
-                  + methodLocation);
-        }
-
-        ConfigType type = configAnnotation.type();
-
-        if (type == ConfigType.AUTO) {
-          type = detectConfigType(parameterTypes[0], methodLocation);
-        }
-
-        //Note: default value is handled by ozone-default.xml. Here we can
-        //use any default.
-        try {
-          switch (type) {
-          case STRING:
-            setterMethod.invoke(configuration, get(key));
-            break;
-          case INT:
-            setterMethod.invoke(configuration,
-                getInt(key, 0));
-            break;
-          case BOOLEAN:
-            setterMethod.invoke(configuration,
-                getBoolean(key, false));
-            break;
-          case LONG:
-            setterMethod.invoke(configuration,
-                getLong(key, 0));
-            break;
-          case TIME:
-            setterMethod.invoke(configuration,
-                getTimeDuration(key, 0, configAnnotation.timeUnit()));
-            break;
-          default:
-            throw new ConfigurationException(
-                "Unsupported ConfigType " + type + " on " + methodLocation);
-          }
-        } catch (InvocationTargetException | IllegalAccessException e) {
-          throw new ConfigurationException(
-              "Can't inject configuration to " + methodLocation, e);
-        }
-
-      }
-    }
-    return configuration;
-
-  }
-
-  private ConfigType detectConfigType(Class<?> parameterType,
-      String methodLocation) {
-    ConfigType type;
-    if (parameterType == String.class) {
-      type = ConfigType.STRING;
-    } else if (parameterType == Integer.class || parameterType == int.class) {
-      type = ConfigType.INT;
-    } else if (parameterType == Long.class || parameterType == long.class) {
-      type = ConfigType.LONG;
-    } else if (parameterType == Boolean.class
-        || parameterType == boolean.class) {
-      type = ConfigType.BOOLEAN;
-    } else {
-      throw new ConfigurationException(
-          "Unsupported configuration type " + parameterType + " in "
-              + methodLocation);
-    }
-    return type;
-  }
-
-  /**
-   * Class to marshall/un-marshall configuration from xml files.
-   */
-  @XmlAccessorType(XmlAccessType.FIELD)
-  @XmlRootElement(name = "configuration")
-  public static class XMLConfiguration {
-
-    @XmlElement(name = "property", type = Property.class)
-    private List<Property> properties = new ArrayList<>();
-
-    public XMLConfiguration() {
-    }
-
-    public XMLConfiguration(List<Property> properties) {
-      this.properties = properties;
-    }
-
-    public List<Property> getProperties() {
-      return properties;
-    }
-
-    public void setProperties(List<Property> properties) {
-      this.properties = properties;
-    }
-  }
-
-  /**
-   * Class to marshall/un-marshall configuration properties from xml files.
-   */
-  @XmlAccessorType(XmlAccessType.FIELD)
-  @XmlRootElement(name = "property")
-  public static class Property implements Comparable<Property> {
-
-    private String name;
-    private String value;
-    private String tag;
-    private String description;
-
-    public String getName() {
-      return name;
-    }
-
-    public void setName(String name) {
-      this.name = name;
-    }
-
-    public String getValue() {
-      return value;
-    }
-
-    public void setValue(String value) {
-      this.value = value;
-    }
-
-    public String getTag() {
-      return tag;
-    }
-
-    public void setTag(String tag) {
-      this.tag = tag;
-    }
-
-    public String getDescription() {
-      return description;
-    }
-
-    public void setDescription(String description) {
-      this.description = description;
-    }
-
-    @Override
-    public int compareTo(Property o) {
-      if (this == o) {
-        return 0;
-      }
-      return this.getName().compareTo(o.getName());
-    }
-
-    @Override
-    public String toString() {
-      return this.getName() + " " + this.getValue() + " " + this.getTag();
-    }
-
-    @Override
-    public int hashCode() {
-      return this.getName().hashCode();
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-      return (obj instanceof Property) && (((Property) obj).getName())
-          .equals(this.getName());
-    }
-  }
-
-  public static void activate() {
-    // adds the default resources
-    Configuration.addDefaultResource("hdfs-default.xml");
-    Configuration.addDefaultResource("hdfs-site.xml");
-    Configuration.addDefaultResource("ozone-default.xml");
-  }
-
-  /**
-   * The super class method getAllPropertiesByTag
-   * does not override values of properties
-   * if there is no tag present in the configs of
-   * newly added resources.
-   *
-   * @param tag
-   * @return Properties that belong to the tag
-   */
-  @Override
-  public Properties getAllPropertiesByTag(String tag) {
-    // Call getProps first to load the newly added resources
-    // before calling super.getAllPropertiesByTag
-    Properties updatedProps = getProps();
-    Properties propertiesByTag = super.getAllPropertiesByTag(tag);
-    Properties props = new Properties();
-    Enumeration properties = propertiesByTag.propertyNames();
-    while (properties.hasMoreElements()) {
-      Object propertyName = properties.nextElement();
-      // get the current value of the property
-      Object value = updatedProps.getProperty(propertyName.toString());
-      if (value != null) {
-        props.put(propertyName, value);
-      }
-    }
-    return props;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/package-info.java

@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.conf;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/FunctionWithServiceException.java

@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.function;
-
-import com.google.protobuf.ServiceException;
-
-/**
- * Functional interface like java.util.function.Function but with
- * checked exception.
- */
-@FunctionalInterface
-public interface FunctionWithServiceException<T, R> {
-
-  /**
-   * Applies this function to the given argument.
-   *
-   * @param t the function argument
-   * @return the function result
-   */
-  R apply(T t) throws ServiceException;
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/function/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Functional interfaces for ozone, similar to java.util.function.
- */
-package org.apache.hadoop.hdds.function;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/package-info.java

@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds;
-
-/**
- * Generic HDDS specific configurator and helper classes.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/DatanodeDetails.java

@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.protocol;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.net.NetConstants;
-import org.apache.hadoop.hdds.scm.net.NodeImpl;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-/**
- * DatanodeDetails class contains details about DataNode like:
- * - UUID of the DataNode.
- * - IP and Hostname details.
- * - Port details to which the DataNode will be listening.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class DatanodeDetails extends NodeImpl implements
-    Comparable<DatanodeDetails> {
-/**
-   * DataNode's unique identifier in the cluster.
-   */
-  private final UUID uuid;
-
-  private String ipAddress;
-  private String hostName;
-  private List<Port> ports;
-  private String certSerialId;
-
-  /**
-   * Constructs DatanodeDetails instance. DatanodeDetails.Builder is used
-   * for instantiating DatanodeDetails.
-   * @param uuid DataNode's UUID
-   * @param ipAddress IP Address of this DataNode
-   * @param hostName DataNode's hostname
-   * @param networkLocation DataNode's network location path
-   * @param ports Ports used by the DataNode
-   * @param certSerialId serial id from SCM issued certificate.
-   */
-  private DatanodeDetails(String uuid, String ipAddress, String hostName,
-      String networkLocation, List<Port> ports, String certSerialId) {
-    super(hostName, networkLocation, NetConstants.NODE_COST_DEFAULT);
-    this.uuid = UUID.fromString(uuid);
-    this.ipAddress = ipAddress;
-    this.hostName = hostName;
-    this.ports = ports;
-    this.certSerialId = certSerialId;
-  }
-
-  protected DatanodeDetails(DatanodeDetails datanodeDetails) {
-    super(datanodeDetails.getHostName(), datanodeDetails.getNetworkLocation(),
-        datanodeDetails.getCost());
-    this.uuid = datanodeDetails.uuid;
-    this.ipAddress = datanodeDetails.ipAddress;
-    this.hostName = datanodeDetails.hostName;
-    this.ports = datanodeDetails.ports;
-    this.setNetworkName(datanodeDetails.getNetworkName());
-  }
-
-  /**
-   * Returns the DataNode UUID.
-   *
-   * @return UUID of DataNode
-   */
-  public UUID getUuid() {
-    return uuid;
-  }
-
-  /**
-   * Returns the string representation of DataNode UUID.
-   *
-   * @return UUID of DataNode
-   */
-  public String getUuidString() {
-    return uuid.toString();
-  }
-
-  /**
-   * Sets the IP address of Datanode.
-   *
-   * @param ip IP Address
-   */
-  public void setIpAddress(String ip) {
-    this.ipAddress = ip;
-  }
-
-  /**
-   * Returns IP address of DataNode.
-   *
-   * @return IP address
-   */
-  public String getIpAddress() {
-    return ipAddress;
-  }
-
-  /**
-   * Sets the Datanode hostname.
-   *
-   * @param host hostname
-   */
-  public void setHostName(String host) {
-    this.hostName = host;
-  }
-
-  /**
-   * Returns Hostname of DataNode.
-   *
-   * @return Hostname
-   */
-  public String getHostName() {
-    return hostName;
-  }
-
-  /**
-   * Sets a DataNode Port.
-   *
-   * @param port DataNode port
-   */
-  public void setPort(Port port) {
-    // If the port is already in the list remove it first and add the
-    // new/updated port value.
-    ports.remove(port);
-    ports.add(port);
-  }
-
-  /**
-   * Returns all the Ports used by DataNode.
-   *
-   * @return DataNode Ports
-   */
-  public List<Port> getPorts() {
-    return ports;
-  }
-
-  /**
-   * Given the name returns port number, null if the asked port is not found.
-   *
-   * @param name Name of the port
-   *
-   * @return Port
-   */
-  public Port getPort(Port.Name name) {
-    for (Port port : ports) {
-      if (port.getName().equals(name)) {
-        return port;
-      }
-    }
-    return null;
-  }
-
-  /**
-   * Returns a DatanodeDetails from the protocol buffers.
-   *
-   * @param datanodeDetailsProto - protoBuf Message
-   * @return DatanodeDetails
-   */
-  public static DatanodeDetails getFromProtoBuf(
-      HddsProtos.DatanodeDetailsProto datanodeDetailsProto) {
-    DatanodeDetails.Builder builder = newBuilder();
-    builder.setUuid(datanodeDetailsProto.getUuid());
-    if (datanodeDetailsProto.hasIpAddress()) {
-      builder.setIpAddress(datanodeDetailsProto.getIpAddress());
-    }
-    if (datanodeDetailsProto.hasHostName()) {
-      builder.setHostName(datanodeDetailsProto.getHostName());
-    }
-    if (datanodeDetailsProto.hasCertSerialId()) {
-      builder.setCertSerialId(datanodeDetailsProto.getCertSerialId());
-    }
-    for (HddsProtos.Port port : datanodeDetailsProto.getPortsList()) {
-      builder.addPort(newPort(
-          Port.Name.valueOf(port.getName().toUpperCase()), port.getValue()));
-    }
-    if (datanodeDetailsProto.hasNetworkName()) {
-      builder.setNetworkName(datanodeDetailsProto.getNetworkName());
-    }
-    if (datanodeDetailsProto.hasNetworkLocation()) {
-      builder.setNetworkLocation(datanodeDetailsProto.getNetworkLocation());
-    }
-    return builder.build();
-  }
-
-  /**
-   * Returns a DatanodeDetails protobuf message from a datanode ID.
-   * @return HddsProtos.DatanodeDetailsProto
-   */
-  public HddsProtos.DatanodeDetailsProto getProtoBufMessage() {
-    HddsProtos.DatanodeDetailsProto.Builder builder =
-        HddsProtos.DatanodeDetailsProto.newBuilder()
-            .setUuid(getUuidString());
-    if (ipAddress != null) {
-      builder.setIpAddress(ipAddress);
-    }
-    if (hostName != null) {
-      builder.setHostName(hostName);
-    }
-    if (certSerialId != null) {
-      builder.setCertSerialId(certSerialId);
-    }
-    if (!Strings.isNullOrEmpty(getNetworkName())) {
-      builder.setNetworkName(getNetworkName());
-    }
-    if (!Strings.isNullOrEmpty(getNetworkLocation())) {
-      builder.setNetworkLocation(getNetworkLocation());
-    }
-
-    for (Port port : ports) {
-      builder.addPorts(HddsProtos.Port.newBuilder()
-          .setName(port.getName().toString())
-          .setValue(port.getValue())
-          .build());
-    }
-    return builder.build();
-  }
-
-  @Override
-  public String toString() {
-    return uuid.toString() + "{" +
-        "ip: " +
-        ipAddress +
-        ", host: " +
-        hostName +
-        ", networkLocation: " +
-        getNetworkLocation() +
-        ", certSerialId: " + certSerialId +
-        "}";
-  }
-
-  @Override
-  public int compareTo(DatanodeDetails that) {
-    return this.getUuid().compareTo(that.getUuid());
-  }
-
-  @Override
-  public boolean equals(Object obj) {
-    return obj instanceof DatanodeDetails &&
-        uuid.equals(((DatanodeDetails) obj).uuid);
-  }
-
-  @Override
-  public int hashCode() {
-    return uuid.hashCode();
-  }
-
-  /**
-   * Returns DatanodeDetails.Builder instance.
-   *
-   * @return DatanodeDetails.Builder
-   */
-  public static Builder newBuilder() {
-    return new Builder();
-  }
-
-  /**
-   * Builder class for building DatanodeDetails.
-   */
-  public static final class Builder {
-    private String id;
-    private String ipAddress;
-    private String hostName;
-    private String networkName;
-    private String networkLocation;
-    private List<Port> ports;
-    private String certSerialId;
-
-    /**
-     * Default private constructor. To create Builder instance use
-     * DatanodeDetails#newBuilder.
-     */
-    private Builder() {
-      ports = new ArrayList<>();
-    }
-
-    /**
-     * Sets the DatanodeUuid.
-     *
-     * @param uuid DatanodeUuid
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setUuid(String uuid) {
-      this.id = uuid;
-      return this;
-    }
-
-    /**
-     * Sets the IP address of DataNode.
-     *
-     * @param ip address
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setIpAddress(String ip) {
-      this.ipAddress = ip;
-      return this;
-    }
-
-    /**
-     * Sets the hostname of DataNode.
-     *
-     * @param host hostname
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setHostName(String host) {
-      this.hostName = host;
-      return this;
-    }
-
-    /**
-     * Sets the network name of DataNode.
-     *
-     * @param name network name
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setNetworkName(String name) {
-      this.networkName = name;
-      return this;
-    }
-
-    /**
-     * Sets the network location of DataNode.
-     *
-     * @param loc location
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setNetworkLocation(String loc) {
-      this.networkLocation = loc;
-      return this;
-    }
-
-    /**
-     * Adds a DataNode Port.
-     *
-     * @param port DataNode port
-     *
-     * @return DatanodeDetails.Builder
-     */
-    public Builder addPort(Port port) {
-      this.ports.add(port);
-      return this;
-    }
-
-    /**
-     * Adds certificate serial id.
-     *
-     * @param certId Serial id of SCM issued certificate.
-     *
-     * @return DatanodeDetails.Builder
-     */
-    public Builder setCertSerialId(String certId) {
-      this.certSerialId = certId;
-      return this;
-    }
-
-    /**
-     * Builds and returns DatanodeDetails instance.
-     *
-     * @return DatanodeDetails
-     */
-    public DatanodeDetails build() {
-      Preconditions.checkNotNull(id);
-      if (networkLocation == null) {
-        networkLocation = NetConstants.DEFAULT_RACK;
-      }
-      DatanodeDetails dn = new DatanodeDetails(id, ipAddress, hostName,
-          networkLocation, ports, certSerialId);
-      if (networkName != null) {
-        dn.setNetworkName(networkName);
-      }
-      return dn;
-    }
-  }
-
-  /**
-   * Constructs a new Port with name and value.
-   *
-   * @param name Name of the port
-   * @param value Port number
-   *
-   * @return {@code Port} instance
-   */
-  public static Port newPort(Port.Name name, Integer value) {
-    return new Port(name, value);
-  }
-
-  /**
-   * Container to hold DataNode Port details.
-   */
-  public static final class Port {
-
-    /**
-     * Ports that are supported in DataNode.
-     */
-    public enum Name {
-      STANDALONE, RATIS, REST
-    }
-
-    private Name name;
-    private Integer value;
-
-    /**
-     * Private constructor for constructing Port object. Use
-     * DatanodeDetails#newPort to create a new Port object.
-     *
-     * @param name
-     * @param value
-     */
-    private Port(Name name, Integer value) {
-      this.name = name;
-      this.value = value;
-    }
-
-    /**
-     * Returns the name of the port.
-     *
-     * @return Port name
-     */
-    public Name getName() {
-      return name;
-    }
-
-    /**
-     * Returns the port number.
-     *
-     * @return Port number
-     */
-    public Integer getValue() {
-      return value;
-    }
-
-    @Override
-    public int hashCode() {
-      return name.hashCode();
-    }
-
-    /**
-     * Ports are considered equal if they have the same name.
-     *
-     * @param anObject
-     *          The object to compare this {@code Port} against
-     * @return {@code true} if the given object represents a {@code Port}
-               and has the same name, {@code false} otherwise
-     */
-    @Override
-    public boolean equals(Object anObject) {
-      if (this == anObject) {
-        return true;
-      }
-      if (anObject instanceof Port) {
-        return name.equals(((Port) anObject).name);
-      }
-      return false;
-    }
-  }
-
-  /**
-   * Returns serial id of SCM issued certificate.
-   *
-   * @return certificate serial id
-   */
-  public String getCertSerialId() {
-    return certSerialId;
-  }
-
-  /**
-   * Set certificate serial id of SCM issued certificate.
-   *
-   */
-  public void setCertSerialId(String certSerialId) {
-    this.certSerialId = certSerialId;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/SCMSecurityProtocol.java

@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.protocol;
-
-import java.io.IOException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.security.KerberosInfo;
-
-/**
- * The protocol used to perform security related operations with SCM.
- */
-@KerberosInfo(
-    serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
-@InterfaceAudience.Private
-public interface SCMSecurityProtocol {
-
-  @SuppressWarnings("checkstyle:ConstantName")
-  /**
-   * Version 1: Initial version.
-   */
-  long versionID = 1L;
-
-  /**
-   * Get SCM signed certificate for DataNode.
-   *
-   * @param dataNodeDetails - DataNode Details.
-   * @param certSignReq     - Certificate signing request.
-   * @return byte[]         - SCM signed certificate.
-   */
-  String getDataNodeCertificate(
-      DatanodeDetailsProto dataNodeDetails,
-      String certSignReq) throws IOException;
-
-  /**
-   * Get SCM signed certificate for OM.
-   *
-   * @param omDetails       - DataNode Details.
-   * @param certSignReq     - Certificate signing request.
-   * @return String         - pem encoded SCM signed
-   *                          certificate.
-   */
-  String getOMCertificate(OzoneManagerDetailsProto omDetails,
-      String certSignReq) throws IOException;
-
-  /**
-   * Get SCM signed certificate for given certificate serial id if it exists.
-   * Throws exception if it's not found.
-   *
-   * @param certSerialId    - Certificate serial id.
-   * @return String         - pem encoded SCM signed
-   *                          certificate with given cert id if it
-   *                          exists.
-   */
-  String getCertificate(String certSerialId) throws IOException;
-
-  /**
-   * Get CA certificate.
-   *
-   * @return String         - pem encoded CA certificate.
-   */
-  String getCACertificate() throws IOException;
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocol/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains HDDS protocol related classes.
- */
-package org.apache.hadoop.hdds.protocol;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java

@@ -1,213 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.protocolPB;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.function.Consumer;
-
-import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityRequest.Builder;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityResponse;
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.Type;
-import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import static org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetOMCertRequestProto;
-
-/**
- * This class is the client-side translator that forwards requests for
- * {@link SCMSecurityProtocol} to the {@link SCMSecurityProtocolPB} proxy.
- */
-public class SCMSecurityProtocolClientSideTranslatorPB implements
-    SCMSecurityProtocol, ProtocolTranslator, Closeable {
-
-  /**
-   * RpcController is not used and hence is set to null.
-   */
-  private static final RpcController NULL_RPC_CONTROLLER = null;
-  private final SCMSecurityProtocolPB rpcProxy;
-
-  public SCMSecurityProtocolClientSideTranslatorPB(
-      SCMSecurityProtocolPB rpcProxy) {
-    this.rpcProxy = rpcProxy;
-  }
-
-  /**
-   * Helper method to wrap the request and send the message.
-   */
-  private SCMSecurityResponse submitRequest(
-      SCMSecurityProtocolProtos.Type type,
-      Consumer<Builder> builderConsumer) throws IOException {
-    final SCMSecurityResponse response;
-    try {
-
-      Builder builder = SCMSecurityRequest.newBuilder()
-          .setCmdType(type)
-          .setTraceID(TracingUtil.exportCurrentSpan());
-      builderConsumer.accept(builder);
-      SCMSecurityRequest wrapper = builder.build();
-
-      response = rpcProxy.submitRequest(NULL_RPC_CONTROLLER, wrapper);
-    } catch (ServiceException ex) {
-      throw ProtobufHelper.getRemoteException(ex);
-    }
-    return response;
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated
-   * with it. If the stream is already closed then invoking this
-   * method has no effect.
-   *
-   * <p> As noted in {@link AutoCloseable#close()}, cases where the
-   * close may fail require careful attention. It is strongly advised
-   * to relinquish the underlying resources and to internally
-   * <em>mark</em> the {@code Closeable} as closed, prior to throwing
-   * the {@code IOException}.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override
-  public void close() throws IOException {
-    RPC.stopProxy(rpcProxy);
-  }
-
-  /**
-   * Get SCM signed certificate for DataNode.
-   *
-   * @param dataNodeDetails - DataNode Details.
-   * @param certSignReq     - Certificate signing request.
-   * @return byte[]         - SCM signed certificate.
-   */
-  @Override
-  public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
-      String certSignReq) throws IOException {
-    return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
-        .getX509Certificate();
-  }
-
-  /**
-   * Get SCM signed certificate for OM.
-   *
-   * @param omDetails   - OzoneManager Details.
-   * @param certSignReq - Certificate signing request.
-   * @return byte[]         - SCM signed certificate.
-   */
-  @Override
-  public String getOMCertificate(OzoneManagerDetailsProto omDetails,
-      String certSignReq) throws IOException {
-    return getOMCertChain(omDetails, certSignReq).getX509Certificate();
-  }
-
-  /**
-   * Get SCM signed certificate for OM.
-   *
-   * @param omDetails   - OzoneManager Details.
-   * @param certSignReq - Certificate signing request.
-   * @return byte[]         - SCM signed certificate.
-   */
-  public SCMGetCertResponseProto getOMCertChain(
-      OzoneManagerDetailsProto omDetails, String certSignReq)
-      throws IOException {
-    SCMGetOMCertRequestProto request = SCMGetOMCertRequestProto
-        .newBuilder()
-        .setCSR(certSignReq)
-        .setOmDetails(omDetails)
-        .build();
-    return submitRequest(Type.GetOMCertificate,
-        builder -> builder.setGetOMCertRequest(request))
-        .getGetCertResponseProto();
-  }
-
-  /**
-   * Get SCM signed certificate with given serial id. Throws exception if
-   * certificate is not found.
-   *
-   * @param certSerialId - Certificate serial id.
-   * @return string         - pem encoded certificate.
-   */
-  @Override
-  public String getCertificate(String certSerialId) throws IOException {
-    SCMGetCertificateRequestProto request = SCMGetCertificateRequestProto
-        .newBuilder()
-        .setCertSerialId(certSerialId)
-        .build();
-    return submitRequest(Type.GetCertificate,
-        builder -> builder.setGetCertificateRequest(request))
-        .getGetCertResponseProto()
-        .getX509Certificate();
-  }
-
-  /**
-   * Get SCM signed certificate for Datanode.
-   *
-   * @param dnDetails   - Datanode Details.
-   * @param certSignReq - Certificate signing request.
-   * @return byte[]         - SCM signed certificate.
-   */
-  public SCMGetCertResponseProto getDataNodeCertificateChain(
-      DatanodeDetailsProto dnDetails, String certSignReq)
-      throws IOException {
-
-    SCMGetDataNodeCertRequestProto request =
-        SCMGetDataNodeCertRequestProto.newBuilder()
-            .setCSR(certSignReq)
-            .setDatanodeDetails(dnDetails)
-            .build();
-    return submitRequest(Type.GetDataNodeCertificate,
-        builder -> builder.setGetDataNodeCertRequest(request))
-        .getGetCertResponseProto();
-  }
-
-  /**
-   * Get CA certificate.
-   *
-   * @return serial   - Root certificate.
-   */
-  @Override
-  public String getCACertificate() throws IOException {
-    SCMGetCACertificateRequestProto protoIns = SCMGetCACertificateRequestProto
-        .getDefaultInstance();
-    return submitRequest(Type.GetCACertificate,
-        builder -> builder.setGetCACertificateRequest(protoIns))
-        .getGetCertResponseProto().getX509Certificate();
-
-  }
-
-  /**
-   * Return the proxy object underlying this protocol translator.
-   *
-   * @return the proxy object underlying this protocol translator.
-   */
-  @Override
-  public Object getUnderlyingProxyObject() {
-    return rpcProxy;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolPB.java

@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.hdds.protocolPB;
-
-import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMSecurityProtocolService;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.security.KerberosInfo;
-
-/**
- * Protocol for security related operations on SCM.
- */
-
-@ProtocolInfo(protocolName =
-    "org.apache.hadoop.hdds.protocol.SCMSecurityProtocol",
-    protocolVersion = 1)
-@KerberosInfo(serverPrincipal = ScmConfigKeys.HDDS_SCM_KERBEROS_PRINCIPAL_KEY)
-public interface SCMSecurityProtocolPB extends
-    SCMSecurityProtocolService.BlockingInterface {
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.protocolPB;
-/**
- * This package contains classes for wiring HDDS protobuf calls to rpc.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/ContainerCommandRequestMessage.java

@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.ratis;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutSmallFileRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.ratis.util.JavaUtils;
-
-import java.util.Objects;
-import java.util.function.Supplier;
-
-/**
- * Implementing the {@link Message} interface
- * for {@link ContainerCommandRequestProto}.
- */
-public final class ContainerCommandRequestMessage implements Message {
-  public static ContainerCommandRequestMessage toMessage(
-      ContainerCommandRequestProto request, String traceId) {
-    final ContainerCommandRequestProto.Builder b
-        = ContainerCommandRequestProto.newBuilder(request);
-    if (traceId != null) {
-      b.setTraceID(traceId);
-    }
-
-    ByteString data = ByteString.EMPTY;
-    if (request.getCmdType() == Type.WriteChunk) {
-      final WriteChunkRequestProto w = request.getWriteChunk();
-      data = w.getData();
-      b.setWriteChunk(w.toBuilder().clearData());
-    } else if (request.getCmdType() == Type.PutSmallFile) {
-      final PutSmallFileRequestProto p = request.getPutSmallFile();
-      data = p.getData();
-      b.setPutSmallFile(p.toBuilder().setData(ByteString.EMPTY));
-    }
-    return new ContainerCommandRequestMessage(b.build(), data);
-  }
-
-  public static ContainerCommandRequestProto toProto(
-      ByteString bytes, RaftGroupId groupId)
-      throws InvalidProtocolBufferException {
-    final int i = 4 + bytes.asReadOnlyByteBuffer().getInt();
-    final ContainerCommandRequestProto header
-        = ContainerCommandRequestProto.parseFrom(bytes.substring(4, i));
-    // TODO: setting pipeline id can be avoided if the client is sending it.
-    //       In such case, just have to validate the pipeline id.
-    final ContainerCommandRequestProto.Builder b = header.toBuilder();
-    if (groupId != null) {
-      b.setPipelineID(groupId.getUuid().toString());
-    }
-    final ByteString data = bytes.substring(i);
-    if (header.getCmdType() == Type.WriteChunk) {
-      b.setWriteChunk(b.getWriteChunkBuilder().setData(data));
-    } else if (header.getCmdType() == Type.PutSmallFile) {
-      b.setPutSmallFile(b.getPutSmallFileBuilder().setData(data));
-    }
-    return b.build();
-  }
-
-  private final ContainerCommandRequestProto header;
-  private final ByteString data;
-  private final Supplier<ByteString> contentSupplier
-      = JavaUtils.memoize(this::buildContent);
-
-  private ContainerCommandRequestMessage(
-      ContainerCommandRequestProto header, ByteString data) {
-    this.header = Objects.requireNonNull(header, "header == null");
-    this.data = Objects.requireNonNull(data, "data == null");
-  }
-
-  private ByteString buildContent() {
-    final ByteString headerBytes = header.toByteString();
-    return RatisHelper.int2ByteString(headerBytes.size())
-        .concat(headerBytes)
-        .concat(data);
-  }
-
-  @Override
-  public ByteString getContent() {
-    return contentSupplier.get();
-  }
-
-  @Override
-  public String toString() {
-    return header + ", data.size=" + data.size();
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java

@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.ratis;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.security.exception.SCMSecurityException;
-import org.apache.hadoop.hdds.security.x509.SecurityConfig;
-import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer;
-import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import org.apache.ratis.RaftConfigKeys;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.client.RaftClientConfigKeys;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.grpc.GrpcFactory;
-import org.apache.ratis.grpc.GrpcTlsConfig;
-import org.apache.ratis.proto.RaftProtos;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.retry.RetryPolicies;
-import org.apache.ratis.retry.RetryPolicy;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.util.SizeInBytes;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Ratis helper methods.
- */
-public interface RatisHelper {
-  Logger LOG = LoggerFactory.getLogger(RatisHelper.class);
-
-  static String toRaftPeerIdString(DatanodeDetails id) {
-    return id.getUuidString();
-  }
-
-  static UUID toDatanodeId(String peerIdString) {
-    return UUID.fromString(peerIdString);
-  }
-
-  static UUID toDatanodeId(RaftPeerId peerId) {
-    return toDatanodeId(peerId.toString());
-  }
-
-  static UUID toDatanodeId(RaftProtos.RaftPeerProto peerId) {
-    return toDatanodeId(RaftPeerId.valueOf(peerId.getId()));
-  }
-
-  static String toRaftPeerAddressString(DatanodeDetails id) {
-    return id.getIpAddress() + ":" +
-        id.getPort(DatanodeDetails.Port.Name.RATIS).getValue();
-  }
-
-  static RaftPeerId toRaftPeerId(DatanodeDetails id) {
-    return RaftPeerId.valueOf(toRaftPeerIdString(id));
-  }
-
-  static RaftPeer toRaftPeer(DatanodeDetails id) {
-    return new RaftPeer(toRaftPeerId(id), toRaftPeerAddressString(id));
-  }
-
-  static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
-    return toRaftPeers(pipeline.getNodes());
-  }
-
-  static <E extends DatanodeDetails> List<RaftPeer> toRaftPeers(
-      List<E> datanodes) {
-    return datanodes.stream().map(RatisHelper::toRaftPeer)
-        .collect(Collectors.toList());
-  }
-
-  /* TODO: use a dummy id for all groups for the moment.
-   *       It should be changed to a unique id for each group.
-   */
-  RaftGroupId DUMMY_GROUP_ID =
-      RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
-
-  RaftGroup EMPTY_GROUP = RaftGroup.valueOf(DUMMY_GROUP_ID,
-      Collections.emptyList());
-
-  static RaftGroup emptyRaftGroup() {
-    return EMPTY_GROUP;
-  }
-
-  static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
-    return peers.isEmpty()? emptyRaftGroup()
-        : RaftGroup.valueOf(DUMMY_GROUP_ID, peers);
-  }
-
-  static RaftGroup newRaftGroup(RaftGroupId groupId,
-      Collection<DatanodeDetails> peers) {
-    final List<RaftPeer> newPeers = peers.stream()
-        .map(RatisHelper::toRaftPeer)
-        .collect(Collectors.toList());
-    return peers.isEmpty() ? RaftGroup.valueOf(groupId, Collections.emptyList())
-        : RaftGroup.valueOf(groupId, newPeers);
-  }
-
-  static RaftGroup newRaftGroup(Pipeline pipeline) {
-    return RaftGroup.valueOf(RaftGroupId.valueOf(pipeline.getId().getId()),
-        toRaftPeers(pipeline));
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,
-      RetryPolicy retryPolicy, int maxOutStandingRequest,
-      GrpcTlsConfig tlsConfig, TimeDuration timeout) throws IOException {
-    return newRaftClient(rpcType, toRaftPeerId(pipeline.getFirstNode()),
-        newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
-            pipeline.getNodes()), retryPolicy, maxOutStandingRequest, tlsConfig,
-        timeout);
-  }
-
-  static TimeDuration getClientRequestTimeout(Configuration conf) {
-    // Set the client requestTimeout
-    final TimeUnit timeUnit =
-        OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
-            .getUnit();
-    final long duration = conf.getTimeDuration(
-        OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY,
-        OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT
-            .getDuration(), timeUnit);
-    final TimeDuration clientRequestTimeout =
-        TimeDuration.valueOf(duration, timeUnit);
-    return clientRequestTimeout;
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
-      RetryPolicy retryPolicy, int maxOutstandingRequests,
-      GrpcTlsConfig tlsConfig, TimeDuration clientRequestTimeout) {
-    return newRaftClient(rpcType, leader.getId(),
-        newRaftGroup(new ArrayList<>(Arrays.asList(leader))), retryPolicy,
-        maxOutstandingRequests, tlsConfig, clientRequestTimeout);
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
-      RetryPolicy retryPolicy, int maxOutstandingRequests,
-      TimeDuration clientRequestTimeout) {
-    return newRaftClient(rpcType, leader.getId(),
-        newRaftGroup(new ArrayList<>(Arrays.asList(leader))), retryPolicy,
-        maxOutstandingRequests, null, clientRequestTimeout);
-  }
-
-  static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,
-      RaftGroup group, RetryPolicy retryPolicy, int maxOutStandingRequest,
-      GrpcTlsConfig tlsConfig, TimeDuration clientRequestTimeout) {
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("newRaftClient: {}, leader={}, group={}",
-          rpcType, leader, group);
-    }
-    final RaftProperties properties = new RaftProperties();
-    RaftConfigKeys.Rpc.setType(properties, rpcType);
-    RaftClientConfigKeys.Rpc
-        .setRequestTimeout(properties, clientRequestTimeout);
-
-    GrpcConfigKeys.setMessageSizeMax(properties,
-        SizeInBytes.valueOf(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE));
-    GrpcConfigKeys.OutputStream.setOutstandingAppendsMax(properties,
-        maxOutStandingRequest);
-
-    RaftClient.Builder builder =  RaftClient.newBuilder()
-        .setRaftGroup(group)
-        .setLeaderId(leader)
-        .setProperties(properties)
-        .setRetryPolicy(retryPolicy);
-
-    // TODO: GRPC TLS only for now, netty/hadoop RPC TLS support later.
-    if (tlsConfig != null && rpcType == SupportedRpcType.GRPC) {
-      builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
-    }
-    return builder.build();
-  }
-
-  // For External gRPC client to server with gRPC TLS.
-  // No mTLS for external client as SCM CA does not issued certificates for them
-  static GrpcTlsConfig createTlsClientConfig(SecurityConfig conf,
-      X509Certificate caCert) {
-    GrpcTlsConfig tlsConfig = null;
-    if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
-      tlsConfig = new GrpcTlsConfig(null, null,
-          caCert, false);
-    }
-    return tlsConfig;
-  }
-
-  // For Internal gRPC client from SCM to DN with gRPC TLS
-  static GrpcTlsConfig createTlsClientConfigForSCM(SecurityConfig conf,
-      CertificateServer certificateServer) throws IOException {
-    if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
-      try {
-        X509Certificate caCert =
-            CertificateCodec.getX509Certificate(
-                certificateServer.getCACertificate());
-        return new GrpcTlsConfig(null, null,
-            caCert, false);
-      } catch (CertificateException ex) {
-        throw new SCMSecurityException("Fail to find SCM CA certificate.", ex);
-      }
-    }
-    return null;
-  }
-
-  // For gRPC server running DN container service with gPRC TLS
-  // No mTLS as the channel is shared for for external client, which
-  // does not have SCM CA issued certificates.
-  // In summary:
-  // authenticate from server to client is via TLS.
-  // authenticate from client to server is via block token (or container token).
-  static GrpcTlsConfig createTlsServerConfigForDN(SecurityConfig conf,
-      CertificateClient caClient)  {
-    if (conf.isSecurityEnabled() && conf.isGrpcTlsEnabled()) {
-      return new GrpcTlsConfig(
-          caClient.getPrivateKey(), caClient.getCertificate(),
-          null, false);
-    }
-    return null;
-  }
-
-  static RetryPolicy createRetryPolicy(Configuration conf) {
-    int maxRetryCount =
-        conf.getInt(OzoneConfigKeys.DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY,
-            OzoneConfigKeys.
-                DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT);
-    long retryInterval = conf.getTimeDuration(OzoneConfigKeys.
-        DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY, OzoneConfigKeys.
-        DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT
-        .toIntExact(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
-    TimeDuration sleepDuration =
-        TimeDuration.valueOf(retryInterval, TimeUnit.MILLISECONDS);
-    RetryPolicy retryPolicy = RetryPolicies
-        .retryUpToMaximumCountWithFixedSleep(maxRetryCount, sleepDuration);
-    return retryPolicy;
-  }
-
-  static Long getMinReplicatedIndex(
-      Collection<RaftProtos.CommitInfoProto> commitInfos) {
-    return commitInfos.stream().map(RaftProtos.CommitInfoProto::getCommitIndex)
-        .min(Long::compareTo).orElse(null);
-  }
-
-  static ByteString int2ByteString(int n) {
-    final ByteString.Output out = ByteString.newOutput();
-    try(DataOutputStream dataOut = new DataOutputStream(out)) {
-      dataOut.writeInt(n);
-    } catch (IOException e) {
-      throw new IllegalStateException(
-          "Failed to write integer n = " + n + " to a ByteString.", e);
-    }
-    return out.toByteString();
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.ratis;
-
-/**
- * This package contains classes related to Apache Ratis.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ByteStringConversion.java

@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations;
-
-import java.nio.ByteBuffer;
-import java.util.function.Function;
-
-/**
- * Helper class to create a conversion function from ByteBuffer to ByteString
- * based on the property
- * {@link OzoneConfigKeys#OZONE_UNSAFEBYTEOPERATIONS_ENABLED} in the
- * Ozone configuration.
- */
-public final class ByteStringConversion {
-  private ByteStringConversion(){} // no instantiation.
-
-  /**
-   * Creates the conversion function to be used to convert ByteBuffers to
-   * ByteString instances to be used in protobuf messages.
-   *
-   * @param config the Ozone configuration
-   * @return the conversion function defined by
-   *          {@link OzoneConfigKeys#OZONE_UNSAFEBYTEOPERATIONS_ENABLED}
-   * @see <pre>ByteBuffer</pre>
-   */
-  public static Function<ByteBuffer, ByteString> createByteBufferConversion(
-      Configuration config){
-    boolean unsafeEnabled =
-        config!=null && config.getBoolean(
-            OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED,
-            OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED_DEFAULT);
-    if (unsafeEnabled) {
-      return buffer -> UnsafeByteOperations.unsafeWrap(buffer);
-    } else {
-      return buffer -> {
-        ByteString retval = ByteString.copyFrom(buffer);
-        buffer.flip();
-        return retval;
-      };
-    }
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -1,375 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
-import org.apache.ratis.util.TimeDuration;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * This class contains constants for configuration keys used in SCM.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class ScmConfigKeys {
-
-  // Location of SCM DB files. For now we just support a single
-  // metadata dir but in future we may support multiple for redundancy or
-  // performance.
-  public static final String OZONE_SCM_DB_DIRS = "ozone.scm.db.dirs";
-
-  public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
-      = "dfs.container.ratis.enabled";
-  public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
-      = false;
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
-      = "dfs.container.ratis.rpc.type";
-  public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
-      = "GRPC";
-  public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
-      = "dfs.container.ratis.num.write.chunk.threads";
-  public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
-      = 60;
-  public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
-      = "dfs.container.ratis.replication.level";
-  public static final ReplicationLevel
-      DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
-  public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
-      = "dfs.container.ratis.num.container.op.executors";
-  public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
-      = 10;
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
-      "dfs.container.ratis.segment.size";
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
-      "1MB";
-  public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
-      "dfs.container.ratis.segment.preallocated.size";
-  public static final String
-      DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "16KB";
-  public static final String
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
-      "dfs.container.ratis.statemachinedata.sync.timeout";
-  public static final TimeDuration
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
-      TimeDuration.valueOf(10, TimeUnit.SECONDS);
-  public static final String
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES =
-      "dfs.container.ratis.statemachinedata.sync.retries";
-  public static final int
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_RETRIES_DEFAULT = -1;
-  public static final String
-      DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS =
-      "dfs.container.ratis.statemachine.max.pending.apply-transactions";
-  // The default value of maximum number of pending state machine apply
-  // transactions is kept same as default snapshot threshold.
-  public static final int
-      DFS_CONTAINER_RATIS_STATEMACHINE_MAX_PENDING_APPLY_TXNS_DEFAULT =
-      100000;
-  public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS =
-      "dfs.container.ratis.log.queue.num-elements";
-  public static final int DFS_CONTAINER_RATIS_LOG_QUEUE_NUM_ELEMENTS_DEFAULT =
-      1024;
-  public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT =
-      "dfs.container.ratis.log.queue.byte-limit";
-  public static final String DFS_CONTAINER_RATIS_LOG_QUEUE_BYTE_LIMIT_DEFAULT =
-      "4GB";
-  public static final String
-      DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS =
-      "dfs.container.ratis.log.appender.queue.num-elements";
-  public static final int
-      DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_NUM_ELEMENTS_DEFAULT = 1;
-  public static final String DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT =
-      "dfs.container.ratis.log.appender.queue.byte-limit";
-  public static final String
-      DFS_CONTAINER_RATIS_LOG_APPENDER_QUEUE_BYTE_LIMIT_DEFAULT = "32MB";
-  public static final String DFS_CONTAINER_RATIS_LOG_PURGE_GAP =
-      "dfs.container.ratis.log.purge.gap";
-  // TODO: Set to 1024 once RATIS issue around purge is fixed.
-  public static final int DFS_CONTAINER_RATIS_LOG_PURGE_GAP_DEFAULT =
-      1000000;
-
-  public static final String DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS =
-      "dfs.container.ratis.leader.num.pending.requests";
-  public static final int
-      DFS_CONTAINER_RATIS_LEADER_NUM_PENDING_REQUESTS_DEFAULT = 4096;
-  // expiry interval stateMachineData cache entry inside containerStateMachine
-  public static final String
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL =
-      "dfs.container.ratis.statemachine.cache.expiry.interval";
-  public static final String
-      DFS_CONTAINER_RATIS_STATEMACHINEDATA_CACHE_EXPIRY_INTERVAL_DEFAULT =
-      "10s";
-  public static final String DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_KEY =
-      "dfs.ratis.client.request.timeout.duration";
-  public static final TimeDuration
-      DFS_RATIS_CLIENT_REQUEST_TIMEOUT_DURATION_DEFAULT =
-      TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
-  public static final String DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_KEY =
-      "dfs.ratis.client.request.max.retries";
-  public static final int DFS_RATIS_CLIENT_REQUEST_MAX_RETRIES_DEFAULT = 180;
-  public static final String DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_KEY =
-      "dfs.ratis.client.request.retry.interval";
-  public static final TimeDuration
-      DFS_RATIS_CLIENT_REQUEST_RETRY_INTERVAL_DEFAULT =
-      TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS);
-  public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
-      "dfs.ratis.server.retry-cache.timeout.duration";
-  public static final TimeDuration
-      DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
-      TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
-  public static final String DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_KEY =
-      "dfs.ratis.server.request.timeout.duration";
-  public static final TimeDuration
-      DFS_RATIS_SERVER_REQUEST_TIMEOUT_DURATION_DEFAULT =
-      TimeDuration.valueOf(3000, TimeUnit.MILLISECONDS);
-  public static final String
-      DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
-      "dfs.ratis.leader.election.minimum.timeout.duration";
-  public static final TimeDuration
-      DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
-      TimeDuration.valueOf(5, TimeUnit.SECONDS);
-
-  public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
-      "dfs.ratis.snapshot.threshold";
-  public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;
-
-  public static final String DFS_RATIS_SERVER_FAILURE_DURATION_KEY =
-      "dfs.ratis.server.failure.duration";
-  public static final TimeDuration
-      DFS_RATIS_SERVER_FAILURE_DURATION_DEFAULT =
-      TimeDuration.valueOf(120, TimeUnit.SECONDS);
-
-  // TODO : this is copied from OzoneConsts, may need to move to a better place
-  public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
-  // 16 MB by default
-  public static final String OZONE_SCM_CHUNK_SIZE_DEFAULT = "16MB";
-
-  public static final String OZONE_SCM_CLIENT_PORT_KEY =
-      "ozone.scm.client.port";
-  public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
-
-  public static final String OZONE_SCM_DATANODE_PORT_KEY =
-      "ozone.scm.datanode.port";
-  public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
-
-  // OZONE_OM_PORT_DEFAULT = 9862
-  public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY =
-      "ozone.scm.block.client.port";
-  public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
-
-  public static final String OZONE_SCM_SECURITY_SERVICE_PORT_KEY =
-      "ozone.scm.security.service.port";
-  public static final int OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT = 9961;
-
-  // Container service client
-  public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
-      "ozone.scm.client.address";
-  public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
-      "ozone.scm.client.bind.host";
-  public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  // Block service client
-  public static final String OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY =
-      "ozone.scm.block.client.address";
-  public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY =
-      "ozone.scm.block.client.bind.host";
-  public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  // SCM Security service address.
-  public static final String OZONE_SCM_SECURITY_SERVICE_ADDRESS_KEY =
-      "ozone.scm.security.service.address";
-  public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_KEY =
-      "ozone.scm.security.service.bind.host";
-  public static final String OZONE_SCM_SECURITY_SERVICE_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
-      "ozone.scm.datanode.address";
-  public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
-      "ozone.scm.datanode.bind.host";
-  public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
-      "0.0.0.0";
-
-  public static final String OZONE_SCM_HTTP_ENABLED_KEY =
-      "ozone.scm.http.enabled";
-  public static final String OZONE_SCM_HTTP_BIND_HOST_KEY =
-      "ozone.scm.http-bind-host";
-  public static final String OZONE_SCM_HTTPS_BIND_HOST_KEY =
-      "ozone.scm.https-bind-host";
-  public static final String OZONE_SCM_HTTP_ADDRESS_KEY =
-      "ozone.scm.http-address";
-  public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
-      "ozone.scm.https-address";
-  public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY =
-      "hdds.scm.kerberos.keytab.file";
-  public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY =
-      "hdds.scm.kerberos.principal";
-  public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
-  public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
-  public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
-
-  public static final String HDDS_REST_HTTP_ADDRESS_KEY =
-      "hdds.rest.http-address";
-  public static final String HDDS_REST_HTTP_ADDRESS_DEFAULT = "0.0.0.0:9880";
-  public static final String HDDS_DATANODE_DIR_KEY = "hdds.datanode.dir";
-  public static final String HDDS_REST_CSRF_ENABLED_KEY =
-      "hdds.rest.rest-csrf.enabled";
-  public static final boolean HDDS_REST_CSRF_ENABLED_DEFAULT = false;
-  public static final String HDDS_REST_NETTY_HIGH_WATERMARK =
-      "hdds.rest.netty.high.watermark";
-  public static final int HDDS_REST_NETTY_HIGH_WATERMARK_DEFAULT = 65536;
-  public static final int HDDS_REST_NETTY_LOW_WATERMARK_DEFAULT = 32768;
-  public static final String HDDS_REST_NETTY_LOW_WATERMARK =
-      "hdds.rest.netty.low.watermark";
-
-  public static final String OZONE_SCM_HANDLER_COUNT_KEY =
-      "ozone.scm.handler.count.key";
-  public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
-
-  public static final String OZONE_SCM_SECURITY_HANDLER_COUNT_KEY =
-      "ozone.scm.security.handler.count.key";
-  public static final int OZONE_SCM_SECURITY_HANDLER_COUNT_DEFAULT = 2;
-
-  public static final String OZONE_SCM_DEADNODE_INTERVAL =
-      "ozone.scm.dead.node.interval";
-  public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
-      "10m";
-
-  public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL =
-      "ozone.scm.heartbeat.thread.interval";
-  public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT =
-      "3s";
-
-  public static final String OZONE_SCM_STALENODE_INTERVAL =
-      "ozone.scm.stale.node.interval";
-  public static final String OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
-      "5m";
-
-  public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
-      "ozone.scm.heartbeat.rpc-timeout";
-  public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
-      "1s";
-
-  /**
-   * Defines how frequently we will log the missing of heartbeat to a specific
-   * SCM. In the default case we will write a warning message for each 10
-   * sequential heart beats that we miss to a specific SCM. This is to avoid
-   * overrunning the log with lots of HB missed Log statements.
-   */
-  public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
-      "ozone.scm.heartbeat.log.warn.interval.count";
-  public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
-      10;
-
-  // ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
-  // Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
-  //
-  // If this key is not specified datanodes will not be able to find
-  // SCM. The SCM membership can be dynamic, so this key should contain
-  // all possible SCM names. Once the SCM leader is discovered datanodes will
-  // get the right list of SCMs to heartbeat to from the leader.
-  // While it is good for the datanodes to know the names of all SCM nodes,
-  // it is sufficient to actually know the name of on working SCM. That SCM
-  // will be able to return the information about other SCMs that are part of
-  // the SCM replicated Log.
-  //
-  //In case of a membership change, any one of the SCM machines will be
-  // able to send back a new list to the datanodes.
-  public static final String OZONE_SCM_NAMES = "ozone.scm.names";
-
-  public static final int OZONE_SCM_DEFAULT_PORT =
-      OZONE_SCM_DATANODE_PORT_DEFAULT;
-  // The path where datanode ID is to be written to.
-  // if this value is not set then container startup will fail.
-  public static final String OZONE_SCM_DATANODE_ID_DIR =
-      "ozone.scm.datanode.id.dir";
-
-  public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
-      "ozone.scm.db.cache.size.mb";
-  public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
-
-  public static final String OZONE_SCM_CONTAINER_SIZE =
-      "ozone.scm.container.size";
-  public static final String OZONE_SCM_CONTAINER_SIZE_DEFAULT = "5GB";
-
-  public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
-      "ozone.scm.container.placement.impl";
-
-  public static final String OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT =
-      "ozone.scm.pipeline.owner.container.count";
-  public static final int OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT = 3;
-
-  public static final String
-      OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY =
-      "ozone.scm.keyvalue.container.deletion-choosing.policy";
-
-  public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT =
-      "ozone.scm.container.creation.lease.timeout";
-
-  public static final String
-      OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
-
-  public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT =
-      "ozone.scm.pipeline.destroy.timeout";
-
-  public static final String OZONE_SCM_PIPELINE_DESTROY_TIMEOUT_DEFAULT =
-      "66s";
-
-  public static final String OZONE_SCM_PIPELINE_CREATION_INTERVAL =
-      "ozone.scm.pipeline.creation.interval";
-  public static final String OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT =
-      "120s";
-
-  public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
-      "ozone.scm.block.deletion.max.retry";
-  public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
-
-  public static final String HDDS_SCM_WATCHER_TIMEOUT =
-      "hdds.scm.watcher.timeout";
-
-  public static final String HDDS_SCM_WATCHER_TIMEOUT_DEFAULT =
-      "10m";
-
-  public static final String
-      HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY =
-      "hdds.scm.http.kerberos.principal";
-  public static final String
-      HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY =
-      "hdds.scm.http.kerberos.keytab";
-
-  // Network topology
-  public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE =
-      "ozone.scm.network.topology.schema.file";
-  public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT =
-      "network-topology-default.xml";
-
-  public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
-  public static final boolean HDDS_TRACING_ENABLED_DEFAULT = true;
-
-  /**
-   * Never constructed.
-   */
-  private ScmConfigKeys() {
-
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmInfo.java

@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-/**
- * ScmInfo wraps the result returned from SCM#getScmInfo which
- * contains clusterId and the SCM Id.
- */
-public final class ScmInfo {
-  private String clusterId;
-  private String scmId;
-
-  /**
-   * Builder for ScmInfo.
-   */
-  public static class Builder {
-    private String clusterId;
-    private String scmId;
-
-    /**
-     * sets the cluster id.
-     * @param cid clusterId to be set
-     * @return Builder for ScmInfo
-     */
-    public Builder setClusterId(String cid) {
-      this.clusterId = cid;
-      return this;
-    }
-
-    /**
-     * sets the scmId.
-     * @param id scmId
-     * @return Builder for scmInfo
-     */
-    public Builder setScmId(String id) {
-      this.scmId = id;
-      return this;
-    }
-
-    public ScmInfo build() {
-      return new ScmInfo(clusterId, scmId);
-    }
-  }
-
-  private ScmInfo(String clusterId, String scmId) {
-    this.clusterId = clusterId;
-    this.scmId = scmId;
-  }
-
-  /**
-   * Gets the clusterId from the Version file.
-   * @return ClusterId
-   */
-  public String getClusterId() {
-    return clusterId;
-  }
-
-  /**
-   * Gets the SCM Id from the Version file.
-   * @return SCM Id
-   */
-  public String getScmId() {
-    return scmId;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientReply.java

@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerCommandResponseProto;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-
-/**
- * This class represents the reply from XceiverClient.
- */
-public class XceiverClientReply {
-
-  private CompletableFuture<ContainerCommandResponseProto> response;
-  private Long logIndex;
-
-  /**
-   * List of datanodes where the command got executed and reply is received.
-   * If there is an exception in the reply, these datanodes will inform
-   * about the servers where there is a failure.
-   */
-  private List<DatanodeDetails> datanodes;
-
-  public XceiverClientReply(
-      CompletableFuture<ContainerCommandResponseProto> response) {
-    this(response, null);
-  }
-
-  public XceiverClientReply(
-      CompletableFuture<ContainerCommandResponseProto> response,
-      List<DatanodeDetails> datanodes) {
-    this.logIndex = (long) 0;
-    this.response = response;
-    this.datanodes = datanodes == null ? new ArrayList<>() : datanodes;
-  }
-
-  public CompletableFuture<ContainerCommandResponseProto> getResponse() {
-    return response;
-  }
-
-  public long getLogIndex() {
-    return logIndex;
-  }
-
-  public void setLogIndex(Long logIndex) {
-    this.logIndex = logIndex;
-  }
-
-  public List<DatanodeDetails> getDatanodes() {
-    return datanodes;
-  }
-
-  public void addDatanode(DatanodeDetails dn) {
-    datanodes.add(dn);
-  }
-
-  public void setResponse(
-      CompletableFuture<ContainerCommandResponseProto> response) {
-    this.response = response;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientSpi.java

@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdds.scm.storage.CheckedBiFunction;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public abstract class XceiverClientSpi implements Closeable {
-
-  final private AtomicInteger referenceCount;
-  private boolean isEvicted;
-
-  XceiverClientSpi() {
-    this.referenceCount = new AtomicInteger(0);
-    this.isEvicted = false;
-  }
-
-  void incrementReference() {
-    this.referenceCount.incrementAndGet();
-  }
-
-  void decrementReference() {
-    this.referenceCount.decrementAndGet();
-    cleanup();
-  }
-
-  void setEvicted() {
-    isEvicted = true;
-    cleanup();
-  }
-
-  // close the xceiverClient only if,
-  // 1) there is no refcount on the client
-  // 2) it has been evicted from the cache.
-  private void cleanup() {
-    if (referenceCount.get() == 0 && isEvicted) {
-      close();
-    }
-  }
-
-  @VisibleForTesting
-  public int getRefcount() {
-    return referenceCount.get();
-  }
-
-  /**
-   * Connects to the leader in the pipeline.
-   */
-  public abstract void connect() throws Exception;
-
-  /**
-   * Connects to the leader in the pipeline using encoded token. To be used
-   * in a secure cluster.
-   */
-  public abstract void connect(String encodedToken) throws Exception;
-
-  @Override
-  public abstract void close();
-
-  /**
-   * Returns the pipeline of machines that host the container used by this
-   * client.
-   *
-   * @return pipeline of machines that host the container
-   */
-  public abstract Pipeline getPipeline();
-
-  /**
-   * Sends a given command to server and gets the reply back.
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  public ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request) throws IOException {
-    try {
-      XceiverClientReply reply;
-      reply = sendCommandAsync(request);
-      ContainerCommandResponseProto responseProto = reply.getResponse().get();
-      return responseProto;
-    } catch (ExecutionException | InterruptedException e) {
-      throw new IOException("Failed to command " + request, e);
-    }
-  }
-
-  /**
-   * Sends a given command to server and gets the reply back along with
-   * the server associated info.
-   * @param request Request
-   * @param validators functions to validate the response
-   * @return Response to the command
-   * @throws IOException
-   */
-  public ContainerCommandResponseProto sendCommand(
-      ContainerCommandRequestProto request, List<CheckedBiFunction> validators)
-      throws IOException {
-    try {
-      XceiverClientReply reply;
-      reply = sendCommandAsync(request);
-      ContainerCommandResponseProto responseProto = reply.getResponse().get();
-      for (CheckedBiFunction function : validators) {
-        function.apply(request, responseProto);
-      }
-      return responseProto;
-    } catch (ExecutionException | InterruptedException e) {
-      throw new IOException("Failed to command " + request, e);
-    }
-  }
-
-  /**
-   * Sends a given command to server gets a waitable future back.
-   *
-   * @param request Request
-   * @return Response to the command
-   * @throws IOException
-   */
-  public abstract XceiverClientReply
-      sendCommandAsync(ContainerCommandRequestProto request)
-      throws IOException, ExecutionException, InterruptedException;
-
-  /**
-   * Returns pipeline Type.
-   *
-   * @return - {Stand_Alone, Ratis or Chained}
-   */
-  public abstract HddsProtos.ReplicationType getPipelineType();
-
-  /**
-   * Check if an specfic commitIndex is replicated to majority/all servers.
-   * @param index index to watch for
-   * @param timeout timeout provided for the watch operation to complete
-   * @return reply containing the min commit index replicated to all or majority
-   *         servers in case of a failure
-   * @throws InterruptedException
-   * @throws ExecutionException
-   * @throws TimeoutException
-   * @throws IOException
-   */
-  public abstract XceiverClientReply watchForCommit(long index, long timeout)
-      throws InterruptedException, ExecutionException, TimeoutException,
-      IOException;
-
-  /**
-   * returns the min commit index replicated to all servers.
-   * @return min commit index replicated to all servers.
-   */
-  public abstract long getReplicatedMinCommitIndex();
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.client;
-
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
-    .ContainerDataProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * The interface to call into underlying container layer.
- *
- * Written as interface to allow easy testing: implement a mock container layer
- * for standalone testing of CBlock API without actually calling into remote
- * containers. Actual container layer can simply re-implement this.
- *
- * NOTE this is temporarily needed class. When SCM containers are full-fledged,
- * this interface will likely be removed.
- */
-@InterfaceStability.Unstable
-public interface ScmClient extends Closeable {
-  /**
-   * Creates a Container on SCM and returns the pipeline.
-   * @return ContainerInfo
-   * @throws IOException
-   */
-  ContainerWithPipeline createContainer(String owner) throws IOException;
-
-  /**
-   * Gets a container by Name -- Throws if the container does not exist.
-   * @param containerId - Container ID
-   * @return Pipeline
-   * @throws IOException
-   */
-  ContainerInfo getContainer(long containerId) throws IOException;
-
-  /**
-   * Gets a container by Name -- Throws if the container does not exist.
-   * @param containerId - Container ID
-   * @return ContainerWithPipeline
-   * @throws IOException
-   */
-  ContainerWithPipeline getContainerWithPipeline(long containerId)
-      throws IOException;
-
-  /**
-   * Close a container.
-   *
-   * @param containerId - ID of the container.
-   * @param pipeline - Pipeline where the container is located.
-   * @throws IOException
-   */
-  void closeContainer(long containerId, Pipeline pipeline) throws IOException;
-
-  /**
-   * Close a container.
-   *
-   * @param containerId - ID of the container.
-   * @throws IOException
-   */
-  void closeContainer(long containerId) throws IOException;
-
-  /**
-   * Deletes an existing container.
-   * @param containerId - ID of the container.
-   * @param pipeline - Pipeline that represents the container.
-   * @param force - true to forcibly delete the container.
-   * @throws IOException
-   */
-  void deleteContainer(long containerId, Pipeline pipeline, boolean force)
-      throws IOException;
-
-  /**
-   * Deletes an existing container.
-   * @param containerId - ID of the container.
-   * @param force - true to forcibly delete the container.
-   * @throws IOException
-   */
-  void deleteContainer(long containerId, boolean force) throws IOException;
-
-  /**
-   * Lists a range of containers and get their info.
-   *
-   * @param startContainerID start containerID.
-   * @param count count must be {@literal >} 0.
-   *
-   * @return a list of pipeline.
-   * @throws IOException
-   */
-  List<ContainerInfo> listContainer(long startContainerID,
-      int count) throws IOException;
-
-  /**
-   * Read meta data from an existing container.
-   * @param containerID - ID of the container.
-   * @param pipeline - Pipeline where the container is located.
-   * @return ContainerInfo
-   * @throws IOException
-   */
-  ContainerDataProto readContainer(long containerID, Pipeline pipeline)
-      throws IOException;
-
-  /**
-   * Read meta data from an existing container.
-   * @param containerID - ID of the container.
-   * @return ContainerInfo
-   * @throws IOException
-   */
-  ContainerDataProto readContainer(long containerID)
-      throws IOException;
-
-  /**
-   * Gets the container size -- Computed by SCM from Container Reports.
-   * @param containerID - ID of the container.
-   * @return number of bytes used by this container.
-   * @throws IOException
-   */
-  long getContainerSize(long containerID) throws IOException;
-
-  /**
-   * Creates a Container on SCM and returns the pipeline.
-   * @param type - Replication Type.
-   * @param replicationFactor - Replication Factor
-   * @return ContainerInfo
-   * @throws IOException - in case of error.
-   */
-  ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
-      HddsProtos.ReplicationFactor replicationFactor,
-      String owner) throws IOException;
-
-  /**
-   * Returns a set of Nodes that meet a query criteria.
-   * @param nodeStatuses - Criteria that we want the node to have.
-   * @param queryScope - Query scope - Cluster or pool.
-   * @param poolName - if it is pool, a pool name is required.
-   * @return A set of nodes that meet the requested criteria.
-   * @throws IOException
-   */
-  List<HddsProtos.Node> queryNode(HddsProtos.NodeState nodeStatuses,
-      HddsProtos.QueryScope queryScope, String poolName) throws IOException;
-
-  /**
-   * Creates a specified replication pipeline.
-   * @param type - Type
-   * @param factor - Replication factor
-   * @param nodePool - Set of machines.
-   * @throws IOException
-   */
-  Pipeline createReplicationPipeline(HddsProtos.ReplicationType type,
-      HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool)
-      throws IOException;
-
-  /**
-   * Returns the list of active Pipelines.
-   *
-   * @return list of Pipeline
-   * @throws IOException in case of any exception
-   */
-  List<Pipeline> listPipelines() throws IOException;
-
-  /**
-   * Activates the pipeline given a pipeline ID.
-   *
-   * @param pipelineID PipelineID to activate.
-   * @throws IOException In case of exception while activating the pipeline
-   */
-  void activatePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
-
-  /**
-   * Deactivates the pipeline given a pipeline ID.
-   *
-   * @param pipelineID PipelineID to deactivate.
-   * @throws IOException In case of exception while deactivating the pipeline
-   */
-  void deactivatePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
-
-  /**
-   * Closes the pipeline given a pipeline ID.
-   *
-   * @param pipelineID PipelineID to close.
-   * @throws IOException In case of exception while closing the pipeline
-   */
-  void closePipeline(HddsProtos.PipelineID pipelineID) throws IOException;
-
-  /**
-   * Check if SCM is in safe mode.
-   *
-   * @return Returns true if SCM is in safe mode else returns false.
-   * @throws IOException
-   */
-  boolean inSafeMode() throws IOException;
-
-  /**
-   * Force SCM out of safe mode.
-   *
-   * @return returns true if operation is successful.
-   * @throws IOException
-   */
-  boolean forceExitSafeMode() throws IOException;
-
-  /**
-   * Start ReplicationManager.
-   */
-  void startReplicationManager() throws IOException;
-
-  /**
-   * Stop ReplicationManager.
-   */
-  void stopReplicationManager() throws IOException;
-
-  /**
-   * Returns ReplicationManager status.
-   *
-   * @return True if ReplicationManager is running, false otherwise.
-   */
-  boolean getReplicationManagerStatus() throws IOException;
-
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/package-info.java

@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.client;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerException.java

@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container;
-
-import java.io.IOException;
-
-/**
- * Signals that ContainerException of some sort has occurred. This is parent
- * of all the exceptions thrown by ContainerManager.
- */
-public class ContainerException extends IOException {
-
-  /**
-   * Constructs an {@code ContainerException} with {@code null}
-   * as its error detail message.
-   */
-  public ContainerException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code ContainerException} with the specified detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public ContainerException(String message) {
-    super(message);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerID.java

@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.hdds.scm.container;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.lang3.builder.CompareToBuilder;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-/**
- * Container ID is an integer that is a value between 1..MAX_CONTAINER ID.
- * <p>
- * We are creating a specific type for this to avoid mixing this with
- * normal integers in code.
- */
-public final class ContainerID implements Comparable<ContainerID> {
-
-  private final long id;
-
-  // TODO: make this private.
-  /**
-   * Constructs ContainerID.
-   *
-   * @param id int
-   */
-  public ContainerID(long id) {
-    this.id = id;
-  }
-
-  /**
-   * Factory method for creation of ContainerID.
-   * @param containerID  long
-   * @return ContainerID.
-   */
-  public static ContainerID valueof(final long containerID) {
-    Preconditions.checkState(containerID > 0,
-        "Container ID should be a positive long. "+ containerID);
-    return new ContainerID(containerID);
-  }
-
-  /**
-   * Returns int representation of ID.
-   *
-   * @return int
-   */
-  public long getId() {
-    return id;
-  }
-
-  public byte[] getBytes() {
-    return Longs.toByteArray(id);
-  }
-
-  @Override
-  public boolean equals(final Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    final ContainerID that = (ContainerID) o;
-
-    return new EqualsBuilder()
-        .append(getId(), that.getId())
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(61, 71)
-        .append(getId())
-        .toHashCode();
-  }
-
-  @Override
-  public int compareTo(final ContainerID that) {
-    Preconditions.checkNotNull(that);
-    return new CompareToBuilder()
-        .append(this.getId(), that.getId())
-        .build();
-  }
-
-  @Override
-  public String toString() {
-    return "#" + id;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java

@@ -1,471 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container;
-
-import static java.lang.Math.max;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Preconditions;
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.Arrays;
-import java.util.Comparator;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-import org.apache.hadoop.util.Time;
-
-/**
- * Class wraps ozone container info.
- */
-public class ContainerInfo implements Comparator<ContainerInfo>,
-    Comparable<ContainerInfo>, Externalizable {
-
-  private static final ObjectWriter WRITER;
-  private static final String SERIALIZATION_ERROR_MSG = "Java serialization not"
-      + " supported. Use protobuf instead.";
-
-  static {
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
-    mapper
-        .setVisibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE);
-    WRITER = mapper.writerWithDefaultPrettyPrinter();
-  }
-
-  private HddsProtos.LifeCycleState state;
-  @JsonIgnore
-  private PipelineID pipelineID;
-  private ReplicationFactor replicationFactor;
-  private ReplicationType replicationType;
-  private long usedBytes;
-  private long numberOfKeys;
-  private long lastUsed;
-  // The wall-clock ms since the epoch at which the current state enters.
-  private long stateEnterTime;
-  private String owner;
-  private long containerID;
-  private long deleteTransactionId;
-  // The sequenceId of a close container cannot change, and all the
-  // container replica should have the same sequenceId.
-  private long sequenceId;
-
-  /**
-   * Allows you to maintain private data on ContainerInfo. This is not
-   * serialized via protobuf, just allows us to maintain some private data.
-   */
-  @JsonIgnore
-  private byte[] data;
-
-  @SuppressWarnings("parameternumber")
-  ContainerInfo(
-      long containerID,
-      HddsProtos.LifeCycleState state,
-      PipelineID pipelineID,
-      long usedBytes,
-      long numberOfKeys,
-      long stateEnterTime,
-      String owner,
-      long deleteTransactionId,
-      long sequenceId,
-      ReplicationFactor replicationFactor,
-      ReplicationType repType) {
-    this.containerID = containerID;
-    this.pipelineID = pipelineID;
-    this.usedBytes = usedBytes;
-    this.numberOfKeys = numberOfKeys;
-    this.lastUsed = Time.monotonicNow();
-    this.state = state;
-    this.stateEnterTime = stateEnterTime;
-    this.owner = owner;
-    this.deleteTransactionId = deleteTransactionId;
-    this.sequenceId = sequenceId;
-    this.replicationFactor = replicationFactor;
-    this.replicationType = repType;
-  }
-
-  /**
-   * Needed for serialization findbugs.
-   */
-  public ContainerInfo() {
-  }
-
-  public static ContainerInfo fromProtobuf(HddsProtos.ContainerInfoProto info) {
-    ContainerInfo.Builder builder = new ContainerInfo.Builder();
-    return builder.setPipelineID(
-        PipelineID.getFromProtobuf(info.getPipelineID()))
-        .setUsedBytes(info.getUsedBytes())
-        .setNumberOfKeys(info.getNumberOfKeys())
-        .setState(info.getState())
-        .setStateEnterTime(info.getStateEnterTime())
-        .setOwner(info.getOwner())
-        .setContainerID(info.getContainerID())
-        .setDeleteTransactionId(info.getDeleteTransactionId())
-        .setReplicationFactor(info.getReplicationFactor())
-        .setReplicationType(info.getReplicationType())
-        .build();
-  }
-
-  public long getContainerID() {
-    return containerID;
-  }
-
-  public HddsProtos.LifeCycleState getState() {
-    return state;
-  }
-
-  public void setState(HddsProtos.LifeCycleState state) {
-    this.state = state;
-  }
-
-  public long getStateEnterTime() {
-    return stateEnterTime;
-  }
-
-  public ReplicationFactor getReplicationFactor() {
-    return replicationFactor;
-  }
-
-  public PipelineID getPipelineID() {
-    return pipelineID;
-  }
-
-  public long getUsedBytes() {
-    return usedBytes;
-  }
-
-  public void setUsedBytes(long value) {
-    usedBytes = value;
-  }
-
-  public long getNumberOfKeys() {
-    return numberOfKeys;
-  }
-
-  public void setNumberOfKeys(long value) {
-    numberOfKeys = value;
-  }
-
-  public long getDeleteTransactionId() {
-    return deleteTransactionId;
-  }
-
-  public long getSequenceId() {
-    return sequenceId;
-  }
-
-  public void updateDeleteTransactionId(long transactionId) {
-    deleteTransactionId = max(transactionId, deleteTransactionId);
-  }
-
-  public void updateSequenceId(long sequenceID) {
-    assert (isOpen() || state == HddsProtos.LifeCycleState.QUASI_CLOSED);
-    sequenceId = max(sequenceID, sequenceId);
-  }
-
-  public ContainerID containerID() {
-    return new ContainerID(getContainerID());
-  }
-
-  /**
-   * Gets the last used time from SCM's perspective.
-   *
-   * @return time in milliseconds.
-   */
-  public long getLastUsed() {
-    return lastUsed;
-  }
-
-  public ReplicationType getReplicationType() {
-    return replicationType;
-  }
-
-  public void updateLastUsedTime() {
-    lastUsed = Time.monotonicNow();
-  }
-
-  public HddsProtos.ContainerInfoProto getProtobuf() {
-    HddsProtos.ContainerInfoProto.Builder builder =
-        HddsProtos.ContainerInfoProto.newBuilder();
-    Preconditions.checkState(containerID > 0);
-    return builder.setContainerID(getContainerID())
-        .setUsedBytes(getUsedBytes())
-        .setNumberOfKeys(getNumberOfKeys()).setState(getState())
-        .setStateEnterTime(getStateEnterTime()).setContainerID(getContainerID())
-        .setDeleteTransactionId(getDeleteTransactionId())
-        .setPipelineID(getPipelineID().getProtobuf())
-        .setReplicationFactor(getReplicationFactor())
-        .setReplicationType(getReplicationType())
-        .setOwner(getOwner())
-        .build();
-  }
-
-  public String getOwner() {
-    return owner;
-  }
-
-  public void setOwner(String owner) {
-    this.owner = owner;
-  }
-
-  @Override
-  public String toString() {
-    return "ContainerInfo{"
-        + "id=" + containerID
-        + ", state=" + state
-        + ", pipelineID=" + pipelineID
-        + ", stateEnterTime=" + stateEnterTime
-        + ", owner=" + owner
-        + '}';
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    ContainerInfo that = (ContainerInfo) o;
-
-    return new EqualsBuilder()
-        .append(getContainerID(), that.getContainerID())
-
-        // TODO : Fix this later. If we add these factors some tests fail.
-        // So Commenting this to continue and will enforce this with
-        // Changes in pipeline where we remove Container Name to
-        // SCMContainerinfo from Pipeline.
-        // .append(pipeline.getFactor(), that.pipeline.getFactor())
-        // .append(pipeline.getType(), that.pipeline.getType())
-        .append(owner, that.owner)
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(11, 811)
-        .append(getContainerID())
-        .append(getOwner())
-        .toHashCode();
-  }
-
-  /**
-   * Compares its two arguments for order.  Returns a negative integer, zero, or
-   * a positive integer as the first argument is less than, equal to, or greater
-   * than the second.<p>
-   *
-   * @param o1 the first object to be compared.
-   * @param o2 the second object to be compared.
-   * @return a negative integer, zero, or a positive integer as the first
-   * argument is less than, equal to, or greater than the second.
-   * @throws NullPointerException if an argument is null and this comparator
-   *                              does not permit null arguments
-   * @throws ClassCastException   if the arguments' types prevent them from
-   *                              being compared by this comparator.
-   */
-  @Override
-  public int compare(ContainerInfo o1, ContainerInfo o2) {
-    return Long.compare(o1.getLastUsed(), o2.getLastUsed());
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less than,
-   * equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(ContainerInfo o) {
-    return this.compare(this, o);
-  }
-
-  /**
-   * Returns a JSON string of this object.
-   *
-   * @return String - json string
-   * @throws IOException
-   */
-  public String toJsonString() throws IOException {
-    return WRITER.writeValueAsString(this);
-  }
-
-  /**
-   * Returns private data that is set on this containerInfo.
-   *
-   * @return blob, the user can interpret it any way they like.
-   */
-  public byte[] getData() {
-    if (this.data != null) {
-      return Arrays.copyOf(this.data, this.data.length);
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * Set private data on ContainerInfo object.
-   *
-   * @param data -- private data.
-   */
-  public void setData(byte[] data) {
-    if (data != null) {
-      this.data = Arrays.copyOf(data, data.length);
-    }
-  }
-
-  /**
-   * Throws IOException as default java serialization is not supported. Use
-   * serialization via protobuf instead.
-   *
-   * @param out the stream to write the object to
-   * @throws IOException Includes any I/O exceptions that may occur
-   * @serialData Overriding methods should use this tag to describe
-   * the data layout of this Externalizable object.
-   * List the sequence of element types and, if possible,
-   * relate the element to a public/protected field and/or
-   * method of this Externalizable class.
-   */
-  @Override
-  public void writeExternal(ObjectOutput out) throws IOException {
-    throw new IOException(SERIALIZATION_ERROR_MSG);
-  }
-
-  /**
-   * Throws IOException as default java serialization is not supported. Use
-   * serialization via protobuf instead.
-   *
-   * @param in the stream to read data from in order to restore the object
-   * @throws IOException            if I/O errors occur
-   * @throws ClassNotFoundException If the class for an object being
-   *                                restored cannot be found.
-   */
-  @Override
-  public void readExternal(ObjectInput in)
-      throws IOException, ClassNotFoundException {
-    throw new IOException(SERIALIZATION_ERROR_MSG);
-  }
-
-  /**
-   * Builder class for ContainerInfo.
-   */
-  public static class Builder {
-    private HddsProtos.LifeCycleState state;
-    private long used;
-    private long keys;
-    private long stateEnterTime;
-    private String owner;
-    private long containerID;
-    private long deleteTransactionId;
-    private long sequenceId;
-    private PipelineID pipelineID;
-    private ReplicationFactor replicationFactor;
-    private ReplicationType replicationType;
-
-    public Builder setReplicationType(
-        ReplicationType repType) {
-      this.replicationType = repType;
-      return this;
-    }
-
-    public Builder setPipelineID(PipelineID pipelineId) {
-      this.pipelineID = pipelineId;
-      return this;
-    }
-
-    public Builder setReplicationFactor(ReplicationFactor repFactor) {
-      this.replicationFactor = repFactor;
-      return this;
-    }
-
-    public Builder setContainerID(long id) {
-      Preconditions.checkState(id >= 0);
-      this.containerID = id;
-      return this;
-    }
-
-    public Builder setState(HddsProtos.LifeCycleState lifeCycleState) {
-      this.state = lifeCycleState;
-      return this;
-    }
-
-    public Builder setUsedBytes(long bytesUsed) {
-      this.used = bytesUsed;
-      return this;
-    }
-
-    public Builder setNumberOfKeys(long keyCount) {
-      this.keys = keyCount;
-      return this;
-    }
-
-    public Builder setStateEnterTime(long time) {
-      this.stateEnterTime = time;
-      return this;
-    }
-
-    public Builder setOwner(String containerOwner) {
-      this.owner = containerOwner;
-      return this;
-    }
-
-    public Builder setDeleteTransactionId(long deleteTransactionID) {
-      this.deleteTransactionId = deleteTransactionID;
-      return this;
-    }
-
-    public Builder setSequenceId(long sequenceID) {
-      this.sequenceId = sequenceID;
-      return this;
-    }
-
-    public ContainerInfo build() {
-      return new ContainerInfo(containerID, state, pipelineID,
-          used, keys, stateEnterTime, owner, deleteTransactionId,
-          sequenceId, replicationFactor, replicationType);
-    }
-  }
-
-  /**
-   * Check if a container is in open state, this will check if the
-   * container is either open or closing state. Any containers in these states
-   * is managed as an open container by SCM.
-   */
-  public boolean isOpen() {
-    return state == HddsProtos.LifeCycleState.OPEN
-        || state == HddsProtos.LifeCycleState.CLOSING;
-  }
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerNotFoundException.java

@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container;
-
-/**
- * Signals that a container is missing from ContainerManager.
- */
-public class ContainerNotFoundException extends ContainerException {
-
-  /**
-   * Constructs an {@code ContainerNotFoundException} with {@code null}
-   * as its error detail message.
-   */
-  public ContainerNotFoundException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code ContainerNotFoundException} with the specified
-   * detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public ContainerNotFoundException(String message) {
-    super(message);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReplicaNotFoundException.java

@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container;
-
-/**
- * Signals that a ContainerReplica is missing from the Container in
- * ContainerManager.
- */
-public class ContainerReplicaNotFoundException extends ContainerException {
-
-  /**
-   * Constructs an {@code ContainerReplicaNotFoundException} with {@code null}
-   * as its error detail message.
-   */
-  public ContainerReplicaNotFoundException() {
-    super();
-  }
-
-  /**
-   * Constructs an {@code ContainerReplicaNotFoundException} with the
-   * specified detail message.
-   *
-   * @param message
-   *        The detail message (which is saved for later retrieval
-   *        by the {@link #getMessage()} method)
-   */
-  public ContainerReplicaNotFoundException(String message) {
-    super(message);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/AllocatedBlock.java

@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import org.apache.hadoop.hdds.client.ContainerBlockID;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-
-/**
- * Allocated block wraps the result returned from SCM#allocateBlock which
- * contains a Pipeline and the key.
- */
-public final class AllocatedBlock {
-  private Pipeline pipeline;
-  private ContainerBlockID containerBlockID;
-
-  /**
-   * Builder for AllocatedBlock.
-   */
-  public static class Builder {
-    private Pipeline pipeline;
-    private ContainerBlockID containerBlockID;
-
-    public Builder setPipeline(Pipeline p) {
-      this.pipeline = p;
-      return this;
-    }
-
-    public Builder setContainerBlockID(ContainerBlockID blockId) {
-      this.containerBlockID = blockId;
-      return this;
-    }
-
-    public AllocatedBlock build() {
-      return new AllocatedBlock(pipeline, containerBlockID);
-    }
-  }
-
-  private AllocatedBlock(Pipeline pipeline, ContainerBlockID containerBlockID) {
-    this.pipeline = pipeline;
-    this.containerBlockID = containerBlockID;
-  }
-
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  public ContainerBlockID getBlockID() {
-    return containerBlockID;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/BlockNotCommittedException.java

@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
-/**
- * Exceptions thrown when a block is yet to be committed on the datanode.
- */
-public class BlockNotCommittedException extends StorageContainerException {
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   */
-  public BlockNotCommittedException(String message) {
-    super(message, ContainerProtos.Result.BLOCK_NOT_COMMITTED);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerNotOpenException.java

@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
-/**
- * Exceptions thrown when a write/update opearation is done on non-open
- * container.
- */
-public class ContainerNotOpenException extends StorageContainerException {
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   */
-  public ContainerNotOpenException(String message) {
-    super(message, ContainerProtos.Result.CONTAINER_NOT_OPEN);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ContainerWithPipeline.java

@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import java.util.Comparator;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
-import org.apache.hadoop.hdds.scm.pipeline.UnknownPipelineStateException;
-
-/**
- * Class wraps ozone container info.
- */
-public class ContainerWithPipeline implements Comparator<ContainerWithPipeline>,
-    Comparable<ContainerWithPipeline> {
-
-  private final ContainerInfo containerInfo;
-  private final Pipeline pipeline;
-
-  public ContainerWithPipeline(ContainerInfo containerInfo, Pipeline pipeline) {
-    this.containerInfo = containerInfo;
-    this.pipeline = pipeline;
-  }
-
-  public ContainerInfo getContainerInfo() {
-    return containerInfo;
-  }
-
-  public Pipeline getPipeline() {
-    return pipeline;
-  }
-
-  public static ContainerWithPipeline fromProtobuf(
-      HddsProtos.ContainerWithPipeline allocatedContainer)
-      throws UnknownPipelineStateException {
-    return new ContainerWithPipeline(
-        ContainerInfo.fromProtobuf(allocatedContainer.getContainerInfo()),
-        Pipeline.getFromProtobuf(allocatedContainer.getPipeline()));
-  }
-
-  public HddsProtos.ContainerWithPipeline getProtobuf()
-      throws UnknownPipelineStateException {
-    HddsProtos.ContainerWithPipeline.Builder builder =
-        HddsProtos.ContainerWithPipeline.newBuilder();
-    builder.setContainerInfo(getContainerInfo().getProtobuf())
-        .setPipeline(getPipeline().getProtobufMessage());
-
-    return builder.build();
-  }
-
-
-  @Override
-  public String toString() {
-    return containerInfo.toString() + " | " + pipeline.toString();
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) {
-      return true;
-    }
-
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-
-    ContainerWithPipeline that = (ContainerWithPipeline) o;
-
-    return new EqualsBuilder()
-        .append(getContainerInfo(), that.getContainerInfo())
-        .append(getPipeline(), that.getPipeline())
-        .isEquals();
-  }
-
-  @Override
-  public int hashCode() {
-    return new HashCodeBuilder(11, 811)
-        .append(getContainerInfo())
-        .append(getPipeline())
-        .toHashCode();
-  }
-
-  /**
-   * Compares its two arguments for order.  Returns a negative integer, zero, or
-   * a positive integer as the first argument is less than, equal to, or greater
-   * than the second.<p>
-   *
-   * @param o1 the first object to be compared.
-   * @param o2 the second object to be compared.
-   * @return a negative integer, zero, or a positive integer as the first
-   * argument is less than, equal to, or greater than the second.
-   * @throws NullPointerException if an argument is null and this comparator
-   *                              does not permit null arguments
-   * @throws ClassCastException   if the arguments' types prevent them from
-   *                              being compared by this comparator.
-   */
-  @Override
-  public int compare(ContainerWithPipeline o1, ContainerWithPipeline o2) {
-    return o1.getContainerInfo().compareTo(o2.getContainerInfo());
-  }
-
-  /**
-   * Compares this object with the specified object for order.  Returns a
-   * negative integer, zero, or a positive integer as this object is less than,
-   * equal to, or greater than the specified object.
-   *
-   * @param o the object to be compared.
-   * @return a negative integer, zero, or a positive integer as this object is
-   * less than, equal to, or greater than the specified object.
-   * @throws NullPointerException if the specified object is null
-   * @throws ClassCastException   if the specified object's type prevents it
-   *                              from being compared to this object.
-   */
-  @Override
-  public int compareTo(ContainerWithPipeline o) {
-    return this.compare(this, o);
-  }
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/DeleteBlockResult.java

@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import org.apache.hadoop.hdds.client.BlockID;
-
-import static org.apache.hadoop.hdds.protocol.proto
-    .ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-
-/**
- * Class wraps storage container manager block deletion results.
- */
-public class DeleteBlockResult {
-  private BlockID blockID;
-  private DeleteScmBlockResult.Result result;
-
-  public DeleteBlockResult(final BlockID blockID,
-      final DeleteScmBlockResult.Result result) {
-    this.blockID = blockID;
-    this.result = result;
-  }
-
-  /**
-   * Get block id deleted.
-   * @return block id.
-   */
-  public BlockID getBlockID() {
-    return blockID;
-  }
-
-  /**
-   * Get key deletion result.
-   * @return key deletion result.
-   */
-  public DeleteScmBlockResult.Result getResult() {
-    return result;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/ExcludeList.java

@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
-import org.apache.hadoop.hdds.scm.container.ContainerID;
-import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Collection;
-
-/**
- * This class contains set of dns and containers which ozone client provides
- * to be handed over to SCM when block allocation request comes.
- */
-public class ExcludeList {
-
-  private final List<DatanodeDetails> datanodes;
-  private final List<ContainerID> containerIds;
-  private final List<PipelineID> pipelineIds;
-
-
-  public ExcludeList() {
-    datanodes = new ArrayList<>();
-    containerIds = new ArrayList<>();
-    pipelineIds = new ArrayList<>();
-  }
-
-  public List<ContainerID> getContainerIds() {
-    return containerIds;
-  }
-
-  public List<DatanodeDetails> getDatanodes() {
-    return datanodes;
-  }
-
-  public void addDatanodes(Collection<DatanodeDetails> dns) {
-    datanodes.addAll(dns);
-  }
-
-  public void addDatanode(DatanodeDetails dn) {
-    datanodes.add(dn);
-  }
-
-  public void addConatinerId(ContainerID containerId) {
-    containerIds.add(containerId);
-  }
-
-  public void addPipeline(PipelineID pipelineId) {
-    pipelineIds.add(pipelineId);
-  }
-
-  public List<PipelineID> getPipelineIds() {
-    return pipelineIds;
-  }
-
-  public HddsProtos.ExcludeListProto getProtoBuf() {
-    HddsProtos.ExcludeListProto.Builder builder =
-        HddsProtos.ExcludeListProto.newBuilder();
-    containerIds
-        .forEach(id -> builder.addContainerIds(id.getId()));
-    datanodes.forEach(dn -> {
-      builder.addDatanodes(dn.getUuidString());
-    });
-    pipelineIds.forEach(pipelineID -> {
-      builder.addPipelineIds(pipelineID.getProtobuf());
-    });
-    return builder.build();
-  }
-
-  public static ExcludeList getFromProtoBuf(
-      HddsProtos.ExcludeListProto excludeListProto) {
-    ExcludeList excludeList = new ExcludeList();
-    excludeListProto.getContainerIdsList().forEach(id -> {
-      excludeList.addConatinerId(ContainerID.valueof(id));
-    });
-    DatanodeDetails.Builder builder = DatanodeDetails.newBuilder();
-    excludeListProto.getDatanodesList().forEach(dn -> {
-      builder.setUuid(dn);
-      excludeList.addDatanode(builder.build());
-    });
-    excludeListProto.getPipelineIdsList().forEach(pipelineID -> {
-      excludeList.addPipeline(PipelineID.getFromProtobuf(pipelineID));
-    });
-    return excludeList;
-  }
-
-  public void clear() {
-    datanodes.clear();
-    containerIds.clear();
-    pipelineIds.clear();
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/InvalidContainerStateException.java

@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
-/**
- * Exceptions thrown when a container is in invalid state while doing a I/O.
- */
-public class InvalidContainerStateException extends StorageContainerException {
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   */
-  public InvalidContainerStateException(String message) {
-    super(message, ContainerProtos.Result.INVALID_CONTAINER_STATE);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/StorageContainerException.java

@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
-import java.io.IOException;
-
-/**
- * Exceptions thrown from the Storage Container.
- */
-public class StorageContainerException extends IOException {
-  private ContainerProtos.Result result;
-
-  /**
-   * Constructs an {@code IOException} with {@code null}
-   * as its error detail message.
-   */
-  public StorageContainerException(ContainerProtos.Result result) {
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   * @param result - The result code
-   */
-  public StorageContainerException(String message,
-      ContainerProtos.Result result) {
-    super(message);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message
-   * and cause.
-   * <p>
-   * <p> Note that the detail message associated with {@code cause} is
-   * <i>not</i> automatically incorporated into this exception's detail
-   * message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the {@link #getMessage()} method)
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   *
-   * @param result - The result code
-   * @since 1.6
-   */
-  public StorageContainerException(String message, Throwable cause,
-      ContainerProtos.Result result) {
-    super(message, cause);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified cause and a
-   * detail message of {@code (cause==null ? null : cause.toString())}
-   * (which typically contains the class and detail message of {@code cause}).
-   * This constructor is useful for IO exceptions that are little more
-   * than wrappers for other throwables.
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @param result - The result code
-   * @since 1.6
-   */
-  public StorageContainerException(Throwable cause, ContainerProtos.Result
-      result) {
-    super(cause);
-    this.result = result;
-  }
-
-  /**
-   * Returns Result.
-   *
-   * @return Result.
-   */
-  public ContainerProtos.Result getResult() {
-    return result;
-  }
-
-
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/package-info.java

@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/package-info.java

@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/ContainerPlacementPolicy.java

@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership.  The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A ContainerPlacementPolicy support choosing datanodes to build replication
- * pipeline with specified constraints.
- */
-public interface ContainerPlacementPolicy {
-
-  /**
-   * Given the replication factor and size required, return set of datanodes
-   * that satisfy the nodes and size requirement.
-   *
-   * @param excludedNodes - list of nodes to be excluded.
-   * @param favoredNodes - list of nodes preferred.
-   * @param nodesRequired - number of datanodes required.
-   * @param sizeRequired - size required for the container or block.
-   * @return list of datanodes chosen.
-   * @throws IOException
-   */
-  List<DatanodeDetails> chooseDatanodes(List<DatanodeDetails> excludedNodes,
-      List<DatanodeDetails> favoredNodes, int nodesRequired, long sizeRequired)
-      throws IOException;
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/package-info.java

@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.container.placement.algorithms;
-/**
- Contains container placement policy interface definition.
- **/

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/SCMException.java

@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception thrown by SCM.
- */
-public class SCMException extends IOException {
-  private final ResultCodes result;
-
-  /**
-   * Constructs an {@code IOException} with {@code null}
-   * as its error detail message.
-   */
-  public SCMException(ResultCodes result) {
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   */
-  public SCMException(String message, ResultCodes result) {
-    super(message);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified detail message
-   * and cause.
-   * <p>
-   * <p> Note that the detail message associated with {@code cause} is
-   * <i>not</i> automatically incorporated into this exception's detail
-   * message.
-   *
-   * @param message The detail message (which is saved for later retrieval by
-   * the
-   * {@link #getMessage()} method)
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public SCMException(String message, Throwable cause, ResultCodes result) {
-    super(message, cause);
-    this.result = result;
-  }
-
-  /**
-   * Constructs an {@code IOException} with the specified cause and a
-   * detail message of {@code (cause==null ? null : cause.toString())}
-   * (which typically contains the class and detail message of {@code cause}).
-   * This constructor is useful for IO exceptions that are little more
-   * than wrappers for other throwables.
-   *
-   * @param cause The cause (which is saved for later retrieval by the {@link
-   * #getCause()} method).  (A null value is permitted, and indicates that the
-   * cause is nonexistent or unknown.)
-   * @since 1.6
-   */
-  public SCMException(Throwable cause, ResultCodes result) {
-    super(cause);
-    this.result = result;
-  }
-
-  /**
-   * Returns resultCode.
-   * @return ResultCode
-   */
-  public ResultCodes getResult() {
-    return result;
-  }
-
-  /**
-   * Error codes to make it easy to decode these exceptions.
-   */
-  public enum ResultCodes {
-    OK,
-    FAILED_TO_LOAD_NODEPOOL,
-    FAILED_TO_FIND_NODE_IN_POOL,
-    FAILED_TO_FIND_HEALTHY_NODES,
-    FAILED_TO_FIND_NODES_WITH_SPACE,
-    FAILED_TO_FIND_SUITABLE_NODE,
-    INVALID_CAPACITY,
-    INVALID_BLOCK_SIZE,
-    SAFE_MODE_EXCEPTION,
-    FAILED_TO_LOAD_OPEN_CONTAINER,
-    FAILED_TO_ALLOCATE_CONTAINER,
-    FAILED_TO_CHANGE_CONTAINER_STATE,
-    FAILED_TO_CHANGE_PIPELINE_STATE,
-    CONTAINER_EXISTS,
-    FAILED_TO_FIND_CONTAINER,
-    FAILED_TO_FIND_CONTAINER_WITH_SPACE,
-    BLOCK_EXISTS,
-    FAILED_TO_FIND_BLOCK,
-    IO_EXCEPTION,
-    UNEXPECTED_CONTAINER_STATE,
-    SCM_NOT_INITIALIZED,
-    DUPLICATE_DATANODE,
-    NO_SUCH_DATANODE,
-    NO_REPLICA_FOUND,
-    FAILED_TO_FIND_ACTIVE_PIPELINE,
-    FAILED_TO_INIT_CONTAINER_PLACEMENT_POLICY,
-    FAILED_TO_ALLOCATE_ENOUGH_BLOCKS,
-    INTERNAL_ERROR
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/exceptions/package-info.java

@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.exceptions;
-/**
- Exception objects for the SCM Server.
- */

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNode.java

@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- * The interface defines an inner node in a network topology.
- * An inner node represents network topology entities, such as data center,
- * rack, switch or logical group.
- */
-public interface InnerNode extends Node {
-  /** A factory interface to get new InnerNode instance. */
-  interface Factory<N extends InnerNode> {
-    /** Construct an InnerNode from name, location, parent, level and cost. */
-    N newInnerNode(String name, String location, InnerNode parent, int level,
-        int cost);
-  }
-
-  /**
-   * Add node <i>n</i> to the subtree of this node.
-   * @param n node to be added
-   * @return true if the node is added; false otherwise
-   */
-  boolean add(Node n);
-
-  /**
-   * Remove node <i>n</i> from the subtree of this node.
-   * @param n node to be deleted
-   */
-  void remove(Node n);
-
-  /**
-   * Given a node's string representation, return a reference to the node.
-   * @param loc string location of the format /dc/rack/nodegroup/node
-   * @return null if the node is not found
-   */
-  Node getNode(String loc);
-
-  /**
-   * @return number of its all nodes at level <i>level</i>. Here level is a
-   * relative level. If level is 1, means node itself. If level is 2, means its
-   * direct children, and so on.
-   **/
-  int getNumOfNodes(int level);
-
-  /**
-   * Get <i>leafIndex</i> leaf of this subtree.
-   *
-   * @param leafIndex an indexed leaf of the node
-   * @return the leaf node corresponding to the given index.
-   */
-  Node getLeaf(int leafIndex);
-
-  /**
-   * Get <i>leafIndex</i> leaf of this subtree.
-   *
-   * @param leafIndex ode's index, start from 0, skip the nodes in
-   *                  excludedScope and excludedNodes with ancestorGen
-   * @param excludedScopes the excluded scopes
-   * @param excludedNodes nodes to be excluded. If ancestorGen is not 0,
-   *                      the chosen node will not share same ancestor with
-   *                      those in excluded nodes at the specified generation
-   * @param ancestorGen ignored with value is 0
-   * @return the leaf node corresponding to the given index
-   */
-  Node getLeaf(int leafIndex, List<String> excludedScopes,
-      Collection<Node> excludedNodes, int ancestorGen);
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/InnerNodeImpl.java

@@ -1,509 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR;
-
-/**
- * A thread safe class that implements InnerNode interface.
- */
-public class InnerNodeImpl extends NodeImpl implements InnerNode {
-  protected static class Factory implements InnerNode.Factory<InnerNodeImpl> {
-    protected Factory() {}
-
-    public InnerNodeImpl newInnerNode(String name, String location,
-        InnerNode parent, int level, int cost) {
-      return new InnerNodeImpl(name, location, parent, level, cost);
-    }
-  }
-
-  static final Factory FACTORY = new Factory();
-  // a map of node's network name to Node for quick search and keep
-  // the insert order
-  private final HashMap<String, Node> childrenMap =
-      new LinkedHashMap<String, Node>();
-  // number of descendant leaves under this node
-  private int numOfLeaves;
-  // LOGGER
-  public static final Logger LOG = LoggerFactory.getLogger(InnerNodeImpl.class);
-
-  /**
-   * Construct an InnerNode from its name, network location, parent, level and
-   * its cost.
-   **/
-  protected InnerNodeImpl(String name, String location, InnerNode parent,
-      int level, int cost) {
-    super(name, location, parent, level, cost);
-  }
-
-  /** @return the number of children this node has */
-  private int getNumOfChildren() {
-    return childrenMap.size();
-  }
-
-  /** @return its leaf nodes number */
-  @Override
-  public int getNumOfLeaves() {
-    return numOfLeaves;
-  }
-
-  /**
-   * @return number of its all nodes at level <i>level</i>. Here level is a
-   * relative level. If level is 1, means node itself. If level is 2, means its
-   * direct children, and so on.
-   **/
-  public int getNumOfNodes(int level) {
-    Preconditions.checkArgument(level > 0);
-    int count = 0;
-    if (level == 1) {
-      count += 1;
-    } else if (level == 2) {
-      count += getNumOfChildren();
-    } else {
-      for (Node node: childrenMap.values()) {
-        if (node instanceof InnerNode) {
-          count += ((InnerNode)node).getNumOfNodes(level -1);
-        } else {
-          throw new RuntimeException("Cannot support Level:" + level +
-              " on this node " + this.toString());
-        }
-      }
-    }
-    return count;
-  }
-
-  /**
-   * Judge if this node is the parent of a leave node <i>n</i>.
-   * @return true if this node is the parent of <i>n</i>
-   */
-  private boolean isLeafParent() {
-    if (childrenMap.isEmpty()) {
-      return true;
-    }
-    Node child = childrenMap.values().iterator().next();
-    return child instanceof InnerNode ? false : true;
-  }
-
-  /**
-   * Judge if this node is the parent of node <i>node</i>.
-   * @param node a node
-   * @return true if this node is the parent of <i>n</i>
-   */
-  private boolean isParent(Node node) {
-    return node.getNetworkLocation().equals(this.getNetworkFullPath());
-  }
-
-  /**
-   * Add node <i>node</i> to the subtree of this node.
-   * @param node node to be added
-   * @return true if the node is added, false if is only updated
-   */
-  public boolean add(Node node) {
-    if (!isAncestor(node)) {
-      throw new IllegalArgumentException(node.getNetworkName()
-          + ", which is located at " + node.getNetworkLocation()
-          + ", is not a descendant of " + this.getNetworkFullPath());
-    }
-    if (isParent(node)) {
-      // this node is the parent, then add it directly
-      node.setParent(this);
-      node.setLevel(this.getLevel() + 1);
-      Node current = childrenMap.put(node.getNetworkName(), node);
-      if (current != null) {
-        return false;
-      }
-    } else {
-      // find the next level ancestor node
-      String ancestorName = getNextLevelAncestorName(node);
-      InnerNode childNode = (InnerNode)childrenMap.get(ancestorName);
-      if (childNode == null) {
-        // create a new InnerNode for this ancestor node
-        childNode = createChildNode(ancestorName);
-        childrenMap.put(childNode.getNetworkName(), childNode);
-      }
-      // add node to the subtree of the next ancestor node
-      if (!childNode.add(node)) {
-        return false;
-      }
-    }
-    numOfLeaves++;
-    return true;
-  }
-
-  /**
-   * Remove node <i>node</i> from the subtree of this node.
-   * @param node node to be deleted
-   */
-  public void remove(Node node) {
-    if (!isAncestor(node)) {
-      throw new IllegalArgumentException(node.getNetworkName()
-          + ", which is located at " + node.getNetworkLocation()
-          + ", is not a descendant of " + this.getNetworkFullPath());
-    }
-    if (isParent(node)) {
-      // this node is the parent, remove it directly
-      if (childrenMap.containsKey(node.getNetworkName())) {
-        childrenMap.remove(node.getNetworkName());
-        node.setParent(null);
-      } else {
-        throw new RuntimeException("Should not come to here. Node:" +
-            node.getNetworkFullPath() + ", Parent:" +
-            this.getNetworkFullPath());
-      }
-    } else {
-      // find the next ancestor node
-      String ancestorName = getNextLevelAncestorName(node);
-      InnerNodeImpl childNode = (InnerNodeImpl)childrenMap.get(ancestorName);
-      Preconditions.checkNotNull(childNode, "InnerNode is deleted before leaf");
-      // remove node from the parent node
-      childNode.remove(node);
-      // if the parent node has no children, remove the parent node too
-      if (childNode.getNumOfChildren() == 0) {
-        childrenMap.remove(ancestorName);
-      }
-    }
-    numOfLeaves--;
-  }
-
-  /**
-   * Given a node's string representation, return a reference to the node.
-   * Node can be leaf node or inner node.
-   *
-   * @param loc string location of a node. If loc starts with "/", it's a
-   *            absolute path, otherwise a relative path. Following examples
-   *            are all accepted,
-   *            1.  /dc1/rm1/rack1          -> an inner node
-   *            2.  /dc1/rm1/rack1/node1    -> a leaf node
-   *            3.  rack1/node1             -> a relative path to this node
-   *
-   * @return null if the node is not found
-   */
-  public Node getNode(String loc) {
-    if (loc == null) {
-      return null;
-    }
-
-    String fullPath = this.getNetworkFullPath();
-    if (loc.equalsIgnoreCase(fullPath)) {
-      return this;
-    }
-
-    // remove current node's location from loc when it's a absolute path
-    if (fullPath.equals(NetConstants.PATH_SEPARATOR_STR)) {
-      // current node is ROOT
-      if (loc.startsWith(PATH_SEPARATOR_STR)) {
-        loc = loc.substring(1);
-      }
-    } else if (loc.startsWith(fullPath)) {
-      loc = loc.substring(fullPath.length());
-      // skip the separator "/"
-      loc = loc.substring(1);
-    }
-
-    String[] path = loc.split(PATH_SEPARATOR_STR, 2);
-    Node child = childrenMap.get(path[0]);
-    if (child == null) {
-      return null;
-    }
-    if (path.length == 1){
-      return child;
-    }
-    if (child instanceof InnerNode) {
-      return ((InnerNode)child).getNode(path[1]);
-    } else {
-      return null;
-    }
-  }
-
-  /**
-   * get <i>leafIndex</i> leaf of this subtree.
-   *
-   * @param leafIndex an indexed leaf of the node
-   * @return the leaf node corresponding to the given index.
-   */
-  public Node getLeaf(int leafIndex) {
-    Preconditions.checkArgument(leafIndex >= 0);
-    // children are leaves
-    if (isLeafParent()) {
-      // range check
-      if (leafIndex >= getNumOfChildren()) {
-        return null;
-      }
-      return getChildNode(leafIndex);
-    } else {
-      for(Node node : childrenMap.values()) {
-        InnerNodeImpl child = (InnerNodeImpl)node;
-        int leafCount = child.getNumOfLeaves();
-        if (leafIndex < leafCount) {
-          return child.getLeaf(leafIndex);
-        } else {
-          leafIndex -= leafCount;
-        }
-      }
-      return null;
-    }
-  }
-
-  /**
-   * Get <i>leafIndex</i> leaf of this subtree.
-   *
-   * @param leafIndex node's index, start from 0, skip the nodes in
-   *                 excludedScope and excludedNodes with ancestorGen
-   * @param excludedScopes the exclude scopes
-   * @param excludedNodes nodes to be excluded from. If ancestorGen is not 0,
-   *                      the chosen node will not share same ancestor with
-   *                      those in excluded nodes at the specified generation
-   * @param ancestorGen  apply to excludeNodes, when value is 0, then no same
-   *                    ancestor enforcement on excludedNodes
-   * @return the leaf node corresponding to the given index.
-   * Example:
-   *
-   *                                /  --- root
-   *                              /  \
-   *                             /    \
-   *                            /      \
-   *                           /        \
-   *                         dc1         dc2
-   *                        / \         / \
-   *                       /   \       /   \
-   *                      /     \     /     \
-   *                    rack1 rack2  rack1  rack2
-   *                   / \     / \  / \     / \
-   *                 n1  n2  n3 n4 n5  n6  n7 n8
-   *
-   *   Input:
-   *   leafIndex = 2
-   *   excludedScope = /dc2/rack2
-   *   excludedNodes = {/dc1/rack1/n1}
-   *   ancestorGen = 1
-   *
-   *   Output:
-   *   node /dc1/rack2/n5
-   *
-   *   Explanation:
-   *   Since excludedNodes is n1 and ancestorGen is 1, it means nodes under
-   *   /root/dc1/rack1 are excluded. Given leafIndex start from 0, LeafIndex 2
-   *   means picking the 3th available node, which is n5.
-   *
-   */
-  public Node getLeaf(int leafIndex, List<String> excludedScopes,
-      Collection<Node> excludedNodes, int ancestorGen) {
-    Preconditions.checkArgument(leafIndex >= 0 && ancestorGen >= 0);
-    // come to leaf parent layer
-    if (isLeafParent()) {
-      return getLeafOnLeafParent(leafIndex, excludedScopes, excludedNodes);
-    }
-
-    int maxLevel = NodeSchemaManager.getInstance().getMaxLevel();
-    // this node's children, it's generation as the ancestor of the leaf node
-    int currentGen = maxLevel - this.getLevel() - 1;
-    // build an ancestor(children) to exclude node count map
-    Map<Node, Integer> countMap =
-        getAncestorCountMap(excludedNodes, ancestorGen, currentGen);
-    // nodes covered by excluded scope
-    Map<String, Integer> excludedNodeCount =
-        getExcludedScopeNodeCount(excludedScopes);
-
-    for (Node child : childrenMap.values()) {
-      int leafCount = child.getNumOfLeaves();
-      // skip nodes covered by excluded scopes
-      for (Map.Entry<String, Integer> entry: excludedNodeCount.entrySet()) {
-        if (entry.getKey().startsWith(child.getNetworkFullPath())) {
-          leafCount -= entry.getValue();
-        }
-      }
-      // skip nodes covered by excluded nodes and ancestorGen
-      Integer count = countMap.get(child);
-      if (count != null) {
-        leafCount -= count;
-      }
-      if (leafIndex < leafCount) {
-        return ((InnerNode)child).getLeaf(leafIndex, excludedScopes,
-            excludedNodes, ancestorGen);
-      } else {
-        leafIndex -= leafCount;
-      }
-    }
-    return null;
-  }
-
-  @Override
-  public boolean equals(Object to) {
-    if (to == null) {
-      return false;
-    }
-    if (this == to) {
-      return true;
-    }
-    return this.toString().equals(to.toString());
-  }
-
-  @Override
-  public int hashCode() {
-    return super.hashCode();
-  }
-
-  /**
-   * Get a ancestor to its excluded node count map.
-   *
-   * @param nodes a collection of leaf nodes to exclude
-   * @param genToExclude  the ancestor generation to exclude
-   * @param genToReturn  the ancestor generation to return the count map
-   * @return the map.
-   * example:
-   *
-   *                *  --- root
-   *              /    \
-   *             *      *   -- genToReturn =2
-   *            / \    / \
-   *          *   *   *   *  -- genToExclude = 1
-   *         /\  /\  /\  /\
-   *       *  * * * * * * *  -- nodes
-   */
-  private Map<Node, Integer> getAncestorCountMap(Collection<Node> nodes,
-      int genToExclude, int genToReturn) {
-    Preconditions.checkState(genToExclude >= 0);
-    Preconditions.checkState(genToReturn >= 0);
-
-    if (nodes == null || nodes.size() == 0) {
-      return Collections.emptyMap();
-    }
-    // with the recursive call, genToReturn can be smaller than genToExclude
-    if (genToReturn < genToExclude) {
-      genToExclude = genToReturn;
-    }
-    // ancestorToExclude to ancestorToReturn map
-    HashMap<Node, Node> ancestorMap = new HashMap<>();
-    for (Node node: nodes) {
-      Node ancestorToExclude = node.getAncestor(genToExclude);
-      Node ancestorToReturn = node.getAncestor(genToReturn);
-      if (ancestorToExclude == null || ancestorToReturn == null) {
-        LOG.warn("Ancestor not found, node: " + node.getNetworkFullPath() +
-            ", generation to exclude: " + genToExclude +
-            ", generation to return:" + genToReturn);
-        continue;
-      }
-      ancestorMap.put(ancestorToExclude, ancestorToReturn);
-    }
-    // ancestorToReturn to exclude node count map
-    HashMap<Node, Integer> countMap = new HashMap<>();
-    for (Map.Entry<Node, Node> entry : ancestorMap.entrySet()) {
-      countMap.compute(entry.getValue(),
-          (key, n) -> (n == null ? 0 : n) + entry.getKey().getNumOfLeaves());
-    }
-
-    return countMap;
-  }
-
-  /**
-   *  Get the node with leafIndex, considering skip nodes in excludedScope
-   *  and in excludeNodes list.
-   */
-  private Node getLeafOnLeafParent(int leafIndex, List<String> excludedScopes,
-      Collection<Node> excludedNodes) {
-    Preconditions.checkArgument(isLeafParent() && leafIndex >= 0);
-    if (leafIndex >= getNumOfChildren()) {
-      return null;
-    }
-    for(Node node : childrenMap.values()) {
-      if (excludedNodes != null && excludedNodes.contains(node)) {
-        continue;
-      }
-      if (excludedScopes != null && excludedScopes.size() > 0) {
-        if (excludedScopes.stream().anyMatch(scope ->
-            node.getNetworkFullPath().startsWith(scope))) {
-          continue;
-        }
-      }
-      if (leafIndex == 0) {
-        return node;
-      }
-      leafIndex--;
-    }
-    return null;
-  }
-
-  /**
-   *  Return child's name of this node which is an ancestor of node <i>n</i>.
-   */
-  private String getNextLevelAncestorName(Node n) {
-    int parentPathLen = this.getNetworkFullPath().length();
-    String name = n.getNetworkLocation().substring(parentPathLen);
-    if (name.charAt(0) == PATH_SEPARATOR) {
-      name = name.substring(1);
-    }
-    int index = name.indexOf(PATH_SEPARATOR);
-    if (index != -1) {
-      name = name.substring(0, index);
-    }
-    return name;
-  }
-
-  /**
-   * Creates a child node to be added to the list of children.
-   * @param name The name of the child node
-   * @return A new inner node
-   * @see InnerNodeImpl(String, String, InnerNode, int)
-   */
-  private InnerNodeImpl createChildNode(String name) {
-    int childLevel = this.getLevel() + 1;
-    int cost = NodeSchemaManager.getInstance().getCost(childLevel);
-    return new InnerNodeImpl(name, this.getNetworkFullPath(), this, childLevel,
-        cost);
-  }
-
-  /** Get node with index <i>index</i>. */
-  private Node getChildNode(int index) {
-    Iterator iterator = childrenMap.values().iterator();
-    Node node = null;
-    while(index >= 0 && iterator.hasNext()) {
-      node = (Node)iterator.next();
-      index--;
-    }
-    return node;
-  }
-
-  /** Get how many leaf nodes are covered by the excludedScopes(no overlap). */
-  private Map<String, Integer> getExcludedScopeNodeCount(
-      List<String> excludedScopes) {
-    HashMap<String, Integer> nodeCounts = new HashMap<>();
-    if (excludedScopes == null || excludedScopes.isEmpty()) {
-      return nodeCounts;
-    }
-
-    for (String scope: excludedScopes) {
-      Node excludedScopeNode = getNode(scope);
-      nodeCounts.put(scope, excludedScopeNode == null ? 0 :
-          excludedScopeNode.getNumOfLeaves());
-    }
-    return nodeCounts;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetConstants.java

@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import  org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
-
-/**
- * Class to hold network topology related constants and configurations.
- */
-public final class NetConstants {
-  private NetConstants() {
-    // Prevent instantiation
-  }
-  public final static char PATH_SEPARATOR = '/';
-  /** Path separator as a string. */
-  public final static String PATH_SEPARATOR_STR = "/";
-  public final static String SCOPE_REVERSE_STR = "~";
-  /** string representation of root. */
-  public final static String ROOT = "";
-  public final static int INNER_NODE_COST_DEFAULT = 1;
-  public final static int NODE_COST_DEFAULT = 0;
-  public final static int ANCESTOR_GENERATION_DEFAULT = 0;
-  public final static int ROOT_LEVEL = 1;
-  public final static String NODE_COST_PREFIX = "$";
-  public final static String DEFAULT_RACK = "/default-rack";
-  public final static String DEFAULT_NODEGROUP = "/default-nodegroup";
-  public final static String DEFAULT_DATACENTER = "/default-datacenter";
-  public final static String DEFAULT_REGION = "/default-dataregion";
-
-  // Build-in network topology node schema
-  public static final NodeSchema ROOT_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.ROOT).build();
-
-  public static final NodeSchema REGION_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.INNER_NODE)
-          .setDefaultName(DEFAULT_REGION).build();
-
-  public static final NodeSchema DATACENTER_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.INNER_NODE)
-          .setDefaultName(DEFAULT_DATACENTER).build();
-
-  public static final NodeSchema RACK_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.INNER_NODE)
-          .setDefaultName(DEFAULT_RACK).build();
-
-  public static final NodeSchema NODEGROUP_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.INNER_NODE)
-          .setDefaultName(DEFAULT_NODEGROUP).build();
-
-  public static final NodeSchema LEAF_SCHEMA =
-      new NodeSchema.Builder().setType(LayerType.LEAF_NODE).build();
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetUtils.java

@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import org.apache.commons.collections.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Utility class to facilitate network topology functions.
- */
-public final class NetUtils {
-  public static final Logger LOG = LoggerFactory.getLogger(NetUtils.class);
-  private NetUtils() {
-    // Prevent instantiation
-  }
-  /**
-   * Normalize a path by stripping off any trailing.
-   * {@link NetConstants#PATH_SEPARATOR}
-   * @param path path to normalize.
-   * @return the normalised path
-   * If <i>path</i>is empty or null, then {@link NetConstants#ROOT} is returned
-   */
-  public static String normalize(String path) {
-    if (path == null || path.length() == 0) {
-      return NetConstants.ROOT;
-    }
-
-    if (path.charAt(0) != NetConstants.PATH_SEPARATOR) {
-      throw new IllegalArgumentException(
-          "Network Location path does not start with "
-              + NetConstants.PATH_SEPARATOR_STR + ": " + path);
-    }
-
-    // Remove any trailing NetConstants.PATH_SEPARATOR
-    return path.length() == 1 ? path :
-        path.replaceAll(NetConstants.PATH_SEPARATOR_STR + "+$", "");
-  }
-
-  /**
-   *  Given a network topology location string, return its network topology
-   *  depth, E.g. the depth of /dc1/rack1/ng1/node1 is 5.
-   */
-  public static int locationToDepth(String location) {
-    String newLocation = normalize(location);
-    return newLocation.equals(NetConstants.PATH_SEPARATOR_STR) ? 1 :
-        newLocation.split(NetConstants.PATH_SEPARATOR_STR).length;
-  }
-
-
-  /**
-   *  Remove node from mutableExcludedNodes if it's covered by excludedScope.
-   *  Please noted that mutableExcludedNodes content might be changed after the
-   *  function call.
-   */
-  public static void removeDuplicate(NetworkTopology topology,
-      Collection<Node> mutableExcludedNodes, List<String> mutableExcludedScopes,
-      int ancestorGen) {
-    if (CollectionUtils.isEmpty(mutableExcludedNodes) ||
-        CollectionUtils.isEmpty(mutableExcludedScopes) || topology == null) {
-      return;
-    }
-
-    Iterator<Node> iterator = mutableExcludedNodes.iterator();
-    while (iterator.hasNext() && (!mutableExcludedScopes.isEmpty())) {
-      Node node = iterator.next();
-      Node ancestor = topology.getAncestor(node, ancestorGen);
-      if (ancestor == null) {
-        LOG.warn("Fail to get ancestor generation " + ancestorGen +
-            " of node :" + node);
-        continue;
-      }
-      // excludedScope is child of ancestor
-      List<String> duplicateList = mutableExcludedScopes.stream()
-          .filter(scope -> scope.startsWith(ancestor.getNetworkFullPath()))
-          .collect(Collectors.toList());
-      mutableExcludedScopes.removeAll(duplicateList);
-
-      // ancestor is covered by excludedScope
-      mutableExcludedScopes.stream().forEach(scope -> {
-        if (ancestor.getNetworkFullPath().startsWith(scope)) {
-          // remove exclude node if it's covered by excludedScope
-          iterator.remove();
-        }
-      });
-    }
-  }
-
-  /**
-   *  Remove node from mutableExcludedNodes if it's not part of scope
-   *  Please noted that mutableExcludedNodes content might be changed after the
-   *  function call.
-   */
-  public static void removeOutscope(Collection<Node> mutableExcludedNodes,
-      String scope) {
-    if (CollectionUtils.isEmpty(mutableExcludedNodes) || scope == null) {
-      return;
-    }
-    synchronized (mutableExcludedNodes) {
-      Iterator<Node> iterator = mutableExcludedNodes.iterator();
-      while (iterator.hasNext()) {
-        Node next = iterator.next();
-        if (!next.getNetworkFullPath().startsWith(scope)) {
-          iterator.remove();
-        }
-      }
-    }
-  }
-
-  /**
-   * Get a ancestor list for nodes on generation <i>generation</i>.
-   *
-   * @param nodes a collection of leaf nodes
-   * @param generation  the ancestor generation
-   * @return the ancestor list. If no ancestor is found, then a empty list is
-   * returned.
-   */
-  public static List<Node> getAncestorList(NetworkTopology topology,
-      Collection<Node> nodes, int generation) {
-    List<Node> ancestorList = new ArrayList<>();
-    if (topology == null || CollectionUtils.isEmpty(nodes) ||
-        generation == 0) {
-      return ancestorList;
-    }
-    Iterator<Node> iterator = nodes.iterator();
-    while (iterator.hasNext()) {
-      Node node = iterator.next();
-      Node ancestor = topology.getAncestor(node, generation);
-      if (ancestor == null) {
-        LOG.warn("Fail to get ancestor generation " + generation +
-            " of node :" + node);
-        continue;
-      }
-      if (!ancestorList.contains(ancestor)) {
-        ancestorList.add(ancestor);
-      }
-    }
-    return ancestorList;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopology.java

@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import java.util.Collection;
-import java.util.List;
-
-/**
- * The interface defines a network topology.
- */
-public interface NetworkTopology {
-  /** Exception for invalid network topology detection. */
-  class InvalidTopologyException extends RuntimeException {
-    private static final long serialVersionUID = 1L;
-    public InvalidTopologyException(String msg) {
-      super(msg);
-    }
-  }
-  /**
-   * Add a leaf node. This will be called when a new datanode is added.
-   * @param node node to be added; can be null
-   * @exception IllegalArgumentException if add a node to a leave or node to be
-   * added is not a leaf
-   */
-  void add(Node node);
-
-  /**
-   * Remove a node from the network topology. This will be called when a
-   * existing datanode is removed from the system.
-   * @param node node to be removed; cannot be null
-   */
-  void remove(Node node);
-
-  /**
-   * Check if the tree already contains node <i>node</i>.
-   * @param node a node
-   * @return true if <i>node</i> is already in the tree; false otherwise
-   */
-  boolean contains(Node node);
-
-  /**
-   * Compare the direct parent of each node for equality.
-   * @return true if their parent are the same
-   */
-  boolean isSameParent(Node node1, Node node2);
-
-  /**
-   * Compare the specified ancestor generation of each node for equality.
-   * ancestorGen 1 means parent.
-   * @return true if their specified generation ancestor are equal
-   */
-  boolean isSameAncestor(Node node1, Node node2, int ancestorGen);
-
-  /**
-   * Get the ancestor for node on generation <i>ancestorGen</i>.
-   *
-   * @param node the node to get ancestor
-   * @param ancestorGen  the ancestor generation
-   * @return the ancestor. If no ancestor is found, then null is returned.
-   */
-  Node getAncestor(Node node, int ancestorGen);
-
-  /**
-   * Return the max level of this topology, start from 1 for ROOT. For example,
-   * topology like "/rack/node" has the max level '3'.
-   */
-  int getMaxLevel();
-
-  /**
-   * Given a string representation of a node, return its reference.
-   * @param loc a path string representing a node, can be leaf or inner node
-   * @return a reference to the node; null if the node is not in the tree
-   */
-  Node getNode(String loc);
-
-  /**
-   * Given a string representation of a InnerNode, return its leaf nodes count.
-   * @param loc a path-like string representation of a InnerNode
-   * @return the number of leaf nodes, 0 if it's not an InnerNode or the node
-   * doesn't exist
-   */
-  int getNumOfLeafNode(String loc);
-
-  /**
-   * Return the node numbers at level <i>level</i>.
-   * @param level topology level, start from 1, which means ROOT
-   * @return the number of nodes on the level
-   */
-  int getNumOfNodes(int level);
-
-  /**
-   * Randomly choose a node in the scope.
-   * @param scope range of nodes from which a node will be chosen. If scope
-   *              starts with ~, choose one from the all nodes except for the
-   *              ones in <i>scope</i>; otherwise, choose one from <i>scope</i>.
-   * @return the chosen node
-   */
-  Node chooseRandom(String scope);
-
-  /**
-   * Randomly choose a node in the scope, ano not in the exclude scope.
-   * @param scope range of nodes from which a node will be chosen. cannot start
-   *              with ~
-   * @param excludedScopes the chosen nodes cannot be in these ranges. cannot
-   *                      starts with ~
-   * @return the chosen node
-   */
-  Node chooseRandom(String scope, List<String>  excludedScopes);
-
-  /**
-   * Randomly choose a leaf node from <i>scope</i>.
-   *
-   * If scope starts with ~, choose one from the all nodes except for the
-   * ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
-   * If excludedNodes is given, choose a node that's not in excludedNodes.
-   *
-   * @param scope range of nodes from which a node will be chosen
-   * @param excludedNodes nodes to be excluded
-   *
-   * @return the chosen node
-   */
-  Node chooseRandom(String scope, Collection<Node> excludedNodes);
-
-  /**
-   * Randomly choose a leaf node from <i>scope</i>.
-   *
-   * If scope starts with ~, choose one from the all nodes except for the
-   * ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
-   * If excludedNodes is given, choose a node that's not in excludedNodes.
-   *
-   * @param scope range of nodes from which a node will be chosen
-   * @param excludedNodes nodes to be excluded from.
-   * @param ancestorGen matters when excludeNodes is not null. It means the
-   * ancestor generation that's not allowed to share between chosen node and the
-   * excludedNodes. For example, if ancestorGen is 1, means chosen node
-   * cannot share the same parent with excludeNodes. If value is 2, cannot
-   * share the same grand parent, and so on. If ancestorGen is 0, then no
-   * effect.
-   *
-   * @return the chosen node
-   */
-  Node chooseRandom(String scope, Collection<Node> excludedNodes,
-      int ancestorGen);
-
-  /**
-   * Randomly choose one node from <i>scope</i>, share the same generation
-   * ancestor with <i>affinityNode</i>, and exclude nodes in
-   * <i>excludeScope</i> and <i>excludeNodes</i>.
-   *
-   * @param scope range of nodes from which a node will be chosen, cannot start
-   *              with ~
-   * @param excludedScopes ranges of nodes to be excluded, cannot start with ~
-   * @param excludedNodes nodes to be excluded
-   * @param affinityNode  when not null, the chosen node should share the same
-   *                     ancestor with this node at generation ancestorGen.
-   *                      Ignored when value is null
-   * @param ancestorGen If 0, then no same generation ancestor enforcement on
-   *                     both excludedNodes and affinityNode. If greater than 0,
-   *                     then apply to affinityNode(if not null), or apply to
-   *                     excludedNodes if affinityNode is null
-   * @return the chosen node
-   */
-  Node chooseRandom(String scope, List<String>  excludedScopes,
-      Collection<Node> excludedNodes, Node affinityNode, int ancestorGen);
-
-  /**
-   * Choose the node at index <i>index</i> from <i>scope</i>, share the same
-   * generation ancestor with <i>affinityNode</i>, and exclude nodes in
-   * <i>excludeScope</i> and <i>excludeNodes</i>.
-   *
-   * @param leafIndex node index, exclude nodes in excludedScope and
-   *                  excludedNodes
-   * @param scope range of nodes from which a node will be chosen, cannot start
-   *              with ~
-   * @param excludedScopes ranges of nodes to be excluded, cannot start with ~
-   * @param excludedNodes nodes to be excluded
-   * @param affinityNode  when not null, the chosen node should share the same
-   *                     ancestor with this node at generation ancestorGen.
-   *                      Ignored when value is null
-   * @param ancestorGen If 0, then no same generation ancestor enforcement on
-   *                     both excludedNodes and affinityNode. If greater than 0,
-   *                     then apply to affinityNode(if not null), or apply to
-   *                     excludedNodes if affinityNode is null
-   * @return the chosen node
-   */
-  Node getNode(int leafIndex, String scope, List<String> excludedScopes,
-      Collection<Node> excludedNodes, Node affinityNode, int ancestorGen);
-
-  /** Return the distance cost between two nodes
-   * The distance cost from one node to its parent is it's parent's cost
-   * The distance cost between two nodes is calculated by summing up their
-   * distances cost to their closest common ancestor.
-   * @param node1 one node
-   * @param node2 another node
-   * @return the distance cost between node1 and node2 which is zero if they
-   * are the same or {@link Integer#MAX_VALUE} if node1 or node2 do not belong
-   * to the cluster
-   */
-  int getDistanceCost(Node node1, Node node2);
-
-  /**
-   * Sort nodes array by network distance to <i>reader</i> to reduces network
-   * traffic and improves performance.
-   *
-   * As an additional twist, we also randomize the nodes at each network
-   * distance. This helps with load balancing when there is data skew.
-   *
-   * @param reader    Node where need the data
-   * @param nodes     Available replicas with the requested data
-   * @param activeLen Number of active nodes at the front of the array
-   */
-  List<? extends Node> sortByDistanceCost(Node reader,
-      List<? extends Node> nodes, int activeLen);
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java

@@ -1,798 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.commons.collections.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.TreeMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.SCOPE_REVERSE_STR;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ANCESTOR_GENERATION_DEFAULT;
-
-/**
- * The class represents a cluster of computers with a tree hierarchical
- * network topology. In the network topology, leaves represent data nodes
- * (computers) and inner nodes represent datacenter/core-switches/routers that
- * manages traffic in/out of data centers or racks.
- */
-public class NetworkTopologyImpl implements NetworkTopology{
-  public static final Logger LOG =
-      LoggerFactory.getLogger(NetworkTopology.class);
-
-  /** The Inner node crate factory. */
-  private final InnerNode.Factory factory;
-  /** The root cluster tree. */
-  private final InnerNode clusterTree;
-  /** Depth of all leaf nodes. */
-  private final int maxLevel;
-  /** Schema manager. */
-  private final NodeSchemaManager schemaManager;
-  /** Lock to coordinate cluster tree access. */
-  private ReadWriteLock netlock = new ReentrantReadWriteLock(true);
-
-  public NetworkTopologyImpl(Configuration conf) {
-    schemaManager = NodeSchemaManager.getInstance();
-    schemaManager.init(conf);
-    maxLevel = schemaManager.getMaxLevel();
-    factory = InnerNodeImpl.FACTORY;
-    clusterTree = factory.newInnerNode(ROOT, null, null,
-        NetConstants.ROOT_LEVEL,
-        schemaManager.getCost(NetConstants.ROOT_LEVEL));
-  }
-
-  @VisibleForTesting
-  public NetworkTopologyImpl(NodeSchemaManager manager) {
-    schemaManager = manager;
-    maxLevel = schemaManager.getMaxLevel();
-    factory = InnerNodeImpl.FACTORY;
-    clusterTree = factory.newInnerNode(ROOT, null, null,
-        NetConstants.ROOT_LEVEL,
-        schemaManager.getCost(NetConstants.ROOT_LEVEL));
-  }
-
-  /**
-   * Add a leaf node. This will be called when a new datanode is added.
-   * @param node node to be added; can be null
-   * @exception IllegalArgumentException if add a node to a leave or node to be
-   * added is not a leaf
-   */
-  public void add(Node node) {
-    Preconditions.checkArgument(node != null, "node cannot be null");
-    if (node instanceof InnerNode) {
-      throw new IllegalArgumentException(
-          "Not allowed to add an inner node: "+ node.getNetworkFullPath());
-    }
-    int newDepth = NetUtils.locationToDepth(node.getNetworkLocation()) + 1;
-
-    // Check depth
-    if (maxLevel != newDepth) {
-      throw new InvalidTopologyException("Failed to add " +
-          node.getNetworkFullPath() + ": Its path depth is not " + maxLevel);
-    }
-    netlock.writeLock().lock();
-    boolean add;
-    try {
-      add = clusterTree.add(node);
-    }finally {
-      netlock.writeLock().unlock();
-    }
-
-    if (add) {
-      LOG.info("Added a new node: " + node.getNetworkFullPath());
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("NetworkTopology became:\n{}", this);
-      }
-    }
-  }
-
-  /**
-   * Remove a node from the network topology. This will be called when a
-   * existing datanode is removed from the system.
-   * @param node node to be removed; cannot be null
-   */
-  public void remove(Node node) {
-    Preconditions.checkArgument(node != null, "node cannot be null");
-    if (node instanceof InnerNode) {
-      throw new IllegalArgumentException(
-          "Not allowed to remove an inner node: "+ node.getNetworkFullPath());
-    }
-    netlock.writeLock().lock();
-    try {
-      clusterTree.remove(node);
-    }finally {
-      netlock.writeLock().unlock();
-    }
-    LOG.info("Removed a node: " + node.getNetworkFullPath());
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("NetworkTopology became:\n{}", this);
-    }
-  }
-
-  /**
-   * Check if the tree already contains node <i>node</i>.
-   * @param node a node
-   * @return true if <i>node</i> is already in the tree; false otherwise
-   */
-  public boolean contains(Node node) {
-    Preconditions.checkArgument(node != null, "node cannot be null");
-    netlock.readLock().lock();
-    try {
-      Node parent = node.getParent();
-      while (parent != null && parent != clusterTree) {
-        parent = parent.getParent();
-      }
-      if (parent == clusterTree) {
-        return true;
-      }
-    } finally {
-      netlock.readLock().unlock();
-    }
-    return false;
-  }
-
-  /**
-   * Compare the specified ancestor generation of each node for equality.
-   * @return true if their specified generation ancestor are equal
-   */
-  public boolean isSameAncestor(Node node1, Node node2, int ancestorGen) {
-    if (node1 == null || node2 == null || ancestorGen <= 0) {
-      return false;
-    }
-    netlock.readLock().lock();
-    try {
-      return node1.getAncestor(ancestorGen) == node2.getAncestor(ancestorGen);
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Compare the direct parent of each node for equality.
-   * @return true if their parent are the same
-   */
-  public boolean isSameParent(Node node1, Node node2) {
-    if (node1 == null || node2 == null) {
-      return false;
-    }
-    netlock.readLock().lock();
-    try {
-      node1 = node1.getParent();
-      node2 = node2.getParent();
-      return node1 == node2;
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Get the ancestor for node on generation <i>ancestorGen</i>.
-   *
-   * @param node the node to get ancestor
-   * @param ancestorGen  the ancestor generation
-   * @return the ancestor. If no ancestor is found, then null is returned.
-   */
-  public Node getAncestor(Node node, int ancestorGen) {
-    if (node == null) {
-      return null;
-    }
-    netlock.readLock().lock();
-    try {
-      return node.getAncestor(ancestorGen);
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Given a string representation of a node(leaf or inner), return its
-   * reference.
-   * @param loc a path string representing a node, can be leaf or inner node
-   * @return a reference to the node, null if the node is not in the tree
-   */
-  public Node getNode(String loc) {
-    loc = NetUtils.normalize(loc);
-    netlock.readLock().lock();
-    try {
-      if (!ROOT.equals(loc)) {
-        return clusterTree.getNode(loc);
-      } else {
-        return clusterTree;
-      }
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Given a string representation of Node, return its leaf nodes count.
-   * @param loc a path-like string representation of Node
-   * @return the number of leaf nodes for InnerNode, 1 for leaf node, 0 if node
-   * doesn't exist
-   */
-  public int getNumOfLeafNode(String loc) {
-    netlock.readLock().lock();
-    try {
-      Node node = getNode(loc);
-      if (node != null) {
-        return node.getNumOfLeaves();
-      }
-    } finally {
-      netlock.readLock().unlock();
-    }
-    return 0;
-  }
-
-  /**
-   * Return the max level of this tree, start from 1 for ROOT. For example,
-   * topology like "/rack/node" has the max level '3'.
-   */
-  public int getMaxLevel() {
-    return maxLevel;
-  }
-
-  /**
-   * Return the node numbers at level <i>level</i>.
-   * @param level topology level, start from 1, which means ROOT
-   * @return the number of nodes on the level
-   */
-  public int getNumOfNodes(int level) {
-    Preconditions.checkArgument(level > 0 && level <= maxLevel,
-        "Invalid level");
-    netlock.readLock().lock();
-    try {
-      return clusterTree.getNumOfNodes(level);
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Randomly choose a node in the scope.
-   * @param scope range of nodes from which a node will be chosen. If scope
-   *              starts with ~, choose one from the all nodes except for the
-   *              ones in <i>scope</i>; otherwise, choose one from <i>scope</i>.
-   * @return the chosen node
-   */
-  public Node chooseRandom(String scope) {
-    if (scope == null) {
-      scope = ROOT;
-    }
-    if (scope.startsWith(SCOPE_REVERSE_STR)) {
-      ArrayList<String> excludedScopes = new ArrayList();
-      excludedScopes.add(scope.substring(1));
-      return chooseRandom(ROOT, excludedScopes, null, null,
-          ANCESTOR_GENERATION_DEFAULT);
-    } else {
-      return chooseRandom(scope, null, null, null, ANCESTOR_GENERATION_DEFAULT);
-    }
-  }
-
-  /**
-   * Randomly choose a node in the scope, ano not in the exclude scope.
-   * @param scope range of nodes from which a node will be chosen. cannot start
-   *              with ~
-   * @param excludedScopes the chosen node cannot be in these ranges. cannot
-   *                      starts with ~
-   * @return the chosen node
-   */
-  public Node chooseRandom(String scope, List<String> excludedScopes) {
-    return chooseRandom(scope, excludedScopes, null, null,
-        ANCESTOR_GENERATION_DEFAULT);
-  }
-
-  /**
-   * Randomly choose a leaf node from <i>scope</i>.
-   *
-   * If scope starts with ~, choose one from the all nodes except for the
-   * ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
-   * If excludedNodes is given, choose a node that's not in excludedNodes.
-   *
-   * @param scope range of nodes from which a node will be chosen
-   * @param excludedNodes nodes to be excluded
-   *
-   * @return the chosen node
-   */
-  public Node chooseRandom(String scope, Collection<Node> excludedNodes) {
-    if (scope == null) {
-      scope = ROOT;
-    }
-    if (scope.startsWith(SCOPE_REVERSE_STR)) {
-      ArrayList<String> excludedScopes = new ArrayList();
-      excludedScopes.add(scope.substring(1));
-      return chooseRandom(ROOT, excludedScopes, excludedNodes, null,
-          ANCESTOR_GENERATION_DEFAULT);
-    } else {
-      return chooseRandom(scope, null, excludedNodes, null,
-          ANCESTOR_GENERATION_DEFAULT);
-    }
-  }
-
-  /**
-   * Randomly choose a leaf node from <i>scope</i>.
-   *
-   * If scope starts with ~, choose one from the all nodes except for the
-   * ones in <i>scope</i>; otherwise, choose nodes from <i>scope</i>.
-   * If excludedNodes is given, choose a node that's not in excludedNodes.
-   *
-   * @param scope range of nodes from which a node will be chosen
-   * @param excludedNodes nodes to be excluded from.
-   * @param ancestorGen matters when excludeNodes is not null. It means the
-   * ancestor generation that's not allowed to share between chosen node and the
-   * excludedNodes. For example, if ancestorGen is 1, means chosen node
-   * cannot share the same parent with excludeNodes. If value is 2, cannot
-   * share the same grand parent, and so on. If ancestorGen is 0, then no
-   * effect.
-   *
-   * @return the chosen node
-   */
-  public Node chooseRandom(String scope, Collection<Node> excludedNodes,
-      int ancestorGen) {
-    if (scope == null) {
-      scope = ROOT;
-    }
-    if (scope.startsWith(SCOPE_REVERSE_STR)) {
-      ArrayList<String> excludedScopes = new ArrayList();
-      excludedScopes.add(scope.substring(1));
-      return chooseRandom(ROOT, excludedScopes, excludedNodes, null,
-          ancestorGen);
-    } else {
-      return chooseRandom(scope, null, excludedNodes, null, ancestorGen);
-    }
-  }
-
-  /**
-   * Randomly choose one leaf node from <i>scope</i>, share the same generation
-   * ancestor with <i>affinityNode</i>, and exclude nodes in
-   * <i>excludeScope</i> and <i>excludeNodes</i>.
-   *
-   * @param scope range of nodes from which a node will be chosen, cannot start
-   *              with ~
-   * @param excludedScopes ranges of nodes to be excluded, cannot start with ~
-   * @param excludedNodes nodes to be excluded
-   * @param affinityNode  when not null, the chosen node should share the same
-   *                     ancestor with this node at generation ancestorGen.
-   *                      Ignored when value is null
-   * @param ancestorGen If 0, then no same generation ancestor enforcement on
-   *                     both excludedNodes and affinityNode. If greater than 0,
-   *                     then apply to affinityNode(if not null), or apply to
-   *                     excludedNodes if affinityNode is null
-   * @return the chosen node
-   */
-  public Node chooseRandom(String scope, List<String> excludedScopes,
-      Collection<Node> excludedNodes, Node affinityNode, int ancestorGen) {
-    if (scope == null) {
-      scope = ROOT;
-    }
-
-    checkScope(scope);
-    checkExcludedScopes(excludedScopes);
-    checkAffinityNode(affinityNode);
-    checkAncestorGen(ancestorGen);
-
-    netlock.readLock().lock();
-    try {
-      return chooseNodeInternal(scope, -1, excludedScopes,
-          excludedNodes, affinityNode, ancestorGen);
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Choose the leaf node at index <i>index</i> from <i>scope</i>, share the
-   * same generation ancestor with <i>affinityNode</i>, and exclude nodes in
-   * <i>excludeScope</i> and <i>excludeNodes</i>.
-   *
-   * @param leafIndex node index, exclude nodes in excludedScope and
-   *                  excludedNodes
-   * @param scope range of nodes from which a node will be chosen, cannot start
-   *              with ~
-   * @param excludedScopes ranges of nodes to be excluded, cannot start with ~
-   * @param excludedNodes nodes to be excluded
-   * @param affinityNode  when not null, the chosen node should share the same
-   *                     ancestor with this node at generation ancestorGen.
-   *                      Ignored when value is null
-   * @param ancestorGen If 0, then no same generation ancestor enforcement on
-   *                     both excludedNodes and affinityNode. If greater than 0,
-   *                     then apply to affinityNode(if not null), or apply to
-   *                     excludedNodes if affinityNode is null
-   * @return the chosen node
-   * Example:
-   *
-   *                                /  --- root
-   *                              /  \
-   *                             /    \
-   *                            /      \
-   *                           /        \
-   *                         dc1         dc2
-   *                        / \         / \
-   *                       /   \       /   \
-   *                      /     \     /     \
-   *                    rack1 rack2  rack1  rack2
-   *                   / \     / \  / \     / \
-   *                 n1  n2  n3 n4 n5  n6  n7 n8
-   *
-   *   Input:
-   *   leafIndex = 1
-   *   excludedScope = /dc2
-   *   excludedNodes = {/dc1/rack1/n1}
-   *   affinityNode = /dc1/rack2/n2
-   *   ancestorGen = 2
-   *
-   *   Output:
-   *   node /dc1/rack2/n4
-   *
-   *   Explanation:
-   *   With affinityNode n2 and ancestorGen 2, it means we can only pick node
-   *   from subtree /dc1. LeafIndex 1, so we pick the 2nd available node n4.
-   *
-   */
-  public Node getNode(int leafIndex, String scope, List<String> excludedScopes,
-      Collection<Node> excludedNodes, Node affinityNode, int ancestorGen) {
-    Preconditions.checkArgument(leafIndex >= 0);
-    if (scope == null) {
-      scope = ROOT;
-    }
-    checkScope(scope);
-    checkExcludedScopes(excludedScopes);
-    checkAffinityNode(affinityNode);
-    checkAncestorGen(ancestorGen);
-
-    netlock.readLock().lock();
-    try {
-      return chooseNodeInternal(scope, leafIndex, excludedScopes,
-          excludedNodes, affinityNode, ancestorGen);
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  private Node chooseNodeInternal(String scope, int leafIndex,
-      List<String> excludedScopes, Collection<Node> excludedNodes,
-      Node affinityNode, int ancestorGen) {
-    Preconditions.checkArgument(scope != null);
-
-    String finalScope = scope;
-    if (affinityNode != null && ancestorGen > 0) {
-      Node affinityAncestor = affinityNode.getAncestor(ancestorGen);
-      if (affinityAncestor == null) {
-        throw new IllegalArgumentException("affinityNode " +
-            affinityNode.getNetworkFullPath() + " doesn't have ancestor on" +
-            " generation  " + ancestorGen);
-      }
-      // affinity ancestor should has overlap with scope
-      if (affinityAncestor.getNetworkFullPath().startsWith(scope)){
-        finalScope = affinityAncestor.getNetworkFullPath();
-      } else if (!scope.startsWith(affinityAncestor.getNetworkFullPath())) {
-        return null;
-      }
-      // reset ancestor generation since the new scope is identified now
-      ancestorGen = 0;
-    }
-
-    // check overlap of excludedScopes and finalScope
-    List<String> mutableExcludedScopes = null;
-    if (excludedScopes != null && !excludedScopes.isEmpty()) {
-      mutableExcludedScopes = new ArrayList<>();
-      for (String s: excludedScopes) {
-        // excludeScope covers finalScope
-        if (finalScope.startsWith(s)) {
-          return null;
-        }
-        // excludeScope and finalScope share nothing case
-        if (s.startsWith(finalScope)) {
-          if (!mutableExcludedScopes.stream().anyMatch(
-              e -> s.startsWith(e))) {
-            mutableExcludedScopes.add(s);
-          }
-        }
-      }
-    }
-
-    // clone excludedNodes before remove duplicate in it
-    Collection<Node> mutableExNodes = null;
-
-    // Remove duplicate in excludedNodes
-    if (excludedNodes != null) {
-      mutableExNodes =
-          excludedNodes.stream().distinct().collect(Collectors.toList());
-    }
-
-    // remove duplicate in mutableExNodes and mutableExcludedScopes
-    NetUtils.removeDuplicate(this, mutableExNodes, mutableExcludedScopes,
-        ancestorGen);
-
-    // calculate available node count
-    Node scopeNode = getNode(finalScope);
-    int availableNodes = getAvailableNodesCount(
-        scopeNode.getNetworkFullPath(), mutableExcludedScopes, mutableExNodes,
-        ancestorGen);
-
-    if (availableNodes <= 0) {
-      LOG.warn("No available node in (scope=\"{}\" excludedScope=\"{}\" " +
-              "excludedNodes=\"{}\"  ancestorGen=\"{}\").",
-          scopeNode.getNetworkFullPath(), excludedScopes, excludedNodes,
-          ancestorGen);
-      return null;
-    }
-
-    // scope is a Leaf node
-    if (!(scopeNode instanceof InnerNode)) {
-      return scopeNode;
-    }
-
-    Node ret;
-    int nodeIndex;
-    if (leafIndex >= 0) {
-      nodeIndex = leafIndex % availableNodes;
-      ret = ((InnerNode)scopeNode).getLeaf(nodeIndex, mutableExcludedScopes,
-          mutableExNodes, ancestorGen);
-    } else {
-      nodeIndex = ThreadLocalRandom.current().nextInt(availableNodes);
-      ret = ((InnerNode)scopeNode).getLeaf(nodeIndex, mutableExcludedScopes,
-          mutableExNodes, ancestorGen);
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Choosing node[index={},random={}] from \"{}\" available " +
-              "nodes, scope=\"{}\", excludedScope=\"{}\", excludeNodes=\"{}\".",
-          nodeIndex, (leafIndex == -1 ? "true" : "false"), availableNodes,
-          scopeNode.getNetworkFullPath(), excludedScopes, excludedNodes);
-      LOG.debug("Chosen node = {}", (ret == null ? "not found" :
-          ret.toString()));
-    }
-    return ret;
-  }
-
-  /** Return the distance cost between two nodes
-   * The distance cost from one node to its parent is it's parent's cost
-   * The distance cost between two nodes is calculated by summing up their
-   * distances cost to their closest common ancestor.
-   * @param node1 one node
-   * @param node2 another node
-   * @return the distance cost between node1 and node2 which is zero if they
-   * are the same or {@link Integer#MAX_VALUE} if node1 or node2 do not belong
-   * to the cluster
-   */
-  public int getDistanceCost(Node node1, Node node2) {
-    if ((node1 != null && node2 != null && node1.equals(node2)) ||
-        (node1 == null && node2 == null))  {
-      return 0;
-    }
-    if (node1 == null || node2 == null) {
-      LOG.warn("One of the nodes is a null pointer");
-      return Integer.MAX_VALUE;
-    }
-    int cost = 0;
-    netlock.readLock().lock();
-    try {
-      if ((node1.getAncestor(maxLevel - 1) != clusterTree) ||
-          (node2.getAncestor(maxLevel - 1) != clusterTree)) {
-        LOG.debug("One of the nodes is outside of network topology");
-        return Integer.MAX_VALUE;
-      }
-      int level1 = node1.getLevel();
-      int level2 = node2.getLevel();
-      if (level1 > maxLevel || level2 > maxLevel) {
-        return Integer.MAX_VALUE;
-      }
-      while(level1 > level2 && node1 != null) {
-        node1 = node1.getParent();
-        level1--;
-        cost += node1 == null? 0 : node1.getCost();
-      }
-      while(level2 > level1 && node2 != null) {
-        node2 = node2.getParent();
-        level2--;
-        cost += node2 == null? 0 : node2.getCost();
-      }
-      while(node1 != null && node2 != null && node1 != node2) {
-        node1 = node1.getParent();
-        node2 = node2.getParent();
-        cost += node1 == null? 0 : node1.getCost();
-        cost += node2 == null? 0 : node2.getCost();
-      }
-      return cost;
-    } finally {
-      netlock.readLock().unlock();
-    }
-  }
-
-  /**
-   * Sort nodes array by network distance to <i>reader</i> to reduces network
-   * traffic and improves performance.
-   *
-   * As an additional twist, we also randomize the nodes at each network
-   * distance. This helps with load balancing when there is data skew.
-   *
-   * @param reader    Node where need the data
-   * @param nodes     Available replicas with the requested data
-   * @param activeLen Number of active nodes at the front of the array
-   */
-  public List<? extends Node> sortByDistanceCost(Node reader,
-      List<? extends Node> nodes, int activeLen) {
-    /** Sort weights for the nodes array */
-    if (reader == null) {
-      return nodes;
-    }
-    int[] costs = new int[activeLen];
-    for (int i = 0; i < activeLen; i++) {
-      costs[i] = getDistanceCost(reader, nodes.get(i));
-    }
-    // Add cost/node pairs to a TreeMap to sort
-    TreeMap<Integer, List<Node>> tree = new TreeMap<Integer, List<Node>>();
-    for (int i = 0; i < activeLen; i++) {
-      int cost = costs[i];
-      Node node = nodes.get(i);
-      List<Node> list = tree.get(cost);
-      if (list == null) {
-        list = Lists.newArrayListWithExpectedSize(1);
-        tree.put(cost, list);
-      }
-      list.add(node);
-    }
-
-    List<Node> ret = new ArrayList<>();
-    for (List<Node> list: tree.values()) {
-      if (list != null) {
-        Collections.shuffle(list);
-        for (Node n: list) {
-          ret.add(n);
-        }
-      }
-    }
-
-    Preconditions.checkState(ret.size() == activeLen,
-        "Wrong number of nodes sorted!");
-    return ret;
-  }
-
-  /**
-   * Return the number of leaves in <i>scope</i> but not in
-   * <i>excludedNodes</i> and <i>excludeScope</i>.
-   * @param scope the scope
-   * @param excludedScopes excluded scopes
-   * @param mutableExcludedNodes a list of excluded nodes, content might be
-   *                            changed after the call
-   * @param ancestorGen same generation ancestor prohibit on excludedNodes
-   * @return number of available nodes
-   */
-  private int getAvailableNodesCount(String scope, List<String> excludedScopes,
-      Collection<Node> mutableExcludedNodes, int ancestorGen) {
-    Preconditions.checkArgument(scope != null);
-
-    Node scopeNode = getNode(scope);
-    if (scopeNode == null) {
-      return 0;
-    }
-    NetUtils.removeOutscope(mutableExcludedNodes, scope);
-    List<Node> excludedAncestorList =
-        NetUtils.getAncestorList(this, mutableExcludedNodes, ancestorGen);
-    for (Node ancestor : excludedAncestorList) {
-      if (scope.startsWith(ancestor.getNetworkFullPath())){
-        return 0;
-      }
-    }
-    // number of nodes to exclude
-    int excludedCount = 0;
-    if (excludedScopes != null) {
-      for (String excludedScope: excludedScopes) {
-        Node excludedScopeNode = getNode(excludedScope);
-        if (excludedScopeNode != null) {
-          if (excludedScope.startsWith(scope)) {
-            excludedCount += excludedScopeNode.getNumOfLeaves();
-          } else if (scope.startsWith(excludedScope)) {
-            return 0;
-          }
-        }
-      }
-    }
-    // excludedNodes is not null case
-    if (mutableExcludedNodes != null && (!mutableExcludedNodes.isEmpty())) {
-      if (ancestorGen == 0) {
-        for (Node node: mutableExcludedNodes) {
-          if (contains(node)) {
-            excludedCount++;
-          }
-        }
-      } else {
-        for (Node ancestor : excludedAncestorList) {
-          if (ancestor.getNetworkFullPath().startsWith(scope)) {
-            excludedCount += ancestor.getNumOfLeaves();
-          }
-        }
-      }
-    }
-
-    int availableCount = scopeNode.getNumOfLeaves() - excludedCount;
-    Preconditions.checkState(availableCount >= 0);
-    return availableCount;
-  }
-
-  @Override
-  public String toString() {
-    // print max level
-    StringBuilder tree = new StringBuilder();
-    tree.append("Level: ");
-    tree.append(maxLevel);
-    tree.append("\n");
-    netlock.readLock().lock();
-    try {
-      // print the number of leaves
-      int numOfLeaves = clusterTree.getNumOfLeaves();
-      tree.append("Number of leaves:");
-      tree.append(numOfLeaves);
-      tree.append("\n");
-      // print all nodes
-      for (int i = 0; i < numOfLeaves; i++) {
-        tree.append(clusterTree.getLeaf(i).getNetworkFullPath());
-        tree.append("\n");
-      }
-    } finally {
-      netlock.readLock().unlock();
-    }
-    return tree.toString();
-  }
-
-  private void checkScope(String scope) {
-    if (scope != null && scope.startsWith(SCOPE_REVERSE_STR)) {
-      throw new IllegalArgumentException("scope " + scope +
-          " should not start with " + SCOPE_REVERSE_STR);
-    }
-  }
-
-  private void checkExcludedScopes(List<String> excludedScopes) {
-    if (!CollectionUtils.isEmpty(excludedScopes)) {
-      excludedScopes.stream().forEach(scope -> {
-        if (scope.startsWith(SCOPE_REVERSE_STR)) {
-          throw new IllegalArgumentException("excludedScope " + scope +
-              " cannot start with " + SCOPE_REVERSE_STR);
-        }
-      });
-    }
-  }
-
-  private void checkAffinityNode(Node affinityNode) {
-    if (affinityNode != null && (!contains(affinityNode))) {
-      throw new IllegalArgumentException("Affinity node " +
-          affinityNode.getNetworkFullPath() + " is not a member of topology");
-    }
-  }
-
-  private void checkAncestorGen(int ancestorGen) {
-    if (ancestorGen > (maxLevel - 1) || ancestorGen < 0) {
-      throw new IllegalArgumentException("ancestorGen " + ancestorGen +
-          " exceeds this network topology acceptable level [0, " +
-          (maxLevel - 1) + "]");
-    }
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/Node.java

@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-/**
- * The interface defines a node in a network topology.
- * A node may be a leave representing a data node or an inner
- * node representing a data center or rack.
- * Each node has a name and its location in the network is
- * decided by a string with syntax similar to a file name.
- * For example, a data node's name is hostname:port# and if it's located at
- * rack "orange" in data center "dog", the string representation of its
- * network location will be /dog/orange.
- */
-public interface Node {
-  /** @return the string representation of this node's network location path,
-   *  exclude itself. In another words, its parent's full network location */
-  String getNetworkLocation();
-
-  /**
-   * Set this node's network location.
-   * @param location it's network location
-   */
-  void setNetworkLocation(String location);
-
-  /** @return this node's self name in network topology. This should be node's
-   * IP or hostname.
-   * */
-  String getNetworkName();
-
-  /**
-   * Set this node's name, can be hostname or Ipaddress.
-   * @param name it's network name
-   */
-  void setNetworkName(String name);
-
-  /** @return this node's full path in network topology. It's the concatenation
-   *  of location and name.
-   * */
-  String getNetworkFullPath();
-
-  /** @return this node's parent */
-  InnerNode getParent();
-
-  /**
-   * Set this node's parent.
-   * @param parent the parent
-   */
-  void setParent(InnerNode parent);
-
-  /** @return this node's ancestor, generation 0 is itself, generation 1 is
-   *  node's parent, and so on.*/
-  Node getAncestor(int generation);
-
-  /**
-   * @return this node's level in the tree.
-   * E.g. the root of a tree returns 1 and root's children return 2
-   */
-  int getLevel();
-
-  /**
-   * Set this node's level in the tree.
-   * @param i the level
-   */
-  void setLevel(int i);
-
-  /**
-   * @return this node's cost when network traffic go through it.
-   * E.g. the cost of going cross a switch is 1, and cost of going through a
-   * datacenter can be 5.
-   * Be default the cost of leaf datanode is 0, all other node is 1.
-   */
-  int getCost();
-
-  /** @return the leaf nodes number under this node. */
-  int getNumOfLeaves();
-
-  /**
-   * Judge if this node is an ancestor of node <i>n</i>.
-   * Ancestor includes itself and parents case.
-   *
-   * @param n a node
-   * @return true if this node is an ancestor of <i>n</i>
-   */
-  boolean isAncestor(Node n);
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeImpl.java

@@ -1,222 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import com.google.common.base.Preconditions;
-
-import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT;
-import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
-
-/**
- * A thread safe class that implements interface Node.
- */
-public class NodeImpl implements Node {
-  // host:port#
-  private String name;
-  // string representation of this node's location, such as /dc1/rack1
-  private String location;
-  // location + "/" + name
-  private String path;
-  // which level of the tree the node resides, start from 1 for root
-  private int level;
-  // node's parent
-  private InnerNode parent;
-  // the cost to go through this node
-  private final int cost;
-
-  /**
-   * Construct a node from its name and its location.
-   * @param name this node's name (can be null, must not contain
-   * {@link NetConstants#PATH_SEPARATOR})
-   * @param location this node's location
-   */
-  public NodeImpl(String name, String location, int cost) {
-    if (name != null && name.contains(PATH_SEPARATOR_STR)) {
-      throw new IllegalArgumentException(
-          "Network location name:" + name + " should not contain " +
-              PATH_SEPARATOR_STR);
-    }
-    this.name = (name == null) ? ROOT : name;
-    this.location = NetUtils.normalize(location);
-    this.path = getPath();
-    this.cost = cost;
-  }
-
-  /**
-   * Construct a node from its name and its location.
-   *
-   * @param name     this node's name (can be null, must not contain
-   *                 {@link NetConstants#PATH_SEPARATOR})
-   * @param location this node's location
-   * @param parent   this node's parent node
-   * @param level    this node's level in the tree
-   * @param cost     this node's cost if traffic goes through it
-   */
-  public NodeImpl(String name, String location, InnerNode parent, int level,
-      int cost) {
-    this(name, location, cost);
-    this.parent = parent;
-    this.level = level;
-  }
-
-  /**
-   * @return this node's name
-   */
-  public String getNetworkName() {
-    return name;
-  }
-
-  /**
-   * Set this node's name, can be hostname or Ipaddress.
-   * @param networkName it's network name
-   */
-  public void setNetworkName(String networkName) {
-    this.name = networkName;
-    this.path = getPath();
-  }
-
-  /**
-   * @return this node's network location
-   */
-  public String getNetworkLocation() {
-    return location;
-  }
-
-  /**
-   * Set this node's network location.
-   * @param networkLocation it's network location
-   */
-  @Override
-  public void setNetworkLocation(String networkLocation) {
-    this.location = networkLocation;
-    this.path = getPath();
-  }
-
-  /**
-   * @return this node's full path in network topology. It's the concatenation
-   * of location and name.
-   */
-  public String getNetworkFullPath() {
-    return path;
-  }
-
-  /**
-   * @return this node's parent
-   */
-  public InnerNode getParent() {
-    return parent;
-  }
-
-  /**
-   * @return this node's ancestor, generation 0 is itself, generation 1 is
-   * node's parent, and so on.
-   */
-  public Node getAncestor(int generation) {
-    Preconditions.checkArgument(generation >= 0);
-    Node current = this;
-    while (generation > 0 && current != null) {
-      current = current.getParent();
-      generation--;
-    }
-    return current;
-  }
-
-  /**
-   * Set this node's parent.
-   *
-   * @param parent the parent
-   */
-  public void setParent(InnerNode parent) {
-    this.parent = parent;
-  }
-
-  /**
-   * @return this node's level in the tree.
-   * E.g. the root of a tree returns 0 and its children return 1
-   */
-  public int getLevel() {
-    return this.level;
-  }
-
-  /**
-   * Set this node's level in the tree.
-   *
-   * @param level the level
-   */
-  public void setLevel(int level) {
-    this.level = level;
-  }
-
-  /**
-   * @return this node's cost when network traffic go through it.
-   * E.g. the cost of going cross a switch is 1, and cost of going through a
-   * datacenter is 5.
-   * Be default the cost of leaf datanode is 0, all other inner node is 1.
-   */
-  public int getCost() {
-    return this.cost;
-  }
-
-  /** @return the leaf nodes number under this node. */
-  public int getNumOfLeaves() {
-    return 1;
-  }
-
-  /**
-   * Check if this node is an ancestor of node <i>node</i>. Ancestor includes
-   * itself and parents case;
-   * @param node a node
-   * @return true if this node is an ancestor of <i>node</i>
-   */
-  public boolean isAncestor(Node node) {
-    return this.getNetworkFullPath().equals(PATH_SEPARATOR_STR) ||
-        node.getNetworkLocation().startsWith(this.getNetworkFullPath()) ||
-            node.getNetworkFullPath().equalsIgnoreCase(
-                this.getNetworkFullPath());
-  }
-
-  @Override
-  public boolean equals(Object to) {
-    if (to == null) {
-      return false;
-    }
-    if (this == to) {
-      return true;
-    }
-    return this.toString().equals(to.toString());
-  }
-
-  @Override
-  public int hashCode() {
-    return toString().hashCode();
-  }
-
-  /**
-   * @return this node's path as its string representation
-   */
-  @Override
-  public String toString() {
-    return getNetworkFullPath();
-  }
-
-  private String getPath() {
-    return this.location.equals(PATH_SEPARATOR_STR) ?
-        this.location + this.name :
-        this.location + PATH_SEPARATOR_STR + this.name;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchema.java

@@ -1,183 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import org.apache.hadoop.HadoopIllegalArgumentException;
-
-import java.util.List;
-
-/**
- * Network topology schema to housekeeper relevant information.
- */
-public final class NodeSchema {
-  /**
-   * Network topology layer type enum definition.
-   */
-  public enum LayerType{
-    ROOT("Root", NetConstants.INNER_NODE_COST_DEFAULT),
-    INNER_NODE("InnerNode", NetConstants.INNER_NODE_COST_DEFAULT),
-    LEAF_NODE("Leaf", NetConstants.NODE_COST_DEFAULT);
-
-    private final String description;
-    // default cost
-    private final int cost;
-
-    LayerType(String description, int cost) {
-      this.description = description;
-      this.cost = cost;
-    }
-
-    @Override
-    public String toString() {
-      return description;
-    }
-
-    public int getCost(){
-      return cost;
-    }
-    public static LayerType getType(String typeStr) {
-      for (LayerType type: LayerType.values()) {
-        if (typeStr.equalsIgnoreCase(type.toString())) {
-          return type;
-        }
-      }
-      return null;
-    }
-  }
-
-  // default cost
-  private int cost;
-  // layer Type, mandatory property
-  private LayerType type;
-  // default name, can be null or ""
-  private String defaultName;
-  // layer prefix, can be null or ""
-  private String prefix;
-  // sublayer
-  private List<NodeSchema> sublayer;
-
-  /**
-   * Builder for NodeSchema.
-   */
-  public static class Builder {
-    private int cost = -1;
-    private LayerType type;
-    private String defaultName;
-    private String prefix;
-
-    public Builder setCost(int nodeCost) {
-      this.cost = nodeCost;
-      return this;
-    }
-
-    public Builder setPrefix(String nodePrefix) {
-      this.prefix = nodePrefix;
-      return this;
-    }
-
-    public Builder setType(LayerType nodeType) {
-      this.type = nodeType;
-      return this;
-    }
-
-    public Builder setDefaultName(String nodeDefaultName) {
-      this.defaultName = nodeDefaultName;
-      return this;
-    }
-
-    public NodeSchema build() {
-      if (type == null) {
-        throw new HadoopIllegalArgumentException("Type is mandatory for a " +
-            "network topology node layer definition");
-      }
-      if (cost == -1) {
-        cost = type.getCost();
-      }
-      return new NodeSchema(type, cost, prefix, defaultName);
-    }
-  }
-
-  /**
-   * Constructor.
-   * @param type layer type
-   * @param cost layer's default cost
-   * @param prefix layer's prefix
-   * @param defaultName layer's default name is if specified
-   */
-  public NodeSchema(LayerType type, int cost, String prefix,
-      String defaultName) {
-    this.type = type;
-    this.cost = cost;
-    this.prefix = prefix;
-    this.defaultName = defaultName;
-  }
-
-  /**
-   * Constructor. This constructor is only used when build NodeSchema from
-   * YAML file.
-   */
-  public NodeSchema() {
-    this.type = LayerType.INNER_NODE;
-  }
-
-  public boolean matchPrefix(String name) {
-    if (name == null || name.isEmpty() || prefix == null || prefix.isEmpty()) {
-      return false;
-    }
-    return name.trim().toLowerCase().startsWith(prefix.toLowerCase());
-  }
-
-  public LayerType getType() {
-    return this.type;
-  }
-
-  public void setType(LayerType type) {
-    this.type = type;
-  }
-
-  public String getPrefix() {
-    return this.prefix;
-  }
-
-  public void setPrefix(String prefix) {
-    this.prefix = prefix;
-  }
-
-  public String getDefaultName() {
-    return this.defaultName;
-  }
-
-  public void setDefaultName(String name) {
-    this.defaultName = name;
-  }
-
-  public int getCost() {
-    return this.cost;
-  }
-  public void setCost(int cost) {
-    this.cost = cost;
-  }
-
-  public void setSublayer(List<NodeSchema> sublayer) {
-    this.sublayer = sublayer;
-  }
-
-  public List<NodeSchema> getSublayer() {
-    return sublayer;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaLoader.java

@@ -1,489 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import org.apache.commons.io.FilenameUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.w3c.dom.Text;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import  org.apache.hadoop.hdds.scm.net.NodeSchema.LayerType;
-import org.yaml.snakeyaml.Yaml;
-
-/**
- * A Network topology layer schema loading tool that loads user defined network
- * layer schema data from a XML configuration file.
- */
-public final class NodeSchemaLoader {
-  private static final Logger LOG
-      = LoggerFactory.getLogger(NodeSchemaLoader.class);
-  private static final String CONFIGURATION_TAG = "configuration";
-  private static final String LAYOUT_VERSION_TAG = "layoutversion";
-  private static final String TOPOLOGY_TAG = "topology";
-  private static final String TOPOLOGY_PATH = "path";
-  private static final String TOPOLOGY_ENFORCE_PREFIX = "enforceprefix";
-  private static final String LAYERS_TAG = "layers";
-  private static final String LAYER_TAG = "layer";
-  private static final String LAYER_ID = "id";
-  private static final String LAYER_TYPE = "type";
-  private static final String LAYER_COST = "cost";
-  private static final String LAYER_PREFIX = "prefix";
-  private static final String LAYER_DEFAULT_NAME = "default";
-
-  private static final int LAYOUT_VERSION = 1;
-  private volatile static NodeSchemaLoader instance = null;
-  private NodeSchemaLoader() {}
-
-  public static NodeSchemaLoader getInstance() {
-    if (instance == null) {
-      instance = new NodeSchemaLoader();
-    }
-    return instance;
-  }
-
-  /**
-   * Class to house keep the result of parsing a network topology schema file.
-   */
-  public static class NodeSchemaLoadResult {
-    private List<NodeSchema> schemaList;
-    private boolean enforcePrefix;
-
-    NodeSchemaLoadResult(List<NodeSchema> schemaList, boolean enforcePrefix) {
-      this.schemaList = schemaList;
-      this.enforcePrefix = enforcePrefix;
-    }
-
-    public boolean isEnforePrefix() {
-      return enforcePrefix;
-    }
-
-    public List<NodeSchema> getSchemaList() {
-      return schemaList;
-    }
-  }
-
-  /**
-   * Load user defined network layer schemas from a XML/YAML configuration file.
-   * @param schemaFilePath path of schema file
-   * @return all valid node schemas defined in schema file
-   */
-  public NodeSchemaLoadResult loadSchemaFromFile(String schemaFilePath)
-      throws IllegalArgumentException, FileNotFoundException {
-    try {
-      File schemaFile = new File(schemaFilePath);
-
-      if (schemaFile.exists()) {
-        LOG.info("Load network topology schema file " +
-            schemaFile.getAbsolutePath());
-        try (FileInputStream inputStream = new FileInputStream(schemaFile)) {
-          return loadSchemaFromStream(schemaFilePath, inputStream);
-        }
-      } else {
-        // try to load with classloader
-        ClassLoader classloader =
-            Thread.currentThread().getContextClassLoader();
-        if (classloader == null) {
-          classloader = NodeSchemaLoader.class.getClassLoader();
-        }
-        if (classloader != null) {
-          try (InputStream stream = classloader
-              .getResourceAsStream(schemaFilePath)) {
-            if (stream != null) {
-              LOG.info("Loading file from " + classloader
-                  .getResources(schemaFilePath));
-              return loadSchemaFromStream(schemaFilePath, stream);
-            }
-          }
-        }
-
-      }
-
-      String msg = "Network topology layer schema file " +
-          schemaFilePath + "[" + schemaFile.getAbsolutePath() +
-          "] is not found.";
-      LOG.warn(msg);
-      throw new FileNotFoundException(msg);
-
-    } catch (FileNotFoundException e) {
-      throw e;
-    } catch (ParserConfigurationException | IOException | SAXException e) {
-      throw new IllegalArgumentException("Failed to load network topology node"
-          + " schema file: " + schemaFilePath + " , error:" + e.getMessage(),
-          e);
-    }
-  }
-
-  private NodeSchemaLoadResult loadSchemaFromStream(String schemaFilePath,
-      InputStream stream)
-      throws ParserConfigurationException, SAXException, IOException {
-    if (FilenameUtils.getExtension(schemaFilePath).toLowerCase()
-        .compareTo("yaml") == 0) {
-      return loadSchemaFromYaml(stream);
-    } else {
-      return loadSchema(stream);
-    }
-  }
-
-  /**
-   * Load network topology layer schemas from a XML configuration file.
-   * @param inputStream schema file as an inputStream
-   * @return all valid node schemas defined in schema file
-   * @throws ParserConfigurationException ParserConfigurationException happen
-   * @throws IOException no such schema file
-   * @throws SAXException xml file has some invalid elements
-   * @throws IllegalArgumentException xml file content is logically invalid
-   */
-  private NodeSchemaLoadResult loadSchema(InputStream inputStream) throws
-      ParserConfigurationException, SAXException, IOException {
-    LOG.info("Loading network topology layer schema file");
-    // Read and parse the schema file.
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-    dbf.setIgnoringComments(true);
-    DocumentBuilder builder = dbf.newDocumentBuilder();
-    Document doc = builder.parse(inputStream);
-    Element root = doc.getDocumentElement();
-
-    if (!CONFIGURATION_TAG.equals(root.getTagName())) {
-      throw new IllegalArgumentException("Bad network topology layer schema " +
-          "configuration file: top-level element not <" + CONFIGURATION_TAG +
-          ">");
-    }
-    NodeSchemaLoadResult schemaList;
-    if (root.getElementsByTagName(LAYOUT_VERSION_TAG).getLength() == 1) {
-      if (loadLayoutVersion(root) == LAYOUT_VERSION) {
-        if (root.getElementsByTagName(LAYERS_TAG).getLength() == 1) {
-          Map<String, NodeSchema> schemas = loadLayersSection(root);
-          if (root.getElementsByTagName(TOPOLOGY_TAG).getLength() == 1) {
-            schemaList = loadTopologySection(root, schemas);
-          } else {
-            throw new IllegalArgumentException("Bad network topology layer " +
-                "schema configuration file: no or multiple <" + TOPOLOGY_TAG +
-                "> element");
-          }
-        } else {
-          throw new IllegalArgumentException("Bad network topology layer schema"
-              + " configuration file: no or multiple <" + LAYERS_TAG +
-              ">element");
-        }
-      } else {
-        throw new IllegalArgumentException("The parse failed because of bad "
-            + LAYOUT_VERSION_TAG + " value, expected:" + LAYOUT_VERSION);
-      }
-    } else {
-      throw new IllegalArgumentException("Bad network topology layer schema " +
-          "configuration file: no or multiple <" + LAYOUT_VERSION_TAG +
-          "> elements");
-    }
-    return schemaList;
-  }
-
-  /**
-   * Load network topology layer schemas from a YAML configuration file.
-   * @param schemaFile as inputStream
-   * @return all valid node schemas defined in schema file
-   * @throws ParserConfigurationException ParserConfigurationException happen
-   * @throws IOException no such schema file
-   * @throws SAXException xml file has some invalid elements
-   * @throws IllegalArgumentException xml file content is logically invalid
-   */
-  private NodeSchemaLoadResult loadSchemaFromYaml(InputStream schemaFile) {
-    LOG.info("Loading network topology layer schema file {}", schemaFile);
-    NodeSchemaLoadResult finalSchema;
-
-    try {
-      Yaml yaml = new Yaml();
-      NodeSchema nodeTree;
-
-      nodeTree = yaml.loadAs(schemaFile, NodeSchema.class);
-
-      List<NodeSchema> schemaList = new ArrayList<>();
-      if (nodeTree.getType() != LayerType.ROOT) {
-        throw new IllegalArgumentException("First layer is not a ROOT node."
-            + " schema file.");
-      }
-      schemaList.add(nodeTree);
-      if (nodeTree.getSublayer() != null) {
-        nodeTree = nodeTree.getSublayer().get(0);
-      }
-
-      while (nodeTree != null) {
-        if (nodeTree.getType() == LayerType.LEAF_NODE
-                && nodeTree.getSublayer() != null) {
-          throw new IllegalArgumentException("Leaf node in the middle of path."
-              + " schema file.");
-        }
-        if (nodeTree.getType() == LayerType.ROOT) {
-          throw new IllegalArgumentException("Multiple root nodes are defined."
-              + " schema file.");
-        }
-        schemaList.add(nodeTree);
-        if (nodeTree.getSublayer() != null) {
-          nodeTree = nodeTree.getSublayer().get(0);
-        } else {
-          break;
-        }
-      }
-      finalSchema = new NodeSchemaLoadResult(schemaList, true);
-    } catch (Exception e) {
-      throw new IllegalArgumentException("Fail to load network topology node"
-          + " schema file: " + schemaFile + " , error:"
-          + e.getMessage(), e);
-    }
-
-    return finalSchema;
-  }
-
-  /**
-   * Load layoutVersion from root element in the XML configuration file.
-   * @param root root element
-   * @return layout version
-   */
-  private int loadLayoutVersion(Element root) {
-    int layoutVersion;
-    Text text = (Text) root.getElementsByTagName(LAYOUT_VERSION_TAG)
-        .item(0).getFirstChild();
-    if (text != null) {
-      String value = text.getData().trim();
-      try {
-        layoutVersion = Integer.parseInt(value);
-      } catch (NumberFormatException e) {
-        throw new IllegalArgumentException("Bad " + LAYOUT_VERSION_TAG +
-            " value " + value + " is found. It should be an integer.");
-      }
-    } else {
-      throw new IllegalArgumentException("Value of <" + LAYOUT_VERSION_TAG +
-          "> is null");
-    }
-    return layoutVersion;
-  }
-
-  /**
-   * Load layers from root element in the XML configuration file.
-   * @param root root element
-   * @return A map of node schemas with layer ID and layer schema
-   */
-  private Map<String, NodeSchema> loadLayersSection(Element root) {
-    NodeList elements = root.getElementsByTagName(LAYER_TAG);
-    Map<String, NodeSchema> schemas = new HashMap<String, NodeSchema>();
-    for (int i = 0; i < elements.getLength(); i++) {
-      Node node = elements.item(i);
-      if (node instanceof Element) {
-        Element element = (Element) node;
-        if (LAYER_TAG.equals(element.getTagName())) {
-          String layerId = element.getAttribute(LAYER_ID);
-          NodeSchema schema = parseLayerElement(element);
-          if (!schemas.containsValue(schema)) {
-            schemas.put(layerId, schema);
-          } else {
-            throw new IllegalArgumentException("Repetitive layer in network " +
-                "topology node schema configuration file: " + layerId);
-          }
-        } else {
-          throw new IllegalArgumentException("Bad element in network topology "
-              + "node schema configuration file: " + element.getTagName());
-        }
-      }
-    }
-
-    // Integrity check, only one ROOT and one LEAF is allowed
-    boolean foundRoot = false;
-    boolean foundLeaf = false;
-    for(NodeSchema schema: schemas.values()) {
-      if (schema.getType() == LayerType.ROOT) {
-        if (foundRoot) {
-          throw new IllegalArgumentException("Multiple ROOT layers are found" +
-              " in network topology schema configuration file");
-        } else {
-          foundRoot = true;
-        }
-      }
-      if (schema.getType() == LayerType.LEAF_NODE) {
-        if (foundLeaf) {
-          throw new IllegalArgumentException("Multiple LEAF layers are found" +
-              " in network topology schema configuration file");
-        } else {
-          foundLeaf = true;
-        }
-      }
-    }
-    if (!foundRoot) {
-      throw new IllegalArgumentException("No ROOT layer is found" +
-          " in network topology schema configuration file");
-    }
-    if (!foundLeaf) {
-      throw new IllegalArgumentException("No LEAF layer is found" +
-          " in network topology schema configuration file");
-    }
-    return schemas;
-  }
-
-  /**
-   * Load network topology from root element in the XML configuration file and
-   * sort node schemas according to the topology path.
-   * @param root root element
-   * @param schemas schema map
-   * @return all valid node schemas defined in schema file
-   */
-  private NodeSchemaLoadResult loadTopologySection(Element root,
-      Map<String, NodeSchema> schemas) {
-    NodeList elements = root.getElementsByTagName(TOPOLOGY_TAG)
-        .item(0).getChildNodes();
-    List<NodeSchema> schemaList = new ArrayList<NodeSchema>();
-    boolean enforecePrefix = false;
-    for (int i = 0; i < elements.getLength(); i++) {
-      Node node = elements.item(i);
-      if (node instanceof Element) {
-        Element element = (Element) node;
-        String tagName = element.getTagName();
-        // Get the nonnull text value.
-        Text text = (Text) element.getFirstChild();
-        String value;
-        if (text != null) {
-          value = text.getData().trim();
-          if (value.isEmpty()) {
-            // Element with empty value is ignored
-            continue;
-          }
-        } else {
-          throw new IllegalArgumentException("Value of <" + tagName
-              + "> is null");
-        }
-        if (TOPOLOGY_PATH.equals(tagName)) {
-          if(value.startsWith(NetConstants.PATH_SEPARATOR_STR)) {
-            value = value.substring(1, value.length());
-          }
-          String[] layerIDs = value.split(NetConstants.PATH_SEPARATOR_STR);
-          if (layerIDs == null || layerIDs.length != schemas.size()) {
-            throw new IllegalArgumentException("Topology path depth doesn't "
-                + "match layer element numbers");
-          }
-          for (int j = 0; j < layerIDs.length; j++) {
-            if (schemas.get(layerIDs[j]) == null) {
-              throw new IllegalArgumentException("No layer found for id " +
-                  layerIDs[j]);
-            }
-          }
-          if (schemas.get(layerIDs[0]).getType() != LayerType.ROOT) {
-            throw new IllegalArgumentException("Topology path doesn't start "
-                + "with ROOT layer");
-          }
-          if (schemas.get(layerIDs[layerIDs.length -1]).getType() !=
-              LayerType.LEAF_NODE) {
-            throw new IllegalArgumentException("Topology path doesn't end "
-                + "with LEAF layer");
-          }
-          for (int j = 0; j < layerIDs.length; j++) {
-            schemaList.add(schemas.get(layerIDs[j]));
-          }
-        } else if (TOPOLOGY_ENFORCE_PREFIX.equalsIgnoreCase(tagName)) {
-          enforecePrefix = Boolean.parseBoolean(value);
-        } else {
-          throw new IllegalArgumentException("Unsupported Element <" +
-              tagName + ">");
-        }
-      }
-    }
-    // Integrity check
-    if (enforecePrefix) {
-      // Every InnerNode should have prefix defined
-      for (NodeSchema schema: schemas.values()) {
-        if (schema.getType() == LayerType.INNER_NODE &&
-            schema.getPrefix() == null) {
-          throw new IllegalArgumentException("There is layer without prefix " +
-              "defined while prefix is enforced.");
-        }
-      }
-    }
-    return new NodeSchemaLoadResult(schemaList, enforecePrefix);
-  }
-
-  /**
-   * Load a layer from a layer element in the XML configuration file.
-   * @param element network topology node layer element
-   * @return ECSchema
-   */
-  private NodeSchema parseLayerElement(Element element) {
-    NodeList fields = element.getChildNodes();
-    LayerType type = null;
-    int cost = 0;
-    String prefix = null;
-    String defaultName = null;
-    for (int i = 0; i < fields.getLength(); i++) {
-      Node fieldNode = fields.item(i);
-      if (fieldNode instanceof Element) {
-        Element field = (Element) fieldNode;
-        String tagName = field.getTagName();
-        // Get the nonnull text value.
-        Text text = (Text) field.getFirstChild();
-        String value;
-        if (text != null) {
-          value = text.getData().trim();
-          if (value.isEmpty()) {
-            // Element with empty value is ignored
-            continue;
-          }
-        } else {
-          continue;
-        }
-        if (LAYER_COST.equalsIgnoreCase(tagName)) {
-          cost = Integer.parseInt(value);
-          if (cost < 0) {
-            throw new IllegalArgumentException(
-                "Cost should be positive number or 0");
-          }
-        } else if (LAYER_TYPE.equalsIgnoreCase(tagName)) {
-          type = NodeSchema.LayerType.getType(value);
-          if (type == null) {
-            throw new IllegalArgumentException(
-                "Unsupported layer type:" + value);
-          }
-        } else if (LAYER_PREFIX.equalsIgnoreCase(tagName)) {
-          prefix = value;
-        } else if (LAYER_DEFAULT_NAME.equalsIgnoreCase(tagName)) {
-          defaultName = value;
-        } else {
-          throw new IllegalArgumentException("Unsupported Element <" + tagName
-              + ">");
-        }
-      }
-    }
-    // type is a mandatory property
-    if (type == null) {
-      throw new IllegalArgumentException("Missing type Element");
-    }
-    return new NodeSchema(type, cost, prefix, defaultName);
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NodeSchemaManager.java

@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.scm.net.NodeSchemaLoader.NodeSchemaLoadResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/** The class manages all network topology schemas. */
-
-public final class NodeSchemaManager {
-  private static final Logger LOG = LoggerFactory.getLogger(
-      NodeSchemaManager.class);
-
-  // All schema saved and sorted from ROOT to LEAF node
-  private List<NodeSchema> allSchema;
-  // enforcePrefix only applies to INNER_NODE
-  private boolean enforcePrefix;
-  // max level, includes ROOT level
-  private int maxLevel = -1;
-
-  private volatile static NodeSchemaManager instance = null;
-
-  private NodeSchemaManager() {
-  }
-
-  public static NodeSchemaManager getInstance() {
-    if (instance == null) {
-      instance = new NodeSchemaManager();
-    }
-    return instance;
-  }
-
-  public void init(Configuration conf) {
-    /**
-     * Load schemas from network topology schema configuration file
-     */
-    String schemaFile = conf.get(
-        ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE,
-        ScmConfigKeys.OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT);
-    NodeSchemaLoadResult result;
-    try {
-      result = NodeSchemaLoader.getInstance().loadSchemaFromFile(schemaFile);
-      allSchema = result.getSchemaList();
-      enforcePrefix = result.isEnforePrefix();
-      maxLevel = allSchema.size();
-    } catch (Throwable e) {
-      String msg = "Failed to load schema file:" + schemaFile
-          + ", error: " + e.getMessage();
-      LOG.error(msg, e);
-      throw new RuntimeException(msg, e);
-    }
-  }
-
-  @VisibleForTesting
-  public void init(NodeSchema[] schemas, boolean enforce) {
-    allSchema = new ArrayList<>();
-    allSchema.addAll(Arrays.asList(schemas));
-    enforcePrefix = enforce;
-    maxLevel = schemas.length;
-  }
-
-  public int getMaxLevel() {
-    return maxLevel;
-  }
-
-  public int getCost(int level) {
-    Preconditions.checkArgument(level <= maxLevel &&
-        level >= (NetConstants.ROOT_LEVEL));
-    return allSchema.get(level - NetConstants.ROOT_LEVEL).getCost();
-  }
-
-  /**
-   * Given a incomplete network path, return its complete network path if
-   * possible. E.g. input is 'node1', output is '/rack-default/node1' if this
-   * schema manages ROOT, RACK and LEAF, with prefix defined and enforce prefix
-   * enabled.
-   *
-   * @param path the incomplete input path
-   * @return complete path, null if cannot carry out complete action or action
-   * failed
-   */
-  public String complete(String path) {
-    if (!enforcePrefix) {
-      return null;
-    }
-    String normalizedPath = NetUtils.normalize(path);
-    String[] subPath = normalizedPath.split(NetConstants.PATH_SEPARATOR_STR);
-    if ((subPath.length) == maxLevel) {
-      return path;
-    }
-    StringBuffer newPath = new StringBuffer(NetConstants.ROOT);
-    // skip the ROOT and LEAF layer
-    int i, j;
-    for (i = 1, j = 1; i < subPath.length && j < (allSchema.size() - 1);) {
-      if (allSchema.get(j).matchPrefix(subPath[i])) {
-        newPath.append(NetConstants.PATH_SEPARATOR_STR + subPath[i]);
-        i++;
-        j++;
-      } else {
-        newPath.append(allSchema.get(j).getDefaultName());
-        j++;
-      }
-    }
-    if (i == (subPath.length - 1)) {
-      newPath.append(NetConstants.PATH_SEPARATOR_STR + subPath[i]);
-      return newPath.toString();
-    }
-    return null;
-  }
-}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/package-info.java

@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdds.scm.net;
-/**
- The network topology supported by Ozone.
- */