|
@@ -21,12 +21,21 @@ package org.apache.hadoop.ozone.container.common.impl;
|
|
import com.google.common.annotations.VisibleForTesting;
|
|
import com.google.common.annotations.VisibleForTesting;
|
|
import com.google.common.base.Preconditions;
|
|
import com.google.common.base.Preconditions;
|
|
import com.google.common.collect.Maps;
|
|
import com.google.common.collect.Maps;
|
|
|
|
+import org.apache.hadoop.conf.StorageUnit;
|
|
|
|
+import org.apache.hadoop.hdds.HddsUtils;
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
|
|
|
|
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
|
|
|
|
+import org.apache.hadoop.hdds.protocol.proto
|
|
|
|
+ .StorageContainerDatanodeProtocolProtos.ContainerInfo;
|
|
|
|
+import org.apache.hadoop.hdds.protocol.proto
|
|
|
|
+ .StorageContainerDatanodeProtocolProtos.ContainerAction;
|
|
|
|
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
|
|
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
|
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
|
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.Container;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.Container;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
|
|
|
|
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
|
|
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
|
|
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
@@ -35,11 +44,14 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
.ContainerCommandResponseProto;
|
|
.ContainerCommandResponseProto;
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
.ContainerType;
|
|
.ContainerType;
|
|
|
|
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
|
|
|
|
+ .ContainerLifeCycleState;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
|
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.slf4j.LoggerFactory;
|
|
|
|
|
|
import java.util.Map;
|
|
import java.util.Map;
|
|
|
|
+import java.util.Optional;
|
|
|
|
|
|
/**
|
|
/**
|
|
* Ozone Container dispatcher takes a call from the netty server and routes it
|
|
* Ozone Container dispatcher takes a call from the netty server and routes it
|
|
@@ -53,6 +65,8 @@ public class HddsDispatcher implements ContainerDispatcher {
|
|
private final Configuration conf;
|
|
private final Configuration conf;
|
|
private final ContainerSet containerSet;
|
|
private final ContainerSet containerSet;
|
|
private final VolumeSet volumeSet;
|
|
private final VolumeSet volumeSet;
|
|
|
|
+ private final StateContext context;
|
|
|
|
+ private final float containerCloseThreshold;
|
|
private String scmID;
|
|
private String scmID;
|
|
private ContainerMetrics metrics;
|
|
private ContainerMetrics metrics;
|
|
|
|
|
|
@@ -61,10 +75,11 @@ public class HddsDispatcher implements ContainerDispatcher {
|
|
* XceiverServerHandler.
|
|
* XceiverServerHandler.
|
|
*/
|
|
*/
|
|
public HddsDispatcher(Configuration config, ContainerSet contSet,
|
|
public HddsDispatcher(Configuration config, ContainerSet contSet,
|
|
- VolumeSet volumes) {
|
|
|
|
|
|
+ VolumeSet volumes, StateContext context) {
|
|
this.conf = config;
|
|
this.conf = config;
|
|
this.containerSet = contSet;
|
|
this.containerSet = contSet;
|
|
this.volumeSet = volumes;
|
|
this.volumeSet = volumes;
|
|
|
|
+ this.context = context;
|
|
this.handlers = Maps.newHashMap();
|
|
this.handlers = Maps.newHashMap();
|
|
this.metrics = ContainerMetrics.create(conf);
|
|
this.metrics = ContainerMetrics.create(conf);
|
|
for (ContainerType containerType : ContainerType.values()) {
|
|
for (ContainerType containerType : ContainerType.values()) {
|
|
@@ -72,6 +87,9 @@ public class HddsDispatcher implements ContainerDispatcher {
|
|
Handler.getHandlerForContainerType(
|
|
Handler.getHandlerForContainerType(
|
|
containerType, conf, containerSet, volumeSet, metrics));
|
|
containerType, conf, containerSet, volumeSet, metrics));
|
|
}
|
|
}
|
|
|
|
+ this.containerCloseThreshold = conf.getFloat(
|
|
|
|
+ ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
|
|
|
|
+ ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
|
|
|
|
|
|
}
|
|
}
|
|
|
|
|
|
@@ -113,7 +131,11 @@ public class HddsDispatcher implements ContainerDispatcher {
|
|
} catch (StorageContainerException ex) {
|
|
} catch (StorageContainerException ex) {
|
|
return ContainerUtils.logAndReturnError(LOG, ex, msg);
|
|
return ContainerUtils.logAndReturnError(LOG, ex, msg);
|
|
}
|
|
}
|
|
-
|
|
|
|
|
|
+ // Small performance optimization. We check if the operation is of type
|
|
|
|
+ // write before trying to send CloseContainerAction.
|
|
|
|
+ if (!HddsUtils.isReadOnly(msg)) {
|
|
|
|
+ sendCloseContainerActionIfNeeded(container);
|
|
|
|
+ }
|
|
Handler handler = getHandler(containerType);
|
|
Handler handler = getHandler(containerType);
|
|
if (handler == null) {
|
|
if (handler == null) {
|
|
StorageContainerException ex = new StorageContainerException("Invalid " +
|
|
StorageContainerException ex = new StorageContainerException("Invalid " +
|
|
@@ -130,6 +152,43 @@ public class HddsDispatcher implements ContainerDispatcher {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ /**
|
|
|
|
+ * If the container usage reaches the close threshold we send Close
|
|
|
|
+ * ContainerAction to SCM.
|
|
|
|
+ *
|
|
|
|
+ * @param container current state of container
|
|
|
|
+ */
|
|
|
|
+ private void sendCloseContainerActionIfNeeded(Container container) {
|
|
|
|
+ // We have to find a more efficient way to close a container.
|
|
|
|
+ Boolean isOpen = Optional.ofNullable(container)
|
|
|
|
+ .map(cont -> cont.getContainerState() == ContainerLifeCycleState.OPEN)
|
|
|
|
+ .orElse(Boolean.FALSE);
|
|
|
|
+ if (isOpen) {
|
|
|
|
+ ContainerData containerData = container.getContainerData();
|
|
|
|
+ double containerUsedPercentage = 1.0f * containerData.getBytesUsed() /
|
|
|
|
+ StorageUnit.GB.toBytes(containerData.getMaxSizeGB());
|
|
|
|
+ if (containerUsedPercentage >= containerCloseThreshold) {
|
|
|
|
+
|
|
|
|
+ ContainerInfo containerInfo = ContainerInfo.newBuilder()
|
|
|
|
+ .setContainerID(containerData.getContainerID())
|
|
|
|
+ .setReadCount(containerData.getReadCount())
|
|
|
|
+ .setWriteCount(containerData.getWriteCount())
|
|
|
|
+ .setReadBytes(containerData.getReadBytes())
|
|
|
|
+ .setWriteBytes(containerData.getWriteBytes())
|
|
|
|
+ .setUsed(containerData.getBytesUsed())
|
|
|
|
+ .setState(HddsProtos.LifeCycleState.OPEN)
|
|
|
|
+ .build();
|
|
|
|
+
|
|
|
|
+ ContainerAction action = ContainerAction.newBuilder()
|
|
|
|
+ .setContainer(containerInfo)
|
|
|
|
+ .setAction(ContainerAction.Action.CLOSE)
|
|
|
|
+ .setReason(ContainerAction.Reason.CONTAINER_FULL)
|
|
|
|
+ .build();
|
|
|
|
+ context.addContainerActionIfAbsent(action);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
@Override
|
|
@Override
|
|
public Handler getHandler(ContainerProtos.ContainerType containerType) {
|
|
public Handler getHandler(ContainerProtos.ContainerType containerType) {
|
|
return handlers.get(containerType);
|
|
return handlers.get(containerType);
|