|
@@ -61,6 +61,33 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
}
|
|
|
};
|
|
|
|
|
|
+ private static final ThreadLocal<HashMap<NodeNotChosenReason, Integer>>
|
|
|
+ CHOOSE_RANDOM_REASONS =
|
|
|
+ new ThreadLocal<HashMap<NodeNotChosenReason, Integer>>() {
|
|
|
+ @Override
|
|
|
+ protected HashMap<NodeNotChosenReason, Integer> initialValue() {
|
|
|
+ return new HashMap<>();
|
|
|
+ }
|
|
|
+ };
|
|
|
+
|
|
|
+ private enum NodeNotChosenReason {
|
|
|
+ NOT_IN_SERVICE("the node isn't in service"),
|
|
|
+ NODE_STALE("the node is stale"),
|
|
|
+ NODE_TOO_BUSY("the node is too busy"),
|
|
|
+ TOO_MANY_NODES_ON_RACK("the rack has too many chosen nodes"),
|
|
|
+ NOT_ENOUGH_STORAGE_SPACE("no enough storage space to place the block");
|
|
|
+
|
|
|
+ private final String text;
|
|
|
+
|
|
|
+ NodeNotChosenReason(final String logText) {
|
|
|
+ text = logText;
|
|
|
+ }
|
|
|
+
|
|
|
+ private String getText() {
|
|
|
+ return text;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
protected boolean considerLoad;
|
|
|
protected double considerLoadFactor;
|
|
|
private boolean preferLocalNode;
|
|
@@ -710,6 +737,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
builder.setLength(0);
|
|
|
builder.append("[");
|
|
|
}
|
|
|
+ CHOOSE_RANDOM_REASONS.get().clear();
|
|
|
boolean badTarget = false;
|
|
|
DatanodeStorageInfo firstChosen = null;
|
|
|
while (numOfReplicas > 0) {
|
|
@@ -780,14 +808,24 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
}
|
|
|
if (numOfReplicas>0) {
|
|
|
String detail = enableDebugLogging;
|
|
|
- if (LOG.isDebugEnabled()) {
|
|
|
- if (badTarget && builder != null) {
|
|
|
- detail = builder.toString();
|
|
|
+ if (LOG.isDebugEnabled() && builder != null) {
|
|
|
+ detail = builder.toString();
|
|
|
+ if (badTarget) {
|
|
|
builder.setLength(0);
|
|
|
} else {
|
|
|
+ if (detail.length() > 1) {
|
|
|
+ // only log if there's more than "[", which is always appended at
|
|
|
+ // the beginning of this method.
|
|
|
+ LOG.debug(detail);
|
|
|
+ }
|
|
|
detail = "";
|
|
|
}
|
|
|
}
|
|
|
+ final HashMap<NodeNotChosenReason, Integer> reasonMap =
|
|
|
+ CHOOSE_RANDOM_REASONS.get();
|
|
|
+ if (!reasonMap.isEmpty()) {
|
|
|
+ LOG.info("Not enough replicas was chosen. Reason:{}", reasonMap);
|
|
|
+ }
|
|
|
throw new NotEnoughReplicasException(detail);
|
|
|
}
|
|
|
|
|
@@ -833,19 +871,38 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
if (storage != null) {
|
|
|
results.add(storage);
|
|
|
} else {
|
|
|
- logNodeIsNotChosen(dnd, "no good storage to place the block ");
|
|
|
+ logNodeIsNotChosen(dnd, NodeNotChosenReason.NOT_ENOUGH_STORAGE_SPACE,
|
|
|
+ " for storage type " + storageType);
|
|
|
}
|
|
|
return storage;
|
|
|
}
|
|
|
|
|
|
private static void logNodeIsNotChosen(DatanodeDescriptor node,
|
|
|
- String reason) {
|
|
|
+ NodeNotChosenReason reason) {
|
|
|
+ logNodeIsNotChosen(node, reason, null);
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void logNodeIsNotChosen(DatanodeDescriptor node,
|
|
|
+ NodeNotChosenReason reason, String reasonDetails) {
|
|
|
+ assert reason != null;
|
|
|
if (LOG.isDebugEnabled()) {
|
|
|
// build the error message for later use.
|
|
|
debugLoggingBuilder.get()
|
|
|
.append("\n Datanode ").append(node)
|
|
|
- .append(" is not chosen since ").append(reason).append(".");
|
|
|
+ .append(" is not chosen since ").append(reason.getText());
|
|
|
+ if (reasonDetails != null) {
|
|
|
+ debugLoggingBuilder.get().append(" ").append(reasonDetails);
|
|
|
+ }
|
|
|
+ debugLoggingBuilder.get().append(".");
|
|
|
+ }
|
|
|
+ // always populate reason map to log high level reasons.
|
|
|
+ final HashMap<NodeNotChosenReason, Integer> reasonMap =
|
|
|
+ CHOOSE_RANDOM_REASONS.get();
|
|
|
+ Integer base = reasonMap.get(reason);
|
|
|
+ if (base == null) {
|
|
|
+ base = 0;
|
|
|
}
|
|
|
+ reasonMap.put(reason, base + 1);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -867,13 +924,13 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
boolean avoidStaleNodes) {
|
|
|
// check if the node is (being) decommissioned
|
|
|
if (!node.isInService()) {
|
|
|
- logNodeIsNotChosen(node, "the node isn't in service.");
|
|
|
+ logNodeIsNotChosen(node, NodeNotChosenReason.NOT_IN_SERVICE);
|
|
|
return false;
|
|
|
}
|
|
|
|
|
|
if (avoidStaleNodes) {
|
|
|
if (node.isStale(this.staleInterval)) {
|
|
|
- logNodeIsNotChosen(node, "the node is stale ");
|
|
|
+ logNodeIsNotChosen(node, NodeNotChosenReason.NODE_STALE);
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
@@ -884,8 +941,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
stats.getInServiceXceiverAverage();
|
|
|
final int nodeLoad = node.getXceiverCount();
|
|
|
if (nodeLoad > maxLoad) {
|
|
|
- logNodeIsNotChosen(node, "the node is too busy (load: " + nodeLoad
|
|
|
- + " > " + maxLoad + ") ");
|
|
|
+ logNodeIsNotChosen(node, NodeNotChosenReason.NODE_TOO_BUSY,
|
|
|
+ "(load: " + nodeLoad + " > " + maxLoad + ")");
|
|
|
return false;
|
|
|
}
|
|
|
}
|
|
@@ -900,7 +957,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
|
|
|
}
|
|
|
}
|
|
|
if (counter > maxTargetPerRack) {
|
|
|
- logNodeIsNotChosen(node, "the rack has too many chosen nodes ");
|
|
|
+ logNodeIsNotChosen(node, NodeNotChosenReason.TOO_MANY_NODES_ON_RACK);
|
|
|
return false;
|
|
|
}
|
|
|
|