|
@@ -20,11 +20,10 @@ package org.apache.hadoop.hdfs.server.namenode;
|
|
import org.apache.hadoop.classification.InterfaceAudience;
|
|
import org.apache.hadoop.classification.InterfaceAudience;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.conf.Configuration;
|
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
|
|
+import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
|
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
|
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
|
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
|
|
|
|
|
|
|
|
-import java.util.Arrays;
|
|
|
|
import java.util.Map;
|
|
import java.util.Map;
|
|
import java.util.TreeMap;
|
|
import java.util.TreeMap;
|
|
import java.util.stream.Collectors;
|
|
import java.util.stream.Collectors;
|
|
@@ -40,50 +39,12 @@ import java.util.stream.Collectors;
|
|
@InterfaceAudience.LimitedPrivate({"HDFS"})
|
|
@InterfaceAudience.LimitedPrivate({"HDFS"})
|
|
public final class ErasureCodingPolicyManager {
|
|
public final class ErasureCodingPolicyManager {
|
|
|
|
|
|
- /**
|
|
|
|
- * TODO: HDFS-8095.
|
|
|
|
- */
|
|
|
|
- private static final int DEFAULT_CELLSIZE = 64 * 1024;
|
|
|
|
- private static final ErasureCodingPolicy SYS_POLICY1 =
|
|
|
|
- new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_SCHEMA,
|
|
|
|
- DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_POLICY_ID);
|
|
|
|
- private static final ErasureCodingPolicy SYS_POLICY2 =
|
|
|
|
- new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA,
|
|
|
|
- DEFAULT_CELLSIZE, HdfsConstants.RS_3_2_POLICY_ID);
|
|
|
|
- private static final ErasureCodingPolicy SYS_POLICY3 =
|
|
|
|
- new ErasureCodingPolicy(ErasureCodeConstants.RS_6_3_LEGACY_SCHEMA,
|
|
|
|
- DEFAULT_CELLSIZE, HdfsConstants.RS_6_3_LEGACY_POLICY_ID);
|
|
|
|
- private static final ErasureCodingPolicy SYS_POLICY4 =
|
|
|
|
- new ErasureCodingPolicy(ErasureCodeConstants.XOR_2_1_SCHEMA,
|
|
|
|
- DEFAULT_CELLSIZE, HdfsConstants.XOR_2_1_POLICY_ID);
|
|
|
|
- private static final ErasureCodingPolicy SYS_POLICY5 =
|
|
|
|
- new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA,
|
|
|
|
- DEFAULT_CELLSIZE, HdfsConstants.RS_10_4_POLICY_ID);
|
|
|
|
-
|
|
|
|
- //We may add more later.
|
|
|
|
- private static final ErasureCodingPolicy[] SYS_POLICIES =
|
|
|
|
- new ErasureCodingPolicy[]{SYS_POLICY1, SYS_POLICY2, SYS_POLICY3,
|
|
|
|
- SYS_POLICY4, SYS_POLICY5};
|
|
|
|
-
|
|
|
|
// Supported storage policies for striped EC files
|
|
// Supported storage policies for striped EC files
|
|
private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE =
|
|
private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE =
|
|
new byte[]{
|
|
new byte[]{
|
|
HdfsConstants.HOT_STORAGE_POLICY_ID,
|
|
HdfsConstants.HOT_STORAGE_POLICY_ID,
|
|
HdfsConstants.COLD_STORAGE_POLICY_ID,
|
|
HdfsConstants.COLD_STORAGE_POLICY_ID,
|
|
HdfsConstants.ALLSSD_STORAGE_POLICY_ID};
|
|
HdfsConstants.ALLSSD_STORAGE_POLICY_ID};
|
|
- /**
|
|
|
|
- * All supported policies maintained in NN memory for fast querying,
|
|
|
|
- * identified and sorted by its name.
|
|
|
|
- */
|
|
|
|
- private static final Map<String, ErasureCodingPolicy> SYSTEM_POLICIES_BY_NAME;
|
|
|
|
-
|
|
|
|
- static {
|
|
|
|
- // Create a hashmap of all available policies for quick lookup by name
|
|
|
|
- SYSTEM_POLICIES_BY_NAME = new TreeMap<>();
|
|
|
|
- for (ErasureCodingPolicy policy : SYS_POLICIES) {
|
|
|
|
- SYSTEM_POLICIES_BY_NAME.put(policy.getName(), policy);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
|
|
|
|
/**
|
|
/**
|
|
* All enabled policies maintained in NN memory for fast querying,
|
|
* All enabled policies maintained in NN memory for fast querying,
|
|
@@ -101,9 +62,10 @@ public final class ErasureCodingPolicyManager {
|
|
if (policyName.trim().isEmpty()) {
|
|
if (policyName.trim().isEmpty()) {
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
- ErasureCodingPolicy ecPolicy = SYSTEM_POLICIES_BY_NAME.get(policyName);
|
|
|
|
|
|
+ ErasureCodingPolicy ecPolicy =
|
|
|
|
+ SystemErasureCodingPolicies.getByName(policyName);
|
|
if (ecPolicy == null) {
|
|
if (ecPolicy == null) {
|
|
- String sysPolicies = Arrays.asList(SYS_POLICIES).stream()
|
|
|
|
|
|
+ String sysPolicies = SystemErasureCodingPolicies.getPolicies().stream()
|
|
.map(ErasureCodingPolicy::getName)
|
|
.map(ErasureCodingPolicy::getName)
|
|
.collect(Collectors.joining(", "));
|
|
.collect(Collectors.joining(", "));
|
|
String msg = String.format("EC policy '%s' specified at %s is not a " +
|
|
String msg = String.format("EC policy '%s' specified at %s is not a " +
|
|
@@ -124,35 +86,6 @@ public final class ErasureCodingPolicyManager {
|
|
*/
|
|
*/
|
|
}
|
|
}
|
|
|
|
|
|
- /**
|
|
|
|
- * Get system defined policies.
|
|
|
|
- * @return system policies
|
|
|
|
- */
|
|
|
|
- public static ErasureCodingPolicy[] getSystemPolicies() {
|
|
|
|
- return SYS_POLICIES;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Get a policy by policy ID.
|
|
|
|
- * @return ecPolicy, or null if not found
|
|
|
|
- */
|
|
|
|
- public static ErasureCodingPolicy getPolicyByID(byte id) {
|
|
|
|
- for (ErasureCodingPolicy policy : SYS_POLICIES) {
|
|
|
|
- if (policy.getId() == id) {
|
|
|
|
- return policy;
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
- return null;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- /**
|
|
|
|
- * Get a policy by policy name.
|
|
|
|
- * @return ecPolicy, or null if not found
|
|
|
|
- */
|
|
|
|
- public static ErasureCodingPolicy getPolicyByName(String name) {
|
|
|
|
- return SYSTEM_POLICIES_BY_NAME.get(name);
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
/**
|
|
/**
|
|
* Get the set of enabled policies.
|
|
* Get the set of enabled policies.
|
|
* @return all policies
|
|
* @return all policies
|