HDDS-101. SCM CA: generate CSR for SCM CA clients.

Contributed by Xiaoyu Yao.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificates/CertificateSignRequest.java

@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.apache.logging.log4j.util.Strings;
+import org.bouncycastle.asn1.DEROctetString;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.BasicConstraints;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.GeneralName;
+import org.bouncycastle.asn1.x509.GeneralNames;
+import org.bouncycastle.asn1.x509.KeyUsage;
+import org.bouncycastle.operator.ContentSigner;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCS10CertificationRequestBuilder;
+import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequestBuilder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A certificate sign request object that wraps operations to build a
+ * PKCS10CertificationRequest to CA.
+ */
+public final class CertificateSignRequest {
+  private final KeyPair keyPair;
+  private final SecurityConfig config;
+  private final Extensions extensions;
+  private String subject;
+  private String clusterID;
+  private String scmID;
+
+  /**
+   * Private Ctor for CSR.
+   *
+   * @param subject - Subject
+   * @param scmID - SCM ID
+   * @param clusterID - Cluster ID
+   * @param keyPair - KeyPair
+   * @param config - SCM Config
+   * @param extensions - CSR extensions
+   */
+  private CertificateSignRequest(String subject, String scmID, String clusterID,
+      KeyPair keyPair, SecurityConfig config, Extensions extensions) {
+    this.subject = subject;
+    this.clusterID = clusterID;
+    this.scmID = scmID;
+    this.keyPair = keyPair;
+    this.config = config;
+    this.extensions = extensions;
+  }
+
+  private PKCS10CertificationRequest generateCSR() throws
+      OperatorCreationException {
+    X500Name dnName = SecurityUtil.getDistinguishedName(subject, scmID,
+        clusterID);
+    PKCS10CertificationRequestBuilder p10Builder =
+        new JcaPKCS10CertificationRequestBuilder(dnName, keyPair.getPublic());
+
+    ContentSigner contentSigner =
+        new JcaContentSignerBuilder(config.getSignatureAlgo())
+            .setProvider(config.getProvider())
+            .build(keyPair.getPrivate());
+
+    if (extensions != null) {
+      p10Builder.addAttribute(
+          PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extensions);
+    }
+    return p10Builder.build(contentSigner);
+  }
+
+  /**
+   * Builder class for Certificate Sign Request.
+   */
+  public static class Builder {
+    private String subject;
+    private String clusterID;
+    private String scmID;
+    private KeyPair key;
+    private SecurityConfig config;
+    private List<GeneralName> altNames;
+    private Boolean ca = false;
+
+    public CertificateSignRequest.Builder setConfiguration(
+        Configuration configuration) {
+      this.config = new SecurityConfig(configuration);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setKey(KeyPair keyPair) {
+      this.key = keyPair;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setSubject(String subjectString) {
+      this.subject = subjectString;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setClusterID(String s) {
+      this.clusterID = s;
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setScmID(String s) {
+      this.scmID = s;
+      return this;
+    }
+
+    // Support SAN extenion with DNS and RFC822 Name
+    // other name type will be added as needed.
+    public CertificateSignRequest.Builder addDnsName(String dnsName) {
+      Preconditions.checkNotNull(dnsName, "dnsName cannot be null");
+      this.addAltName(GeneralName.dNSName, dnsName);
+      return this;
+    }
+
+    public CertificateSignRequest.Builder addRfc822Name(String name) {
+      Preconditions.checkNotNull(name, "Rfc822Name cannot be null");
+      this.addAltName(GeneralName.rfc822Name, name);
+      return this;
+    }
+
+    // IP address is subject to change which is optional for now.
+    public CertificateSignRequest.Builder addIpAddress(String ip) {
+      Preconditions.checkNotNull(ip, "Ip address cannot be null");
+      this.addAltName(GeneralName.iPAddress, ip);
+      return this;
+    }
+
+    private CertificateSignRequest.Builder addAltName(int tag, String name) {
+      if (altNames == null) {
+        altNames = new ArrayList<>();
+      }
+      altNames.add(new GeneralName(tag, name));
+      return this;
+    }
+
+    public CertificateSignRequest.Builder setCA(Boolean isCA) {
+      this.ca = isCA;
+      return this;
+    }
+
+    private Extension getKeyUsageExtension() throws IOException {
+      int keyUsageFlag = KeyUsage.digitalSignature | KeyUsage.keyEncipherment
+          | KeyUsage.dataEncipherment | KeyUsage.keyAgreement;
+
+      if (ca) {
+        keyUsageFlag |= KeyUsage.keyCertSign | KeyUsage.cRLSign;
+      }
+      KeyUsage keyUsage = new KeyUsage(keyUsageFlag);
+      return new Extension(Extension.keyUsage, true,
+          new DEROctetString(keyUsage));
+    }
+
+    private Optional<Extension> getSubjectAltNameExtension() throws
+        IOException {
+      if (altNames != null) {
+        return Optional.of(new Extension(Extension.subjectAlternativeName,
+            true, new DEROctetString(new GeneralNames(
+            altNames.toArray(new GeneralName[altNames.size()])))));
+      }
+      return Optional.empty();
+    }
+
+    private Extension getBasicExtension() throws IOException {
+      // We don't set pathLenConstraint means no limit is imposed.
+      return new Extension(Extension.basicConstraints,
+          true, new DEROctetString(new BasicConstraints(ca)));
+    }
+
+    private Extensions createExtensions() throws IOException {
+      List<Extension> extensions = new ArrayList<>();
+
+      // Add basic extension
+      extensions.add(getBasicExtension());
+
+      // Add key usage extension
+      extensions.add(getKeyUsageExtension());
+
+      // Add subject alternate name extension
+      Optional<Extension> san = getSubjectAltNameExtension();
+      if (san.isPresent()) {
+        extensions.add(san.get());
+      }
+
+      return new Extensions(
+          extensions.toArray(new Extension[extensions.size()]));
+    }
+
+    public PKCS10CertificationRequest build() throws SCMSecurityException {
+      Preconditions.checkNotNull(key, "KeyPair cannot be null");
+      Preconditions.checkArgument(Strings.isNotBlank(subject), "Subject " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(clusterID), "Cluster ID " +
+          "cannot be blank");
+      Preconditions.checkArgument(Strings.isNotBlank(scmID), "SCM ID cannot " +
+          "be blank");
+
+      try {
+        CertificateSignRequest csr = new CertificateSignRequest(subject, scmID,
+            clusterID, key, config, createExtensions());
+        return csr.generateCSR();
+      } catch (IOException ioe) {
+        throw new CertificateException(String.format("Unable to create " +
+            "extension for certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)), ioe.getCause());
+      } catch (OperatorCreationException ex) {
+        throw new CertificateException(String.format("Unable to create " +
+            "certificate sign request for %s.", SecurityUtil
+            .getDistinguishedName(subject, scmID, clusterID)),
+            ex.getCause());
+      }
+    }
+  }
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/HDDSKeyGenerator.java

@@ -96,7 +96,7 @@ public class HDDSKeyGenerator {
    */
   public KeyPair generateKey(int size, String algorithm, String provider)
       throws NoSuchProviderException, NoSuchAlgorithmException {
-    LOG.info("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
+    LOG.debug("Generating key pair using size:{}, Algorithm:{}, Provider:{}",
         size, algorithm, provider);
     KeyPairGenerator generator = KeyPairGenerator
         .getInstance(algorithm, provider);

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/keys/SecurityUtil.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.hdds.security.x509.keys;
+
+import org.apache.hadoop.hdds.security.x509.exceptions.CertificateException;
+import org.bouncycastle.asn1.ASN1ObjectIdentifier;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.ASN1Set;
+import org.bouncycastle.asn1.pkcs.Attribute;
+import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+
+/**
+ * Utility functions for Security modules for Ozone.
+ */
+public final class SecurityUtil {
+
+  // Ozone Certificate distinguished format: (CN=Subject,OU=ScmID,O=ClusterID).
+  private static final String DISTINGUISHED_NAME_FORMAT = "CN=%s,OU=%s,O=%s";
+
+  private SecurityUtil() {
+  }
+
+  public static String getDistinguishedNameFormat() {
+    return DISTINGUISHED_NAME_FORMAT;
+  }
+
+  public static X500Name getDistinguishedName(String subject, String scmID,
+      String clusterID) {
+    return new X500Name(String.format(getDistinguishedNameFormat(), subject,
+        scmID, clusterID));
+  }
+
+  // TODO: move the PKCS10CSRValidator class
+  public static Extensions getPkcs9Extensions(PKCS10CertificationRequest csr)
+      throws CertificateException {
+    ASN1Set pkcs9ExtReq = getPkcs9ExtRequest(csr);
+    Object extReqElement = pkcs9ExtReq.getObjects().nextElement();
+    if (extReqElement instanceof Extensions) {
+      return (Extensions) extReqElement;
+    } else {
+      if (extReqElement instanceof ASN1Sequence) {
+        return Extensions.getInstance((ASN1Sequence) extReqElement);
+      } else {
+        throw new CertificateException("Unknown element type :" + extReqElement
+            .getClass().getSimpleName());
+      }
+    }
+  }
+
+  public static ASN1Set getPkcs9ExtRequest(PKCS10CertificationRequest csr)
+      throws CertificateException {
+    for (Attribute attr : csr.getAttributes()) {
+      ASN1ObjectIdentifier oid = attr.getAttrType();
+      if (oid.equals(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest)) {
+        return attr.getAttrValues();
+      }
+    }
+    throw new CertificateException("No PKCS#9 extension found in CSR");
+  }
+}

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/security/x509/certificates/TestCertificateSignRequest.java

@@ -0,0 +1,268 @@
+package org.apache.hadoop.hdds.security.x509.certificates;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.hdds.security.x509.exceptions.SCMSecurityException;
+import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
+import org.apache.hadoop.hdds.security.x509.keys.SecurityUtil;
+import org.bouncycastle.asn1.ASN1Sequence;
+import org.bouncycastle.asn1.x509.Extension;
+import org.bouncycastle.asn1.x509.Extensions;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.bouncycastle.operator.ContentVerifierProvider;
+import org.bouncycastle.operator.OperatorCreationException;
+import org.bouncycastle.operator.jcajce.JcaContentVerifierProviderBuilder;
+import org.bouncycastle.pkcs.PKCS10CertificationRequest;
+import org.bouncycastle.pkcs.PKCSException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.security.KeyPair;
+import java.security.NoSuchAlgorithmException;
+import java.security.NoSuchProviderException;
+import java.util.UUID;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
+
+public class TestCertificateSignRequest {
+
+  private SecurityConfig securityConfig;
+  private static OzoneConfiguration conf = new OzoneConfiguration();
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Before
+  public void init() throws IOException {
+    conf.set(OZONE_METADATA_DIRS, temporaryFolder.newFolder().toString());
+    securityConfig = new SecurityConfig(conf);
+  }
+
+  @Test
+  public void testGenerateCSR() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException,
+      OperatorCreationException, PKCSException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+    Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+    // Verify basic constraints extension
+    Extension basicExt = extensions.getExtension(Extension
+        .basicConstraints);
+    Assert.assertEquals(true, basicExt.isCritical());
+
+    // Verify key usage extension
+    Extension keyUsageExt = extensions.getExtension(Extension.keyUsage);
+    Assert.assertEquals(true, keyUsageExt.isCritical());
+
+
+    // Verify San extension not set
+    Assert.assertEquals(null,
+        extensions.getExtension(Extension.subjectAlternativeName));
+
+    // Verify signature in CSR
+    ContentVerifierProvider verifierProvider =
+        new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+            .getProvider()).build(csr.getSubjectPublicKeyInfo());
+    Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+  }
+
+  @Test
+  public void testGenerateCSRwithSan() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException,
+      OperatorCreationException, PKCSException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+
+    // Multi-home
+    builder.addIpAddress("192.168.1.1");
+    builder.addIpAddress("192.168.2.1");
+
+    builder.addDnsName("dn1.abc.com");
+    builder.addRfc822Name("test@abc.com");
+
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+    Extensions extensions = SecurityUtil.getPkcs9Extensions(csr);
+
+    // Verify key usage extension
+    Extension sanExt = extensions.getExtension(Extension.keyUsage);
+    Assert.assertEquals(true, sanExt.isCritical());
+
+
+    // Verify signature in CSR
+    ContentVerifierProvider verifierProvider =
+        new JcaContentVerifierProviderBuilder().setProvider(securityConfig
+            .getProvider()).build(csr.getSubjectPublicKeyInfo());
+    Assert.assertEquals(true, csr.isSignatureValid(verifierProvider));
+  }
+
+  @Test
+  public void testGenerateCSRWithInvalidParams() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+
+    try {
+      builder.setKey(null);
+      builder.build();
+      Assert.fail("Null Key should have failed.");
+    } catch (NullPointerException | IllegalArgumentException e) {
+      builder.setKey(keyPair);
+    }
+
+    // Now try with blank/null Subject.
+    try {
+      builder.setSubject(null);
+      builder.build();
+      Assert.fail("Null/Blank Subject should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setSubject(subject);
+    }
+
+    try {
+      builder.setSubject("");
+      builder.build();
+      Assert.fail("Null/Blank Subject should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setSubject(subject);
+    }
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setScmID(null);
+      builder.build();
+      Assert.fail("Null/Blank SCM ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setScmID(scmID);
+    }
+
+    // Now try with blank/null SCM ID
+    try {
+      builder.setClusterID(null);
+      builder.build();
+      Assert.fail("Null/Blank Cluster ID should have thrown.");
+    } catch (IllegalArgumentException e) {
+      builder.setClusterID(clusterID);
+    }
+
+    // Now try with invalid IP address
+    try {
+      builder.addIpAddress("255.255.255.*");
+      builder.build();
+      Assert.fail("Invalid ip address");
+    } catch (IllegalArgumentException e) {
+    }
+
+    PKCS10CertificationRequest csr = builder.build();
+
+    // Check the Subject Name is in the expected format.
+    String dnName = String.format(SecurityUtil.getDistinguishedNameFormat(),
+        subject, scmID, clusterID);
+    Assert.assertEquals(csr.getSubject().toString(), dnName);
+
+    // Verify the public key info match
+    byte[] encoded = keyPair.getPublic().getEncoded();
+    SubjectPublicKeyInfo subjectPublicKeyInfo =
+        SubjectPublicKeyInfo.getInstance(ASN1Sequence.getInstance(encoded));
+    SubjectPublicKeyInfo csrPublicKeyInfo = csr.getSubjectPublicKeyInfo();
+    Assert.assertEquals(csrPublicKeyInfo, subjectPublicKeyInfo);
+
+    // Verify CSR with attribute for extensions
+    Assert.assertEquals(1, csr.getAttributes().length);
+  }
+  
+  @Test
+  public void testCsrSerialization() throws NoSuchProviderException,
+      NoSuchAlgorithmException, SCMSecurityException, IOException {
+    String clusterID = UUID.randomUUID().toString();
+    String scmID = UUID.randomUUID().toString();
+    String subject = "DN001";
+    HDDSKeyGenerator keyGen =
+        new HDDSKeyGenerator(securityConfig.getConfiguration());
+    KeyPair keyPair = keyGen.generateKey();
+
+    CertificateSignRequest.Builder builder =
+        new CertificateSignRequest.Builder()
+            .setSubject(subject)
+            .setScmID(scmID)
+            .setClusterID(clusterID)
+            .setKey(keyPair)
+            .setConfiguration(conf);
+    PKCS10CertificationRequest csr = builder.build();
+    byte[] csrBytes = csr.getEncoded();
+
+    // Verify de-serialized CSR matches with the original CSR
+    PKCS10CertificationRequest dsCsr = new PKCS10CertificationRequest(csrBytes);
+    Assert.assertEquals(csr, dsCsr);
+  }
+}