HDDS-516. Implement CopyObject REST endpoint. Contributed by Bharat Viswanadham.

hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot

@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       S3 gateway test with aws cli
+Library             OperatingSystem
+Library             String
+Resource            ../commonlib.robot
+Resource            commonawslib.robot
+Test Setup          Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL}       http://s3g:9878
+${BUCKET}             generated
+${DESTBUCKET}         generated1
+
+
+*** Keywords ***
+Create Dest Bucket
+
+    ${postfix} =         Generate Random String  5  [NUMBERS]
+    Set Suite Variable   ${DESTBUCKET}             destbucket-${postfix}
+    Execute AWSS3APICli  create-bucket --bucket ${DESTBUCKET}
+
+
+*** Test Cases ***
+Copy Object Happy Scenario
+    Run Keyword if    '${DESTBUCKET}' == 'generated1'    Create Dest Bucket
+                        Execute                    date > /tmp/copyfile
+    ${result} =         Execute AWSS3ApiCli        put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile
+    ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${BUCKET} --prefix copyobject/
+                        Should contain             ${result}         f1
+
+    ${result} =         Execute AWSS3ApiCli        copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+    ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${DESTBUCKET} --prefix copyobject/
+                        Should contain             ${result}         f1
+    #copying again will not throw error
+    ${result} =         Execute AWSS3ApiCli        copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1
+    ${result} =         Execute AWSS3ApiCli        list-objects --bucket ${DESTBUCKET} --prefix copyobject/
+                        Should contain             ${result}         f1
+
+Copy Object Where Bucket is not available
+    ${result} =         Execute AWSS3APICli and checkrc        copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1      255
+                        Should contain             ${result}        NoSuchBucket
+    ${result} =         Execute AWSS3APICli and checkrc        copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1  255
+                        Should contain             ${result}        NoSuchBucket
+
+Copy Object Where both source and dest are same
+     ${result} =         Execute AWSS3APICli and checkrc        copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1      255
+                         Should contain             ${result}        InvalidRequest
+
+Copy Object Where Key not available
+    ${result} =         Execute AWSS3APICli and checkrc        copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey       255
+                        Should contain             ${result}        NoSuchKey

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyObjectResponse.java

@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3.endpoint;
+
+import org.apache.hadoop.ozone.s3.commontypes.IsoDateAdapter;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import java.time.Instant;
+
+/**
+ * Copy object Response.
+ */
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlRootElement(name = "ListAllMyBucketsResult",
+    namespace = "http://s3.amazonaws.com/doc/2006-03-01/")
+public class CopyObjectResponse {
+
+  @XmlJavaTypeAdapter(IsoDateAdapter.class)
+  @XmlElement(name = "LastModified")
+  private Instant lastModified;
+
+  @XmlElement(name = "ETag")
+  private String eTag;
+
+
+  public Instant getLastModified() {
+    return lastModified;
+  }
+
+  public void setLastModified(Instant lastModified) {
+    this.lastModified = lastModified;
+  }
+
+  public String getETag() {
+    return eTag;
+  }
+
+  public void setETag(String tag) {
+    this.eTag = tag;
+  }
+
+
+}

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -41,11 +41,8 @@ import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.io.OzoneInputStream;
@@ -56,6 +53,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
 
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.ozone.web.utils.OzoneUtils;
 import org.apache.http.HttpStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -97,18 +95,27 @@ public class ObjectEndpoint extends EndpointBase {
           ReplicationType replicationType,
       @DefaultValue("ONE") @QueryParam("replicationFactor")
           ReplicationFactor replicationFactor,
-      @DefaultValue("32 * 1024 * 1024") @QueryParam("chunkSize")
-          String chunkSize,
       @HeaderParam("Content-Length") long length,
       InputStream body) throws IOException, OS3Exception {
 
+    OzoneOutputStream output = null;
     try {
-      Configuration config = new OzoneConfiguration();
-      config.set(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize);
+      String copyHeader = headers.getHeaderString("x-amz-copy-source");
 
+      if (copyHeader != null) {
+        //Copy object, as copy source available.
+        CopyObjectResponse copyObjectResponse = copyObject(
+            copyHeader, bucketName, keyPath, replicationType,
+            replicationFactor);
+        return Response.status(Status.OK).entity(copyObjectResponse).header(
+            "Connection", "close").build();
+      }
+
+      // Normal put object
       OzoneBucket bucket = getBucket(bucketName);
-      OzoneOutputStream output = bucket
-          .createKey(keyPath, length, replicationType, replicationFactor);
+
+      output = bucket.createKey(keyPath, length, replicationType,
+          replicationFactor);
 
       if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"
           .equals(headers.getHeaderString("x-amz-content-sha256"))) {
@@ -116,13 +123,16 @@ public class ObjectEndpoint extends EndpointBase {
       }
 
       IOUtils.copy(body, output);
-      output.close();
 
       return Response.ok().status(HttpStatus.SC_OK)
           .build();
     } catch (IOException ex) {
       LOG.error("Exception occurred in PutObject", ex);
       throw ex;
+    } finally {
+      if (output != null) {
+        output.close();
+      }
     }
   }
 
@@ -239,4 +249,86 @@ public class ObjectEndpoint extends EndpointBase {
   public void setHeaders(HttpHeaders headers) {
     this.headers = headers;
   }
+
+  private CopyObjectResponse copyObject(String copyHeader,
+                                        String destBucket,
+                                        String destkey,
+                                        ReplicationType replicationType,
+                                        ReplicationFactor replicationFactor)
+      throws OS3Exception, IOException {
+
+    if (copyHeader.startsWith("/")) {
+      copyHeader = copyHeader.substring(1);
+    }
+    int pos = copyHeader.indexOf("/");
+    if (pos == -1) {
+      OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+          .INVALID_ARGUMENT, copyHeader);
+      ex.setErrorMessage("Copy Source must mention the source bucket and " +
+          "key: sourcebucket/sourcekey");
+      throw ex;
+    }
+    String sourceBucket = copyHeader.substring(0, pos);
+    String sourceKey = copyHeader.substring(pos + 1);
+
+    OzoneInputStream sourceInputStream = null;
+    OzoneOutputStream destOutputStream = null;
+    boolean closed = false;
+    try {
+      // Checking whether we trying to copying to it self.
+      if (sourceBucket.equals(destBucket)) {
+        if (sourceKey.equals(destkey)) {
+          OS3Exception ex = S3ErrorTable.newError(S3ErrorTable
+              .INVALID_REQUEST, copyHeader);
+          ex.setErrorMessage("This copy request is illegal because it is " +
+              "trying to copy an object to it self itself without changing " +
+              "the object's metadata, storage class, website redirect " +
+              "location or encryption attributes.");
+          throw ex;
+        }
+      }
+
+      OzoneBucket sourceOzoneBucket = getBucket(sourceBucket);
+      OzoneBucket destOzoneBucket = getBucket(destBucket);
+
+      OzoneKeyDetails sourceKeyDetails = sourceOzoneBucket.getKey(sourceKey);
+      long sourceKeyLen = sourceKeyDetails.getDataSize();
+
+      sourceInputStream = sourceOzoneBucket.readKey(sourceKey);
+
+      destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen,
+          replicationType, replicationFactor);
+
+      IOUtils.copy(sourceInputStream, destOutputStream);
+
+      // Closing here, as if we don't call close this key will not commit in
+      // OM, and getKey fails.
+      sourceInputStream.close();
+      destOutputStream.close();
+      closed = true;
+
+      OzoneKeyDetails destKeyDetails = destOzoneBucket.getKey(destkey);
+
+      CopyObjectResponse copyObjectResponse = new CopyObjectResponse();
+      copyObjectResponse.setETag(OzoneUtils.getRequestID());
+      copyObjectResponse.setLastModified(Instant.ofEpochMilli(destKeyDetails
+          .getModificationTime()));
+      return copyObjectResponse;
+    } catch (IOException ex) {
+      if (ex.getMessage().contains("KEY_NOT_FOUND")) {
+        throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_KEY, sourceKey);
+      }
+      LOG.error("Exception occurred in PutObject", ex);
+      throw ex;
+    } finally {
+      if (!closed) {
+        if (sourceInputStream != null) {
+          sourceInputStream.close();
+        }
+        if (destOutputStream != null) {
+          destOutputStream.close();
+        }
+      }
+    }
+  }
 }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java

@@ -52,6 +52,12 @@ public final class S3ErrorTable {
   public static final OS3Exception NO_SUCH_KEY = new OS3Exception(
       "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND);
 
+  public static final OS3Exception INVALID_ARGUMENT = new OS3Exception(
+      "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST);
+
+  public static final OS3Exception INVALID_REQUEST = new OS3Exception(
+      "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST);
+
   /**
    * Create a new instance of Error.
    * @param e Error Template

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java

@@ -38,6 +38,8 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
+
+import static org.junit.Assert.fail;
 import static org.mockito.Mockito.when;
 
 /**
@@ -48,6 +50,9 @@ public class TestPutObject {
   private String userName = "ozone";
   private String bucketName = "b1";
   private String keyName = "key1";
+  private String destBucket = "b2";
+  private String destkey = "key2";
+  private String nonexist = "nonexist";
   private OzoneClientStub clientStub;
   private ObjectStore objectStoreStub;
   private ObjectEndpoint objectEndpoint;
@@ -60,6 +65,7 @@ public class TestPutObject {
 
     // Create bucket
     objectStoreStub.createS3Bucket(userName, bucketName);
+    objectStoreStub.createS3Bucket("ozone1", destBucket);
 
     // Create PutObject and setClient to OzoneClientStub
     objectEndpoint = new ObjectEndpoint();
@@ -75,8 +81,8 @@ public class TestPutObject {
 
     //WHEN
     Response response = objectEndpoint.put(bucketName, keyName,
-        ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "32 * 1024 * 1024",
-        CONTENT.length(), body);
+        ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+        body);
 
     //THEN
     String volumeName = clientStub.getObjectStore()
@@ -109,7 +115,6 @@ public class TestPutObject {
     Response response = objectEndpoint.put(bucketName, keyName,
         ReplicationType.STAND_ALONE,
         ReplicationFactor.ONE,
-        "32 * 1024 * 1024",
         chunkedContent.length(),
         new ByteArrayInputStream(chunkedContent.getBytes()));
 
@@ -125,4 +130,107 @@ public class TestPutObject {
     Assert.assertEquals(200, response.getStatus());
     Assert.assertEquals("1234567890abcde", keyContent);
   }
+
+  @Test
+  public void testCopyObject() throws IOException, OS3Exception {
+    // Put object in to source bucket
+    HttpHeaders headers = Mockito.mock(HttpHeaders.class);
+    ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes());
+    objectEndpoint.setHeaders(headers);
+    keyName = "sourceKey";
+
+    Response response = objectEndpoint.put(bucketName, keyName,
+        ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+        body);
+
+    String volumeName = clientStub.getObjectStore().getOzoneVolumeName(
+        bucketName);
+
+    OzoneInputStream ozoneInputStream = clientStub.getObjectStore().getVolume(
+        volumeName).getBucket(bucketName).readKey(keyName);
+
+    String keyContent = IOUtils.toString(ozoneInputStream, Charset.forName(
+        "UTF-8"));
+
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(CONTENT, keyContent);
+
+
+    // Add copy header, and then call put
+    when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+        bucketName  + "/" + keyName);
+
+    response = objectEndpoint.put(destBucket, destkey,
+        ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+        body);
+
+    // Check destination key and response
+    volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket);
+    ozoneInputStream = clientStub.getObjectStore().getVolume(volumeName)
+        .getBucket(destBucket).readKey(destkey);
+
+    keyContent = IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8"));
+
+    Assert.assertEquals(200, response.getStatus());
+    Assert.assertEquals(CONTENT, keyContent);
+
+    // source and dest same
+    try {
+      objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE,
+          ReplicationFactor.ONE, CONTENT.length(), body);
+      fail("test copy object failed");
+    } catch (OS3Exception ex) {
+      Assert.assertTrue(ex.getErrorMessage().contains("This copy request is " +
+          "illegal"));
+    }
+
+    // source bucket not found
+    try {
+      when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+          nonexist + "/"  + keyName);
+      response = objectEndpoint.put(destBucket, destkey,
+          ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+          body);
+      fail("test copy object failed");
+    } catch (OS3Exception ex) {
+      Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
+    }
+
+    // dest bucket not found
+    try {
+      when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+          bucketName + "/" + keyName);
+      response = objectEndpoint.put(nonexist, destkey,
+          ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+          body);
+      fail("test copy object failed");
+    } catch (OS3Exception ex) {
+      Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
+    }
+
+    //Both source and dest bucket not found
+    try {
+      when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+          nonexist + "/" + keyName);
+      response = objectEndpoint.put(nonexist, destkey,
+          ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+          body);
+      fail("test copy object failed");
+    } catch (OS3Exception ex) {
+      Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
+    }
+
+    // source key not found
+    try {
+      when(headers.getHeaderString("x-amz-copy-source")).thenReturn(
+          bucketName + "/" + nonexist);
+      response = objectEndpoint.put("nonexistent", keyName,
+          ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(),
+          body);
+      fail("test copy object failed");
+    } catch (OS3Exception ex) {
+      Assert.assertTrue(ex.getCode().contains("NoSuchBucket"));
+    }
+
+  }
 }