HDDS-525. Support virtual-hosted style URLs. Contributed by Bharat Viswanadham

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -1035,12 +1035,12 @@
 
   <property>
     <name>hadoop.tags.custom</name>
-    <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
+    <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
   </property>
 
   <property>
     <name>ozone.tags.system</name>
-    <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
+    <value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY</value>
   </property>
 
 
@@ -1222,4 +1222,78 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.s3g.authentication.kerberos.principal</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The server principal used by Ozone S3Gateway server. This is
+      typically set to
+      HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
+      HTTP/ by convention.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.domain.name</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>List of Ozone S3Gateway domain names. If multiple
+      domain names to be provided, they should be a "," seperated.
+      This parameter is only required when virtual host style pattern is
+      followed.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.http-address</name>
+    <value>0.0.0.0:9878</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The address and the base port where the Ozone S3Gateway
+      Server will
+      listen on.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.http-bind-host</name>
+    <value>0.0.0.0</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTP server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.http-address.
+      This is useful for making the Ozone S3Gateway HTTP server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.http.enabled</name>
+    <value>true</value>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The boolean which enables the Ozone S3Gateway server
+      .</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.https-address</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>Ozone S3Gateway serverHTTPS server address and port
+      .</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.https-bind-host</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The actual address the HTTPS server will bind to. If this optional address
+      is set, it overrides only the hostname portion of ozone.s3g.https-address.
+      This is useful for making the Ozone S3Gateway HTTPS server listen on all
+      interfaces by setting it to 0.0.0.0.</description>
+  </property>
+
+  <property>
+    <name>ozone.s3g.keytab.file</name>
+    <value/>
+    <tag>OZONE, S3GATEWAY</tag>
+    <description>The keytab file used by the S3Gateway server to login as its
+      service principal. </description>
+  </property>
+
+
 </configuration>

hadoop-ozone/integration-test/pom.xml

@@ -42,6 +42,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-objectstore-service</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-ozone-s3gateway</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-ozone-client</artifactId>

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java

@@ -21,6 +21,7 @@ import org.apache.hadoop.conf.TestConfigurationFieldsBase;
 import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.ozone.om.OMConfigKeys;
 import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.ozone.s3.S3GatewayConfigKeys;
 
 /**
  * Tests if configuration constants documented in ozone-defaults.xml.
@@ -32,7 +33,8 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
     xmlFilename = new String("ozone-default.xml");
     configurationClasses =
         new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
-            OMConfigKeys.class, HddsConfigKeys.class};
+            OMConfigKeys.class, HddsConfigKeys.class,
+            S3GatewayConfigKeys.class};
     errorIfMissingConfigProps = true;
     errorIfMissingXmlProps = true;
     xmlPropsToSkipCompare.add("hadoop.tags.custom");

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java

@@ -44,6 +44,7 @@ public final class S3GatewayConfigKeys {
   public static final int OZONE_S3G_HTTPS_BIND_PORT_DEFAULT = 9879;
   public static final String OZONE_S3G_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
       "ozone.s3g.authentication.kerberos.principal";
+  public static final String OZONE_S3G_DOMAIN_NAME = "ozone.s3g.domain.name";
 
   /**
    * Never constructed.

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/VirtualHostStyleFilter.java

@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import javax.inject.Inject;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.ext.Provider;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME;
+
+/**
+ * Filter used to convert virtual host style pattern to path style pattern.
+ */
+
+@Provider
+@PreMatching
+public class VirtualHostStyleFilter implements ContainerRequestFilter {
+
+  private static final Logger LOG = LoggerFactory.getLogger(
+      VirtualHostStyleFilter.class);
+  private static final Pattern URL_SCHEME_PATTERN = Pattern.compile("" +
+      "(?<bucket>(.+))\\.(?<volume>(.+))\\.");
+
+  @Inject
+  private OzoneConfiguration conf;
+  private String[] domains;
+
+  @Override
+  public void filter(ContainerRequestContext requestContext) throws
+      IOException {
+    domains = conf.getTrimmedStrings(OZONE_S3G_DOMAIN_NAME);
+
+    if (domains.length == 0) {
+      // domains is not configured, might be it is path style.
+      // So, do not continue further, just return.
+      return;
+    }
+    //Get the value of the host
+    String host = requestContext.getHeaderString(HttpHeaders.HOST);
+    String domain = getDomainName(host);
+
+    if (domain == null) {
+      throw getException("Invalid S3 Gateway request {" + requestContext
+          .getUriInfo().getRequestUri().toString() + " }: No matching domain " +
+          "{" + Arrays.toString(domains) + "} for the host {" + host  + "}");
+    }
+
+    LOG.debug("Http header host name is {}", host);
+    LOG.debug("Domain name matched is {}", domain);
+
+    //Check if we have a Virtual Host style request, host length greater than
+    // address length means it is virtual host style, we need to convert to
+    // path style.
+    if (host.length() > domain.length()) {
+      String bothNames = host.substring(0, host.length() - domain.length());
+      LOG.debug("Both volume name and bucket name is {}", bothNames);
+      Matcher matcher = URL_SCHEME_PATTERN.matcher(bothNames);
+
+      if (!matcher.matches()) {
+        throw getException("Invalid S3 Gateway request {" + requestContext
+            .getUriInfo().getRequestUri().toString() +"}:" +" Host: {" + host
+            + " is in invalid format");
+      }
+
+      String bucketStr = matcher.group("bucket");
+      String volumeStr = matcher.group("volume");
+
+      LOG.debug("bucket {}, volumeStr {}", bucketStr, volumeStr);
+
+      URI baseURI = requestContext.getUriInfo().getBaseUri();
+      String currentPath = requestContext.getUriInfo().getPath();
+      String newPath = String.format("%s/%s", volumeStr, bucketStr);
+      if (currentPath != null) {
+        newPath += String.format("%s", currentPath);
+      }
+      URI requestAddr = UriBuilder.fromUri(baseURI).path(newPath).build();
+      requestContext.setRequestUri(baseURI, requestAddr);
+    }
+  }
+
+  private InvalidRequestException getException(String message) {
+    return new InvalidRequestException(message);
+  }
+
+  @VisibleForTesting
+  public void setConfiguration(OzoneConfiguration config) {
+    this.conf = config;
+  }
+
+
+  /**
+   * This method finds the longest match with the domain name.
+   * @param host
+   * @return domain name matched with the host. if none of them are matching,
+   * return null.
+   */
+  private String getDomainName(String host) {
+    String match = null;
+    int length=0;
+    for (String domainVal : domains) {
+      if (host.endsWith(domainVal)) {
+        int len = domainVal.length();
+        if (len > length) {
+          length = len;
+          match = domainVal;
+        }
+      }
+    }
+    return match;
+  }
+
+}

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/TestVirtualHostStyleFilter.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.ozone.s3;
+
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.glassfish.jersey.internal.PropertiesDelegate;
+import org.glassfish.jersey.server.ContainerRequest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.SecurityContext;
+import java.net.URI;
+
+/**
+ * This class test virtual host style mapping conversion to path style.
+ */
+public class TestVirtualHostStyleFilter {
+
+  private static OzoneConfiguration conf;
+  private static String s3HttpAddr;
+
+  @Before
+  public void setup() {
+    conf = new OzoneConfiguration();
+    s3HttpAddr = "localhost:9878";
+    conf.set(S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY, s3HttpAddr);
+    conf.set(S3GatewayConfigKeys.OZONE_S3G_DOMAIN_NAME, s3HttpAddr);
+  }
+
+  /**
+   * Create containerRequest object.
+   * @return ContainerRequest
+   * @throws Exception
+   */
+  public ContainerRequest createContainerRequest(String host, String path,
+                                                 boolean virtualHostStyle)
+      throws Exception {
+    URI baseUri = new URI("http://" + s3HttpAddr);
+    URI virtualHostStyleUri;
+    if (path == null) {
+      virtualHostStyleUri = new URI("http://" + s3HttpAddr);
+    } else {
+      virtualHostStyleUri = new URI("http://" + s3HttpAddr + path);
+    }
+    URI pathStyleUri = new URI("http://" + s3HttpAddr + path);
+    String httpMethod = "DELETE";
+    SecurityContext securityContext = Mockito.mock(SecurityContext.class);
+    PropertiesDelegate propertiesDelegate = Mockito.mock(PropertiesDelegate
+        .class);
+    ContainerRequest containerRequest;
+    if (virtualHostStyle) {
+      containerRequest = new ContainerRequest(baseUri, virtualHostStyleUri,
+          httpMethod, securityContext, propertiesDelegate);
+      containerRequest.header(HttpHeaders.HOST, host);
+    } else {
+      containerRequest = new ContainerRequest(baseUri, pathStyleUri,
+          httpMethod, securityContext, propertiesDelegate);
+      containerRequest.header(HttpHeaders.HOST, host);
+    }
+    return containerRequest;
+  }
+
+  @Test
+  public void testVirtualHostStyle() throws  Exception {
+    VirtualHostStyleFilter virtualHostStyleFilter =
+        new VirtualHostStyleFilter();
+    virtualHostStyleFilter.setConfiguration(conf);
+
+    ContainerRequest containerRequest = createContainerRequest("mybucket" +
+            ".myvolume.localhost:9878", "/myfile", true);
+    virtualHostStyleFilter.filter(containerRequest);
+    URI expected = new URI("http://" + s3HttpAddr +
+        "/myvolume/mybucket/myfile");
+    Assert.assertEquals(expected, containerRequest.getRequestUri());
+  }
+
+  @Test
+  public void testPathStyle() throws Exception {
+
+    VirtualHostStyleFilter virtualHostStyleFilter =
+        new VirtualHostStyleFilter();
+    virtualHostStyleFilter.setConfiguration(conf);
+
+    ContainerRequest containerRequest = createContainerRequest(s3HttpAddr,
+        "/myvolume/mybucket/myfile", false);
+    virtualHostStyleFilter.filter(containerRequest);
+    URI expected = new URI("http://" + s3HttpAddr +
+        "/myvolume/mybucket/myfile");
+    Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+  }
+
+  @Test
+  public void testVirtualHostStyleWithCreateBucketRequest() throws Exception {
+
+    VirtualHostStyleFilter virtualHostStyleFilter =
+        new VirtualHostStyleFilter();
+    virtualHostStyleFilter.setConfiguration(conf);
+
+    ContainerRequest containerRequest = createContainerRequest("mybucket" +
+        ".myvolume.localhost:9878", null, true);
+    virtualHostStyleFilter.filter(containerRequest);
+    URI expected = new URI("http://" + s3HttpAddr +
+        "/myvolume/mybucket");
+    Assert.assertEquals(expected, containerRequest.getRequestUri());
+
+  }
+
+  @Test
+  public void testVirtualHostStyleWithNoMatchingDomain() throws Exception {
+
+    VirtualHostStyleFilter virtualHostStyleFilter =
+        new VirtualHostStyleFilter();
+    virtualHostStyleFilter.setConfiguration(conf);
+
+    ContainerRequest containerRequest = createContainerRequest("mybucket" +
+        ".myvolume.localhost:9999", null, true);
+    try {
+      virtualHostStyleFilter.filter(containerRequest);
+    } catch (InvalidRequestException ex) {
+      GenericTestUtils.assertExceptionContains("No matching domain", ex);
+    }
+
+  }
+
+  @Test
+  public void testVirtualHostStyleWithoutVolumeName() throws Exception {
+
+    VirtualHostStyleFilter virtualHostStyleFilter =
+        new VirtualHostStyleFilter();
+    virtualHostStyleFilter.setConfiguration(conf);
+
+    ContainerRequest containerRequest = createContainerRequest("mybucket." +
+        ".localhost:9878", null, true);
+    try {
+      virtualHostStyleFilter.filter(containerRequest);
+    } catch (InvalidRequestException ex) {
+      GenericTestUtils.assertExceptionContains("invalid format", ex);
+    }
+
+  }
+
+}

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/package-info.java

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Unit tests for the bucket related rest endpoints.
+ */
+package org.apache.hadoop.ozone.s3;