HADOOP-8545. Filesystem Implementation for OpenStack Swift

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1526848 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml

@@ -348,4 +348,20 @@
        <Method name="waitForServiceToStop" />
        <Bug code="JLM" />
      </Match>
- </FindBugsFilter>
+
+  <!--
+  OpenStack Swift FS module -closes streams in a different method
+  from where they are opened.
+  -->
+    <Match>
+      <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
+      <Method name="uploadFileAttempt"/>
+      <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
+    </Match>
+    <Match>
+      <Class name="org.apache.hadoop.fs.swift.snative.SwiftNativeOutputStream"/>
+      <Method name="uploadFilePartAttempt"/>
+      <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
+    </Match>
+
+</FindBugsFilter>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Seekable.java

@@ -22,7 +22,9 @@ import java.io.*;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
-/** Stream that permits seeking. */
+/**
+ *  Stream that permits seeking.
+ */
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public interface Seekable {

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -492,6 +492,11 @@
   </description>
 </property>
 
+<property>
+  <name>fs.swift.impl</name>
+  <value>org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem</value>
+  <description>The implementation class of the OpenStack Swift Filesystem</description>
+</property>
 
 <property>
   <name>fs.automatic.close</name>

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java

@@ -944,14 +944,20 @@ public abstract class FSMainOperationsBaseTest extends FileSystemTestHelper {
       rename(src, dst, false, true, false, Rename.NONE);
       Assert.fail("Expected exception was not thrown");
     } catch (IOException e) {
-      Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
+      IOException ioException = unwrapException(e);
+      if (!(ioException instanceof FileNotFoundException)) {
+        throw ioException;
+      }
     }
 
     try {
       rename(src, dst, false, true, false, Rename.OVERWRITE);
       Assert.fail("Expected exception was not thrown");
     } catch (IOException e) {
-      Assert.assertTrue(unwrapException(e) instanceof FileNotFoundException);
+      IOException ioException = unwrapException(e);
+      if (!(ioException instanceof FileNotFoundException)) {
+        throw ioException;
+      }
     }
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java

@@ -270,7 +270,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
   
   public void testWriteInNonExistentDirectory() throws IOException {
     Path path = path("/test/hadoop/file");
-    assertFalse("Parent doesn't exist", fs.exists(path.getParent()));
+    assertFalse("Parent exists", fs.exists(path.getParent()));
     createFile(path);
     
     assertTrue("Exists", fs.exists(path));
@@ -280,7 +280,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
 
   public void testDeleteNonExistentFile() throws IOException {
     Path path = path("/test/hadoop/file");    
-    assertFalse("Doesn't exist", fs.exists(path));
+    assertFalse("Path exists: " + path, fs.exists(path));
     assertFalse("No deletion", fs.delete(path, true));
   }
   

hadoop-project/pom.xml

@@ -305,6 +305,12 @@
         <version>${project.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-openstack</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
@@ -335,6 +341,11 @@
         <artifactId>commons-httpclient</artifactId>
         <version>3.1</version>
       </dependency>
+      <dependency>
+        <groupId>org.apache.httpcomponents</groupId>
+        <artifactId>httpclient</artifactId>
+        <version>4.2.5</version>
+      </dependency>
       <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>

hadoop-tools/hadoop-openstack/pom.xml

@@ -0,0 +1,153 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <relativePath>../../hadoop-project</relativePath>
+  </parent>
+  <artifactId>hadoop-openstack</artifactId>
+  <version>3.0.0-SNAPSHOT</version>
+  <name>Apache Hadoop OpenStack support</name>
+  <description>
+    This module contains code to support integration with OpenStack.
+    Currently this consists of a filesystem client to read data from
+    and write data to an OpenStack Swift object store.
+  </description>
+  <packaging>jar</packaging>
+
+  <properties>
+    <file.encoding>UTF-8</file.encoding>
+    <downloadSources>true</downloadSources>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <profiles>
+    <profile>
+      <id>tests-off</id>
+      <activation>
+        <file>
+          <missing>src/test/resources/auth-keys.xml</missing>
+        </file>
+      </activation>
+      <properties>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
+    <profile>
+      <id>tests-on</id>
+      <activation>
+        <file>
+          <exists>src/test/resources/auth-keys.xml</exists>
+        </file>
+      </activation>
+      <properties>
+        <maven.test.skip>false</maven.test.skip>
+      </properties>
+    </profile>
+
+  </profiles>
+
+  <build>
+    <plugins>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-site-plugin</artifactId>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.maven.doxia</groupId>
+            <artifactId>doxia-module-markdown</artifactId>
+            <version>1.3</version>
+          </dependency>
+        </dependencies>
+        <configuration>
+          <inputEncoding>UTF-8</inputEncoding>
+          <outputEncoding>UTF-8</outputEncoding>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-project-info-reports-plugin</artifactId>
+
+        <configuration>
+          <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
+          <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>compile</scope>
+      <type>test-jar</type>
+    </dependency>
+
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-mapper-asl</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.codehaus.jackson</groupId>
+      <artifactId>jackson-core-asl</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!-- Used for loading test resources and converting a File to byte[] -->
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <scope>compile</scope>
+    </dependency>
+
+    <!-- Used for mocking dependencies -->
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.8.5</version>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyAuthenticationRequest.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * Class that represents authentication request to Openstack Keystone.
+ * Contains basic authentication information.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS
+ */
+public class ApiKeyAuthenticationRequest extends AuthenticationRequest {
+  /**
+   * Credentials for login
+   */
+  private ApiKeyCredentials apiKeyCredentials;
+
+  /**
+   * API key auth
+   * @param tenantName tenant
+   * @param apiKeyCredentials credentials
+   */
+  public ApiKeyAuthenticationRequest(String tenantName, ApiKeyCredentials apiKeyCredentials) {
+    this.tenantName = tenantName;
+    this.apiKeyCredentials = apiKeyCredentials;
+  }
+
+  /**
+   * @return credentials for login into Keystone
+   */
+  @JsonProperty("RAX-KSKEY:apiKeyCredentials")
+  public ApiKeyCredentials getApiKeyCredentials() {
+    return apiKeyCredentials;
+  }
+
+  /**
+   * @param apiKeyCredentials credentials for login into Keystone
+   */
+  public void setApiKeyCredentials(ApiKeyCredentials apiKeyCredentials) {
+    this.apiKeyCredentials = apiKeyCredentials;
+  }
+
+  @Override
+  public String toString() {
+    return "Auth as " +
+           "tenant '" + tenantName + "' "
+           + apiKeyCredentials;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/ApiKeyCredentials.java

@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+
+/**
+ * Describes credentials to log in Swift using Keystone authentication.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class ApiKeyCredentials {
+  /**
+   * user login
+   */
+  private String username;
+
+  /**
+   * user password
+   */
+  private String apikey;
+
+  /**
+   * default constructor
+   */
+  public ApiKeyCredentials() {
+  }
+
+  /**
+   * @param username user login
+   * @param apikey user api key
+   */
+  public ApiKeyCredentials(String username, String apikey) {
+    this.username = username;
+    this.apikey = apikey;
+  }
+
+  /**
+   * @return user api key
+   */
+  public String getApiKey() {
+    return apikey;
+  }
+
+  /**
+   * @param apikey user api key
+   */
+  public void setApiKey(String apikey) {
+    this.apikey = apikey;
+  }
+
+  /**
+   * @return login
+   */
+  public String getUsername() {
+    return username;
+  }
+
+  /**
+   * @param username login
+   */
+  public void setUsername(String username) {
+    this.username = username;
+  }
+
+  @Override
+  public String toString() {
+    return "user " +
+           "'" + username + '\'' +
+           " with key of length " + ((apikey == null) ? 0 : apikey.length());
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequest.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * Class that represents authentication request to Openstack Keystone.
+ * Contains basic authentication information.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class AuthenticationRequest {
+
+  /**
+   * tenant name
+   */
+  protected String tenantName;
+
+  public AuthenticationRequest() {
+  }
+
+  /**
+   * @return tenant name for Keystone authorization
+   */
+  public String getTenantName() {
+    return tenantName;
+  }
+
+  /**
+   * @param tenantName tenant name for authorization
+   */
+  public void setTenantName(String tenantName) {
+    this.tenantName = tenantName;
+  }
+
+  @Override
+  public String toString() {
+    return "AuthenticationRequest{" +
+           "tenantName='" + tenantName + '\'' +
+           '}';
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationRequestWrapper.java

@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * This class is used for correct hierarchy mapping of
+ * Keystone authentication model and java code.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class AuthenticationRequestWrapper {
+  /**
+   * authentication request
+   */
+  private AuthenticationRequest auth;
+
+  /**
+   * default constructor used for json parsing
+   */
+  public AuthenticationRequestWrapper() {
+  }
+
+  /**
+   * @param auth authentication requests
+   */
+  public AuthenticationRequestWrapper(AuthenticationRequest auth) {
+    this.auth = auth;
+  }
+
+  /**
+   * @return authentication request
+   */
+  public AuthenticationRequest getAuth() {
+    return auth;
+  }
+
+  /**
+   * @param auth authentication request
+   */
+  public void setAuth(AuthenticationRequest auth) {
+    this.auth = auth;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationResponse.java

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
+import org.apache.hadoop.fs.swift.auth.entities.Catalog;
+import org.apache.hadoop.fs.swift.auth.entities.User;
+
+import java.util.List;
+
+/**
+ * Response from KeyStone deserialized into AuthenticationResponse class.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class AuthenticationResponse {
+  private Object metadata;
+  private List<Catalog> serviceCatalog;
+  private User user;
+  private AccessToken token;
+
+  public Object getMetadata() {
+    return metadata;
+  }
+
+  public void setMetadata(Object metadata) {
+    this.metadata = metadata;
+  }
+
+  public List<Catalog> getServiceCatalog() {
+    return serviceCatalog;
+  }
+
+  public void setServiceCatalog(List<Catalog> serviceCatalog) {
+    this.serviceCatalog = serviceCatalog;
+  }
+
+  public User getUser() {
+    return user;
+  }
+
+  public void setUser(User user) {
+    this.user = user;
+  }
+
+  public AccessToken getToken() {
+    return token;
+  }
+
+  public void setToken(AccessToken token) {
+    this.token = token;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/AuthenticationWrapper.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * This class is used for correct hierarchy mapping of
+ * Keystone authentication model and java code
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class AuthenticationWrapper {
+
+  /**
+   * authentication response field
+   */
+  private AuthenticationResponse access;
+
+  /**
+   * @return authentication response
+   */
+  public AuthenticationResponse getAccess() {
+    return access;
+  }
+
+  /**
+   * @param access sets authentication response
+   */
+  public void setAccess(AuthenticationResponse access) {
+    this.access = access;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeyStoneAuthRequest.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * Class that represents authentication to OpenStack Keystone.
+ * Contains basic authentication information.
+ * Used when {@link ApiKeyAuthenticationRequest} is not applicable.
+ * (problem with different Keystone installations/versions/modifications)
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class KeyStoneAuthRequest extends AuthenticationRequest {
+
+  /**
+   * Credentials for Keystone authentication
+   */
+  private KeystoneApiKeyCredentials apiAccessKeyCredentials;
+
+  /**
+   * @param tenant                  Keystone tenant name for authentication
+   * @param apiAccessKeyCredentials Credentials for authentication
+   */
+  public KeyStoneAuthRequest(String tenant, KeystoneApiKeyCredentials apiAccessKeyCredentials) {
+    this.apiAccessKeyCredentials = apiAccessKeyCredentials;
+    this.tenantName = tenant;
+  }
+
+  public KeystoneApiKeyCredentials getApiAccessKeyCredentials() {
+    return apiAccessKeyCredentials;
+  }
+
+  public void setApiAccessKeyCredentials(KeystoneApiKeyCredentials apiAccessKeyCredentials) {
+    this.apiAccessKeyCredentials = apiAccessKeyCredentials;
+  }
+
+  @Override
+  public String toString() {
+    return "KeyStoneAuthRequest as " +
+            "tenant '" + tenantName + "' "
+            + apiAccessKeyCredentials;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/KeystoneApiKeyCredentials.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * Class for Keystone authentication.
+ * Used when {@link ApiKeyCredentials} is not applicable
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class KeystoneApiKeyCredentials {
+
+  /**
+   * User access key
+   */
+  private String accessKey;
+
+  /**
+   * User access secret
+   */
+  private String secretKey;
+
+  public KeystoneApiKeyCredentials(String accessKey, String secretKey) {
+    this.accessKey = accessKey;
+    this.secretKey = secretKey;
+  }
+
+  public String getAccessKey() {
+    return accessKey;
+  }
+
+  public void setAccessKey(String accessKey) {
+    this.accessKey = accessKey;
+  }
+
+  public String getSecretKey() {
+    return secretKey;
+  }
+
+  public void setSecretKey(String secretKey) {
+    this.secretKey = secretKey;
+  }
+
+  @Override
+  public String toString() {
+    return "user " +
+            "'" + accessKey + '\'' +
+            " with key of length " + ((secretKey == null) ? 0 : secretKey.length());
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordAuthenticationRequest.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * Class that represents authentication request to Openstack Keystone.
+ * Contains basic authentication information.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class PasswordAuthenticationRequest extends AuthenticationRequest {
+  /**
+   * Credentials for login
+   */
+  private PasswordCredentials passwordCredentials;
+
+  /**
+   * @param tenantName tenant
+   * @param passwordCredentials password credentials
+   */
+  public PasswordAuthenticationRequest(String tenantName, PasswordCredentials passwordCredentials) {
+    this.tenantName = tenantName;
+    this.passwordCredentials = passwordCredentials;
+  }
+
+  /**
+   * @return credentials for login into Keystone
+   */
+  public PasswordCredentials getPasswordCredentials() {
+    return passwordCredentials;
+  }
+
+  /**
+   * @param passwordCredentials credentials for login into Keystone
+   */
+  public void setPasswordCredentials(PasswordCredentials passwordCredentials) {
+    this.passwordCredentials = passwordCredentials;
+  }
+
+  @Override
+  public String toString() {
+    return "Authenticate as " +
+           "tenant '" + tenantName + "' "
+           + passwordCredentials;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/PasswordCredentials.java

@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+
+/**
+ * Describes credentials to log in Swift using Keystone authentication.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class PasswordCredentials {
+  /**
+   * user login
+   */
+  private String username;
+
+  /**
+   * user password
+   */
+  private String password;
+
+  /**
+   * default constructor
+   */
+  public PasswordCredentials() {
+  }
+
+  /**
+   * @param username user login
+   * @param password user password
+   */
+  public PasswordCredentials(String username, String password) {
+    this.username = username;
+    this.password = password;
+  }
+
+  /**
+   * @return user password
+   */
+  public String getPassword() {
+    return password;
+  }
+
+  /**
+   * @param password user password
+   */
+  public void setPassword(String password) {
+    this.password = password;
+  }
+
+  /**
+   * @return login
+   */
+  public String getUsername() {
+    return username;
+  }
+
+  /**
+   * @param username login
+   */
+  public void setUsername(String username) {
+    this.username = username;
+  }
+
+  @Override
+  public String toString() {
+    return "user '" + username + '\'' +
+            " with password of length " + ((password == null) ? 0 : password.length());
+  }
+}
+

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/Roles.java

@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth;
+
+/**
+ * Describes user roles in Openstack system.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+public class Roles {
+  /**
+   * role name
+   */
+  private String name;
+
+  /**
+   * This field user in RackSpace auth model
+   */
+  private String id;
+
+  /**
+   * This field user in RackSpace auth model
+   */
+  private String description;
+
+  /**
+   * Service id used in HP public Cloud
+   */
+  private String serviceId;
+
+  /**
+   * Service id used in HP public Cloud
+   */
+  private String tenantId;
+
+  /**
+   * @return role name
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * @param name role name
+   */
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getId() {
+    return id;
+  }
+
+  public void setId(String id) {
+    this.id = id;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  public String getServiceId() {
+    return serviceId;
+  }
+
+  public void setServiceId(String serviceId) {
+    this.serviceId = serviceId;
+  }
+
+  public String getTenantId() {
+    return tenantId;
+  }
+
+  public void setTenantId(String tenantId) {
+    this.tenantId = tenantId;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/AccessToken.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth.entities;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+/**
+ * Access token representation of Openstack Keystone authentication.
+ * Class holds token id, tenant and expiration time.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ *
+ * Example:
+ * <pre>
+ * "token" : {
+ *   "RAX-AUTH:authenticatedBy" : [ "APIKEY" ],
+ *   "expires" : "2013-07-12T05:19:24.685-05:00",
+ *   "id" : "8bbea4215113abdab9d4c8fb0d37",
+ *   "tenant" : { "id" : "01011970",
+ *   "name" : "77777"
+ *   }
+ *  }
+ * </pre>
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+
+public class AccessToken {
+  /**
+   * token expiration time
+   */
+  private String expires;
+  /**
+   * token id
+   */
+  private String id;
+  /**
+   * tenant name for whom id is attached
+   */
+  private Tenant tenant;
+
+  /**
+   * @return token expiration time
+   */
+  public String getExpires() {
+    return expires;
+  }
+
+  /**
+   * @param expires the token expiration time
+   */
+  public void setExpires(String expires) {
+    this.expires = expires;
+  }
+
+  /**
+   * @return token value
+   */
+  public String getId() {
+    return id;
+  }
+
+  /**
+   * @param id token value
+   */
+  public void setId(String id) {
+    this.id = id;
+  }
+
+  /**
+   * @return tenant authenticated in Openstack Keystone
+   */
+  public Tenant getTenant() {
+    return tenant;
+  }
+
+  /**
+   * @param tenant tenant authenticated in Openstack Keystone
+   */
+  public void setTenant(Tenant tenant) {
+    this.tenant = tenant;
+  }
+
+  @Override
+  public String toString() {
+    return "AccessToken{" +
+            "id='" + id + '\'' +
+            ", tenant=" + tenant +
+            ", expires='" + expires + '\'' +
+            '}';
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Catalog.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth.entities;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+import java.util.List;
+
+/**
+ * Describes Openstack Swift REST endpoints.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+
+public class Catalog {
+  /**
+   * List of valid swift endpoints
+   */
+  private List<Endpoint> endpoints;
+  /**
+   * endpoint links are additional information description
+   * which aren't used in Hadoop and Swift integration scope
+   */
+  private List<Object> endpoints_links;
+  /**
+   * Openstack REST service name. In our case name = "keystone"
+   */
+  private String name;
+
+  /**
+   * Type of REST service. In our case type = "identity"
+   */
+  private String type;
+
+  /**
+   * @return List of endpoints
+   */
+  public List<Endpoint> getEndpoints() {
+    return endpoints;
+  }
+
+  /**
+   * @param endpoints list of endpoints
+   */
+  public void setEndpoints(List<Endpoint> endpoints) {
+    this.endpoints = endpoints;
+  }
+
+  /**
+   * @return list of endpoint links
+   */
+  public List<Object> getEndpoints_links() {
+    return endpoints_links;
+  }
+
+  /**
+   * @param endpoints_links list of endpoint links
+   */
+  public void setEndpoints_links(List<Object> endpoints_links) {
+    this.endpoints_links = endpoints_links;
+  }
+
+  /**
+   * @return name of Openstack REST service
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * @param name of Openstack REST service
+   */
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  /**
+   * @return type of Openstack REST service
+   */
+  public String getType() {
+    return type;
+  }
+
+  /**
+   * @param type of REST service
+   */
+  public void setType(String type) {
+    this.type = type;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Endpoint.java

@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth.entities;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+import java.net.URI;
+
+/**
+ * Openstack Swift endpoint description.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+
+public class Endpoint {
+
+  /**
+   * endpoint id
+   */
+  private String id;
+
+  /**
+   * Keystone admin URL
+   */
+  private URI adminURL;
+
+  /**
+   * Keystone internal URL
+   */
+  private URI internalURL;
+
+  /**
+   * public accessible URL
+   */
+  private URI publicURL;
+
+  /**
+   * public accessible URL#2
+   */
+  private URI publicURL2;
+
+  /**
+   * Openstack region name
+   */
+  private String region;
+
+  /**
+   * This field is used in RackSpace authentication model
+   */
+  private String tenantId;
+
+  /**
+   * This field user in RackSpace auth model
+   */
+  private String versionId;
+
+  /**
+   * This field user in RackSpace auth model
+   */
+  private String versionInfo;
+
+  /**
+   * This field user in RackSpace auth model
+   */
+  private String versionList;
+
+
+  /**
+   * @return endpoint id
+   */
+  public String getId() {
+    return id;
+  }
+
+  /**
+   * @param id endpoint id
+   */
+  public void setId(String id) {
+    this.id = id;
+  }
+
+  /**
+   * @return Keystone admin URL
+   */
+  public URI getAdminURL() {
+    return adminURL;
+  }
+
+  /**
+   * @param adminURL Keystone admin URL
+   */
+  public void setAdminURL(URI adminURL) {
+    this.adminURL = adminURL;
+  }
+
+  /**
+   * @return internal Keystone
+   */
+  public URI getInternalURL() {
+    return internalURL;
+  }
+
+  /**
+   * @param internalURL Keystone internal URL
+   */
+  public void setInternalURL(URI internalURL) {
+    this.internalURL = internalURL;
+  }
+
+  /**
+   * @return public accessible URL
+   */
+  public URI getPublicURL() {
+    return publicURL;
+  }
+
+  /**
+   * @param publicURL public URL
+   */
+  public void setPublicURL(URI publicURL) {
+    this.publicURL = publicURL;
+  }
+
+  public URI getPublicURL2() {
+    return publicURL2;
+  }
+
+  public void setPublicURL2(URI publicURL2) {
+    this.publicURL2 = publicURL2;
+  }
+
+  /**
+   * @return Openstack region name
+   */
+  public String getRegion() {
+    return region;
+  }
+
+  /**
+   * @param region Openstack region name
+   */
+  public void setRegion(String region) {
+    this.region = region;
+  }
+
+  public String getTenantId() {
+    return tenantId;
+  }
+
+  public void setTenantId(String tenantId) {
+    this.tenantId = tenantId;
+  }
+
+  public String getVersionId() {
+    return versionId;
+  }
+
+  public void setVersionId(String versionId) {
+    this.versionId = versionId;
+  }
+
+  public String getVersionInfo() {
+    return versionInfo;
+  }
+
+  public void setVersionInfo(String versionInfo) {
+    this.versionInfo = versionInfo;
+  }
+
+  public String getVersionList() {
+    return versionList;
+  }
+
+  public void setVersionList(String versionList) {
+    this.versionList = versionList;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/Tenant.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth.entities;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+/**
+ * Tenant is abstraction in Openstack which describes all account
+ * information and user privileges in system.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Tenant {
+
+  /**
+   * tenant id
+   */
+  private String id;
+
+  /**
+   * tenant short description which Keystone returns
+   */
+  private String description;
+
+  /**
+   * boolean enabled user account or no
+   */
+  private boolean enabled;
+
+  /**
+   * tenant human readable name
+   */
+  private String name;
+
+  /**
+   * @return tenant name
+   */
+  public String getName() {
+    return name;
+  }
+
+  /**
+   * @param name tenant name
+   */
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  /**
+   * @return true if account enabled and false otherwise
+   */
+  public boolean isEnabled() {
+    return enabled;
+  }
+
+  /**
+   * @param enabled enable or disable
+   */
+  public void setEnabled(boolean enabled) {
+    this.enabled = enabled;
+  }
+
+  /**
+   * @return account short description
+   */
+  public String getDescription() {
+    return description;
+  }
+
+  /**
+   * @param description set account description
+   */
+  public void setDescription(String description) {
+    this.description = description;
+  }
+
+  /**
+   * @return set tenant id
+   */
+  public String getId() {
+    return id;
+  }
+
+  /**
+   * @param id tenant id
+   */
+  public void setId(String id) {
+    this.id = id;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/auth/entities/User.java

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.auth.entities;
+
+import org.apache.hadoop.fs.swift.auth.Roles;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+import java.util.List;
+
+/**
+ * Describes user entity in Keystone
+ * In different Swift installations User is represented differently.
+ * To avoid any JSON deserialization failures this entity is ignored.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class User {
+
+  /**
+   * user id in Keystone
+   */
+  private String id;
+
+  /**
+   * user human readable name
+   */
+  private String name;
+
+  /**
+   * user roles in Keystone
+   */
+  private List<Roles> roles;
+
+  /**
+   * links to user roles
+   */
+  private List<Object> roles_links;
+
+  /**
+   * human readable username in Keystone
+   */
+  private String username;
+
+  /**
+   * @return user id
+   */
+  public String getId() {
+    return id;
+  }
+
+  /**
+   * @param id user id
+   */
+  public void setId(String id) {
+    this.id = id;
+  }
+
+
+  /**
+   * @return user name
+   */
+  public String getName() {
+    return name;
+  }
+
+
+  /**
+   * @param name user name
+   */
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  /**
+   * @return user roles
+   */
+  public List<Roles> getRoles() {
+    return roles;
+  }
+
+  /**
+   * @param roles sets user roles
+   */
+  public void setRoles(List<Roles> roles) {
+    this.roles = roles;
+  }
+
+  /**
+   * @return user roles links
+   */
+  public List<Object> getRoles_links() {
+    return roles_links;
+  }
+
+  /**
+   * @param roles_links user roles links
+   */
+  public void setRoles_links(List<Object> roles_links) {
+    this.roles_links = roles_links;
+  }
+
+  /**
+   * @return username
+   */
+  public String getUsername() {
+    return username;
+  }
+
+  /**
+   * @param username human readable user name
+   */
+  public void setUsername(String username) {
+    this.username = username;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftAuthenticationFailedException.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import org.apache.commons.httpclient.HttpMethod;
+
+import java.net.URI;
+
+/**
+ * An exception raised when an authentication request was rejected
+ */
+public class SwiftAuthenticationFailedException extends SwiftInvalidResponseException {
+
+  public SwiftAuthenticationFailedException(String message,
+                                            int statusCode,
+                                            String operation,
+                                            URI uri) {
+    super(message, statusCode, operation, uri);
+  }
+
+  public SwiftAuthenticationFailedException(String message,
+                                            String operation,
+                                            URI uri,
+                                            HttpMethod method) {
+    super(message, operation, uri, method);
+  }
+
+  @Override
+  public String exceptionTitle() {
+    return "Authentication Failure";
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftBadRequestException.java

@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import org.apache.commons.httpclient.HttpMethod;
+
+import java.net.URI;
+
+/**
+ * Thrown to indicate that data locality can't be calculated or requested path is incorrect.
+ * Data locality can't be calculated if Openstack Swift version is old.
+ */
+public class SwiftBadRequestException extends SwiftInvalidResponseException {
+
+  public SwiftBadRequestException(String message,
+                                  String operation,
+                                  URI uri,
+                                  HttpMethod method) {
+    super(message, operation, uri, method);
+  }
+
+  public SwiftBadRequestException(String message,
+                                  int statusCode,
+                                  String operation,
+                                  URI uri) {
+    super(message, statusCode, operation, uri);
+  }
+
+  @Override
+  public String exceptionTitle() {
+    return "BadRequest";
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConfigurationException.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Exception raised to indicate there is some problem with how the Swift FS
+ * is configured
+ */
+public class SwiftConfigurationException extends SwiftException {
+  public SwiftConfigurationException(String message) {
+    super(message);
+  }
+
+  public SwiftConfigurationException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionClosedException.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Exception raised when an attempt is made to use a closed stream
+ */
+public class SwiftConnectionClosedException extends SwiftException {
+
+  public static final String MESSAGE =
+    "Connection to Swift service has been closed";
+
+  public SwiftConnectionClosedException() {
+    super(MESSAGE);
+  }
+
+  public SwiftConnectionClosedException(String reason) {
+    super(MESSAGE + ": " + reason);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftConnectionException.java

@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Thrown to indicate that connection is lost or failed to be made
+ */
+public class SwiftConnectionException extends SwiftException {
+  public SwiftConnectionException() {
+  }
+
+  public SwiftConnectionException(String message) {
+    super(message);
+  }
+
+  public SwiftConnectionException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftException.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import java.io.IOException;
+
+/**
+ * A Swift-specific exception -subclasses exist
+ * for various specific problems.
+ */
+public class SwiftException extends IOException {
+  public SwiftException() {
+    super();
+  }
+
+  public SwiftException(String message) {
+    super(message);
+  }
+
+  public SwiftException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public SwiftException(Throwable cause) {
+    super(cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInternalStateException.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * The internal state of the Swift client is wrong -presumably a sign
+ * of some bug
+ */
+public class SwiftInternalStateException extends SwiftException {
+
+  public SwiftInternalStateException(String message) {
+    super(message);
+  }
+
+  public SwiftInternalStateException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public SwiftInternalStateException(Throwable cause) {
+    super(cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftInvalidResponseException.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import org.apache.commons.httpclient.HttpMethod;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Exception raised when the HTTP code is invalid. The status code,
+ * method name and operation URI are all in the response.
+ */
+public class SwiftInvalidResponseException extends SwiftConnectionException {
+
+  public final int statusCode;
+  public final String operation;
+  public final URI uri;
+  public final String body;
+
+  public SwiftInvalidResponseException(String message,
+                                       int statusCode,
+                                       String operation,
+                                       URI uri) {
+    super(message);
+    this.statusCode = statusCode;
+    this.operation = operation;
+    this.uri = uri;
+    this.body = "";
+  }
+
+  public SwiftInvalidResponseException(String message,
+                                       String operation,
+                                       URI uri,
+                                       HttpMethod method) {
+    super(message);
+    this.statusCode = method.getStatusCode();
+    this.operation = operation;
+    this.uri = uri;
+    String bodyAsString;
+    try {
+      bodyAsString = method.getResponseBodyAsString();
+      if (bodyAsString == null) {
+        bodyAsString = "";
+      }
+    } catch (IOException e) {
+      bodyAsString = "";
+    }
+    this.body = bodyAsString;
+  }
+
+  public int getStatusCode() {
+    return statusCode;
+  }
+
+  public String getOperation() {
+    return operation;
+  }
+
+  public URI getUri() {
+    return uri;
+  }
+
+  public String getBody() {
+    return body;
+  }
+
+  /**
+   * Override point: title of an exception -this is used in the
+   * toString() method.
+   * @return the new exception title
+   */
+  public String exceptionTitle() {
+    return "Invalid Response";
+  }
+
+  /**
+   * Build a description that includes the exception title, the URI,
+   * the message, the status code -and any body of the response
+   * @return the string value for display
+   */
+  @Override
+  public String toString() {
+    StringBuilder msg = new StringBuilder();
+    msg.append(exceptionTitle());
+    msg.append(": ");
+    msg.append(getMessage());
+    msg.append("  ");
+    msg.append(operation);
+    msg.append(" ");
+    msg.append(uri);
+    msg.append(" => ");
+    msg.append(statusCode);
+    if (body != null && !body.isEmpty()) {
+      msg.append(" : ");
+      msg.append(body);
+    }
+
+    return msg.toString();
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftJsonMarshallingException.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Exception raised when the J/O mapping fails.
+ */
+public class SwiftJsonMarshallingException extends SwiftException {
+
+  public SwiftJsonMarshallingException(String message) {
+    super(message);
+  }
+
+  public SwiftJsonMarshallingException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftNotDirectoryException.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Exception raised when an operation is meant to work on a directory, but
+ * the target path is not a directory
+ */
+public class SwiftNotDirectoryException extends SwiftException {
+  private final Path path;
+
+  public SwiftNotDirectoryException(Path path) {
+    this(path, "");
+  }
+
+  public SwiftNotDirectoryException(Path path,
+                                    String message) {
+    super(path.toString() + message);
+    this.path = path;
+  }
+
+  public Path getPath() {
+    return path;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftOperationFailedException.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Used to relay exceptions upstream from the inner implementation
+ * to the public API, where it is downgraded to a log+failure.
+ * Making it visible internally aids testing
+ */
+public class SwiftOperationFailedException extends SwiftException {
+
+  public SwiftOperationFailedException(String message) {
+    super(message);
+  }
+
+  public SwiftOperationFailedException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftPathExistsException.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Exception raised when trying to create a file that already exists
+ * and the overwrite flag is set to false.
+ */
+public class SwiftPathExistsException extends SwiftException {
+  public SwiftPathExistsException(String message) {
+    super(message);
+  }
+
+  public SwiftPathExistsException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftThrottledRequestException.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+import org.apache.commons.httpclient.HttpMethod;
+
+import java.net.URI;
+
+/**
+ * Exception raised if a Swift endpoint returned a HTTP response indicating
+ * the caller is being throttled.
+ */
+public class SwiftThrottledRequestException extends
+                                            SwiftInvalidResponseException {
+  public SwiftThrottledRequestException(String message,
+                                        String operation,
+                                        URI uri,
+                                        HttpMethod method) {
+    super(message, operation, uri, method);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/exceptions/SwiftUnsupportedFeatureException.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.exceptions;
+
+/**
+ * Exception raised on an unsupported feature in the FS API -such as
+ * <code>append()</code>
+ */
+public class SwiftUnsupportedFeatureException extends SwiftException {
+
+  public SwiftUnsupportedFeatureException(String message) {
+    super(message);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/CopyMethod.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
+
+/**
+ * Implementation for SwiftRestClient to make copy requests.
+ * COPY is a method that came with WebDAV (RFC2518), and is not something that
+ * can be handled by all proxies en-route to a filesystem.
+ */
+class CopyMethod extends EntityEnclosingMethod {
+
+  public CopyMethod(String uri) {
+    super(uri);
+  }
+
+  /**
+   * @return http method name
+   */
+  @Override
+  public String getName() {
+    return "COPY";
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/ExceptionDiags.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.net.ConnectException;
+import java.net.NoRouteToHostException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+
+/**
+ * Variant of Hadoop Netutils exception wrapping with URI awareness and
+ * available in branch-1 too.
+ */
+public class ExceptionDiags {
+  private static final Log LOG = LogFactory.getLog(ExceptionDiags.class);
+
+  /** text to point users elsewhere: {@value} */
+  private static final String FOR_MORE_DETAILS_SEE
+    = " For more details see:  ";
+  /** text included in wrapped exceptions if the host is null: {@value} */
+  public static final String UNKNOWN_HOST = "(unknown)";
+  /** Base URL of the Hadoop Wiki: {@value} */
+  public static final String HADOOP_WIKI = "http://wiki.apache.org/hadoop/";
+
+  /**
+   * Take an IOException and a URI, wrap it where possible with
+   * something that includes the URI
+   *
+   * @param dest target URI
+   * @param operation operation
+   * @param exception the caught exception.
+   * @return an exception to throw
+   */
+  public static IOException wrapException(final String dest,
+                                          final String operation,
+                                          final IOException exception) {
+    String action = operation + " " + dest;
+    String xref = null;
+
+    if (exception instanceof ConnectException) {
+      xref = "ConnectionRefused";
+    } else if (exception instanceof UnknownHostException) {
+      xref = "UnknownHost";
+    } else if (exception instanceof SocketTimeoutException) {
+      xref = "SocketTimeout";
+    } else if (exception instanceof NoRouteToHostException) {
+      xref = "NoRouteToHost";
+    }
+    String msg = action
+                 + " failed on exception: "
+                 + exception;
+    if (xref != null) {
+       msg = msg + ";" + see(xref);
+    }
+    return wrapWithMessage(exception, msg);
+  }
+
+  private static String see(final String entry) {
+    return FOR_MORE_DETAILS_SEE + HADOOP_WIKI + entry;
+  }
+
+  @SuppressWarnings("unchecked")
+  private static <T extends IOException> T wrapWithMessage(
+    T exception, String msg) {
+    Class<? extends Throwable> clazz = exception.getClass();
+    try {
+      Constructor<? extends Throwable> ctor =
+        clazz.getConstructor(String.class);
+      Throwable t = ctor.newInstance(msg);
+      return (T) (t.initCause(exception));
+    } catch (Throwable e) {
+      LOG.warn("Unable to wrap exception of type " +
+               clazz + ": it has no (String) constructor", e);
+      return exception;
+    }
+  }
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpBodyContent.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.http;
+
+/**
+ * Response tuple from GET operations; combines the input stream with the content length
+ */
+public class HttpBodyContent {
+  private final long contentLength;
+  private final HttpInputStreamWithRelease inputStream;
+
+  /**
+   * build a body response
+   * @param inputStream input stream from the operatin
+   * @param contentLength length of content; may be -1 for "don't know"
+   */
+  public HttpBodyContent(HttpInputStreamWithRelease inputStream,
+                         long contentLength) {
+    this.contentLength = contentLength;
+    this.inputStream = inputStream;
+  }
+
+  public long getContentLength() {
+    return contentLength;
+  }
+
+  public HttpInputStreamWithRelease getInputStream() {
+    return inputStream;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/HttpInputStreamWithRelease.java

@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+
+/**
+ * This replaces the input stream release class from JetS3t and AWS;
+ * # Failures in the constructor are relayed up instead of simply logged.
+ * # it is set up to be more robust at teardown
+ * # release logic is thread safe
+ * Note that the thread safety of the inner stream contains no thread
+ * safety guarantees -this stream is not to be read across streams.
+ * The thread safety logic here is to ensure that even if somebody ignores
+ * that rule, the release code does not get entered twice -and that
+ * any release in one thread is picked up by read operations in all others.
+ */
+public class HttpInputStreamWithRelease extends InputStream {
+
+  private static final Log LOG =
+    LogFactory.getLog(HttpInputStreamWithRelease.class);
+  private final URI uri;
+  private HttpMethod method;
+  //flag to say the stream is released -volatile so that read operations
+  //pick it up even while unsynchronized.
+  private volatile boolean released;
+  //volatile flag to verify that data is consumed.
+  private volatile boolean dataConsumed;
+  private InputStream inStream;
+  /**
+   * In debug builds, this is filled in with the construction-time
+   * stack, which is then included in logs from the finalize(), method.
+   */
+  private final Exception constructionStack;
+
+  /**
+   * Why the stream is closed
+   */
+  private String reasonClosed = "unopened";
+
+  public HttpInputStreamWithRelease(URI uri, HttpMethod method) throws
+                                                                IOException {
+    this.uri = uri;
+    this.method = method;
+    constructionStack = LOG.isDebugEnabled() ? new Exception("stack") : null;
+    if (method == null) {
+      throw new IllegalArgumentException("Null 'method' parameter ");
+    }
+    try {
+      inStream = method.getResponseBodyAsStream();
+    } catch (IOException e) {
+      inStream = new ByteArrayInputStream(new byte[]{});
+      throw releaseAndRethrow("getResponseBodyAsStream() in constructor -" + e, e);
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    release("close()", null);
+  }
+
+  /**
+   * Release logic
+   * @param reason reason for release (used in debug messages)
+   * @param ex exception that is a cause -null for non-exceptional releases
+   * @return true if the release took place here
+   * @throws IOException if the abort or close operations failed.
+   */
+  private synchronized boolean release(String reason, Exception ex) throws
+                                                                   IOException {
+    if (!released) {
+      reasonClosed = reason;
+      try {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Releasing connection to " + uri + ":  " + reason, ex);
+        }
+        if (method != null) {
+          if (!dataConsumed) {
+            method.abort();
+          }
+          method.releaseConnection();
+        }
+        if (inStream != null) {
+          //this guard may seem un-needed, but a stack trace seen
+          //on the JetS3t predecessor implied that it
+          //is useful
+          inStream.close();
+        }
+        return true;
+      } finally {
+        //if something went wrong here, we do not want the release() operation
+        //to try and do anything in advance.
+        released = true;
+        dataConsumed = true;
+      }
+    } else {
+      return false;
+    }
+  }
+
+  /**
+   * Release the method, using the exception as a cause
+   * @param operation operation that failed
+   * @param ex the exception which triggered it.
+   * @return the exception to throw
+   */
+  private IOException releaseAndRethrow(String operation, IOException ex) {
+    try {
+      release(operation, ex);
+    } catch (IOException ioe) {
+      LOG.debug("Exception during release: " + operation + " - " + ioe, ioe);
+      //make this the exception if there was none before
+      if (ex == null) {
+        ex = ioe;
+      }
+    }
+    return ex;
+  }
+
+  /**
+   * Assume that the connection is not released: throws an exception if it is
+   * @throws SwiftConnectionClosedException
+   */
+  private synchronized void assumeNotReleased() throws SwiftConnectionClosedException {
+    if (released || inStream == null) {
+      throw new SwiftConnectionClosedException(reasonClosed);
+    }
+  }
+
+  @Override
+  public int available() throws IOException {
+    assumeNotReleased();
+    try {
+      return inStream.available();
+    } catch (IOException e) {
+      throw releaseAndRethrow("available() failed -" + e, e);
+    }
+  }
+
+  @Override
+  public int read() throws IOException {
+    assumeNotReleased();
+    int read = 0;
+    try {
+      read = inStream.read();
+    } catch (EOFException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("EOF exception " + e, e);
+      }
+      read = -1;
+    } catch (IOException e) {
+      throw releaseAndRethrow("read()", e);
+    }
+    if (read < 0) {
+      dataConsumed = true;
+      release("read() -all data consumed", null);
+    }
+    return read;
+  }
+
+  @Override
+  public int read(byte[] b, int off, int len) throws IOException {
+    SwiftUtils.validateReadArgs(b, off, len);
+    //if the stream is already closed, then report an exception.
+    assumeNotReleased();
+    //now read in a buffer, reacting differently to different operations
+    int read;
+    try {
+      read = inStream.read(b, off, len);
+    } catch (EOFException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("EOF exception " + e, e);
+      }
+      read = -1;
+    } catch (IOException e) {
+      throw releaseAndRethrow("read(b, off, " + len + ")", e);
+    }
+    if (read < 0) {
+      dataConsumed = true;
+      release("read() -all data consumed", null);
+    }
+    return read;
+  }
+
+  /**
+   * Finalizer does release the stream, but also logs at WARN level
+   * including the URI at fault
+   */
+  @Override
+  protected void finalize() {
+    try {
+      if (release("finalize()", constructionStack)) {
+        LOG.warn("input stream of " + uri
+                 + " not closed properly -cleaned up in finalize()");
+      }
+    } catch (Exception e) {
+      //swallow anything that failed here
+      LOG.warn("Exception while releasing " + uri + "in finalizer",
+               e);
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "HttpInputStreamWithRelease working with " + uri
+      +" released=" + released
+      +" dataConsumed=" + dataConsumed;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/RestClientBindings.java

@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+
+import java.net.URI;
+import java.util.Properties;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;
+
+/**
+ * This class implements the binding logic between Hadoop configurations
+ * and the swift rest client.
+ * <p/>
+ * The swift rest client takes a Properties instance containing
+ * the string values it uses to bind to a swift endpoint.
+ * <p/>
+ * This class extracts the values for a specific filesystem endpoint
+ * and then builds an appropriate Properties file.
+ */
+public final class RestClientBindings {
+  private static final Log LOG = LogFactory.getLog(RestClientBindings.class);
+
+  public static final String E_INVALID_NAME = "Invalid swift hostname '%s':" +
+          " hostname must in form container.service";
+
+  /**
+   * Public for testing : build the full prefix for use in resolving
+   * configuration items
+   *
+   * @param service service to use
+   * @return the prefix string <i>without any trailing "."</i>
+   */
+  public static String buildSwiftInstancePrefix(String service) {
+    return SWIFT_SERVICE_PREFIX + service;
+  }
+
+  /**
+   * Raise an exception for an invalid service name
+   *
+   * @param hostname hostname that was being parsed
+   * @return an exception to throw
+   */
+  private static SwiftConfigurationException invalidName(String hostname) {
+    return new SwiftConfigurationException(
+            String.format(E_INVALID_NAME, hostname));
+  }
+
+  /**
+   * Get the container name from the hostname -the single element before the
+   * first "." in the hostname
+   *
+   * @param hostname hostname to split
+   * @return the container
+   * @throws SwiftConfigurationException
+   */
+  public static String extractContainerName(String hostname) throws
+          SwiftConfigurationException {
+    int i = hostname.indexOf(".");
+    if (i <= 0) {
+      throw invalidName(hostname);
+    }
+    return hostname.substring(0, i);
+  }
+
+  public static String extractContainerName(URI uri) throws
+          SwiftConfigurationException {
+    return extractContainerName(uri.getHost());
+  }
+
+  /**
+   * Get the service name from a longer hostname string
+   *
+   * @param hostname hostname
+   * @return the separated out service name
+   * @throws SwiftConfigurationException if the hostname was invalid
+   */
+  public static String extractServiceName(String hostname) throws
+          SwiftConfigurationException {
+    int i = hostname.indexOf(".");
+    if (i <= 0) {
+      throw invalidName(hostname);
+    }
+    String service = hostname.substring(i + 1);
+    if (service.isEmpty() || service.contains(".")) {
+      //empty service contains dots in -not currently supported
+      throw invalidName(hostname);
+    }
+    return service;
+  }
+
+  public static String extractServiceName(URI uri) throws
+          SwiftConfigurationException {
+    return extractServiceName(uri.getHost());
+  }
+
+  /**
+   * Build a properties instance bound to the configuration file -using
+   * the filesystem URI as the source of the information.
+   *
+   * @param fsURI filesystem URI
+   * @param conf  configuration
+   * @return a properties file with the instance-specific properties extracted
+   *         and bound to the swift client properties.
+   * @throws SwiftConfigurationException if the configuration is invalid
+   */
+  public static Properties bind(URI fsURI, Configuration conf) throws
+          SwiftConfigurationException {
+    String host = fsURI.getHost();
+    if (host == null || host.isEmpty()) {
+      //expect shortnames -> conf names
+      throw invalidName(host);
+    }
+
+    String container = extractContainerName(host);
+    String service = extractServiceName(host);
+
+    //build filename schema
+    String prefix = buildSwiftInstancePrefix(service);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Filesystem " + fsURI
+              + " is using configuration keys " + prefix);
+    }
+    Properties props = new Properties();
+    props.setProperty(SWIFT_SERVICE_PROPERTY, service);
+    props.setProperty(SWIFT_CONTAINER_PROPERTY, container);
+    copy(conf, prefix + DOT_AUTH_URL, props, SWIFT_AUTH_PROPERTY, true);
+    copy(conf, prefix + DOT_USERNAME, props, SWIFT_USERNAME_PROPERTY, true);
+    copy(conf, prefix + DOT_APIKEY, props, SWIFT_APIKEY_PROPERTY, false);
+    copy(conf, prefix + DOT_PASSWORD, props, SWIFT_PASSWORD_PROPERTY,
+            props.contains(SWIFT_APIKEY_PROPERTY) ? true : false);
+    copy(conf, prefix + DOT_TENANT, props, SWIFT_TENANT_PROPERTY, false);
+    copy(conf, prefix + DOT_REGION, props, SWIFT_REGION_PROPERTY, false);
+    copy(conf, prefix + DOT_HTTP_PORT, props, SWIFT_HTTP_PORT_PROPERTY, false);
+    copy(conf, prefix +
+            DOT_HTTPS_PORT, props, SWIFT_HTTPS_PORT_PROPERTY, false);
+
+    copyBool(conf, prefix + DOT_PUBLIC, props, SWIFT_PUBLIC_PROPERTY, false);
+    copyBool(conf, prefix + DOT_LOCATION_AWARE, props,
+             SWIFT_LOCATION_AWARE_PROPERTY, false);
+
+    return props;
+  }
+
+  /**
+   * Extract a boolean value from the configuration and copy it to the
+   * properties instance.
+   * @param conf     source configuration
+   * @param confKey  key in the configuration file
+   * @param props    destination property set
+   * @param propsKey key in the property set
+   * @param defVal default value
+   */
+  private static void copyBool(Configuration conf,
+                               String confKey,
+                               Properties props,
+                               String propsKey,
+                               boolean defVal) {
+    boolean b = conf.getBoolean(confKey, defVal);
+    props.setProperty(propsKey, Boolean.toString(b));
+  }
+
+  private static void set(Properties props, String key, String optVal) {
+    if (optVal != null) {
+      props.setProperty(key, optVal);
+    }
+  }
+
+  /**
+   * Copy a (trimmed) property from the configuration file to the properties file.
+   * <p/>
+   * If marked as required and not found in the configuration, an
+   * exception is raised.
+   * If not required -and missing- then the property will not be set.
+   * In this case, if the property is already in the Properties instance,
+   * it will remain untouched.
+   *
+   * @param conf     source configuration
+   * @param confKey  key in the configuration file
+   * @param props    destination property set
+   * @param propsKey key in the property set
+   * @param required is the property required
+   * @throws SwiftConfigurationException if the property is required but was
+   *                                     not found in the configuration instance.
+   */
+  public static void copy(Configuration conf, String confKey, Properties props,
+                          String propsKey,
+                          boolean required) throws SwiftConfigurationException {
+    //TODO: replace. version compatibility issue conf.getTrimmed fails with NoSuchMethodError
+    String val = conf.get(confKey);
+    if (val != null) {
+      val = val.trim();
+    }
+    if (required && val == null) {
+      throw new SwiftConfigurationException(
+              "Missing mandatory configuration option: "
+                      +
+                      confKey);
+    }
+    set(props, propsKey, val);
+  }
+
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftProtocolConstants.java

@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.hadoop.util.VersionInfo;
+
+/**
+ * Constants used in the Swift REST protocol,
+ * and in the properties used to configure the {@link SwiftRestClient}.
+ */
+public class SwiftProtocolConstants {
+  /**
+   * Swift-specific header for authentication: {@value}
+   */
+  public static final String HEADER_AUTH_KEY = "X-Auth-Token";
+
+  /**
+   * Default port used by Swift for HTTP
+   */
+  public static final int SWIFT_HTTP_PORT = 8080;
+
+  /**
+   * Default port used by Swift Auth for HTTPS
+   */
+  public static final int SWIFT_HTTPS_PORT = 443;
+
+  /** HTTP standard {@value} header */
+  public static final String HEADER_RANGE = "Range";
+
+  /** HTTP standard {@value} header */
+  public static final String HEADER_DESTINATION = "Destination";
+
+  /** HTTP standard {@value} header */
+  public static final String HEADER_LAST_MODIFIED = "Last-Modified";
+
+  /** HTTP standard {@value} header */
+  public static final String HEADER_CONTENT_LENGTH = "Content-Length";
+
+  /** HTTP standard {@value} header */
+  public static final String HEADER_CONTENT_RANGE = "Content-Range";
+
+  /**
+   * Patten for range headers
+   */
+  public static final String SWIFT_RANGE_HEADER_FORMAT_PATTERN = "bytes=%d-%d";
+
+  /**
+   *  section in the JSON catalog provided after auth listing the swift FS:
+   *  {@value}
+   */
+  public static final String SERVICE_CATALOG_SWIFT = "swift";
+  /**
+   *  section in the JSON catalog provided after auth listing the cloudfiles;
+   * this is an alternate catalog entry name
+   *  {@value}
+   */
+  public static final String SERVICE_CATALOG_CLOUD_FILES = "cloudFiles";
+  /**
+   *  section in the JSON catalog provided after auth listing the object store;
+   * this is an alternate catalog entry name
+   *  {@value}
+   */
+  public static final String SERVICE_CATALOG_OBJECT_STORE = "object-store";
+
+  /**
+   * entry in the swift catalog defining the prefix used to talk to objects
+   *  {@value}
+   */
+  public static final String SWIFT_OBJECT_AUTH_ENDPOINT =
+          "/object_endpoint/";
+  /**
+   * Swift-specific header: object manifest used in the final upload
+   * of a multipart operation: {@value}
+   */
+  public static final String X_OBJECT_MANIFEST = "X-Object-Manifest";
+  /**
+   * Swift-specific header -#of objects in a container: {@value}
+   */
+  public static final String X_CONTAINER_OBJECT_COUNT =
+          "X-Container-Object-Count";
+  /**
+   * Swift-specific header: no. of bytes used in a container {@value}
+   */
+  public static final String X_CONTAINER_BYTES_USED = "X-Container-Bytes-Used";
+
+  /**
+   * Header to set when requesting the latest version of a file: : {@value}
+   */
+  public static final String X_NEWEST = "X-Newest";
+
+  /**
+   * throttled response sent by some endpoints.
+   */
+  public static final int SC_THROTTLED_498 = 498;
+  /**
+   * W3C recommended status code for throttled operations
+   */
+  public static final int SC_TOO_MANY_REQUESTS_429 = 429;
+
+  public static final String FS_SWIFT = "fs.swift";
+
+  /**
+   * Prefix for all instance-specific values in the configuration: {@value}
+   */
+  public static final String SWIFT_SERVICE_PREFIX = FS_SWIFT + ".service.";
+
+  /**
+   * timeout for all connections: {@value}
+   */
+  public static final String SWIFT_CONNECTION_TIMEOUT =
+          FS_SWIFT + ".connect.timeout";
+
+  /**
+   * timeout for all connections: {@value}
+   */
+  public static final String SWIFT_SOCKET_TIMEOUT =
+          FS_SWIFT + ".socket.timeout";
+
+  /**
+   * the default socket timeout in millis {@value}.
+   * This controls how long the connection waits for responses from
+   * servers.
+   */
+  public static final int DEFAULT_SOCKET_TIMEOUT = 60000;
+
+  /**
+   * connection retry count for all connections: {@value}
+   */
+  public static final String SWIFT_RETRY_COUNT =
+          FS_SWIFT + ".connect.retry.count";
+
+  /**
+   * delay in millis between bulk (delete, rename, copy operations: {@value}
+   */
+  public static final String SWIFT_THROTTLE_DELAY =
+          FS_SWIFT + ".connect.throttle.delay";
+
+  /**
+   * the default throttle delay in millis {@value}
+   */
+  public static final int DEFAULT_THROTTLE_DELAY = 0;
+
+  /**
+   * blocksize for all filesystems: {@value}
+   */
+  public static final String SWIFT_BLOCKSIZE =
+          FS_SWIFT + ".blocksize";
+
+  /**
+   * the default blocksize for filesystems in KB: {@value}
+   */
+  public static final int DEFAULT_SWIFT_BLOCKSIZE = 32 * 1024;
+
+  /**
+   * partition size for all filesystems in KB: {@value}
+   */
+  public static final String SWIFT_PARTITION_SIZE =
+    FS_SWIFT + ".partsize";
+
+  /**
+   * The default partition size for uploads: {@value}
+   */
+  public static final int DEFAULT_SWIFT_PARTITION_SIZE = 4608*1024;
+
+  /**
+   * request size for reads in KB: {@value}
+   */
+  public static final String SWIFT_REQUEST_SIZE =
+    FS_SWIFT + ".requestsize";
+
+  /**
+   * The default reqeuest size for reads: {@value}
+   */
+  public static final int DEFAULT_SWIFT_REQUEST_SIZE = 64;
+
+
+  public static final String HEADER_USER_AGENT="User-Agent";
+
+  /**
+   * The user agent sent in requests.
+   */
+  public static final String SWIFT_USER_AGENT= "Apache Hadoop Swift Client "
+                                               + VersionInfo.getBuildVersion();
+
+  /**
+   * Key for passing the service name as a property -not read from the
+   * configuration : {@value}
+   */
+  public static final String DOT_SERVICE = ".SERVICE-NAME";
+
+  /**
+   * Key for passing the container name as a property -not read from the
+   * configuration : {@value}
+   */
+  public static final String DOT_CONTAINER = ".CONTAINER-NAME";
+
+  public static final String DOT_AUTH_URL = ".auth.url";
+  public static final String DOT_TENANT = ".tenant";
+  public static final String DOT_USERNAME = ".username";
+  public static final String DOT_PASSWORD = ".password";
+  public static final String DOT_HTTP_PORT = ".http.port";
+  public static final String DOT_HTTPS_PORT = ".https.port";
+  public static final String DOT_REGION = ".region";
+  public static final String DOT_PROXY_HOST = ".proxy.host";
+  public static final String DOT_PROXY_PORT = ".proxy.port";
+  public static final String DOT_LOCATION_AWARE = ".location-aware";
+  public static final String DOT_APIKEY = ".apikey";
+  public static final String DOT_USE_APIKEY = ".useApikey";
+
+  /**
+   * flag to say use public URL
+   */
+  public static final String DOT_PUBLIC = ".public";
+
+  public static final String SWIFT_SERVICE_PROPERTY = FS_SWIFT + DOT_SERVICE;
+  public static final String SWIFT_CONTAINER_PROPERTY = FS_SWIFT + DOT_CONTAINER;
+
+  public static final String SWIFT_AUTH_PROPERTY = FS_SWIFT + DOT_AUTH_URL;
+  public static final String SWIFT_TENANT_PROPERTY = FS_SWIFT + DOT_TENANT;
+  public static final String SWIFT_USERNAME_PROPERTY = FS_SWIFT + DOT_USERNAME;
+  public static final String SWIFT_PASSWORD_PROPERTY = FS_SWIFT + DOT_PASSWORD;
+  public static final String SWIFT_APIKEY_PROPERTY = FS_SWIFT + DOT_APIKEY;
+  public static final String SWIFT_HTTP_PORT_PROPERTY = FS_SWIFT + DOT_HTTP_PORT;
+  public static final String SWIFT_HTTPS_PORT_PROPERTY = FS_SWIFT
+          + DOT_HTTPS_PORT;
+  public static final String SWIFT_REGION_PROPERTY = FS_SWIFT + DOT_REGION;
+  public static final String SWIFT_PUBLIC_PROPERTY = FS_SWIFT + DOT_PUBLIC;
+
+  public static final String SWIFT_USE_API_KEY_PROPERTY = FS_SWIFT + DOT_USE_APIKEY;
+
+  public static final String SWIFT_LOCATION_AWARE_PROPERTY = FS_SWIFT +
+            DOT_LOCATION_AWARE;
+
+  public static final String SWIFT_PROXY_HOST_PROPERTY = FS_SWIFT + DOT_PROXY_HOST;
+  public static final String SWIFT_PROXY_PORT_PROPERTY = FS_SWIFT + DOT_PROXY_PORT;
+  public static final String HTTP_ROUTE_DEFAULT_PROXY =
+    "http.route.default-proxy";
+  /**
+   * Topology to return when a block location is requested
+   */
+  public static final String TOPOLOGY_PATH = "/swift/unknown";
+  /**
+   * Block location to return when a block location is requested
+   */
+  public static final String BLOCK_LOCATION = "/default-rack/swift";
+  /**
+   * Default number of attempts to retry a connect request: {@value}
+   */
+  static final int DEFAULT_RETRY_COUNT = 3;
+  /**
+   * Default timeout in milliseconds for connection requests: {@value}
+   */
+  static final int DEFAULT_CONNECT_TIMEOUT = 15000;
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/http/SwiftRestClient.java

@@ -0,0 +1,1877 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpHost;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.HttpMethodBase;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.httpclient.methods.DeleteMethod;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.methods.HeadMethod;
+import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
+import org.apache.commons.httpclient.methods.PostMethod;
+import org.apache.commons.httpclient.methods.PutMethod;
+import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.httpclient.params.HttpConnectionParams;
+import org.apache.commons.httpclient.params.HttpMethodParams;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.auth.ApiKeyAuthenticationRequest;
+import org.apache.hadoop.fs.swift.auth.ApiKeyCredentials;
+import org.apache.hadoop.fs.swift.auth.AuthenticationRequest;
+import org.apache.hadoop.fs.swift.auth.AuthenticationRequestWrapper;
+import org.apache.hadoop.fs.swift.auth.AuthenticationResponse;
+import org.apache.hadoop.fs.swift.auth.AuthenticationWrapper;
+import org.apache.hadoop.fs.swift.auth.KeyStoneAuthRequest;
+import org.apache.hadoop.fs.swift.auth.KeystoneApiKeyCredentials;
+import org.apache.hadoop.fs.swift.auth.PasswordAuthenticationRequest;
+import org.apache.hadoop.fs.swift.auth.PasswordCredentials;
+import org.apache.hadoop.fs.swift.auth.entities.AccessToken;
+import org.apache.hadoop.fs.swift.auth.entities.Catalog;
+import org.apache.hadoop.fs.swift.auth.entities.Endpoint;
+import org.apache.hadoop.fs.swift.exceptions.SwiftAuthenticationFailedException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftThrottledRequestException;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.DurationStatsTable;
+import org.apache.hadoop.fs.swift.util.JSONUtil;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.http.conn.params.ConnRoutePNames;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.commons.httpclient.HttpStatus.*;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.*;
+
+/**
+ * This implements the client-side of the Swift REST API
+ *
+ * The core actions put, get and query data in the Swift object store,
+ * after authenticationg the client.
+ *
+ * <b>Logging:</b>
+ *
+ * Logging at DEBUG level displays detail about the actions of this
+ * client, including HTTP requests and responses -excluding authentication
+ * details.
+ */
+public final class SwiftRestClient {
+  private static final Log LOG = LogFactory.getLog(SwiftRestClient.class);
+
+  /**
+   * Header that says "use newest version" -ensures that
+   * the query doesn't pick up older versions served by
+   * an eventually consistent filesystem (except in the special case
+   * of a network partition, at which point no guarantees about
+   * consistency can be made.
+   */
+  public static final Header NEWEST =
+          new Header(SwiftProtocolConstants.X_NEWEST, "true");
+
+  /**
+   * the authentication endpoint as supplied in the configuration
+   */
+  private final URI authUri;
+
+  /**
+   * Swift region. Some OpenStack installations has more than one region.
+   * In this case user can specify the region with which Hadoop will be working
+   */
+  private final String region;
+
+  /**
+   * tenant name
+   */
+  private final String tenant;
+
+  /**
+   * username name
+   */
+  private final String username;
+
+  /**
+   * user password
+   */
+  private final String password;
+
+  /**
+   * user api key
+   */
+  private final String apiKey;
+
+  /**
+   * The authentication request used to authenticate with Swift
+   */
+  private final AuthenticationRequest authRequest;
+
+  /**
+   * This auth request is similar to @see authRequest,
+   * with one difference: it has another json representation when
+   * authRequest one is not applicable
+   */
+  private AuthenticationRequest keystoneAuthRequest;
+
+  private boolean useKeystoneAuthentication = false;
+
+    /**
+   * The container this client is working with
+   */
+  private final String container;
+  private final String serviceDescription;
+
+  /**
+   * Access token (Secret)
+   */
+  private AccessToken token;
+
+  /**
+   * Endpoint for swift operations, obtained after authentication
+   */
+  private URI endpointURI;
+
+  /**
+   * URI under which objects can be found.
+   * This is set when the user is authenticated -the URI
+   * is returned in the body of the success response.
+   */
+  private URI objectLocationURI;
+
+  private final URI filesystemURI;
+
+  /**
+   * The name of the service provider
+   */
+  private final String serviceProvider;
+
+  /**
+   * Should the public swift endpoint be used, rather than the in-cluster one?
+   */
+  private final boolean usePublicURL;
+
+  /**
+   * Number of times to retry a connection
+   */
+  private final int retryCount;
+
+  /**
+   * How long (in milliseconds) should a connection be attempted
+   */
+  private final int connectTimeout;
+
+  /**
+   * How long (in milliseconds) should a connection be attempted
+   */
+  private final int socketTimeout;
+
+  /**
+   * How long (in milliseconds) between bulk operations
+   */
+  private final int throttleDelay;
+
+  /**
+  * the name of a proxy host (can be null, in which case there is no proxy)
+   */
+  private String proxyHost;
+
+  /**
+   * The port of a proxy. This is ignored if {@link #proxyHost} is null
+   */
+  private int proxyPort;
+
+  /**
+   * Flag to indicate whether or not the client should
+   * query for file location data.
+   */
+  private final boolean locationAware;
+
+  private final int partSizeKB;
+  /**
+   * The blocksize of this FS
+   */
+  private final int blocksizeKB;
+  private final int bufferSizeKB;
+
+  private final DurationStatsTable durationStats = new DurationStatsTable();
+  /**
+   * objects query endpoint. This is synchronized
+   * to handle a simultaneous update of all auth data in one
+   * go.
+   */
+  private synchronized URI getEndpointURI() {
+    return endpointURI;
+  }
+
+  /**
+   * object location endpoint
+   */
+  private synchronized URI getObjectLocationURI() {
+    return objectLocationURI;
+  }
+
+  /**
+   * token for Swift communication
+   */
+  private synchronized AccessToken getToken() {
+    return token;
+  }
+
+  /**
+   * Setter of authentication and endpoint details.
+   * Being synchronized guarantees that all three fields are set up together.
+   * It is up to the reader to read all three fields in their own
+   * synchronized block to be sure that they are all consistent.
+   *
+   * @param endpoint endpoint URI
+   * @param objectLocation object location URI
+   * @param authToken auth token
+   */
+  private void setAuthDetails(URI endpoint,
+                              URI objectLocation,
+                              AccessToken authToken) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(String.format("setAuth: endpoint=%s; objectURI=%s; token=%s",
+              endpoint, objectLocation, authToken));
+    }
+    synchronized (this) {
+      endpointURI = endpoint;
+      objectLocationURI = objectLocation;
+      token = authToken;
+    }
+  }
+
+
+  /**
+   * Base class for all Swift REST operations
+   *
+   * @param <M> method
+   * @param <R> result
+   */
+  private static abstract class HttpMethodProcessor<M extends HttpMethod, R> {
+    public final M createMethod(String uri) throws IOException {
+      final M method = doCreateMethod(uri);
+      setup(method);
+      return method;
+    }
+
+    /**
+     * Override it to return some result after method is executed.
+     */
+    public abstract R extractResult(M method) throws IOException;
+
+    /**
+     * Factory method to create a REST method against the given URI
+     *
+     * @param uri target
+     * @return method to invoke
+     */
+    protected abstract M doCreateMethod(String uri);
+
+    /**
+     * Override port to set up the method before it is executed.
+     */
+    protected void setup(M method) throws IOException {
+    }
+
+    /**
+     * Override point: what are the status codes that this operation supports
+     *
+     * @return an array with the permitted status code(s)
+     */
+    protected int[] getAllowedStatusCodes() {
+      return new int[]{
+              SC_OK,
+              SC_CREATED,
+              SC_ACCEPTED,
+              SC_NO_CONTENT,
+              SC_PARTIAL_CONTENT,
+      };
+    }
+  }
+
+  private static abstract class GetMethodProcessor<R> extends HttpMethodProcessor<GetMethod, R> {
+    @Override
+    protected final GetMethod doCreateMethod(String uri) {
+      return new GetMethod(uri);
+    }
+  }
+
+  private static abstract class PostMethodProcessor<R> extends HttpMethodProcessor<PostMethod, R> {
+    @Override
+    protected final PostMethod doCreateMethod(String uri) {
+      return new PostMethod(uri);
+    }
+  }
+
+  /**
+   * There's a special type for auth messages, so that low-level
+   * message handlers can react to auth failures differently from everything
+   * else.
+   */
+  private static class AuthPostMethod extends PostMethod {
+
+
+    private AuthPostMethod(String uri) {
+      super(uri);
+    }
+  }
+
+  /**
+   * Generate an auth message
+   * @param <R> response
+   */
+  private static abstract class AuthMethodProcessor<R> extends
+                                                       HttpMethodProcessor<AuthPostMethod, R> {
+    @Override
+    protected final AuthPostMethod doCreateMethod(String uri) {
+      return new AuthPostMethod(uri);
+    }
+  }
+
+  private static abstract class PutMethodProcessor<R> extends HttpMethodProcessor<PutMethod, R> {
+    @Override
+    protected final PutMethod doCreateMethod(String uri) {
+      return new PutMethod(uri);
+    }
+
+    /**
+     * Override point: what are the status codes that this operation supports
+     *
+     * @return the list of status codes to accept
+     */
+    @Override
+    protected int[] getAllowedStatusCodes() {
+      return new int[]{
+              SC_OK,
+              SC_CREATED,
+              SC_NO_CONTENT,
+              SC_ACCEPTED,
+      };
+    }
+  }
+
+  /**
+   * Create operation
+   *
+   * @param <R>
+   */
+  private static abstract class CopyMethodProcessor<R> extends HttpMethodProcessor<CopyMethod, R> {
+    @Override
+    protected final CopyMethod doCreateMethod(String uri) {
+      return new CopyMethod(uri);
+    }
+
+    /**
+     * The only allowed status code is 201:created
+     * @return an array with the permitted status code(s)
+     */
+    @Override
+    protected int[] getAllowedStatusCodes() {
+      return new int[]{
+              SC_CREATED
+      };
+    }
+  }
+
+  /**
+   * Delete operation
+   *
+   * @param <R>
+   */
+  private static abstract class DeleteMethodProcessor<R> extends HttpMethodProcessor<DeleteMethod, R> {
+    @Override
+    protected final DeleteMethod doCreateMethod(String uri) {
+      return new DeleteMethod(uri);
+    }
+
+    @Override
+    protected int[] getAllowedStatusCodes() {
+      return new int[]{
+              SC_OK,
+              SC_ACCEPTED,
+              SC_NO_CONTENT,
+              SC_NOT_FOUND
+      };
+    }
+  }
+
+  private static abstract class HeadMethodProcessor<R> extends HttpMethodProcessor<HeadMethod, R> {
+    @Override
+    protected final HeadMethod doCreateMethod(String uri) {
+      return new HeadMethod(uri);
+    }
+  }
+
+
+  /**
+   * Create a Swift Rest Client instance.
+   *
+   * @param filesystemURI filesystem URI
+   * @param conf The configuration to use to extract the binding
+   * @throws SwiftConfigurationException the configuration is not valid for
+   * defining a rest client against the service
+   */
+  private SwiftRestClient(URI filesystemURI,
+                          Configuration conf)
+      throws SwiftConfigurationException {
+    this.filesystemURI = filesystemURI;
+    Properties props = RestClientBindings.bind(filesystemURI, conf);
+    String stringAuthUri = getOption(props, SWIFT_AUTH_PROPERTY);
+    username = getOption(props, SWIFT_USERNAME_PROPERTY);
+    password = props.getProperty(SWIFT_PASSWORD_PROPERTY);
+    apiKey = props.getProperty(SWIFT_APIKEY_PROPERTY);
+    //optional
+    region = props.getProperty(SWIFT_REGION_PROPERTY);
+    //tenant is optional
+    tenant = props.getProperty(SWIFT_TENANT_PROPERTY);
+    //service is used for diagnostics
+    serviceProvider = props.getProperty(SWIFT_SERVICE_PROPERTY);
+    container = props.getProperty(SWIFT_CONTAINER_PROPERTY);
+    String isPubProp = props.getProperty(SWIFT_PUBLIC_PROPERTY, "false");
+    usePublicURL = "true".equals(isPubProp);
+
+        if (apiKey == null && password == null) {
+            throw new SwiftConfigurationException(
+                    "Configuration for " + filesystemURI +" must contain either "
+                            + SWIFT_PASSWORD_PROPERTY + " or "
+                            + SWIFT_APIKEY_PROPERTY);
+        }
+        //create the (reusable) authentication request
+        if (password != null) {
+            authRequest = new PasswordAuthenticationRequest(tenant,
+                    new PasswordCredentials(
+                            username,
+                            password));
+        } else {
+            authRequest = new ApiKeyAuthenticationRequest(tenant,
+                    new ApiKeyCredentials(
+                            username, apiKey));
+            keystoneAuthRequest = new KeyStoneAuthRequest(tenant,
+                    new KeystoneApiKeyCredentials(username, apiKey));
+    }
+    locationAware = "true".equals(
+      props.getProperty(SWIFT_LOCATION_AWARE_PROPERTY, "false"));
+
+    //now read in properties that are shared across all connections
+
+    //connection and retries
+    try {
+      retryCount = conf.getInt(SWIFT_RETRY_COUNT, DEFAULT_RETRY_COUNT);
+      connectTimeout = conf.getInt(SWIFT_CONNECTION_TIMEOUT,
+                                   DEFAULT_CONNECT_TIMEOUT);
+      socketTimeout = conf.getInt(SWIFT_SOCKET_TIMEOUT,
+                                   DEFAULT_SOCKET_TIMEOUT);
+
+      throttleDelay = conf.getInt(SWIFT_THROTTLE_DELAY,
+                                  DEFAULT_THROTTLE_DELAY);
+
+      //proxy options
+      proxyHost = conf.get(SWIFT_PROXY_HOST_PROPERTY);
+      proxyPort = conf.getInt(SWIFT_PROXY_PORT_PROPERTY, 8080);
+
+      blocksizeKB = conf.getInt(SWIFT_BLOCKSIZE,
+                                DEFAULT_SWIFT_BLOCKSIZE);
+      if (blocksizeKB <= 0) {
+        throw new SwiftConfigurationException("Invalid blocksize set in "
+                          + SWIFT_BLOCKSIZE
+                          + ": " + blocksizeKB);
+      }
+      partSizeKB = conf.getInt(SWIFT_PARTITION_SIZE,
+                               DEFAULT_SWIFT_PARTITION_SIZE);
+      if (partSizeKB <=0) {
+        throw new SwiftConfigurationException("Invalid partition size set in "
+                                              + SWIFT_PARTITION_SIZE
+                                              + ": " + partSizeKB);
+      }
+
+      bufferSizeKB = conf.getInt(SWIFT_REQUEST_SIZE,
+                                 DEFAULT_SWIFT_REQUEST_SIZE);
+      if (bufferSizeKB <=0) {
+        throw new SwiftConfigurationException("Invalid buffer size set in "
+                          + SWIFT_REQUEST_SIZE
+                          + ": " + bufferSizeKB);
+      }
+    } catch (NumberFormatException e) {
+      //convert exceptions raised parsing ints and longs into
+      // SwiftConfigurationException instances
+      throw new SwiftConfigurationException(e.toString(), e);
+    }
+    //everything you need for diagnostics. The password is omitted.
+    serviceDescription = String.format(
+      "Service={%s} container={%s} uri={%s}"
+      + " tenant={%s} user={%s} region={%s}"
+      + " publicURL={%b}"
+      + " location aware={%b}"
+      + " partition size={%d KB}, buffer size={%d KB}"
+      + " block size={%d KB}"
+      + " connect timeout={%d}, retry count={%d}"
+      + " socket timeout={%d}"
+      + " throttle delay={%d}"
+      ,
+      serviceProvider,
+      container,
+      stringAuthUri,
+      tenant,
+      username,
+      region != null ? region : "(none)",
+      usePublicURL,
+      locationAware,
+      partSizeKB,
+      bufferSizeKB,
+      blocksizeKB,
+      connectTimeout,
+      retryCount,
+      socketTimeout,
+      throttleDelay
+      );
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(serviceDescription);
+    }
+    try {
+      this.authUri = new URI(stringAuthUri);
+    } catch (URISyntaxException e) {
+      throw new SwiftConfigurationException("The " + SWIFT_AUTH_PROPERTY
+              + " property was incorrect: "
+              + stringAuthUri, e);
+    }
+  }
+
+  /**
+   * Get a mandatory configuration option
+   *
+   * @param props property set
+   * @param key   key
+   * @return value of the configuration
+   * @throws SwiftConfigurationException if there was no match for the key
+   */
+  private static String getOption(Properties props, String key) throws
+          SwiftConfigurationException {
+    String val = props.getProperty(key);
+    if (val == null) {
+      throw new SwiftConfigurationException("Undefined property: " + key);
+    }
+    return val;
+  }
+
+  /**
+   * Make an HTTP GET request to Swift to get a range of data in the object.
+   *
+   * @param path   path to object
+   * @param offset offset from file beginning
+   * @param length file length
+   * @return The input stream -which must be closed afterwards.
+   * @throws IOException Problems
+   * @throws SwiftException swift specific error
+   * @throws FileNotFoundException path is not there
+   */
+  public HttpBodyContent getData(SwiftObjectPath path,
+                                 long offset,
+                                 long length) throws IOException {
+    if (offset < 0) {
+      throw new SwiftException("Invalid offset: " + offset
+                            + " in getDataAsInputStream( path=" + path
+                            + ", offset=" + offset
+                            + ", length =" + length + ")");
+    }
+    if (length <= 0) {
+      throw new SwiftException("Invalid length: " + length
+                + " in getDataAsInputStream( path="+ path
+                            + ", offset=" + offset
+                            + ", length ="+ length + ")");
+    }
+
+    final String range = String.format(SWIFT_RANGE_HEADER_FORMAT_PATTERN,
+            offset,
+            offset + length - 1);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("getData:" + range);
+    }
+
+    return getData(path,
+                   new Header(HEADER_RANGE, range),
+                   SwiftRestClient.NEWEST);
+  }
+
+  /**
+   * Returns object length
+   *
+   * @param uri file URI
+   * @return object length
+   * @throws SwiftException on swift-related issues
+   * @throws IOException on network/IO problems
+   */
+  public long getContentLength(URI uri) throws IOException {
+    preRemoteCommand("getContentLength");
+    return perform("getContentLength", uri, new HeadMethodProcessor<Long>() {
+      @Override
+      public Long extractResult(HeadMethod method) throws IOException {
+        return method.getResponseContentLength();
+      }
+
+      @Override
+      protected void setup(HeadMethod method) throws IOException {
+        super.setup(method);
+        method.addRequestHeader(NEWEST);
+      }
+    });
+  }
+
+  /**
+   * Get the length of the remote object
+   * @param path object to probe
+   * @return the content length
+   * @throws IOException on any failure
+   */
+  public long getContentLength(SwiftObjectPath path) throws IOException {
+    return getContentLength(pathToURI(path));
+  }
+
+  /**
+   * Get the path contents as an input stream.
+   * <b>Warning:</b> this input stream must be closed to avoid
+   * keeping Http connections open.
+   *
+   * @param path path to file
+   * @param requestHeaders http headers
+   * @return byte[] file data or null if the object was not found
+   * @throws IOException on IO Faults
+   * @throws FileNotFoundException if there is nothing at the path
+   */
+  public HttpBodyContent getData(SwiftObjectPath path,
+                                 final Header... requestHeaders)
+          throws IOException {
+    preRemoteCommand("getData");
+    return doGet(pathToURI(path),
+            requestHeaders);
+  }
+
+  /**
+   * Returns object location as byte[]
+   *
+   * @param path path to file
+   * @param requestHeaders http headers
+   * @return byte[] file data or null if the object was not found
+   * @throws IOException on IO Faults
+   */
+  public byte[] getObjectLocation(SwiftObjectPath path,
+                                  final Header... requestHeaders) throws IOException {
+    if (!isLocationAware()) {
+      //if the filesystem is not location aware, do not ask for this information
+      return null;
+    }
+    preRemoteCommand("getObjectLocation");
+    try {
+      return perform("getObjectLocation", pathToObjectLocation(path),
+              new GetMethodProcessor<byte[]>() {
+                @Override
+                protected int[] getAllowedStatusCodes() {
+                  return new int[]{
+                    SC_OK,
+                    SC_FORBIDDEN,
+                    SC_NO_CONTENT
+                  };
+                }
+
+                @Override
+                public byte[] extractResult(GetMethod method) throws
+                        IOException {
+
+                  //TODO: remove SC_NO_CONTENT if it depends on Swift versions
+                  if (method.getStatusCode() == SC_NOT_FOUND
+                      || method.getStatusCode() == SC_FORBIDDEN ||
+                          method.getStatusCode() == SC_NO_CONTENT ||
+                          method.getResponseBodyAsStream() == null) {
+                    return null;
+                  }
+                  final InputStream responseBodyAsStream = method.getResponseBodyAsStream();
+                  final byte[] locationData = new byte[1024];
+
+                  return responseBodyAsStream.read(locationData) > 0 ? locationData : null;
+                }
+
+                @Override
+                     protected void setup(GetMethod method)
+                       throws SwiftInternalStateException {
+                  setHeaders(method, requestHeaders);
+                }
+              });
+    } catch (IOException e) {
+      LOG.warn("Failed to get the location of " + path + ": " + e, e);
+      return null;
+    }
+  }
+
+  /**
+   * Create the URI needed to query the location of an object
+   * @param path object path to retrieve information about
+   * @return the URI for the location operation
+   * @throws SwiftException if the URI could not be constructed
+   */
+  private URI pathToObjectLocation(SwiftObjectPath path) throws SwiftException {
+    URI uri;
+    String dataLocationURI = objectLocationURI.toString();
+    try {
+      if (path.toString().startsWith("/")) {
+        dataLocationURI = dataLocationURI.concat(path.toUriPath());
+      } else {
+        dataLocationURI = dataLocationURI.concat("/").concat(path.toUriPath());
+      }
+
+      uri = new URI(dataLocationURI);
+    } catch (URISyntaxException e) {
+      throw new SwiftException(e);
+    }
+    return uri;
+  }
+
+  /**
+   * Find objects under a prefix
+   *
+   * @param path path prefix
+   * @param requestHeaders optional request headers
+   * @return byte[] file data or null if the object was not found
+   * @throws IOException on IO Faults
+   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
+   * the directory is empty
+   */
+  public byte[] findObjectsByPrefix(SwiftObjectPath path,
+                                    final Header... requestHeaders) throws IOException {
+    preRemoteCommand("findObjectsByPrefix");
+    URI uri;
+    String dataLocationURI = getEndpointURI().toString();
+    try {
+      String object = path.getObject();
+      if (object.startsWith("/")) {
+        object = object.substring(1);
+      }
+      object = encodeUrl(object);
+      dataLocationURI = dataLocationURI.concat("/")
+              .concat(path.getContainer())
+              .concat("/?prefix=")
+              .concat(object)
+      ;
+      uri = new URI(dataLocationURI);
+    } catch (URISyntaxException e) {
+      throw new SwiftException("Bad URI: " + dataLocationURI, e);
+    }
+
+    return perform("findObjectsByPrefix", uri, new GetMethodProcessor<byte[]>() {
+      @Override
+      public byte[] extractResult(GetMethod method) throws IOException {
+        if (method.getStatusCode() == SC_NOT_FOUND) {
+          //no result
+          throw new FileNotFoundException("Not found " + method.getURI());
+        }
+        return method.getResponseBody();
+      }
+
+      @Override
+      protected int[] getAllowedStatusCodes() {
+        return new int[]{
+                SC_OK,
+                SC_NOT_FOUND
+        };
+      }
+
+      @Override
+      protected void setup(GetMethod method) throws
+                     SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Find objects in a directory
+   *
+   * @param path path prefix
+   * @param requestHeaders optional request headers
+   * @return byte[] file data or null if the object was not found
+   * @throws IOException on IO Faults
+   * @throws FileNotFoundException if nothing is at the end of the URI -that is,
+   * the directory is empty
+   */
+  public byte[] listDeepObjectsInDirectory(SwiftObjectPath path,
+                                           boolean listDeep,
+                                       final Header... requestHeaders)
+          throws IOException {
+    preRemoteCommand("listDeepObjectsInDirectory");
+
+    String endpoint = getEndpointURI().toString();
+    StringBuilder dataLocationURI = new StringBuilder();
+    dataLocationURI.append(endpoint);
+    String object = path.getObject();
+    if (object.startsWith("/")) {
+      object = object.substring(1);
+    }
+    if (!object.endsWith("/")) {
+      object = object.concat("/");
+    }
+
+    if (object.equals("/")) {
+      object = "";
+    }
+
+    dataLocationURI = dataLocationURI.append("/")
+            .append(path.getContainer())
+            .append("/?prefix=")
+            .append(object)
+            .append("&format=json");
+
+    //in listing deep set param to false
+    if (listDeep == false) {
+        dataLocationURI.append("&delimiter=/");
+    }
+
+    return findObjects(dataLocationURI.toString(), requestHeaders);
+  }
+
+  /**
+   * Find objects in a location
+   * @param location URI
+   * @param requestHeaders optional request headers
+   * @return the body of te response
+   * @throws IOException IO problems
+   */
+  private byte[] findObjects(String location, final Header[] requestHeaders) throws
+          IOException {
+    URI uri;
+    preRemoteCommand("findObjects");
+    try {
+      uri = new URI(location);
+    } catch (URISyntaxException e) {
+      throw new SwiftException("Bad URI: " + location, e);
+    }
+
+    return perform("findObjects", uri, new GetMethodProcessor<byte[]>() {
+      @Override
+      public byte[] extractResult(GetMethod method) throws IOException {
+        if (method.getStatusCode() == SC_NOT_FOUND) {
+          //no result
+          throw new FileNotFoundException("Not found " + method.getURI());
+        }
+        return method.getResponseBody();
+      }
+
+      @Override
+      protected int[] getAllowedStatusCodes() {
+        return new int[]{
+                SC_OK,
+                SC_NOT_FOUND
+        };
+      }
+
+      @Override
+      protected void setup(GetMethod method)
+        throws SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Copy an object. This is done by sending a COPY method to the filesystem
+   * which is required to handle this WebDAV-level extension to the
+   * base HTTP operations.
+   *
+   * @param src source path
+   * @param dst destination path
+   * @param headers any headers
+   * @return true if the status code was considered successful
+   * @throws IOException on IO Faults
+   */
+  public boolean copyObject(SwiftObjectPath src, final SwiftObjectPath dst,
+                            final Header... headers) throws IOException {
+
+    preRemoteCommand("copyObject");
+
+    return perform("copy", pathToURI(src), new CopyMethodProcessor<Boolean>() {
+      @Override
+      public Boolean extractResult(CopyMethod method) throws IOException {
+        return method.getStatusCode() != SC_NOT_FOUND;
+      }
+
+      @Override
+      protected void setup(CopyMethod method) throws
+                     SwiftInternalStateException {
+        setHeaders(method, headers);
+        method.addRequestHeader(HEADER_DESTINATION, dst.toUriPath());
+      }
+    });
+  }
+
+  /**
+   * Uploads file as Input Stream to Swift.
+   * The data stream will be closed after the request.
+   *
+   * @param path path to Swift
+   * @param data object data
+   * @param length length of data
+   * @param requestHeaders http headers
+   * @throws IOException on IO Faults
+   */
+  public void upload(SwiftObjectPath path,
+                     final InputStream data,
+                     final long length,
+                     final Header... requestHeaders)
+          throws IOException {
+    preRemoteCommand("upload");
+
+    try {
+      perform("upload", pathToURI(path), new PutMethodProcessor<byte[]>() {
+        @Override
+        public byte[] extractResult(PutMethod method) throws IOException {
+          return method.getResponseBody();
+        }
+
+        @Override
+        protected void setup(PutMethod method) throws
+                        SwiftInternalStateException {
+          method.setRequestEntity(new InputStreamRequestEntity(data, length));
+          setHeaders(method, requestHeaders);
+        }
+      });
+    } finally {
+      data.close();
+    }
+
+  }
+
+
+  /**
+   * Deletes object from swift.
+   * The result is true if this operation did the deletion.
+   *
+   * @param path           path to file
+   * @param requestHeaders http headers
+   * @throws IOException on IO Faults
+   */
+  public boolean delete(SwiftObjectPath path, final Header... requestHeaders) throws IOException {
+    preRemoteCommand("delete");
+
+    return perform("", pathToURI(path), new DeleteMethodProcessor<Boolean>() {
+      @Override
+      public Boolean extractResult(DeleteMethod method) throws IOException {
+        return method.getStatusCode() == SC_NO_CONTENT;
+      }
+
+      @Override
+      protected void setup(DeleteMethod method) throws
+                    SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Issue a head request
+   * @param reason reason -used in logs
+   * @param path path to query
+   * @param requestHeaders request header
+   * @return the response headers. This may be an empty list
+   * @throws IOException IO problems
+   * @throws FileNotFoundException if there is nothing at the end
+   */
+  public Header[] headRequest(String reason,
+                              SwiftObjectPath path,
+                              final Header... requestHeaders)
+          throws IOException {
+
+    preRemoteCommand("headRequest: "+ reason);
+    return perform(reason, pathToURI(path), new HeadMethodProcessor<Header[]>() {
+      @Override
+      public Header[] extractResult(HeadMethod method) throws IOException {
+        if (method.getStatusCode() == SC_NOT_FOUND) {
+          throw new FileNotFoundException("Not Found " + method.getURI());
+        }
+
+        return method.getResponseHeaders();
+      }
+
+      @Override
+      protected void setup(HeadMethod method) throws
+                    SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Issue a put request
+   * @param path path
+   * @param requestHeaders optional headers
+   * @return the HTTP response
+   * @throws IOException any problem
+   */
+  public int putRequest(SwiftObjectPath path, final Header... requestHeaders)
+          throws IOException {
+
+    preRemoteCommand("putRequest");
+    return perform(pathToURI(path), new PutMethodProcessor<Integer>() {
+
+      @Override
+      public Integer extractResult(PutMethod method) throws IOException {
+        return method.getStatusCode();
+      }
+
+      @Override
+      protected void setup(PutMethod method) throws
+                    SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Authenticate to Openstack Keystone
+   * As well as returning the access token, the member fields {@link #token},
+   * {@link #endpointURI} and {@link #objectLocationURI} are set up for re-use.
+   * <p/>
+   * This method is re-entrant -if more than one thread attempts to authenticate
+   * neither will block -but the field values with have those of the last caller.
+   * <p/>
+   *
+   * @return authenticated access token
+   */
+  public AccessToken authenticate() throws IOException {
+    final AuthenticationRequest authenticationRequest;
+    if (useKeystoneAuthentication) {
+      authenticationRequest = keystoneAuthRequest;
+    } else {
+      authenticationRequest = authRequest;
+    }
+
+    LOG.debug("started authentication");
+    return perform("authentication",
+                   authUri,
+                   new AuthenticationPost(authenticationRequest));
+  }
+
+  private class AuthenticationPost extends AuthMethodProcessor<AccessToken> {
+    final AuthenticationRequest authenticationRequest;
+
+    private AuthenticationPost(AuthenticationRequest authenticationRequest) {
+      this.authenticationRequest = authenticationRequest;
+    }
+
+    @Override
+    protected void setup(AuthPostMethod method) throws IOException {
+
+      method.setRequestEntity(getAuthenticationRequst(authenticationRequest));
+    }
+
+    /**
+     * specification says any of the 2xxs are OK, so list all
+     * the standard ones
+     * @return a set of 2XX status codes.
+     */
+    @Override
+    protected int[] getAllowedStatusCodes() {
+      return new int[]{
+        SC_OK,
+        SC_BAD_REQUEST,
+        SC_CREATED,
+        SC_ACCEPTED,
+        SC_NON_AUTHORITATIVE_INFORMATION,
+        SC_NO_CONTENT,
+        SC_RESET_CONTENT,
+        SC_PARTIAL_CONTENT,
+        SC_MULTI_STATUS,
+        SC_UNAUTHORIZED //if request unauthorized, try another method
+      };
+    }
+
+    @Override
+    public AccessToken extractResult(AuthPostMethod method) throws IOException {
+      //initial check for failure codes leading to authentication failures
+      if (method.getStatusCode() == SC_BAD_REQUEST) {
+        throw new SwiftAuthenticationFailedException(
+          authenticationRequest.toString(), "POST", authUri, method);
+      }
+
+      final AuthenticationResponse access =
+        JSONUtil.toObject(method.getResponseBodyAsString(),
+                          AuthenticationWrapper.class).getAccess();
+      final List<Catalog> serviceCatalog = access.getServiceCatalog();
+      //locate the specific service catalog that defines Swift; variations
+      //in the name of this add complexity to the search
+      boolean catalogMatch = false;
+      StringBuilder catList = new StringBuilder();
+      StringBuilder regionList = new StringBuilder();
+
+      //these fields are all set together at the end of the operation
+      URI endpointURI = null;
+      URI objectLocation;
+      Endpoint swiftEndpoint = null;
+      AccessToken accessToken;
+
+      for (Catalog catalog : serviceCatalog) {
+        String name = catalog.getName();
+        String type = catalog.getType();
+        String descr = String.format("[%s: %s]; ", name, type);
+        catList.append(descr);
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Catalog entry " + descr);
+        }
+        if (name.equals(SERVICE_CATALOG_SWIFT)
+            || name.equals(SERVICE_CATALOG_CLOUD_FILES)
+            || type.equals(SERVICE_CATALOG_OBJECT_STORE)) {
+          //swift is found
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("Found swift catalog as " + name + " => " + type);
+          }
+          //now go through the endpoints
+          for (Endpoint endpoint : catalog.getEndpoints()) {
+            String endpointRegion = endpoint.getRegion();
+            URI publicURL = endpoint.getPublicURL();
+            URI internalURL = endpoint.getInternalURL();
+            descr = String.format("[%s => %s / %s]; ",
+                                  endpointRegion,
+                                  publicURL,
+                                  internalURL);
+            regionList.append(descr);
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Endpoint " + descr);
+            }
+            if (region == null || endpointRegion.equals(region)) {
+              endpointURI = usePublicURL ? publicURL : internalURL;
+              swiftEndpoint = endpoint;
+              break;
+            }
+          }
+        }
+      }
+      if (endpointURI == null) {
+        String message = "Could not find swift service from auth URL "
+                         + authUri
+                         + " and region '" + region + "'. "
+                         + "Categories: " + catList
+                         + ((regionList.length() > 0) ?
+                            ("regions: " + regionList)
+                                                      : "No regions");
+        throw new SwiftInvalidResponseException(message,
+                                                SC_OK,
+                                                "authenticating",
+                                                authUri);
+
+      }
+
+
+      accessToken = access.getToken();
+      String path = SWIFT_OBJECT_AUTH_ENDPOINT
+                    + swiftEndpoint.getTenantId();
+      String host = endpointURI.getHost();
+      try {
+        objectLocation = new URI(endpointURI.getScheme(),
+                                 null,
+                                 host,
+                                 endpointURI.getPort(),
+                                 path,
+                                 null,
+                                 null);
+      } catch (URISyntaxException e) {
+        throw new SwiftException("object endpoint URI is incorrect: "
+                                 + endpointURI
+                                 + " + " + path,
+                                 e);
+      }
+      setAuthDetails(endpointURI, objectLocation, accessToken);
+
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("authenticated against " + endpointURI);
+      }
+      createDefaultContainer();
+      return accessToken;
+    }
+  }
+
+  private StringRequestEntity getAuthenticationRequst(AuthenticationRequest authenticationRequest)
+    throws IOException {
+    final String data = JSONUtil.toJSON(new AuthenticationRequestWrapper(
+            authenticationRequest));
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Authenticating with " + authenticationRequest);
+    }
+    return toJsonEntity(data);
+  }
+
+  /**
+   * create default container if it doesn't exist for Hadoop Swift integration.
+   * non-reentrant, as this should only be needed once.
+   *
+   * @throws IOException IO problems.
+   */
+  private synchronized void createDefaultContainer() throws IOException {
+    createContainer(container);
+  }
+
+  /**
+   * Create a container -if it already exists, do nothing
+   *
+   * @param containerName the container name
+   * @throws IOException IO problems
+   * @throws SwiftBadRequestException invalid container name
+   * @throws SwiftInvalidResponseException error from the server
+   */
+  public void createContainer(String containerName) throws IOException {
+    SwiftObjectPath objectPath = new SwiftObjectPath(containerName, "");
+    try {
+      //see if the data is there
+      headRequest("createContainer", objectPath, NEWEST);
+    } catch (FileNotFoundException ex) {
+      int status = 0;
+      try {
+        status = putRequest(objectPath);
+      } catch (FileNotFoundException e) {
+        //triggered by a very bad container name.
+        //re-insert the 404 result into the status
+        status = SC_NOT_FOUND;
+      }
+      if (status == SC_BAD_REQUEST) {
+        throw new SwiftBadRequestException(
+          "Bad request -authentication failure or bad container name?",
+          status,
+          "PUT",
+          null);
+      }
+      if (!isStatusCodeExpected(status,
+              SC_OK,
+              SC_CREATED,
+              SC_ACCEPTED,
+              SC_NO_CONTENT)) {
+        throw new SwiftInvalidResponseException("Couldn't create container "
+                + containerName +
+                " for storing data in Swift." +
+                " Try to create container " +
+                containerName + " manually ",
+                status,
+                "PUT",
+                null);
+      } else {
+        throw ex;
+      }
+    }
+  }
+
+  /**
+   * Trigger an initial auth operation if some of the needed
+   * fields are missing
+   *
+   * @throws IOException on problems
+   */
+  private void authIfNeeded() throws IOException {
+    if (getEndpointURI() == null) {
+      authenticate();
+    }
+  }
+
+  /**
+   * Pre-execution actions to be performed by methods. Currently this
+   * <ul>
+   *   <li>Logs the operation at TRACE</li>
+   *   <li>Authenticates the client -if needed</li>
+   * </ul>
+   * @throws IOException
+   */
+  private void preRemoteCommand(String operation) throws IOException {
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("Executing " + operation);
+    }
+    authIfNeeded();
+  }
+
+
+  /**
+   * Performs the HTTP request, validates the response code and returns
+   * the received data. HTTP Status codes are converted into exceptions.
+   *
+   * @param uri URI to source
+   * @param processor HttpMethodProcessor
+   * @param <M> method
+   * @param <R> result type
+   * @return result of HTTP request
+   * @throws IOException IO problems
+   * @throws SwiftBadRequestException the status code indicated "Bad request"
+   * @throws SwiftInvalidResponseException the status code is out of range
+   * for the action (excluding 404 responses)
+   * @throws SwiftInternalStateException the internal state of this client
+   * is invalid
+   * @throws FileNotFoundException a 404 response was returned
+   */
+  private <M extends HttpMethod, R> R perform(URI uri,
+                      HttpMethodProcessor<M, R> processor)
+    throws IOException,
+           SwiftBadRequestException,
+           SwiftInternalStateException,
+           SwiftInvalidResponseException,
+           FileNotFoundException {
+    return perform("",uri, processor);
+  }
+
+  /**
+   * Performs the HTTP request, validates the response code and returns
+   * the received data. HTTP Status codes are converted into exceptions.
+   * @param reason why is this operation taking place. Used for statistics
+   * @param uri URI to source
+   * @param processor HttpMethodProcessor
+   * @param <M> method
+   * @param <R> result type
+   * @return result of HTTP request
+   * @throws IOException IO problems
+   * @throws SwiftBadRequestException the status code indicated "Bad request"
+   * @throws SwiftInvalidResponseException the status code is out of range
+   * for the action (excluding 404 responses)
+   * @throws SwiftInternalStateException the internal state of this client
+   * is invalid
+   * @throws FileNotFoundException a 404 response was returned
+   */
+  private <M extends HttpMethod, R> R perform(String reason,
+                                              URI uri,
+                                              HttpMethodProcessor<M, R> processor)
+      throws IOException, SwiftBadRequestException, SwiftInternalStateException,
+            SwiftInvalidResponseException, FileNotFoundException {
+    checkNotNull(uri);
+    checkNotNull(processor);
+
+    final M method = processor.createMethod(uri.toString());
+
+    //retry policy
+    HttpMethodParams methodParams = method.getParams();
+    methodParams.setParameter(HttpMethodParams.RETRY_HANDLER,
+            new DefaultHttpMethodRetryHandler(
+                    retryCount, false));
+    methodParams.setIntParameter(HttpConnectionParams.CONNECTION_TIMEOUT,
+                                 connectTimeout);
+    methodParams.setSoTimeout(socketTimeout);
+    method.addRequestHeader(HEADER_USER_AGENT, SWIFT_USER_AGENT);
+    Duration duration = new Duration();
+    boolean success = false;
+    try {
+      int statusCode = 0;
+      try {
+        statusCode = exec(method);
+      } catch (IOException e) {
+        //rethrow with extra diagnostics and wiki links
+        throw ExceptionDiags.wrapException(uri.toString(), method.getName(), e);
+      }
+
+      //look at the response and see if it was valid or not.
+      //Valid is more than a simple 200; even 404 "not found" is considered
+      //valid -which it is for many methods.
+
+      //validate the allowed status code for this operation
+      int[] allowedStatusCodes = processor.getAllowedStatusCodes();
+      boolean validResponse = isStatusCodeExpected(statusCode,
+              allowedStatusCodes);
+
+      if (!validResponse) {
+        IOException ioe = buildException(uri, method, statusCode);
+        throw ioe;
+      }
+
+      R r = processor.extractResult(method);
+      success = true;
+      return r;
+    } catch (IOException e) {
+      //release the connection -always
+      method.releaseConnection();
+      throw e;
+    } finally {
+      duration.finished();
+      durationStats.add(method.getName()+" " + reason, duration, success);
+    }
+  }
+
+  /**
+   * Build an exception from a failed operation. This can include generating
+   * specific exceptions (e.g. FileNotFound), as well as the default
+   * {@link SwiftInvalidResponseException}.
+   *
+   * @param uri URI for operation
+   * @param method operation that failed
+   * @param statusCode status code
+   * @param <M> method type
+   * @return an exception to throw
+   */
+  private <M extends HttpMethod> IOException buildException(URI uri,
+                                                            M method,
+                                                            int statusCode) {
+    IOException fault;
+
+    //log the failure @debug level
+    String errorMessage = String.format("Method %s on %s failed, status code: %d," +
+            " status line: %s",
+            method.getName(),
+            uri,
+            statusCode,
+            method.getStatusLine()
+    );
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(errorMessage);
+    }
+    //send the command
+    switch (statusCode) {
+      case SC_NOT_FOUND:
+        fault = new FileNotFoundException("Operation " + method.getName()
+                + " on " + uri);
+        break;
+
+      case SC_BAD_REQUEST:
+        //bad HTTP request
+        fault =  new SwiftBadRequestException(
+          "Bad request against " + uri,
+          method.getName(),
+          uri,
+          method);
+        break;
+
+      case SC_REQUESTED_RANGE_NOT_SATISFIABLE:
+        //out of range
+        StringBuilder errorText = new StringBuilder(method.getStatusText());
+        //get the requested length
+        Header requestContentLen = method.getRequestHeader(HEADER_CONTENT_LENGTH);
+        if (requestContentLen!=null) {
+          errorText.append(" requested ").append(requestContentLen.getValue());
+        }
+        //and the result
+        Header availableContentRange = method.getResponseHeader(
+          HEADER_CONTENT_RANGE);
+        if (requestContentLen!=null) {
+          errorText.append(" available ").append(availableContentRange.getValue());
+        }
+        fault = new EOFException(errorText.toString());
+        break;
+
+      case SC_UNAUTHORIZED:
+        //auth failure; should only happen on the second attempt
+        fault  = new SwiftAuthenticationFailedException(
+                        "Operation not authorized- current access token ="
+                            + getToken(),
+                        method.getName(),
+                        uri,
+                        method);
+        break;
+
+      case SwiftProtocolConstants.SC_TOO_MANY_REQUESTS_429:
+      case SwiftProtocolConstants.SC_THROTTLED_498:
+        //response code that may mean the client is being throttled
+        fault  = new SwiftThrottledRequestException(
+                        "Client is being throttled: too many requests",
+                        method.getName(),
+                        uri,
+                        method);
+        break;
+
+      default:
+        //return a generic invalid HTTP response
+        fault = new SwiftInvalidResponseException(
+                errorMessage,
+                method.getName(),
+                uri,
+                method);
+    }
+
+    return fault;
+  }
+
+  /**
+   * Exec a GET request and return the input stream of the response
+   *
+   * @param uri URI to GET
+   * @param requestHeaders request headers
+   * @return the input stream. This must be closed to avoid log errors
+   * @throws IOException
+   */
+  private HttpBodyContent doGet(final URI uri, final Header... requestHeaders) throws IOException {
+    return perform("", uri, new GetMethodProcessor<HttpBodyContent>() {
+      @Override
+      public HttpBodyContent extractResult(GetMethod method) throws IOException {
+        return
+          new HttpBodyContent(
+            new HttpInputStreamWithRelease(uri, method), method.getResponseContentLength()
+          );
+      }
+
+      @Override
+      protected void setup(GetMethod method) throws
+                    SwiftInternalStateException {
+        setHeaders(method, requestHeaders);
+      }
+    });
+  }
+
+  /**
+   * Create an instance against a specific FS URI,
+   *
+   * @param filesystemURI filesystem to bond to
+   * @param config source of configuration data
+   * @return REST client instance
+   * @throws IOException on instantiation problems
+   */
+  public static SwiftRestClient getInstance(URI filesystemURI,
+                                            Configuration config) throws IOException {
+    return new SwiftRestClient(filesystemURI, config);
+  }
+
+
+  /**
+   * Convert the (JSON) data to a string request as UTF-8
+   *
+   * @param data data
+   * @return the data
+   * @throws SwiftException if for some very unexpected reason it's impossible
+   * to convert the data to UTF-8.
+   */
+  private static StringRequestEntity toJsonEntity(String data) throws
+          SwiftException {
+    StringRequestEntity entity;
+    try {
+      entity = new StringRequestEntity(data, "application/json", "UTF-8");
+    } catch (UnsupportedEncodingException e) {
+      throw new SwiftException("Could not encode data as UTF-8", e);
+    }
+    return entity;
+  }
+
+  /**
+   * Converts Swift path to URI to make request.
+   * This is public for unit testing
+   *
+   * @param path path to object
+   * @param endpointURI damain url e.g. http://domain.com
+   * @return valid URI for object
+   */
+  public static URI pathToURI(SwiftObjectPath path,
+                              URI endpointURI) throws SwiftException {
+    checkNotNull(endpointURI, "Null Endpoint -client is not authenticated");
+
+    String dataLocationURI = endpointURI.toString();
+    try {
+
+      dataLocationURI = SwiftUtils.joinPaths(dataLocationURI, encodeUrl(path.toUriPath()));
+      return new URI(dataLocationURI);
+    } catch (URISyntaxException e) {
+      throw new SwiftException("Failed to create URI from " + dataLocationURI, e);
+    }
+  }
+
+  /**
+   * Encode the URL. This extends {@link URLEncoder#encode(String, String)}
+   * with a replacement of + with %20.
+   * @param url URL string
+   * @return an encoded string
+   * @throws SwiftException if the URL cannot be encoded
+   */
+  private static String encodeUrl(String url) throws SwiftException {
+    if (url.matches(".*\\s+.*")) {
+      try {
+        url = URLEncoder.encode(url, "UTF-8");
+        url = url.replace("+", "%20");
+      } catch (UnsupportedEncodingException e) {
+        throw new SwiftException("failed to encode URI", e);
+      }
+    }
+
+    return url;
+  }
+
+  /**
+   * Convert a swift path to a URI relative to the current endpoint.
+   *
+   * @param path path
+   * @return an path off the current endpoint URI.
+   * @throws SwiftException
+   */
+  private URI pathToURI(SwiftObjectPath path) throws SwiftException {
+    return pathToURI(path, getEndpointURI());
+  }
+
+  /**
+   * Add the headers to the method, and the auth token (which must be set
+   * @param method method to update
+   * @param requestHeaders the list of headers
+   * @throws SwiftInternalStateException not yet authenticated
+   */
+  private void setHeaders(HttpMethodBase method, Header[] requestHeaders)
+      throws SwiftInternalStateException {
+      for (Header header : requestHeaders) {
+        method.addRequestHeader(header);
+      }
+    setAuthToken(method, getToken());
+  }
+
+
+  /**
+   * Set the auth key header of the method to the token ID supplied
+   *
+   * @param method method
+   * @param accessToken access token
+   * @throws SwiftInternalStateException if the client is not yet authenticated
+   */
+  private void setAuthToken(HttpMethodBase method, AccessToken accessToken)
+      throws SwiftInternalStateException {
+    checkNotNull(accessToken,"Not authenticated");
+    method.addRequestHeader(HEADER_AUTH_KEY, accessToken.getId());
+  }
+
+  /**
+   * Execute a method in a new HttpClient instance.
+   * If the auth failed, authenticate then retry the method.
+   *
+   * @param method methot to exec
+   * @param <M> Method type
+   * @return the status code
+   * @throws IOException on any failure
+   */
+  private <M extends HttpMethod> int exec(M method) throws IOException {
+    final HttpClient client = new HttpClient();
+    if (proxyHost != null) {
+      client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY,
+              new HttpHost(proxyHost, proxyPort));
+    }
+
+    int statusCode = execWithDebugOutput(method, client);
+
+    if ((statusCode == HttpStatus.SC_UNAUTHORIZED
+            || statusCode == HttpStatus.SC_BAD_REQUEST)
+            && method instanceof AuthPostMethod
+            && !useKeystoneAuthentication) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Operation failed with status " + method.getStatusCode() +
+                 " attempting keystone auth");
+      }
+      //if rackspace key authentication failed - try custom Keystone authentication
+      useKeystoneAuthentication = true;
+      final AuthPostMethod authentication = (AuthPostMethod) method;
+      //replace rackspace auth with keystone one
+      authentication.setRequestEntity(getAuthenticationRequst(keystoneAuthRequest));
+      statusCode = execWithDebugOutput(method, client);
+    }
+
+    if (statusCode == HttpStatus.SC_UNAUTHORIZED ) {
+      //unauthed -or the auth uri rejected it.
+
+      if (method instanceof AuthPostMethod) {
+          //unauth response from the AUTH URI itself.
+          throw new SwiftAuthenticationFailedException(authRequest.toString(),
+                                                       "auth",
+                                                       authUri,
+                                                       method);
+      }
+      //any other URL: try again
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Reauthenticating");
+      }
+      //re-auth, this may recurse into the same dir
+      authenticate();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Retrying original request");
+      }
+      statusCode = execWithDebugOutput(method, client);
+    }
+    return statusCode;
+  }
+
+  /**
+   * Execute the request with the request and response logged at debug level
+   * @param method method to execute
+   * @param client client to use
+   * @param <M> method type
+   * @return the status code
+   * @throws IOException any failure reported by the HTTP client.
+   */
+  private <M extends HttpMethod> int execWithDebugOutput(M method,
+                                                         HttpClient client) throws
+          IOException {
+    if (LOG.isDebugEnabled()) {
+      StringBuilder builder = new StringBuilder(
+              method.getName() + " " + method.getURI() + "\n");
+      for (Header header : method.getRequestHeaders()) {
+        builder.append(header.toString());
+      }
+      LOG.debug(builder);
+    }
+    int statusCode = client.executeMethod(method);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Status code = " + statusCode);
+    }
+    return statusCode;
+  }
+
+  /**
+   * Ensures that an object reference passed as a parameter to the calling
+   * method is not null.
+   *
+   * @param reference an object reference
+   * @return the non-null reference that was validated
+   * @throws NullPointerException if {@code reference} is null
+   */
+  private static <T> T checkNotNull(T reference) throws
+            SwiftInternalStateException {
+    return checkNotNull(reference, "Null Reference");
+  }
+
+  private static <T> T checkNotNull(T reference, String message) throws
+            SwiftInternalStateException {
+    if (reference == null) {
+      throw new SwiftInternalStateException(message);
+    }
+    return reference;
+  }
+
+  /**
+   * Check for a status code being expected -takes a list of expected values
+   *
+   * @param status received status
+   * @param expected expected value
+   * @return true iff status is an element of [expected]
+   */
+  private boolean isStatusCodeExpected(int status, int... expected) {
+    for (int code : expected) {
+      if (status == code) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+
+  @Override
+  public String toString() {
+    return "Swift client: " + serviceDescription;
+  }
+
+  /**
+   * Get the region which this client is bound to
+   * @return the region
+   */
+  public String getRegion() {
+    return region;
+  }
+
+  /**
+   * Get the tenant to which this client is bound
+   * @return the tenant
+   */
+  public String getTenant() {
+    return tenant;
+  }
+
+  /**
+   * Get the username this client identifies itself as
+   * @return the username
+   */
+  public String getUsername() {
+    return username;
+  }
+
+  /**
+   * Get the container to which this client is bound
+   * @return the container
+   */
+  public String getContainer() {
+    return container;
+  }
+
+  /**
+   * Is this client bound to a location aware Swift blobstore
+   * -that is, can you query for the location of partitions
+   * @return true iff the location of multipart file uploads
+   * can be determined.
+   */
+  public boolean isLocationAware() {
+    return locationAware;
+  }
+
+  /**
+   * Get the blocksize of this filesystem
+   * @return a blocksize >0
+   */
+  public long getBlocksizeKB() {
+    return blocksizeKB;
+  }
+
+  /**
+   * Get the partition size in KB
+   * @return the partition size
+   */
+  public int getPartSizeKB() {
+    return partSizeKB;
+  }
+
+  /**
+   * Get the buffer size in KB
+   * @return the buffer size wanted for reads
+   */
+  public int getBufferSizeKB() {
+    return bufferSizeKB;
+  }
+
+  public int getProxyPort() {
+    return proxyPort;
+  }
+
+  public String getProxyHost() {
+    return proxyHost;
+  }
+
+  public int getRetryCount() {
+    return retryCount;
+  }
+
+  public int getConnectTimeout() {
+    return connectTimeout;
+  }
+
+  public boolean isUsePublicURL() {
+    return usePublicURL;
+  }
+
+  public int getThrottleDelay() {
+    return throttleDelay;
+  }
+
+  /**
+   * Get the current operation statistics
+   * @return a snapshot of the statistics
+   */
+
+  public List<DurationStats> getOperationStatistics() {
+    return durationStats.getDurationStatistics();
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/package.html

@@ -0,0 +1,81 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+        "http://www.w3.org/TR/html4/loose.dtd">
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<html>
+<head>
+    <title>Swift Filesystem Client for Apache Hadoop</title>
+</head>
+<body>
+
+<h1>
+    Swift Filesystem Client for Apache Hadoop
+</h1>
+
+<h2>Introduction</h2>
+
+<div>This package provides support in Apache Hadoop for the OpenStack Swift
+    Key-Value store, allowing client applications -including MR Jobs- to
+    read and write data in Swift.
+</div>
+
+<div>Design Goals</div>
+<ol>
+    <li>Give clients access to SwiftFS files, similar to S3n:</li>
+    <li>maybe: support a Swift Block store -- at least until Swift's
+        support for &gt;5GB files has stabilized.
+    </li>
+    <li>Support for data-locality if the Swift FS provides file location information</li>
+    <li>Support access to multiple Swift filesystems in the same client/task.</li>
+    <li>Authenticate using the Keystone APIs.</li>
+    <li>Avoid dependency on unmaintained libraries.</li>
+</ol>
+
+
+<h2>Supporting multiple Swift Filesystems</h2>
+
+The goal of supporting multiple swift filesystems simultaneously changes how
+clusters are named and authenticated. In Hadoop's S3 and S3N filesystems, the "bucket" into
+which objects are stored is directly named in the URL, such as
+<code>s3n://bucket/object1</code>. The Hadoop configuration contains a
+single set of login credentials for S3 (username and key), which are used to
+authenticate the HTTP operations.
+
+For swift, we need to know not only which "container" name, but which credentials
+to use to authenticate with it -and which URL to use for authentication.
+
+This has led to a different design pattern from S3, as instead of simple bucket names,
+the hostname of an S3 container is two-level, the name of the service provider
+being the second path: <code>swift://bucket.service/</code>
+
+The <code>service</code> portion of this domainame is used as a reference into
+the client settings -and so identify the service provider of that container.
+
+
+<h2>Testing</h2>
+
+<div>
+    The client code can be tested against public or private Swift instances; the
+    public services are (at the time of writing -January 2013-), Rackspace and
+    HP Cloud. Testing against both instances is how interoperability
+    can be verified.
+</div>
+
+</body>
+</html>

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/StrictBufferedFSInputStream.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.hadoop.fs.BufferedFSInputStream;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+
+import java.io.IOException;
+
+/**
+ * Add stricter compliance with the evolving FS specifications
+ */
+public class StrictBufferedFSInputStream extends BufferedFSInputStream {
+
+  public StrictBufferedFSInputStream(FSInputStream in,
+                                     int size) {
+    super(in, size);
+  }
+
+  @Override
+  public void seek(long pos) throws IOException {
+    if (pos < 0) {
+      throw new IOException("Negative position");
+    }
+    if (in == null) {
+      throw new SwiftConnectionClosedException("Stream closed");
+    }
+    super.seek(pos);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftFileStatus.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * A subclass of {@link FileStatus} that contains the
+ * Swift-specific rules of when a file is considered to be a directory.
+ */
+public class SwiftFileStatus extends FileStatus {
+
+  public SwiftFileStatus() {
+  }
+
+  public SwiftFileStatus(long length,
+                         boolean isdir,
+                         int block_replication,
+                         long blocksize, long modification_time, Path path) {
+    super(length, isdir, block_replication, blocksize, modification_time, path);
+  }
+
+  public SwiftFileStatus(long length,
+                         boolean isdir,
+                         int block_replication,
+                         long blocksize,
+                         long modification_time,
+                         long access_time,
+                         FsPermission permission,
+                         String owner, String group, Path path) {
+    super(length, isdir, block_replication, blocksize, modification_time,
+            access_time, permission, owner, group, path);
+  }
+
+  //HDFS2+ only
+
+  public SwiftFileStatus(long length,
+                         boolean isdir,
+                         int block_replication,
+                         long blocksize,
+                         long modification_time,
+                         long access_time,
+                         FsPermission permission,
+                         String owner, String group, Path symlink, Path path) {
+    super(length, isdir, block_replication, blocksize, modification_time,
+          access_time, permission, owner, group, symlink, path);
+  }
+
+  /**
+   * Declare that the path represents a directory, which in the
+   * SwiftNativeFileSystem means "is a directory or a 0 byte file"
+   *
+   * @return true if the status is considered to be a file
+   */
+  @Override
+  public boolean isDir() {
+    return super.isDirectory() || getLen() == 0;
+  }
+
+  /**
+   * A entry is a file if it is not a directory.
+   * By implementing it <i>and not marking as an override</i> this
+   * subclass builds and runs in both Hadoop versions.
+   * @return the opposite value to {@link #isDir()}
+   */
+  @Override
+  public boolean isFile() {
+    return !isDir();
+  }
+
+  /**
+   * Directory test
+   * @return true if the file is considered to be a directory
+   */
+  public boolean isDirectory() {
+    return isDir();
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder sb = new StringBuilder();
+    sb.append(getClass().getSimpleName());
+    sb.append("{ ");
+    sb.append("path=").append(getPath());
+    sb.append("; isDirectory=").append(isDir());
+    sb.append("; length=").append(getLen());
+    sb.append("; blocksize=").append(getBlockSize());
+    sb.append("; modification_time=").append(getModificationTime());
+    sb.append("}");
+    return sb.toString();
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystem.java

@@ -0,0 +1,714 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftPathExistsException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftUnsupportedFeatureException;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.hadoop.util.Progressable;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Swift file system implementation. Extends Hadoop FileSystem
+ */
+public class SwiftNativeFileSystem extends FileSystem {
+
+  /** filesystem prefix: {@value} */
+  public static final String SWIFT = "swift";
+  private static final Log LOG =
+          LogFactory.getLog(SwiftNativeFileSystem.class);
+
+  /**
+   * path to user work directory for storing temporary files
+   */
+  private Path workingDir;
+
+  /**
+   * Swift URI
+   */
+  private URI uri;
+
+  /**
+   * reference to swiftFileSystemStore
+   */
+  private SwiftNativeFileSystemStore store;
+
+  /**
+   * Default constructor for Hadoop
+   */
+  public SwiftNativeFileSystem() {
+    // set client in initialize()
+  }
+
+  /**
+   * This constructor used for testing purposes
+   */
+  public SwiftNativeFileSystem(SwiftNativeFileSystemStore store) {
+    this.store = store;
+  }
+
+  /**
+   * This is for testing
+   * @return the inner store class
+   */
+  public SwiftNativeFileSystemStore getStore() {
+    return store;
+  }
+
+  @Override
+  public String getScheme() {
+    return SWIFT;
+  }
+
+  /**
+   * default class initialization
+   *
+   * @param fsuri path to Swift
+   * @param conf  Hadoop configuration
+   * @throws IOException
+   */
+  @Override
+  public void initialize(URI fsuri, Configuration conf) throws IOException {
+    super.initialize(fsuri, conf);
+
+    setConf(conf);
+    if (store == null) {
+      store = new SwiftNativeFileSystemStore();
+    }
+    this.uri = fsuri;
+    String username = System.getProperty("user.name");
+    this.workingDir = new Path("/user", username)
+      .makeQualified(uri, new Path(username));
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Initializing SwiftNativeFileSystem against URI " + uri
+              + " and working dir " + workingDir);
+    }
+    store.initialize(uri, conf);
+    LOG.debug("SwiftFileSystem initialized");
+  }
+
+  /**
+   * @return path to Swift
+   */
+  @Override
+  public URI getUri() {
+
+    return uri;
+  }
+
+  @Override
+  public String toString() {
+    return "Swift FileSystem " + store;
+  }
+
+  /**
+   * Path to user working directory
+   *
+   * @return Hadoop path
+   */
+  @Override
+  public Path getWorkingDirectory() {
+    return workingDir;
+  }
+
+  /**
+   * @param dir user working directory
+   */
+  @Override
+  public void setWorkingDirectory(Path dir) {
+    workingDir = makeAbsolute(dir);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("SwiftFileSystem.setWorkingDirectory to " + dir);
+    }
+  }
+
+  /**
+   * Return a file status object that represents the path.
+   *
+   * @param path The path we want information from
+   * @return a FileStatus object
+   */
+  @Override
+  public FileStatus getFileStatus(Path path) throws IOException {
+    Path absolutePath = makeAbsolute(path);
+    return store.getObjectMetadata(absolutePath);
+  }
+
+  /**
+   * The blocksize of this filesystem is set by the property
+   * SwiftProtocolConstants.SWIFT_BLOCKSIZE;the default is the value of
+   * SwiftProtocolConstants.DEFAULT_SWIFT_BLOCKSIZE;
+   * @return the blocksize for this FS.
+   */
+  @Override
+  public long getDefaultBlockSize() {
+    return store.getBlocksize();
+  }
+
+  /**
+   * The blocksize for this filesystem.
+   * @see #getDefaultBlockSize()
+   * @param f path of file
+   * @return the blocksize for the path
+   */
+  @Override
+  public long getDefaultBlockSize(Path f) {
+    return store.getBlocksize();
+  }
+
+  @Override
+  public long getBlockSize(Path path) throws IOException {
+    return store.getBlocksize();
+  }
+
+  @Override
+  public boolean isFile(Path f) throws IOException {
+    try {
+      FileStatus fileStatus = getFileStatus(f);
+      return !SwiftUtils.isDirectory(fileStatus);
+    } catch (FileNotFoundException e) {
+      return false;               // f does not exist
+    }
+  }
+
+  @Override
+  public boolean isDirectory(Path f) throws IOException {
+
+    try {
+      FileStatus fileStatus = getFileStatus(f);
+      return SwiftUtils.isDirectory(fileStatus);
+    } catch (FileNotFoundException e) {
+      return false;               // f does not exist
+    }
+  }
+
+  /**
+   * Return an array containing hostnames, offset and size of
+   * portions of the given file.  For a nonexistent
+   * file or regions, null will be returned.
+   * <p/>
+   * This call is most helpful with DFS, where it returns
+   * hostnames of machines that contain the given file.
+   * <p/>
+   * The FileSystem will simply return an elt containing 'localhost'.
+   */
+  @Override
+  public BlockLocation[] getFileBlockLocations(FileStatus file,
+                                               long start,
+                                               long len) throws IOException {
+    //argument checks
+    if (file == null) {
+      return null;
+    }
+
+    if (start < 0 || len < 0) {
+      throw new IllegalArgumentException("Negative start or len parameter" +
+                                         " to getFileBlockLocations");
+    }
+    if (file.getLen() <= start) {
+      return new BlockLocation[0];
+    }
+
+    // Check if requested file in Swift is more than 5Gb. In this case
+    // each block has its own location -which may be determinable
+    // from the Swift client API, depending on the remote server
+    final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath(),
+                                                             false,
+                                                             true);
+    List<URI> locations = new ArrayList<URI>();
+    if (listOfFileBlocks.length > 1) {
+      for (FileStatus fileStatus : listOfFileBlocks) {
+        if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
+                .equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
+          continue;
+        }
+        locations.addAll(store.getObjectLocation(fileStatus.getPath()));
+      }
+    } else {
+      locations = store.getObjectLocation(file.getPath());
+    }
+
+    if (locations.isEmpty()) {
+      LOG.debug("No locations returned for " + file.getPath());
+      //no locations were returned for the object
+      //fall back to the superclass
+
+      String[] name = {SwiftProtocolConstants.BLOCK_LOCATION};
+      String[] host = { "localhost" };
+      String[] topology={SwiftProtocolConstants.TOPOLOGY_PATH};
+      return new BlockLocation[] {
+        new BlockLocation(name, host, topology,0, file.getLen())
+      };
+    }
+
+    final String[] names = new String[locations.size()];
+    final String[] hosts = new String[locations.size()];
+    int i = 0;
+    for (URI location : locations) {
+      hosts[i] = location.getHost();
+      names[i] = location.getAuthority();
+      i++;
+    }
+    return new BlockLocation[]{
+            new BlockLocation(names, hosts, 0, file.getLen())
+    };
+  }
+
+  /**
+   * Create the parent directories.
+   * As an optimization, the entire hierarchy of parent
+   * directories is <i>Not</i> polled. Instead
+   * the tree is walked up from the last to the first,
+   * creating directories until one that exists is found.
+   *
+   * This strategy means if a file is created in an existing directory,
+   * one quick poll sufficies.
+   *
+   * There is a big assumption here: that all parent directories of an existing
+   * directory also exists.
+   * @param path path to create.
+   * @param permission to apply to files
+   * @return true if the operation was successful
+   * @throws IOException on a problem
+   */
+  @Override
+  public boolean mkdirs(Path path, FsPermission permission) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("SwiftFileSystem.mkdirs: " + path);
+    }
+    Path directory = makeAbsolute(path);
+
+    //build a list of paths to create
+    List<Path> paths = new ArrayList<Path>();
+    while (shouldCreate(directory)) {
+      //this directory needs creation, add to the list
+      paths.add(0, directory);
+      //now see if the parent needs to be created
+      directory = directory.getParent();
+    }
+
+    //go through the list of directories to create
+    for (Path p : paths) {
+      if (isNotRoot(p)) {
+        //perform a mkdir operation without any polling of
+        //the far end first
+        forceMkdir(p);
+      }
+    }
+
+    //if an exception was not thrown, this operation is considered
+    //a success
+    return true;
+  }
+
+  private boolean isNotRoot(Path absolutePath) {
+    return !isRoot(absolutePath);
+  }
+
+  private boolean isRoot(Path absolutePath) {
+    return absolutePath.getParent() == null;
+  }
+
+  /**
+   * internal implementation of directory creation.
+   *
+   * @param path path to file
+   * @return boolean file is created; false: no need to create
+   * @throws IOException if specified path is file instead of directory
+   */
+  private boolean mkdir(Path path) throws IOException {
+    Path directory = makeAbsolute(path);
+    boolean shouldCreate = shouldCreate(directory);
+    if (shouldCreate) {
+      forceMkdir(directory);
+    }
+    return shouldCreate;
+  }
+
+  /**
+   * Should mkdir create this directory?
+   * If the directory is root : false
+   * If the entry exists and is a directory: false
+   * If the entry exists and is a file: exception
+   * else: true
+   * @param directory path to query
+   * @return true iff the directory should be created
+   * @throws IOException IO problems
+   * @throws SwiftNotDirectoryException if the path references a file
+   */
+  private boolean shouldCreate(Path directory) throws IOException {
+    FileStatus fileStatus;
+    boolean shouldCreate;
+    if (isRoot(directory)) {
+      //its the base dir, bail out immediately
+      return false;
+    }
+    try {
+      //find out about the path
+      fileStatus = getFileStatus(directory);
+
+      if (!SwiftUtils.isDirectory(fileStatus)) {
+        //if it's a file, raise an error
+        throw new SwiftNotDirectoryException(directory,
+                String.format(": can't mkdir since it exists and is not a directory: %s",
+                        fileStatus));
+      } else {
+        //path exists, and it is a directory
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("skipping mkdir(" + directory + ") as it exists already");
+        }
+        shouldCreate = false;
+      }
+    } catch (FileNotFoundException e) {
+      shouldCreate = true;
+    }
+    return shouldCreate;
+  }
+
+  /**
+   * mkdir of a directory -irrespective of what was there underneath.
+   * There are no checks for the directory existing, there not
+   * being a path there, etc. etc. Those are assumed to have
+   * taken place already
+   * @param absolutePath path to create
+   * @throws IOException IO problems
+   */
+  private void forceMkdir(Path absolutePath) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Making dir '" + absolutePath + "' in Swift");
+    }
+    //file is not found: it must be created
+    store.createDirectory(absolutePath);
+  }
+
+  /**
+   * List the statuses of the files/directories in the given path if the path is
+   * a directory.
+   *
+   * @param path given path
+   * @return the statuses of the files/directories in the given path
+   * @throws IOException
+   */
+  @Override
+  public FileStatus[] listStatus(Path path) throws IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("SwiftFileSystem.listStatus for: " + path);
+    }
+    return store.listSubPaths(makeAbsolute(path), false, true);
+  }
+
+  /**
+   * This optional operation is not supported
+   */
+  @Override
+  public FSDataOutputStream append(Path f, int bufferSize, Progressable progress)
+      throws IOException {
+    LOG.debug("SwiftFileSystem.append");
+    throw new SwiftUnsupportedFeatureException("Not supported: append()");
+  }
+
+  /**
+   * @param permission Currently ignored.
+   */
+  @Override
+  public FSDataOutputStream create(Path file, FsPermission permission,
+                                   boolean overwrite, int bufferSize,
+                                   short replication, long blockSize,
+                                   Progressable progress)
+          throws IOException {
+    LOG.debug("SwiftFileSystem.create");
+
+    FileStatus fileStatus = null;
+    Path absolutePath = makeAbsolute(file);
+    try {
+      fileStatus = getFileStatus(absolutePath);
+    } catch (FileNotFoundException e) {
+      //the file isn't there.
+    }
+
+    if (fileStatus != null) {
+      //the path exists -action depends on whether or not it is a directory,
+      //and what the overwrite policy is.
+
+      //What is clear at this point is that if the entry exists, there's
+      //no need to bother creating any parent entries
+      if (fileStatus.isDirectory()) {
+        //here someone is trying to create a file over a directory
+
+/*    we can't throw an exception here as there is no easy way to distinguish
+     a file from the dir
+
+        throw new SwiftPathExistsException("Cannot create a file over a directory:"
+                                           + file);
+ */
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Overwriting either an empty file or a directory");
+        }
+      }
+      if (overwrite) {
+        //overwrite set -> delete the object.
+        store.delete(absolutePath, true);
+      } else {
+        throw new SwiftPathExistsException("Path exists: " + file);
+      }
+    } else {
+      // destination does not exist -trigger creation of the parent
+      Path parent = file.getParent();
+      if (parent != null) {
+        if (!mkdirs(parent)) {
+          throw new SwiftOperationFailedException(
+            "Mkdirs failed to create " + parent);
+        }
+      }
+    }
+
+    SwiftNativeOutputStream out = createSwiftOutputStream(file);
+    return new FSDataOutputStream(out, statistics);
+  }
+
+  /**
+   * Create the swift output stream
+   * @param path path to write to
+   * @return the new file
+   * @throws IOException
+   */
+  protected SwiftNativeOutputStream createSwiftOutputStream(Path path) throws
+                                                                       IOException {
+    long partSizeKB = getStore().getPartsizeKB();
+    return new SwiftNativeOutputStream(getConf(),
+            getStore(),
+            path.toUri().toString(),
+            partSizeKB);
+  }
+
+  /**
+   * Opens an FSDataInputStream at the indicated Path.
+   *
+   * @param path       the file name to open
+   * @param bufferSize the size of the buffer to be used.
+   * @return the input stream
+   * @throws FileNotFoundException if the file is not found
+   * @throws IOException any IO problem
+   */
+  @Override
+  public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+    int bufferSizeKB = getStore().getBufferSizeKB();
+    long readBlockSize = bufferSizeKB * 1024L;
+    return open(path, bufferSize, readBlockSize);
+  }
+
+  /**
+   * Low-level operation to also set the block size for this operation
+   * @param path       the file name to open
+   * @param bufferSize the size of the buffer to be used.
+   * @param readBlockSize how big should the read blockk/buffer size be?
+   * @return the input stream
+   * @throws FileNotFoundException if the file is not found
+   * @throws IOException any IO problem
+   */
+  public FSDataInputStream open(Path path,
+                                int bufferSize,
+                                long readBlockSize) throws IOException {
+    if (readBlockSize <= 0) {
+      throw new SwiftConfigurationException("Bad remote buffer size");
+    }
+    Path absolutePath = makeAbsolute(path);
+    return new FSDataInputStream(
+            new StrictBufferedFSInputStream(
+                    new SwiftNativeInputStream(store,
+                                       statistics,
+                                       absolutePath,
+                                       readBlockSize),
+                    bufferSize));
+  }
+
+  /**
+   * Renames Path src to Path dst. On swift this uses copy-and-delete
+   * and <i>is not atomic</i>.
+   *
+   * @param src path
+   * @param dst path
+   * @return true if directory renamed, false otherwise
+   * @throws IOException on problems
+   */
+  @Override
+  public boolean rename(Path src, Path dst) throws IOException {
+
+    try {
+      store.rename(makeAbsolute(src), makeAbsolute(dst));
+      //success
+      return true;
+    } catch (SwiftOperationFailedException e) {
+      //downgrade to a failure
+      return false;
+    } catch (FileNotFoundException e) {
+      //downgrade to a failure
+      return false;
+    }
+  }
+
+
+  /**
+   * Delete a file or directory
+   *
+   * @param path      the path to delete.
+   * @param recursive if path is a directory and set to
+   *                  true, the directory is deleted else throws an exception if the
+   *                  directory is not empty
+   *                  case of a file the recursive can be set to either true or false.
+   * @return true if the object was deleted
+   * @throws IOException IO problems
+   */
+  @Override
+  public boolean delete(Path path, boolean recursive) throws IOException {
+    try {
+      return store.delete(path, recursive);
+    } catch (FileNotFoundException e) {
+      //base path was not found.
+      return false;
+    }
+  }
+
+  /**
+   * Delete a file.
+   * This method is abstract in Hadoop 1.x; in 2.x+ it is non-abstract
+   * and deprecated
+   */
+  @Override
+  public boolean delete(Path f) throws IOException {
+    return delete(f, true);
+  }
+
+  /**
+   * Makes path absolute
+   *
+   * @param path path to file
+   * @return absolute path
+   */
+  protected Path makeAbsolute(Path path) {
+    if (path.isAbsolute()) {
+      return path;
+    }
+    return new Path(workingDir, path);
+  }
+
+  /**
+   * Get the current operation statistics
+   * @return a snapshot of the statistics
+   */
+  public List<DurationStats> getOperationStatistics() {
+    return store.getOperationStatistics();
+  }
+
+  /**
+   * Low level method to do a deep listing of all entries, not stopping
+   * at the next directory entry. This is to let tests be confident that
+   * recursive deletes &c really are working.
+   * @param path path to recurse down
+   * @param newest ask for the newest data, potentially slower than not.
+   * @return a potentially empty array of file status
+   * @throws IOException any problem
+   */
+  @InterfaceAudience.Private
+  public FileStatus[] listRawFileStatus(Path path, boolean newest) throws IOException {
+    return store.listSubPaths(makeAbsolute(path), true, newest);
+  }
+
+  /**
+   * Get the number of partitions written by an output stream
+   * This is for testing
+   * @param outputStream output stream
+   * @return the #of partitions written by that stream
+   */
+  @InterfaceAudience.Private
+  public static int getPartitionsWritten(FSDataOutputStream outputStream) {
+    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+    return snos.getPartitionsWritten();
+  }
+
+  private static SwiftNativeOutputStream getSwiftNativeOutputStream(
+    FSDataOutputStream outputStream) {
+    OutputStream wrappedStream = outputStream.getWrappedStream();
+    return (SwiftNativeOutputStream) wrappedStream;
+  }
+
+  /**
+   * Get the size of partitions written by an output stream
+   * This is for testing
+   *
+   * @param outputStream output stream
+   * @return partition size in bytes
+   */
+  @InterfaceAudience.Private
+  public static long getPartitionSize(FSDataOutputStream outputStream) {
+    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+    return snos.getFilePartSize();
+  }
+
+  /**
+   * Get the the number of bytes written to an output stream
+   * This is for testing
+   *
+   * @param outputStream output stream
+   * @return partition size in bytes
+   */
+  @InterfaceAudience.Private
+  public static long getBytesWritten(FSDataOutputStream outputStream) {
+    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+    return snos.getBytesWritten();
+  }
+
+  /**
+   * Get the the number of bytes uploaded by an output stream
+   * to the swift cluster.
+   * This is for testing
+   *
+   * @param outputStream output stream
+   * @return partition size in bytes
+   */
+  @InterfaceAudience.Private
+  public static long getBytesUploaded(FSDataOutputStream outputStream) {
+    SwiftNativeOutputStream snos = getSwiftNativeOutputStream(outputStream);
+    return snos.getBytesUploaded();
+  }
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeFileSystemStore.java

@@ -0,0 +1,977 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.httpclient.HttpStatus;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInvalidResponseException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.http.HttpBodyContent;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.JSONUtil;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.codehaus.jackson.map.type.CollectionType;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * File system store implementation.
+ * Makes REST requests, parses data from responses
+ */
+public class SwiftNativeFileSystemStore {
+  private static final Pattern URI_PATTERN = Pattern.compile("\"\\S+?\"");
+  private static final String PATTERN = "EEE, d MMM yyyy hh:mm:ss zzz";
+  private static final Log LOG =
+          LogFactory.getLog(SwiftNativeFileSystemStore.class);
+  private URI uri;
+  private SwiftRestClient swiftRestClient;
+
+  /**
+   * Initalize the filesystem store -this creates the REST client binding.
+   *
+   * @param fsURI         URI of the filesystem, which is used to map to the filesystem-specific
+   *                      options in the configuration file
+   * @param configuration configuration
+   * @throws IOException on any failure.
+   */
+  public void initialize(URI fsURI, Configuration configuration) throws IOException {
+    this.uri = fsURI;
+    this.swiftRestClient = SwiftRestClient.getInstance(fsURI, configuration);
+  }
+
+  @Override
+  public String toString() {
+    return "SwiftNativeFileSystemStore with "
+            + swiftRestClient;
+  }
+
+  /**
+   * Get the default blocksize of this (bound) filesystem
+   * @return the blocksize returned for all FileStatus queries,
+   * which is used by the MapReduce splitter.
+   */
+  public long getBlocksize() {
+    return 1024L * swiftRestClient.getBlocksizeKB();
+  }
+
+  public long getPartsizeKB() {
+    return swiftRestClient.getPartSizeKB();
+  }
+
+  public int getBufferSizeKB() {
+    return swiftRestClient.getBufferSizeKB();
+  }
+
+  public int getThrottleDelay() {
+    return swiftRestClient.getThrottleDelay();
+  }
+  /**
+   * Upload a file/input stream of a specific length.
+   *
+   * @param path        destination path in the swift filesystem
+   * @param inputStream input data. This is closed afterwards, always
+   * @param length      length of the data
+   * @throws IOException on a problem
+   */
+  public void uploadFile(Path path, InputStream inputStream, long length)
+          throws IOException {
+      swiftRestClient.upload(toObjectPath(path), inputStream, length);
+  }
+
+  /**
+   * Upload part of a larger file.
+   *
+   * @param path        destination path
+   * @param partNumber  item number in the path
+   * @param inputStream input data
+   * @param length      length of the data
+   * @throws IOException on a problem
+   */
+  public void uploadFilePart(Path path, int partNumber,
+                             InputStream inputStream, long length)
+          throws IOException {
+
+    String stringPath = path.toUri().toString();
+    String partitionFilename = SwiftUtils.partitionFilenameFromNumber(
+      partNumber);
+    if (stringPath.endsWith("/")) {
+      stringPath = stringPath.concat(partitionFilename);
+    } else {
+      stringPath = stringPath.concat("/").concat(partitionFilename);
+    }
+
+    swiftRestClient.upload(
+      new SwiftObjectPath(toDirPath(path).getContainer(), stringPath),
+            inputStream,
+            length);
+  }
+
+  /**
+   * Tell the Swift server to expect a multi-part upload by submitting
+   * a 0-byte file with the X-Object-Manifest header
+   *
+   * @param path path of final final
+   * @throws IOException
+   */
+  public void createManifestForPartUpload(Path path) throws IOException {
+    String pathString = toObjectPath(path).toString();
+    if (!pathString.endsWith("/")) {
+      pathString = pathString.concat("/");
+    }
+    if (pathString.startsWith("/")) {
+      pathString = pathString.substring(1);
+    }
+
+    swiftRestClient.upload(toObjectPath(path),
+            new ByteArrayInputStream(new byte[0]),
+            0,
+            new Header(SwiftProtocolConstants.X_OBJECT_MANIFEST, pathString));
+  }
+
+  /**
+   * Get the metadata of an object
+   *
+   * @param path path
+   * @return file metadata. -or null if no headers were received back from the server.
+   * @throws IOException           on a problem
+   * @throws FileNotFoundException if there is nothing at the end
+   */
+  public SwiftFileStatus getObjectMetadata(Path path) throws IOException {
+    return getObjectMetadata(path, true);
+  }
+
+  /**
+   * Get the HTTP headers, in case you really need the low-level
+   * metadata
+   * @param path path to probe
+   * @param newest newest or oldest?
+   * @return the header list
+   * @throws IOException IO problem
+   * @throws FileNotFoundException if there is nothing at the end
+   */
+  public Header[] getObjectHeaders(Path path, boolean newest)
+    throws IOException, FileNotFoundException {
+    SwiftObjectPath objectPath = toObjectPath(path);
+    return stat(objectPath, newest);
+  }
+
+  /**
+   * Get the metadata of an object
+   *
+   * @param path path
+   * @param newest flag to say "set the newest header", otherwise take any entry
+   * @return file metadata. -or null if no headers were received back from the server.
+   * @throws IOException           on a problem
+   * @throws FileNotFoundException if there is nothing at the end
+   */
+  public SwiftFileStatus getObjectMetadata(Path path, boolean newest)
+    throws IOException, FileNotFoundException {
+
+    SwiftObjectPath objectPath = toObjectPath(path);
+    final Header[] headers = stat(objectPath, newest);
+    //no headers is treated as a missing file
+    if (headers.length == 0) {
+      throw new FileNotFoundException("Not Found " + path.toUri());
+    }
+
+    boolean isDir = false;
+    long length = 0;
+    long lastModified = 0 ;
+    for (Header header : headers) {
+      String headerName = header.getName();
+      if (headerName.equals(SwiftProtocolConstants.X_CONTAINER_OBJECT_COUNT) ||
+              headerName.equals(SwiftProtocolConstants.X_CONTAINER_BYTES_USED)) {
+        length = 0;
+        isDir = true;
+      }
+      if (SwiftProtocolConstants.HEADER_CONTENT_LENGTH.equals(headerName)) {
+        length = Long.parseLong(header.getValue());
+      }
+      if (SwiftProtocolConstants.HEADER_LAST_MODIFIED.equals(headerName)) {
+        final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(PATTERN);
+        try {
+          lastModified = simpleDateFormat.parse(header.getValue()).getTime();
+        } catch (ParseException e) {
+          throw new SwiftException("Failed to parse " + header.toString(), e);
+        }
+      }
+    }
+    if (lastModified == 0) {
+      lastModified = System.currentTimeMillis();
+    }
+
+    Path correctSwiftPath = getCorrectSwiftPath(path);
+    return new SwiftFileStatus(length,
+                               isDir,
+                               1,
+                               getBlocksize(),
+                               lastModified,
+                               correctSwiftPath);
+  }
+
+  private Header[] stat(SwiftObjectPath objectPath, boolean newest) throws
+                                                                    IOException {
+    Header[] headers;
+    if (newest) {
+      headers = swiftRestClient.headRequest("getObjectMetadata-newest",
+                                            objectPath, SwiftRestClient.NEWEST);
+    } else {
+      headers = swiftRestClient.headRequest("getObjectMetadata",
+                                            objectPath);
+    }
+    return headers;
+  }
+
+  /**
+   * Get the object as an input stream
+   *
+   * @param path object path
+   * @return the input stream -this must be closed to terminate the connection
+   * @throws IOException           IO problems
+   * @throws FileNotFoundException path doesn't resolve to an object
+   */
+  public HttpBodyContent getObject(Path path) throws IOException {
+    return swiftRestClient.getData(toObjectPath(path),
+                                   SwiftRestClient.NEWEST);
+  }
+
+  /**
+   * Get the input stream starting from a specific point.
+   *
+   * @param path           path to object
+   * @param byteRangeStart starting point
+   * @param length         no. of bytes
+   * @return an input stream that must be closed
+   * @throws IOException IO problems
+   */
+  public HttpBodyContent getObject(Path path, long byteRangeStart, long length)
+          throws IOException {
+    return swiftRestClient.getData(
+      toObjectPath(path), byteRangeStart, length);
+  }
+
+  /**
+   * List a directory.
+   * This is O(n) for the number of objects in this path.
+   *
+   *
+   *
+   * @param path working path
+   * @param listDeep ask for all the data
+   * @param newest ask for the newest data
+   * @return Collection of file statuses
+   * @throws IOException IO problems
+   * @throws FileNotFoundException if the path does not exist
+   */
+  private List<FileStatus> listDirectory(SwiftObjectPath path,
+                                         boolean listDeep,
+                                         boolean newest) throws IOException {
+    final byte[] bytes;
+    final ArrayList<FileStatus> files = new ArrayList<FileStatus>();
+    final Path correctSwiftPath = getCorrectSwiftPath(path);
+    try {
+      bytes = swiftRestClient.listDeepObjectsInDirectory(path, listDeep);
+    } catch (FileNotFoundException e) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("" +
+                "File/Directory not found " + path);
+      }
+      if (SwiftUtils.isRootDir(path)) {
+        return Collections.emptyList();
+      } else {
+        throw e;
+      }
+    } catch (SwiftInvalidResponseException e) {
+      //bad HTTP error code
+      if (e.getStatusCode() == HttpStatus.SC_NO_CONTENT) {
+        //this can come back on a root list if the container is empty
+        if (SwiftUtils.isRootDir(path)) {
+          return Collections.emptyList();
+        } else {
+          //NO_CONTENT returned on something other than the root directory;
+          //see if it is there, and convert to empty list or not found
+          //depending on whether the entry exists.
+          FileStatus stat = getObjectMetadata(correctSwiftPath, newest);
+
+          if (stat.isDirectory()) {
+            //it's an empty directory. state that
+            return Collections.emptyList();
+          } else {
+            //it's a file -return that as the status
+            files.add(stat);
+            return files;
+          }
+        }
+      } else {
+        //a different status code: rethrow immediately
+        throw e;
+      }
+    }
+
+    final CollectionType collectionType = JSONUtil.getJsonMapper().getTypeFactory().
+            constructCollectionType(List.class, SwiftObjectFileStatus.class);
+
+    final List<SwiftObjectFileStatus> fileStatusList =
+            JSONUtil.toObject(new String(bytes), collectionType);
+
+    //this can happen if user lists file /data/files/file
+    //in this case swift will return empty array
+    if (fileStatusList.isEmpty()) {
+      SwiftFileStatus objectMetadata = getObjectMetadata(correctSwiftPath,
+                                                         newest);
+      if (objectMetadata.isFile()) {
+        files.add(objectMetadata);
+      }
+
+      return files;
+    }
+
+    for (SwiftObjectFileStatus status : fileStatusList) {
+      if (status.getName() != null) {
+          files.add(new SwiftFileStatus(status.getBytes(),
+                  status.getBytes() == 0,
+                  1,
+                  getBlocksize(),
+                  status.getLast_modified().getTime(),
+                  getCorrectSwiftPath(new Path(status.getName()))));
+      }
+    }
+
+    return files;
+  }
+
+  /**
+   * List all elements in this directory
+   *
+   *
+   *
+   * @param path     path to work with
+   * @param recursive do a recursive get
+   * @param newest ask for the newest, or can some out of date data work?
+   * @return the file statuses, or an empty array if there are no children
+   * @throws IOException           on IO problems
+   * @throws FileNotFoundException if the path is nonexistent
+   */
+  public FileStatus[] listSubPaths(Path path,
+                                   boolean recursive,
+                                   boolean newest) throws IOException {
+    final Collection<FileStatus> fileStatuses;
+    fileStatuses = listDirectory(toDirPath(path), recursive, newest);
+    return fileStatuses.toArray(new FileStatus[fileStatuses.size()]);
+  }
+
+  /**
+   * Create a directory
+   *
+   * @param path path
+   * @throws IOException
+   */
+  public void createDirectory(Path path) throws IOException {
+    innerCreateDirectory(toDirPath(path));
+  }
+
+  /**
+   * The inner directory creation option. This only creates
+   * the dir at the given path, not any parent dirs.
+   * @param swiftObjectPath swift object path at which a 0-byte blob should be
+   * put
+   * @throws IOException IO problems
+   */
+  private void innerCreateDirectory(SwiftObjectPath swiftObjectPath)
+          throws IOException {
+
+    swiftRestClient.putRequest(swiftObjectPath);
+  }
+
+  private SwiftObjectPath toDirPath(Path path) throws
+          SwiftConfigurationException {
+    return SwiftObjectPath.fromPath(uri, path, false);
+  }
+
+  private SwiftObjectPath toObjectPath(Path path) throws
+          SwiftConfigurationException {
+    return SwiftObjectPath.fromPath(uri, path);
+  }
+
+  /**
+   * Try to find the specific server(s) on which the data lives
+   * @param path path to probe
+   * @return a possibly empty list of locations
+   * @throws IOException on problems determining the locations
+   */
+  public List<URI> getObjectLocation(Path path) throws IOException {
+    final byte[] objectLocation;
+    objectLocation = swiftRestClient.getObjectLocation(toObjectPath(path));
+    if (objectLocation == null || objectLocation.length == 0) {
+      //no object location, return an empty list
+      return new LinkedList<URI>();
+    }
+    return extractUris(new String(objectLocation), path);
+  }
+
+  /**
+   * deletes object from Swift
+   *
+   * @param path path to delete
+   * @return true if the path was deleted by this specific operation.
+   * @throws IOException on a failure
+   */
+  public boolean deleteObject(Path path) throws IOException {
+    SwiftObjectPath swiftObjectPath = toObjectPath(path);
+    if (!SwiftUtils.isRootDir(swiftObjectPath)) {
+      return swiftRestClient.delete(swiftObjectPath);
+    } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Not deleting root directory entry");
+      }
+      return true;
+    }
+  }
+
+  /**
+   * deletes a directory from Swift. This is not recursive
+   *
+   * @param path path to delete
+   * @return true if the path was deleted by this specific operation -or
+   *         the path was root and not acted on.
+   * @throws IOException on a failure
+   */
+  public boolean rmdir(Path path) throws IOException {
+    return deleteObject(path);
+  }
+
+  /**
+   * Does the object exist
+   *
+   * @param path object path
+   * @return true if the metadata of an object could be retrieved
+   * @throws IOException IO problems other than FileNotFound, which
+   *                     is downgraded to an object does not exist return code
+   */
+  public boolean objectExists(Path path) throws IOException {
+    return objectExists(toObjectPath(path));
+  }
+
+  /**
+   * Does the object exist
+   *
+   * @param path swift object path
+   * @return true if the metadata of an object could be retrieved
+   * @throws IOException IO problems other than FileNotFound, which
+   *                     is downgraded to an object does not exist return code
+   */
+  public boolean objectExists(SwiftObjectPath path) throws IOException {
+    try {
+      Header[] headers = swiftRestClient.headRequest("objectExists",
+                                                     path,
+                                                     SwiftRestClient.NEWEST);
+      //no headers is treated as a missing file
+      return headers.length != 0;
+    } catch (FileNotFoundException e) {
+      return false;
+    }
+  }
+
+  /**
+   * Rename through copy-and-delete. this is a consequence of the
+   * Swift filesystem using the path as the hash
+   * into the Distributed Hash Table, "the ring" of filenames.
+   * <p/>
+   * Because of the nature of the operation, it is not atomic.
+   *
+   * @param src source file/dir
+   * @param dst destination
+   * @throws IOException                   IO failure
+   * @throws SwiftOperationFailedException if the rename failed
+   * @throws FileNotFoundException         if the source directory is missing, or
+   *                                       the parent directory of the destination
+   */
+  public void rename(Path src, Path dst)
+    throws FileNotFoundException, SwiftOperationFailedException, IOException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("mv " + src + " " + dst);
+    }
+    boolean renamingOnToSelf = src.equals(dst);
+
+    SwiftObjectPath srcObject = toObjectPath(src);
+    SwiftObjectPath destObject = toObjectPath(dst);
+
+    if (SwiftUtils.isRootDir(srcObject)) {
+      throw new SwiftOperationFailedException("cannot rename root dir");
+    }
+
+    final SwiftFileStatus srcMetadata;
+    srcMetadata = getObjectMetadata(src);
+    SwiftFileStatus dstMetadata;
+    try {
+      dstMetadata = getObjectMetadata(dst);
+    } catch (FileNotFoundException e) {
+      //destination does not exist.
+      LOG.debug("Destination does not exist");
+      dstMetadata = null;
+    }
+
+    //check to see if the destination parent directory exists
+    Path srcParent = src.getParent();
+    Path dstParent = dst.getParent();
+    //skip the overhead of a HEAD call if the src and dest share the same
+    //parent dir (in which case the dest dir exists), or the destination
+    //directory is root, in which case it must also exist
+    if (dstParent != null && !dstParent.equals(srcParent)) {
+      try {
+        getObjectMetadata(dstParent);
+      } catch (FileNotFoundException e) {
+        //destination parent doesn't exist; bail out
+        LOG.debug("destination parent directory " + dstParent + " doesn't exist");
+        throw e;
+      }
+    }
+
+    boolean destExists = dstMetadata != null;
+    boolean destIsDir = destExists && SwiftUtils.isDirectory(dstMetadata);
+    //calculate the destination
+    SwiftObjectPath destPath;
+
+    //enum the child entries and everything underneath
+    List<FileStatus> childStats = listDirectory(srcObject, true, true);
+    boolean srcIsFile = !srcMetadata.isDir();
+    if (srcIsFile) {
+
+      //source is a simple file OR a partitioned file
+      // outcomes:
+      // #1 dest exists and is file: fail
+      // #2 dest exists and is dir: destination path becomes under dest dir
+      // #3 dest does not exist: use dest as name
+      if (destExists) {
+
+        if (destIsDir) {
+          //outcome #2 -move to subdir of dest
+          destPath = toObjectPath(new Path(dst, src.getName()));
+        } else {
+          //outcome #1 dest it's a file: fail if differeent
+          if (!renamingOnToSelf) {
+            throw new SwiftOperationFailedException(
+                    "cannot rename a file over one that already exists");
+          } else {
+            //is mv self self where self is a file. this becomes a no-op
+            LOG.debug("Renaming file onto self: no-op => success");
+            return;
+          }
+        }
+      } else {
+        //outcome #3 -new entry
+        destPath = toObjectPath(dst);
+      }
+      int childCount = childStats.size();
+      //here there is one of:
+      // - a single object ==> standard file
+      // ->
+      if (childCount == 0) {
+        copyThenDeleteObject(srcObject, destPath);
+      } else {
+        //do the copy
+        SwiftUtils.debug(LOG, "Source file appears to be partitioned." +
+                              " copying file and deleting children");
+
+        copyObject(srcObject, destPath);
+        for (FileStatus stat : childStats) {
+          SwiftUtils.debug(LOG, "Deleting partitioned file %s ", stat);
+          deleteObject(stat.getPath());
+        }
+
+        swiftRestClient.delete(srcObject);
+      }
+    } else {
+
+      //here the source exists and is a directory
+      // outcomes (given we know the parent dir exists if we get this far)
+      // #1 destination is a file: fail
+      // #2 destination is a directory: create a new dir under that one
+      // #3 destination doesn't exist: create a new dir with that name
+      // #3 and #4 are only allowed if the dest path is not == or under src
+
+
+      if (destExists && !destIsDir) {
+        // #1 destination is a file: fail
+        throw new SwiftOperationFailedException(
+                "the source is a directory, but not the destination");
+      }
+      Path targetPath;
+      if (destExists) {
+        // #2 destination is a directory: create a new dir under that one
+        targetPath = new Path(dst, src.getName());
+      } else {
+        // #3 destination doesn't exist: create a new dir with that name
+        targetPath = dst;
+      }
+      SwiftObjectPath targetObjectPath = toObjectPath(targetPath);
+      //final check for any recursive operations
+      if (srcObject.isEqualToOrParentOf(targetObjectPath)) {
+        //you can't rename a directory onto itself
+        throw new SwiftOperationFailedException(
+          "cannot move a directory under itself");
+      }
+
+
+      LOG.info("mv  " + srcObject + " " + targetPath);
+
+      logDirectory("Directory to copy ", srcObject, childStats);
+
+      // iterative copy of everything under the directory.
+      // by listing all children this can be done iteratively
+      // rather than recursively -everything in this list is either a file
+      // or a 0-byte-len file pretending to be a directory.
+      String srcURI = src.toUri().toString();
+      int prefixStripCount = srcURI.length() + 1;
+      for (FileStatus fileStatus : childStats) {
+        Path copySourcePath = fileStatus.getPath();
+        String copySourceURI = copySourcePath.toUri().toString();
+
+        String copyDestSubPath = copySourceURI.substring(prefixStripCount);
+
+        Path copyDestPath = new Path(targetPath, copyDestSubPath);
+        if (LOG.isTraceEnabled()) {
+          //trace to debug some low-level rename path problems; retained
+          //in case they ever come back.
+          LOG.trace("srcURI=" + srcURI
+                  + "; copySourceURI=" + copySourceURI
+                  + "; copyDestSubPath=" + copyDestSubPath
+                  + "; copyDestPath=" + copyDestPath);
+        }
+        SwiftObjectPath copyDestination = toObjectPath(copyDestPath);
+
+        try {
+          copyThenDeleteObject(toObjectPath(copySourcePath),
+                  copyDestination);
+        } catch (FileNotFoundException e) {
+          LOG.info("Skipping rename of " + copySourcePath);
+        }
+        //add a throttle delay
+        throttle();
+      }
+      //now rename self. If missing, create the dest directory and warn
+      if (!SwiftUtils.isRootDir(srcObject)) {
+        try {
+          copyThenDeleteObject(srcObject,
+                  targetObjectPath);
+        } catch (FileNotFoundException e) {
+          //create the destination directory
+          LOG.warn("Source directory deleted during rename", e);
+          innerCreateDirectory(destObject);
+        }
+      }
+    }
+  }
+
+  /**
+   * Debug action to dump directory statuses to the debug log
+   *
+   * @param message    explanation
+   * @param objectPath object path (can be null)
+   * @param statuses   listing output
+   */
+  private void logDirectory(String message, SwiftObjectPath objectPath,
+                            Iterable<FileStatus> statuses) {
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(message + ": listing of " + objectPath);
+      for (FileStatus fileStatus : statuses) {
+        LOG.debug(fileStatus.getPath());
+      }
+    }
+  }
+
+  public void copy(Path srcKey, Path dstKey) throws IOException {
+    SwiftObjectPath srcObject = toObjectPath(srcKey);
+    SwiftObjectPath destObject = toObjectPath(dstKey);
+    swiftRestClient.copyObject(srcObject, destObject);
+  }
+
+
+  /**
+   * Copy an object then, if the copy worked, delete it.
+   * If the copy failed, the source object is not deleted.
+   *
+   * @param srcObject  source object path
+   * @param destObject destination object path
+   * @throws IOException IO problems
+
+   */
+  private void copyThenDeleteObject(SwiftObjectPath srcObject,
+                                    SwiftObjectPath destObject) throws
+          IOException {
+
+
+    //do the copy
+    copyObject(srcObject, destObject);
+    //getting here means the copy worked
+    swiftRestClient.delete(srcObject);
+  }
+  /**
+   * Copy an object
+   * @param srcObject  source object path
+   * @param destObject destination object path
+   * @throws IOException IO problems
+   */
+  private void copyObject(SwiftObjectPath srcObject,
+                                    SwiftObjectPath destObject) throws
+          IOException {
+    if (srcObject.isEqualToOrParentOf(destObject)) {
+      throw new SwiftException(
+        "Can't copy " + srcObject + " onto " + destObject);
+    }
+    //do the copy
+    boolean copySucceeded = swiftRestClient.copyObject(srcObject, destObject);
+    if (!copySucceeded) {
+      throw new SwiftException("Copy of " + srcObject + " to "
+              + destObject + "failed");
+    }
+  }
+
+  /**
+   * Take a Hadoop path and return one which uses the URI prefix and authority
+   * of this FS. It doesn't make a relative path absolute
+   * @param path path in
+   * @return path with a URI bound to this FS
+   * @throws SwiftException URI cannot be created.
+   */
+  public Path getCorrectSwiftPath(Path path) throws
+          SwiftException {
+    try {
+      final URI fullUri = new URI(uri.getScheme(),
+              uri.getAuthority(),
+              path.toUri().getPath(),
+              null,
+              null);
+
+      return new Path(fullUri);
+    } catch (URISyntaxException e) {
+      throw new SwiftException("Specified path " + path + " is incorrect", e);
+    }
+  }
+
+  /**
+   * Builds a hadoop-Path from a swift path, inserting the URI authority
+   * of this FS instance
+   * @param path swift object path
+   * @return Hadoop path
+   * @throws SwiftException if the URI couldn't be created.
+   */
+  private Path getCorrectSwiftPath(SwiftObjectPath path) throws
+          SwiftException {
+    try {
+      final URI fullUri = new URI(uri.getScheme(),
+              uri.getAuthority(),
+              path.getObject(),
+              null,
+              null);
+
+      return new Path(fullUri);
+    } catch (URISyntaxException e) {
+      throw new SwiftException("Specified path " + path + " is incorrect", e);
+    }
+  }
+
+
+  /**
+   * extracts URIs from json
+   * @param json json to parse
+   * @param path path (used in exceptions)
+   * @return URIs
+   * @throws SwiftOperationFailedException on any problem parsing the JSON
+   */
+  public static List<URI> extractUris(String json, Path path) throws
+                                                   SwiftOperationFailedException {
+    final Matcher matcher = URI_PATTERN.matcher(json);
+    final List<URI> result = new ArrayList<URI>();
+    while (matcher.find()) {
+      final String s = matcher.group();
+      final String uri = s.substring(1, s.length() - 1);
+      try {
+        URI createdUri = URI.create(uri);
+        result.add(createdUri);
+      } catch (IllegalArgumentException e) {
+        //failure to create the URI, which means this is bad JSON. Convert
+        //to an exception with useful text
+        throw new SwiftOperationFailedException(
+          String.format(
+            "could not convert \"%s\" into a URI." +
+            " source: %s " +
+            " first JSON: %s",
+            uri, path, json.substring(0, 256)));
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Insert a throttled wait if the throttle delay >0
+   * @throws InterruptedIOException if interrupted during sleep
+   */
+  public void throttle() throws InterruptedIOException {
+    int throttleDelay = getThrottleDelay();
+    if (throttleDelay > 0) {
+      try {
+        Thread.sleep(throttleDelay);
+      } catch (InterruptedException e) {
+        //convert to an IOE
+        throw (InterruptedIOException) new InterruptedIOException(e.toString())
+          .initCause(e);
+      }
+    }
+  }
+
+  /**
+   * Get the current operation statistics
+   * @return a snapshot of the statistics
+   */
+  public List<DurationStats> getOperationStatistics() {
+    return swiftRestClient.getOperationStatistics();
+  }
+
+
+  /**
+   * Delete the entire tree. This is an internal one with slightly different
+   * behavior: if an entry is missing, a {@link FileNotFoundException} is
+   * raised. This lets the caller distinguish a file not found with
+   * other reasons for failure, so handles race conditions in recursive
+   * directory deletes better.
+   * <p/>
+   * The problem being addressed is: caller A requests a recursive directory
+   * of directory /dir ; caller B requests a delete of a file /dir/file,
+   * between caller A enumerating the files contents, and requesting a delete
+   * of /dir/file. We want to recognise the special case
+   * "directed file is no longer there" and not convert that into a failure
+   *
+   * @param absolutePath  the path to delete.
+   * @param recursive if path is a directory and set to
+   *                  true, the directory is deleted else throws an exception if the
+   *                  directory is not empty
+   *                  case of a file the recursive can be set to either true or false.
+   * @return true if the object was deleted
+   * @throws IOException           IO problems
+   * @throws FileNotFoundException if a file/dir being deleted is not there -
+   *                               this includes entries below the specified path, (if the path is a dir
+   *                               and recursive is true)
+   */
+  public boolean delete(Path absolutePath, boolean recursive) throws IOException {
+    Path swiftPath = getCorrectSwiftPath(absolutePath);
+    SwiftUtils.debug(LOG, "Deleting path '%s' recursive=%b",
+                     absolutePath,
+                     recursive);
+    boolean askForNewest = true;
+    SwiftFileStatus fileStatus = getObjectMetadata(swiftPath, askForNewest);
+
+    //ask for the file/dir status, but don't demand the newest, as we
+    //don't mind if the directory has changed
+    //list all entries under this directory.
+    //this will throw FileNotFoundException if the file isn't there
+    FileStatus[] statuses = listSubPaths(absolutePath, true, askForNewest);
+    if (statuses == null) {
+      //the directory went away during the non-atomic stages of the operation.
+      // Return false as it was not this thread doing the deletion.
+      SwiftUtils.debug(LOG, "Path '%s' has no status -it has 'gone away'",
+                       absolutePath,
+                       recursive);
+      return false;
+    }
+    int filecount = statuses.length;
+    SwiftUtils.debug(LOG, "Path '%s' %d status entries'",
+                     absolutePath,
+                     filecount);
+
+    if (filecount == 0) {
+      //it's an empty directory or a path
+      rmdir(absolutePath);
+      return true;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      SwiftUtils.debug(LOG, SwiftUtils.fileStatsToString(statuses, "\n"));
+    }
+
+    if (filecount == 1 && swiftPath.equals(statuses[0].getPath())) {
+      // 1 entry => simple file and it is the target
+      //simple file: delete it
+      SwiftUtils.debug(LOG, "Deleting simple file %s", absolutePath);
+      deleteObject(absolutePath);
+      return true;
+    }
+
+    //>1 entry implies directory with children. Run through them,
+    // but first check for the recursive flag and reject it *unless it looks
+    // like a partitioned file (len > 0 && has children)
+    if (!fileStatus.isDir()) {
+      LOG.debug("Multiple child entries but entry has data: assume partitioned");
+    } else if (!recursive) {
+      //if there are children, unless this is a recursive operation, fail immediately
+      throw new SwiftOperationFailedException("Directory " + fileStatus
+                                              + " is not empty: "
+                                              + SwiftUtils.fileStatsToString(
+                                                        statuses, "; "));
+    }
+
+    //delete the entries. including ourself.
+    for (FileStatus entryStatus : statuses) {
+      Path entryPath = entryStatus.getPath();
+      try {
+        boolean deleted = deleteObject(entryPath);
+        if (!deleted) {
+          SwiftUtils.debug(LOG, "Failed to delete entry '%s'; continuing",
+                           entryPath);
+        }
+      } catch (FileNotFoundException e) {
+        //the path went away -race conditions.
+        //do not fail, as the outcome is still OK.
+        SwiftUtils.debug(LOG, "Path '%s' is no longer present; continuing",
+                         entryPath);
+      }
+      throttle();
+    }
+    //now delete self
+    SwiftUtils.debug(LOG, "Deleting base entry %s", absolutePath);
+    deleteObject(absolutePath);
+
+    return true;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeInputStream.java

@@ -0,0 +1,379 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.http.HttpBodyContent;
+import org.apache.hadoop.fs.swift.http.HttpInputStreamWithRelease;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * The input stream from remote Swift blobs.
+ * The class attempts to be buffer aware, and react to a forward seek operation
+ * by trying to scan ahead through the current block of data to find it.
+ * This accelerates some operations that do a lot of seek()/read() actions,
+ * including work (such as in the MR engine) that do a seek() immediately after
+ * an open().
+ */
+class SwiftNativeInputStream extends FSInputStream {
+
+  private static final Log LOG = LogFactory.getLog(SwiftNativeInputStream.class);
+
+  /**
+   *  range requested off the server: {@value}
+   */
+  private final long bufferSize;
+
+  /**
+   * File nativeStore instance
+   */
+  private final SwiftNativeFileSystemStore nativeStore;
+
+  /**
+   * Hadoop statistics. Used to get info about number of reads, writes, etc.
+   */
+  private final FileSystem.Statistics statistics;
+
+  /**
+   * Data input stream
+   */
+  private HttpInputStreamWithRelease httpStream;
+
+  /**
+   * File path
+   */
+  private final Path path;
+
+  /**
+   * Current position
+   */
+  private long pos = 0;
+
+  /**
+   * Length of the file picked up at start time
+   */
+  private long contentLength = -1;
+
+  /**
+   * Why the stream is closed
+   */
+  private String reasonClosed = "unopened";
+
+  /**
+   * Offset in the range requested last
+   */
+  private long rangeOffset = 0;
+
+  public SwiftNativeInputStream(SwiftNativeFileSystemStore storeNative,
+      FileSystem.Statistics statistics, Path path, long bufferSize)
+          throws IOException {
+    this.nativeStore = storeNative;
+    this.statistics = statistics;
+    this.path = path;
+    if (bufferSize <= 0) {
+      throw new IllegalArgumentException("Invalid buffer size");
+    }
+    this.bufferSize = bufferSize;
+    //initial buffer fill
+    this.httpStream = storeNative.getObject(path).getInputStream();
+    //fillBuffer(0);
+  }
+
+  /**
+   * Move to a new position within the file relative to where the pointer is now.
+   * Always call from a synchronized clause
+   * @param offset offset
+   */
+  private synchronized void incPos(int offset) {
+    pos += offset;
+    rangeOffset += offset;
+    SwiftUtils.trace(LOG, "Inc: pos=%d bufferOffset=%d", pos, rangeOffset);
+  }
+
+  /**
+   * Update the start of the buffer; always call from a sync'd clause
+   * @param seekPos position sought.
+   * @param contentLength content length provided by response (may be -1)
+   */
+  private synchronized void updateStartOfBufferPosition(long seekPos,
+                                                        long contentLength) {
+    //reset the seek pointer
+    pos = seekPos;
+    //and put the buffer offset to 0
+    rangeOffset = 0;
+    this.contentLength = contentLength;
+    SwiftUtils.trace(LOG, "Move: pos=%d; bufferOffset=%d; contentLength=%d",
+                     pos,
+                     rangeOffset,
+                     contentLength);
+  }
+
+  @Override
+  public synchronized int read() throws IOException {
+    verifyOpen();
+    int result = -1;
+    try {
+      result = httpStream.read();
+    } catch (IOException e) {
+      String msg = "IOException while reading " + path
+                   + ": " +e + ", attempting to reopen.";
+      LOG.debug(msg, e);
+      if (reopenBuffer()) {
+        result = httpStream.read();
+      }
+    }
+    if (result != -1) {
+      incPos(1);
+    }
+    if (statistics != null && result != -1) {
+      statistics.incrementBytesRead(1);
+    }
+    return result;
+  }
+
+  @Override
+  public synchronized int read(byte[] b, int off, int len) throws IOException {
+    SwiftUtils.debug(LOG, "read(buffer, %d, %d)", off, len);
+    SwiftUtils.validateReadArgs(b, off, len);
+    int result = -1;
+    try {
+      verifyOpen();
+      result = httpStream.read(b, off, len);
+    } catch (IOException e) {
+      //other IO problems are viewed as transient and re-attempted
+      LOG.info("Received IOException while reading '" + path +
+               "', attempting to reopen: " + e);
+      LOG.debug("IOE on read()" + e, e);
+      if (reopenBuffer()) {
+        result = httpStream.read(b, off, len);
+      }
+    }
+    if (result > 0) {
+      incPos(result);
+      if (statistics != null) {
+        statistics.incrementBytesRead(result);
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Re-open the buffer
+   * @return true iff more data could be added to the buffer
+   * @throws IOException if not
+   */
+  private boolean reopenBuffer() throws IOException {
+    innerClose("reopening buffer to trigger refresh");
+    boolean success = false;
+    try {
+      fillBuffer(pos);
+      success =  true;
+    } catch (EOFException eof) {
+      //the EOF has been reached
+      this.reasonClosed = "End of file";
+    }
+    return success;
+  }
+
+  /**
+   * close the stream. After this the stream is not usable -unless and until
+   * it is re-opened (which can happen on some of the buffer ops)
+   * This method is thread-safe and idempotent.
+   *
+   * @throws IOException on IO problems.
+   */
+  @Override
+  public synchronized void close() throws IOException {
+    innerClose("closed");
+  }
+
+  private void innerClose(String reason) throws IOException {
+    try {
+      if (httpStream != null) {
+        reasonClosed = reason;
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Closing HTTP input stream : " + reason);
+        }
+        httpStream.close();
+      }
+    } finally {
+      httpStream = null;
+    }
+  }
+
+  /**
+   * Assume that the connection is not closed: throws an exception if it is
+   * @throws SwiftConnectionClosedException
+   */
+  private void verifyOpen() throws SwiftConnectionClosedException {
+    if (httpStream == null) {
+      throw new SwiftConnectionClosedException(reasonClosed);
+    }
+  }
+
+  @Override
+  public synchronized String toString() {
+    return "SwiftNativeInputStream" +
+           " position=" + pos
+           + " buffer size = " + bufferSize
+           + " "
+           + (httpStream != null ? httpStream.toString()
+                                 : (" no input stream: " + reasonClosed));
+  }
+
+  /**
+   * Treats any finalize() call without the input stream being closed
+   * as a serious problem, logging at error level
+   * @throws Throwable n/a
+   */
+  @Override
+  protected void finalize() throws Throwable {
+    if (httpStream != null) {
+      LOG.error(
+        "Input stream is leaking handles by not being closed() properly: "
+        + httpStream.toString());
+    }
+  }
+
+  /**
+   * Read through the specified number of bytes.
+   * The implementation iterates a byte a time, which may seem inefficient
+   * compared to the read(bytes[]) method offered by input streams.
+   * However, if you look at the code that implements that method, it comes
+   * down to read() one char at a time -only here the return value is discarded.
+   *
+   *<p/>
+   * This is a no-op if the stream is closed
+   * @param bytes number of bytes to read.
+   * @throws IOException IO problems
+   * @throws SwiftException if a read returned -1.
+   */
+  private int chompBytes(long bytes) throws IOException {
+    int count = 0;
+    if (httpStream != null) {
+      int result;
+      for (long i = 0; i < bytes; i++) {
+        result = httpStream.read();
+        if (result < 0) {
+          throw new SwiftException("Received error code while chomping input");
+        }
+        count ++;
+        incPos(1);
+      }
+    }
+    return count;
+  }
+
+  /**
+   * Seek to an offset. If the data is already in the buffer, move to it
+   * @param targetPos target position
+   * @throws IOException on any problem
+   */
+  @Override
+  public synchronized void seek(long targetPos) throws IOException {
+    if (targetPos < 0) {
+      throw new IOException("Negative Seek offset not supported");
+    }
+    //there's some special handling of near-local data
+    //as the seek can be omitted if it is in/adjacent
+    long offset = targetPos - pos;
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Seek to " + targetPos + "; current pos =" + pos
+                + "; offset="+offset);
+    }
+    if (offset == 0) {
+      LOG.debug("seek is no-op");
+      return;
+    }
+
+    if (offset < 0) {
+      LOG.debug("seek is backwards");
+    } else if ((rangeOffset + offset < bufferSize)) {
+      //if the seek is in  range of that requested, scan forwards
+      //instead of closing and re-opening a new HTTP connection
+      SwiftUtils.debug(LOG,
+                       "seek is within current stream"
+                       + "; pos= %d ; targetPos=%d; "
+                       + "offset= %d ; bufferOffset=%d",
+                       pos, targetPos, offset, rangeOffset);
+      try {
+        LOG.debug("chomping ");
+        chompBytes(offset);
+      } catch (IOException e) {
+        //this is assumed to be recoverable with a seek -or more likely to fail
+        LOG.debug("while chomping ",e);
+      }
+      if (targetPos - pos == 0) {
+        LOG.trace("chomping successful");
+        return;
+      }
+      LOG.trace("chomping failed");
+    } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Seek is beyond buffer size of " + bufferSize);
+      }
+    }
+
+    innerClose("seeking to " + targetPos);
+    fillBuffer(targetPos);
+  }
+
+  /**
+   * Fill the buffer from the target position
+   * If the target position == current position, the
+   * read still goes ahead; this is a way of handling partial read failures
+   * @param targetPos target position
+   * @throws IOException IO problems on the read
+   */
+  private void fillBuffer(long targetPos) throws IOException {
+    long length = targetPos + bufferSize;
+    SwiftUtils.debug(LOG, "Fetching %d bytes starting at %d", length, targetPos);
+    HttpBodyContent blob = nativeStore.getObject(path, targetPos, length);
+    httpStream = blob.getInputStream();
+    updateStartOfBufferPosition(targetPos, blob.getContentLength());
+  }
+
+  @Override
+  public synchronized long getPos() throws IOException {
+    return pos;
+  }
+
+  /**
+   * This FS doesn't explicitly support multiple data sources, so
+   * return false here.
+   * @param targetPos the desired target position
+   * @return true if a new source of the data has been set up
+   * as the source of future reads
+   * @throws IOException IO problems
+   */
+  @Override
+  public boolean seekToNewSource(long targetPos) throws IOException {
+    return false;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftNativeOutputStream.java

@@ -0,0 +1,388 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftInternalStateException;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * Output stream, buffers data on local disk.
+ * Writes to Swift on the close() method, unless the
+ * file is significantly large that it is being written as partitions.
+ * In this case, the first partition is written on the first write that puts
+ * data over the partition, as may later writes. The close() then causes
+ * the final partition to be written, along with a partition manifest.
+ */
+class SwiftNativeOutputStream extends OutputStream {
+  public static final int ATTEMPT_LIMIT = 3;
+  private long filePartSize;
+  private static final Log LOG =
+          LogFactory.getLog(SwiftNativeOutputStream.class);
+  private Configuration conf;
+  private String key;
+  private File backupFile;
+  private OutputStream backupStream;
+  private SwiftNativeFileSystemStore nativeStore;
+  private boolean closed;
+  private int partNumber;
+  private long blockOffset;
+  private long bytesWritten;
+  private long bytesUploaded;
+  private boolean partUpload = false;
+  final byte[] oneByte = new byte[1];
+
+  /**
+   * Create an output stream
+   * @param conf configuration to use
+   * @param nativeStore native store to write through
+   * @param key the key to write
+   * @param partSizeKB the partition size
+   * @throws IOException
+   */
+  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+  public SwiftNativeOutputStream(Configuration conf,
+                                 SwiftNativeFileSystemStore nativeStore,
+                                 String key,
+                                 long partSizeKB) throws IOException {
+    this.conf = conf;
+    this.key = key;
+    this.backupFile = newBackupFile();
+    this.nativeStore = nativeStore;
+    this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile));
+    this.partNumber = 1;
+    this.blockOffset = 0;
+    this.filePartSize = 1024L * partSizeKB;
+  }
+
+  private File newBackupFile() throws IOException {
+    File dir = new File(conf.get("hadoop.tmp.dir"));
+    if (!dir.mkdirs() && !dir.exists()) {
+      throw new SwiftException("Cannot create Swift buffer directory: " + dir);
+    }
+    File result = File.createTempFile("output-", ".tmp", dir);
+    result.deleteOnExit();
+    return result;
+  }
+
+  /**
+   * Flush the local backing stream.
+   * This does not trigger a flush of data to the remote blobstore.
+   * @throws IOException
+   */
+  @Override
+  public void flush() throws IOException {
+    backupStream.flush();
+  }
+
+  /**
+   * check that the output stream is open
+   *
+   * @throws SwiftException if it is not
+   */
+  private synchronized void verifyOpen() throws SwiftException {
+    if (closed) {
+      throw new SwiftException("Output stream is closed");
+    }
+  }
+
+  /**
+   * Close the stream. This will trigger the upload of all locally cached
+   * data to the remote blobstore.
+   * @throws IOException IO problems uploading the data.
+   */
+  @Override
+  public synchronized void close() throws IOException {
+    if (closed) {
+      return;
+    }
+
+    try {
+      closed = true;
+      //formally declare as closed.
+      backupStream.close();
+      backupStream = null;
+      Path keypath = new Path(key);
+      if (partUpload) {
+        partUpload(true);
+        nativeStore.createManifestForPartUpload(keypath);
+      } else {
+        uploadOnClose(keypath);
+      }
+    } finally {
+      delete(backupFile);
+      backupFile = null;
+    }
+    assert backupStream == null: "backup stream has been reopened";
+  }
+
+  /**
+   * Upload a file when closed, either in one go, or, if the file is
+   * already partitioned, by uploading the remaining partition and a manifest.
+   * @param keypath key as a path
+   * @throws IOException IO Problems
+   */
+  private void uploadOnClose(Path keypath) throws IOException {
+    boolean uploadSuccess = false;
+    int attempt = 0;
+    while (!uploadSuccess) {
+      try {
+        ++attempt;
+        bytesUploaded += uploadFileAttempt(keypath, attempt);
+        uploadSuccess = true;
+      } catch (IOException e) {
+        LOG.info("Upload failed " + e, e);
+        if (attempt > ATTEMPT_LIMIT) {
+          throw e;
+        }
+      }
+    }
+}
+
+  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+  private long uploadFileAttempt(Path keypath, int attempt) throws IOException {
+    long uploadLen = backupFile.length();
+    SwiftUtils.debug(LOG, "Closing write of file %s;" +
+                          " localfile=%s of length %d - attempt %d",
+                     key,
+                     backupFile,
+                     uploadLen,
+                     attempt);
+
+    nativeStore.uploadFile(keypath,
+                           new FileInputStream(backupFile),
+                           uploadLen);
+    return uploadLen;
+  }
+
+  @Override
+  protected void finalize() throws Throwable {
+    if(!closed) {
+      LOG.warn("stream not closed");
+    }
+    if (backupFile != null) {
+      LOG.warn("Leaking backing file " + backupFile);
+    }
+  }
+
+  private void delete(File file) {
+    if (file != null) {
+      SwiftUtils.debug(LOG, "deleting %s", file);
+      if (!file.delete()) {
+        LOG.warn("Could not delete " + file);
+      }
+    }
+  }
+
+  @Override
+  public void write(int b) throws IOException {
+    //insert to a one byte array
+    oneByte[0] = (byte) b;
+    //then delegate to the array writing routine
+    write(oneByte, 0, 1);
+  }
+
+  @Override
+  public synchronized void write(byte[] buffer, int offset, int len) throws
+                                                                     IOException {
+    //validate args
+    if (offset < 0 || len < 0 || (offset + len) > buffer.length) {
+      throw new IndexOutOfBoundsException("Invalid offset/length for write");
+    }
+    //validate the output stream
+    verifyOpen();
+    SwiftUtils.debug(LOG, " write(offset=%d, len=%d)", offset, len);
+
+    // if the size of file is greater than the partition limit
+    while (blockOffset + len >= filePartSize) {
+      // - then partition the blob and upload as many partitions
+      // are needed.
+      //how many bytes to write for this partition.
+      int subWriteLen = (int) (filePartSize - blockOffset);
+      if (subWriteLen < 0 || subWriteLen > len) {
+        throw new SwiftInternalStateException("Invalid subwrite len: "
+                                              + subWriteLen
+                                              + " -buffer len: " + len);
+      }
+      writeToBackupStream(buffer, offset, subWriteLen);
+      //move the offset along and length down
+      offset += subWriteLen;
+      len -= subWriteLen;
+      //now upload the partition that has just been filled up
+      // (this also sets blockOffset=0)
+      partUpload(false);
+    }
+    //any remaining data is now written
+    writeToBackupStream(buffer, offset, len);
+  }
+
+  /**
+   * Write to the backup stream.
+   * Guarantees:
+   * <ol>
+   *   <li>backupStream is open</li>
+   *   <li>blockOffset + len &lt; filePartSize</li>
+   * </ol>
+   * @param buffer buffer to write
+   * @param offset offset in buffer
+   * @param len length of write.
+   * @throws IOException backup stream write failing
+   */
+  private void writeToBackupStream(byte[] buffer, int offset, int len) throws
+                                                                       IOException {
+    assert len >= 0  : "remainder to write is negative";
+    SwiftUtils.debug(LOG," writeToBackupStream(offset=%d, len=%d)", offset, len);
+    if (len == 0) {
+      //no remainder -downgrade to noop
+      return;
+    }
+
+    //write the new data out to the backup stream
+    backupStream.write(buffer, offset, len);
+    //increment the counters
+    blockOffset += len;
+    bytesWritten += len;
+  }
+
+  /**
+   * Upload a single partition. This deletes the local backing-file,
+   * and re-opens it to create a new one.
+   * @param closingUpload is this the final upload of an upload
+   * @throws IOException on IO problems
+   */
+  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+  private void partUpload(boolean closingUpload) throws IOException {
+    if (backupStream != null) {
+      backupStream.close();
+    }
+
+    if (closingUpload && partUpload && backupFile.length() == 0) {
+      //skipping the upload if
+      // - it is close time
+      // - the final partition is 0 bytes long
+      // - one part has already been written
+      SwiftUtils.debug(LOG, "skipping upload of 0 byte final partition");
+      delete(backupFile);
+    } else {
+      partUpload = true;
+      boolean uploadSuccess = false;
+      int attempt = 0;
+      while(!uploadSuccess) {
+        try {
+          ++attempt;
+          bytesUploaded += uploadFilePartAttempt(attempt);
+          uploadSuccess = true;
+        } catch (IOException e) {
+          LOG.info("Upload failed " + e, e);
+          if (attempt > ATTEMPT_LIMIT) {
+            throw e;
+          }
+        }
+      }
+      delete(backupFile);
+      partNumber++;
+      blockOffset = 0;
+      if (!closingUpload) {
+        //if not the final upload, create a new output stream
+        backupFile = newBackupFile();
+        backupStream =
+          new BufferedOutputStream(new FileOutputStream(backupFile));
+      }
+    }
+  }
+
+  @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+  private long uploadFilePartAttempt(int attempt) throws IOException {
+    long uploadLen = backupFile.length();
+    SwiftUtils.debug(LOG, "Uploading part %d of file %s;" +
+                          " localfile=%s of length %d  - attempt %d",
+                     partNumber,
+                     key,
+                     backupFile,
+                     uploadLen,
+                     attempt);
+    nativeStore.uploadFilePart(new Path(key),
+                               partNumber,
+                               new FileInputStream(backupFile),
+                               uploadLen);
+    return uploadLen;
+  }
+
+  /**
+   * Get the file partition size
+   * @return the partition size
+   */
+  long getFilePartSize() {
+    return filePartSize;
+  }
+
+  /**
+   * Query the number of partitions written
+   * This is intended for testing
+   * @return the of partitions already written to the remote FS
+   */
+  synchronized int getPartitionsWritten() {
+    return partNumber - 1;
+  }
+
+  /**
+   * Get the number of bytes written to the output stream.
+   * This should always be less than or equal to bytesUploaded.
+   * @return the number of bytes written to this stream
+   */
+  long getBytesWritten() {
+    return bytesWritten;
+  }
+
+  /**
+   * Get the number of bytes uploaded to remote Swift cluster.
+   * bytesUploaded -bytesWritten = the number of bytes left to upload
+   * @return the number of bytes written to the remote endpoint
+   */
+  long getBytesUploaded() {
+    return bytesUploaded;
+  }
+
+  @Override
+  public String toString() {
+    return "SwiftNativeOutputStream{" +
+           ", key='" + key + '\'' +
+           ", backupFile=" + backupFile +
+           ", closed=" + closed +
+           ", filePartSize=" + filePartSize +
+           ", partNumber=" + partNumber +
+           ", blockOffset=" + blockOffset +
+           ", partUpload=" + partUpload +
+           ", nativeStore=" + nativeStore +
+           ", bytesWritten=" + bytesWritten +
+           ", bytesUploaded=" + bytesUploaded +
+           '}';
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/snative/SwiftObjectFileStatus.java

@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.snative;
+
+import java.util.Date;
+
+/**
+ * Java mapping of Swift JSON file status.
+ * THIS FILE IS MAPPED BY JACKSON TO AND FROM JSON.
+ * DO NOT RENAME OR MODIFY FIELDS AND THEIR ACCESSORS.
+ */
+
+class SwiftObjectFileStatus {
+  private long bytes;
+  private String content_type;
+  private String hash;
+  private Date last_modified;
+  private String name;
+  private String subdir;
+
+  SwiftObjectFileStatus() {
+  }
+
+  SwiftObjectFileStatus(long bytes, String content_type, String hash,
+                        Date last_modified, String name) {
+    this.bytes = bytes;
+    this.content_type = content_type;
+    this.hash = hash;
+    this.last_modified = last_modified;
+    this.name = name;
+  }
+
+  public long getBytes() {
+    return bytes;
+  }
+
+  public void setBytes(long bytes) {
+    this.bytes = bytes;
+  }
+
+  public String getContent_type() {
+    return content_type;
+  }
+
+  public void setContent_type(String content_type) {
+    this.content_type = content_type;
+  }
+
+  public String getHash() {
+    return hash;
+  }
+
+  public void setHash(String hash) {
+    this.hash = hash;
+  }
+
+  public Date getLast_modified() {
+    return last_modified;
+  }
+
+  public void setLast_modified(Date last_modified) {
+    this.last_modified = last_modified;
+  }
+
+  public String getName() {
+    return pathToRootPath(name);
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getSubdir() {
+    return pathToRootPath(subdir);
+  }
+
+  public void setSubdir(String subdir) {
+    this.subdir = subdir;
+  }
+
+  /**
+   * If path doesn't starts with '/'
+   * method will concat '/'
+   *
+   * @param path specified path
+   * @return root path string
+   */
+  private String pathToRootPath(String path) {
+    if (path == null) {
+      return null;
+    }
+
+    if (path.startsWith("/")) {
+      return path;
+    }
+
+    return "/".concat(path);
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/Duration.java

@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+public class Duration {
+
+  private final long started;
+  private long finished;
+
+  public Duration() {
+    started = time();
+    finished = started;
+  }
+
+  private long time() {
+    return System.currentTimeMillis();
+  }
+
+  public void finished() {
+    finished = time();
+  }
+
+  public String getDurationString() {
+    return humanTime(value());
+  }
+
+  public static String humanTime(long time) {
+    long seconds = (time / 1000);
+    long minutes = (seconds / 60);
+    return String.format("%d:%02d:%03d", minutes, seconds % 60, time % 1000);
+  }
+
+  @Override
+  public String toString() {
+    return getDurationString();
+  }
+
+  public long value() {
+    return finished -started;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStats.java

@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+/**
+ * Build ongoing statistics from duration data
+ */
+public class DurationStats {
+
+  final String operation;
+  int n;
+  long sum;
+  long min;
+  long max;
+  double mean, m2;
+
+  /**
+   * Construct statistics for a given operation.
+   * @param operation operation
+   */
+  public DurationStats(String operation) {
+    this.operation = operation;
+    reset();
+  }
+
+  /**
+   * construct from anothr stats entry;
+   * all value are copied.
+   * @param that the source statistics
+   */
+  public DurationStats(DurationStats that) {
+    operation = that.operation;
+    n = that.n;
+    sum = that.sum;
+    min = that.min;
+    max = that.max;
+    mean = that.mean;
+    m2 = that.m2;
+  }
+
+  /**
+   * Add a duration
+   * @param duration the new duration
+   */
+  public void add(Duration duration) {
+    add(duration.value());
+  }
+
+  /**
+   * Add a number
+   * @param x the number
+   */
+  public void add(long x) {
+    n++;
+    sum += x;
+    double delta = x - mean;
+    mean += delta / n;
+    m2 += delta * (x - mean);
+    if (x < min) {
+      min = x;
+    }
+    if (x > max) {
+      max = x;
+    }
+  }
+
+  /**
+   * Reset the data
+   */
+  public void reset() {
+    n = 0;
+    sum = 0;
+    sum = 0;
+    min = 10000000;
+    max = 0;
+    mean = 0;
+    m2 = 0;
+  }
+
+  /**
+   * Get the number of entries sampled
+   * @return the number of durations added
+   */
+  public int getCount() {
+    return n;
+  }
+
+  /**
+   * Get the sum of all durations
+   * @return all the durations
+   */
+  public long getSum() {
+    return sum;
+  }
+
+  /**
+   * Get the arithmetic mean of the aggregate statistics
+   * @return the arithmetic mean
+   */
+  public double getArithmeticMean() {
+    return mean;
+  }
+
+  /**
+   * Variance, sigma^2
+   * @return variance, or, if no samples are there, 0.
+   */
+  public double getVariance() {
+    return n > 0 ? (m2 / (n - 1)) : 0;
+  }
+
+  /**
+   * Get the std deviation, sigma
+   * @return the stddev, 0 may mean there are no samples.
+   */
+  public double getDeviation() {
+    double variance = getVariance();
+    return (variance > 0) ? Math.sqrt(variance) : 0;
+  }
+
+  /**
+   * Covert to a useful string
+   * @return a human readable summary
+   */
+  @Override
+  public String toString() {
+    return String.format(
+      "%s count=%d total=%.3fs mean=%.3fs stddev=%.3fs min=%.3fs max=%.3fs",
+      operation,
+      n,
+      sum / 1000.0,
+      mean / 1000.0,
+      getDeviation() / 1000000.0,
+      min / 1000.0,
+      max / 1000.0);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/DurationStatsTable.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift.util;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Build a duration stats table to which you can add statistics.
+ * Designed to be multithreaded
+ */
+public class DurationStatsTable {
+
+  private Map<String,DurationStats> statsTable
+    = new HashMap<String, DurationStats>(6);
+
+  /**
+   * Add an operation
+   * @param operation operation name
+   * @param duration duration
+   */
+  public void add(String operation, Duration duration, boolean success) {
+    DurationStats durationStats;
+    String key = operation;
+    if (!success) {
+      key += "-FAIL";
+    }
+    synchronized (this) {
+      durationStats = statsTable.get(key);
+      if (durationStats == null) {
+        durationStats = new DurationStats(key);
+        statsTable.put(key, durationStats);
+      }
+    }
+    synchronized (durationStats) {
+      durationStats.add(duration);
+    }
+  }
+
+  /**
+   * Get the current duration statistics
+   * @return a snapshot of the statistics
+   */
+   public synchronized List<DurationStats> getDurationStatistics() {
+     List<DurationStats> results = new ArrayList<DurationStats>(statsTable.size());
+     for (DurationStats stat: statsTable.values()) {
+       results.add(new DurationStats(stat));
+     }
+     return results;
+   }
+
+  /**
+   * reset the values of the statistics. This doesn't delete them, merely zeroes them.
+   */
+  public synchronized void reset() {
+    for (DurationStats stat : statsTable.values()) {
+      stat.reset();
+    }
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/JSONUtil.java

@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.swift.exceptions.SwiftJsonMarshallingException;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.type.CollectionType;
+import org.codehaus.jackson.type.TypeReference;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+
+
+public class JSONUtil {
+  private static ObjectMapper jsonMapper = new ObjectMapper();
+
+  /**
+   * Private constructor.
+   */
+  private JSONUtil() {
+  }
+
+  /**
+   * Converting object to JSON string. If errors appears throw
+   * MeshinException runtime exception.
+   *
+   * @param object The object to convert.
+   * @return The JSON string representation.
+   * @throws IOException IO issues
+   * @throws SwiftJsonMarshallingException failure to generate JSON
+   */
+  public static String toJSON(Object object) throws
+                                             IOException {
+    Writer json = new StringWriter();
+    try {
+      jsonMapper.writeValue(json, object);
+      return json.toString();
+    } catch (JsonGenerationException e) {
+      throw new SwiftJsonMarshallingException(e.toString(), e);
+    } catch (JsonMappingException e) {
+      throw new SwiftJsonMarshallingException(e.toString(), e);
+    }
+  }
+
+  /**
+   * Convert string representation to object. If errors appears throw
+   * Exception runtime exception.
+   *
+   * @param value The JSON string.
+   * @param klazz The class to convert.
+   * @return The Object of the given class.
+   */
+  public static <T> T toObject(String value, Class<T> klazz) throws
+                                                             IOException {
+    try {
+      return jsonMapper.readValue(value, klazz);
+    } catch (JsonGenerationException e) {
+      throw new SwiftJsonMarshallingException(e.toString()
+                                              + " source: " + value,
+                                              e);
+    } catch (JsonMappingException e) {
+      throw new SwiftJsonMarshallingException(e.toString()
+                                              + " source: " + value,
+                                              e);
+    }
+  }
+
+  /**
+   * @param value         json string
+   * @param typeReference class type reference
+   * @param <T>           type
+   * @return deserialized  T object
+   */
+  public static <T> T toObject(String value,
+                               final TypeReference<T> typeReference)
+            throws IOException {
+    try {
+      return jsonMapper.readValue(value, typeReference);
+    } catch (JsonGenerationException e) {
+      throw new SwiftJsonMarshallingException("Error generating response", e);
+    } catch (JsonMappingException e) {
+      throw new SwiftJsonMarshallingException("Error generating response", e);
+    }
+  }
+
+  /**
+   * @param value          json string
+   * @param collectionType class describing how to deserialize collection of objects
+   * @param <T>            type
+   * @return deserialized  T object
+   */
+  public static <T> T toObject(String value,
+                               final CollectionType collectionType)
+              throws IOException {
+    try {
+      return jsonMapper.readValue(value, collectionType);
+    } catch (JsonGenerationException e) {
+      throw new SwiftJsonMarshallingException(e.toString()
+                                              + " source: " + value,
+                                              e);
+    } catch (JsonMappingException e) {
+      throw new SwiftJsonMarshallingException(e.toString()
+                                              + " source: " + value,
+                                              e);
+    }
+  }
+
+  public static ObjectMapper getJsonMapper() {
+    return jsonMapper;
+  }
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftObjectPath.java

@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+
+import java.net.URI;
+import java.util.regex.Pattern;
+
+/**
+ * Swift hierarchy mapping of (container, path)
+ */
+public final class SwiftObjectPath {
+  private static final Pattern PATH_PART_PATTERN = Pattern.compile(".*/AUTH_\\w*/");
+
+  /**
+   * Swift container
+   */
+  private final String container;
+
+  /**
+   * swift object
+   */
+  private final String object;
+
+  private final String uriPath;
+
+  /**
+   * Build an instance from a (host, object) pair
+   *
+   * @param container container name
+   * @param object    object ref underneath the container
+   */
+  public SwiftObjectPath(String container, String object) {
+
+    this.container = container;
+    this.object = object;
+    uriPath = buildUriPath();
+  }
+
+  public String getContainer() {
+    return container;
+  }
+
+  public String getObject() {
+    return object;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof SwiftObjectPath)) return false;
+    final SwiftObjectPath that = (SwiftObjectPath) o;
+    return this.toUriPath().equals(that.toUriPath());
+  }
+
+  @Override
+  public int hashCode() {
+    int result = container.hashCode();
+    result = 31 * result + object.hashCode();
+    return result;
+  }
+
+  private String buildUriPath() {
+    return SwiftUtils.joinPaths(container, object);
+  }
+
+  public String toUriPath() {
+    return uriPath;
+  }
+
+  @Override
+  public String toString() {
+    return toUriPath();
+  }
+
+  /**
+   * Test for the object matching a path, ignoring the container
+   * value.
+   *
+   * @param path path string
+   * @return true iff the object's name matches the path
+   */
+  public boolean objectMatches(String path) {
+    return object.equals(path);
+  }
+
+
+  /**
+   * Query to see if the possibleChild object is a child path of this.
+   * object.
+   *
+   * The test is done by probing for the path of the this object being
+   * at the start of the second -with a trailing slash, and both
+   * containers being equal
+   *
+   * @param possibleChild possible child dir
+   * @return true iff the possibleChild is under this object
+   */
+  public boolean isEqualToOrParentOf(SwiftObjectPath possibleChild) {
+    String origPath = toUriPath();
+    String path = origPath;
+    if (!path.endsWith("/")) {
+      path = path + "/";
+    }
+    String childPath = possibleChild.toUriPath();
+    return childPath.equals(origPath) || childPath.startsWith(path);
+  }
+
+  /**
+   * Create a path tuple of (container, path), where the container is
+   * chosen from the host of the URI.
+   *
+   * @param uri  uri to start from
+   * @param path path underneath
+   * @return a new instance.
+   * @throws SwiftConfigurationException if the URI host doesn't parse into
+   *                                     container.service
+   */
+  public static SwiftObjectPath fromPath(URI uri,
+                                         Path path)
+          throws SwiftConfigurationException {
+    return fromPath(uri, path, false);
+  }
+
+  /**
+   * Create a path tuple of (container, path), where the container is
+   * chosen from the host of the URI.
+   * A trailing slash can be added to the path. This is the point where
+   * these /-es need to be appended, because when you construct a {@link Path}
+   * instance, {@link Path#normalizePath(String, String)} is called
+   * -which strips off any trailing slash.
+   *
+   * @param uri              uri to start from
+   * @param path             path underneath
+   * @param addTrailingSlash should a trailing slash be added if there isn't one.
+   * @return a new instance.
+   * @throws SwiftConfigurationException if the URI host doesn't parse into
+   *                                     container.service
+   */
+  public static SwiftObjectPath fromPath(URI uri,
+                                         Path path,
+                                         boolean addTrailingSlash)
+          throws SwiftConfigurationException {
+
+    String url =
+            path.toUri().getPath().replaceAll(PATH_PART_PATTERN.pattern(), "");
+    //add a trailing slash if needed
+    if (addTrailingSlash && !url.endsWith("/")) {
+      url += "/";
+    }
+
+    String container = uri.getHost();
+    if (container == null) {
+      //no container, not good: replace with ""
+      container = "";
+    } else if (container.contains(".")) {
+      //its a container.service URI. Strip the container
+      container = RestClientBindings.extractContainerName(container);
+    }
+    return new SwiftObjectPath(container, url);
+  }
+
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftTestUtils.java

@@ -0,0 +1,544 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+/**
+ * Utilities used across test cases
+ */
+public class SwiftTestUtils extends org.junit.Assert {
+
+  private static final Log LOG =
+    LogFactory.getLog(SwiftTestUtils.class);
+
+  public static final String TEST_FS_SWIFT = "test.fs.swift.name";
+  public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+  /**
+   * Get the test URI
+   * @param conf configuration
+   * @throws SwiftConfigurationException missing parameter or bad URI
+   */
+  public static URI getServiceURI(Configuration conf) throws
+                                                      SwiftConfigurationException {
+    String instance = conf.get(TEST_FS_SWIFT);
+    if (instance == null) {
+      throw new SwiftConfigurationException(
+        "Missing configuration entry " + TEST_FS_SWIFT);
+    }
+    try {
+      return new URI(instance);
+    } catch (URISyntaxException e) {
+      throw new SwiftConfigurationException("Bad URI: " + instance);
+    }
+  }
+
+  public static boolean hasServiceURI(Configuration conf) {
+    String instance = conf.get(TEST_FS_SWIFT);
+    return instance != null;
+  }
+
+  /**
+   * Assert that a property in the property set matches the expected value
+   * @param props property set
+   * @param key property name
+   * @param expected expected value. If null, the property must not be in the set
+   */
+  public static void assertPropertyEquals(Properties props,
+                                          String key,
+                                          String expected) {
+    String val = props.getProperty(key);
+    if (expected == null) {
+      assertNull("Non null property " + key + " = " + val, val);
+    } else {
+      assertEquals("property " + key + " = " + val,
+                          expected,
+                          val);
+    }
+  }
+
+  /**
+   *
+   * Write a file and read it in, validating the result. Optional flags control
+   * whether file overwrite operations should be enabled, and whether the
+   * file should be deleted afterwards.
+   *
+   * If there is a mismatch between what was written and what was expected,
+   * a small range of bytes either side of the first error are logged to aid
+   * diagnosing what problem occurred -whether it was a previous file
+   * or a corrupting of the current file. This assumes that two
+   * sequential runs to the same path use datasets with different character
+   * moduli.
+   *
+   * @param fs filesystem
+   * @param path path to write to
+   * @param len length of data
+   * @param overwrite should the create option allow overwrites?
+   * @param delete should the file be deleted afterwards? -with a verification
+   * that it worked. Deletion is not attempted if an assertion has failed
+   * earlier -it is not in a <code>finally{}</code> block.
+   * @throws IOException IO problems
+   */
+  public static void writeAndRead(FileSystem fs,
+                                  Path path,
+                                  byte[] src,
+                                  int len,
+                                  int blocksize,
+                                  boolean overwrite,
+                                  boolean delete) throws IOException {
+    fs.mkdirs(path.getParent());
+
+    writeDataset(fs, path, src, len, blocksize, overwrite);
+
+    byte[] dest = readDataset(fs, path, len);
+
+    compareByteArrays(src, dest, len);
+
+    if (delete) {
+      boolean deleted = fs.delete(path, false);
+      assertTrue("Deleted", deleted);
+      assertPathDoesNotExist(fs, "Cleanup failed", path);
+    }
+  }
+
+  /**
+   * Write a file.
+   * Optional flags control
+   * whether file overwrite operations should be enabled
+   * @param fs filesystem
+   * @param path path to write to
+   * @param len length of data
+   * @param overwrite should the create option allow overwrites?
+   * @throws IOException IO problems
+   */
+  public static void writeDataset(FileSystem fs,
+                                   Path path,
+                                   byte[] src,
+                                   int len,
+                                   int blocksize,
+                                   boolean overwrite) throws IOException {
+    assertTrue(
+      "Not enough data in source array to write " + len + " bytes",
+      src.length >= len);
+    FSDataOutputStream out = fs.create(path,
+                                       overwrite,
+                                       fs.getConf()
+                                         .getInt(IO_FILE_BUFFER_SIZE,
+                                                 4096),
+                                       (short) 1,
+                                       blocksize);
+    out.write(src, 0, len);
+    out.close();
+    assertFileHasLength(fs, path, len);
+  }
+
+  /**
+   * Read the file and convert to a byte dataaset
+   * @param fs filesystem
+   * @param path path to read from
+   * @param len length of data to read
+   * @return the bytes
+   * @throws IOException IO problems
+   */
+  public static byte[] readDataset(FileSystem fs, Path path, int len)
+      throws IOException {
+    FSDataInputStream in = fs.open(path);
+    byte[] dest = new byte[len];
+    try {
+      in.readFully(0, dest);
+    } finally {
+      in.close();
+    }
+    return dest;
+  }
+
+  /**
+   * Assert that tthe array src[0..len] and dest[] are equal
+   * @param src source data
+   * @param dest actual
+   * @param len length of bytes to compare
+   */
+  public static void compareByteArrays(byte[] src,
+                                       byte[] dest,
+                                       int len) {
+    assertEquals("Number of bytes read != number written",
+                        len, dest.length);
+    int errors = 0;
+    int first_error_byte = -1;
+    for (int i = 0; i < len; i++) {
+      if (src[i] != dest[i]) {
+        if (errors == 0) {
+          first_error_byte = i;
+        }
+        errors++;
+      }
+    }
+
+    if (errors > 0) {
+      String message = String.format(" %d errors in file of length %d",
+                                     errors, len);
+      LOG.warn(message);
+      // the range either side of the first error to print
+      // this is a purely arbitrary number, to aid user debugging
+      final int overlap = 10;
+      for (int i = Math.max(0, first_error_byte - overlap);
+           i < Math.min(first_error_byte + overlap, len);
+           i++) {
+        byte actual = dest[i];
+        byte expected = src[i];
+        String letter = toChar(actual);
+        String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+        if (expected != actual) {
+          line = String.format("[%04d] %2x %s -expected %2x %s\n",
+                               i,
+                               actual,
+                               letter,
+                               expected,
+                               toChar(expected));
+        }
+        LOG.warn(line);
+      }
+      fail(message);
+    }
+  }
+
+  /**
+   * Convert a byte to a character for printing. If the
+   * byte value is < 32 -and hence unprintable- the byte is
+   * returned as a two digit hex value
+   * @param b byte
+   * @return the printable character string
+   */
+  public static String toChar(byte b) {
+    if (b >= 0x20) {
+      return Character.toString((char) b);
+    } else {
+      return String.format("%02x", b);
+    }
+  }
+
+  public static String toChar(byte[] buffer) {
+    StringBuilder builder = new StringBuilder(buffer.length);
+    for (byte b : buffer) {
+      builder.append(toChar(b));
+    }
+    return builder.toString();
+  }
+
+  public static byte[] toAsciiByteArray(String s) {
+    char[] chars = s.toCharArray();
+    int len = chars.length;
+    byte[] buffer = new byte[len];
+    for (int i = 0; i < len; i++) {
+      buffer[i] = (byte) (chars[i] & 0xff);
+    }
+    return buffer;
+  }
+
+  public static void cleanupInTeardown(FileSystem fileSystem,
+                                       String cleanupPath) {
+    cleanup("TEARDOWN", fileSystem, cleanupPath);
+  }
+
+  public static void cleanup(String action,
+                             FileSystem fileSystem,
+                             String cleanupPath) {
+    noteAction(action);
+    try {
+      if (fileSystem != null) {
+        fileSystem.delete(new Path(cleanupPath).makeQualified(fileSystem),
+                          true);
+      }
+    } catch (Exception e) {
+      LOG.error("Error deleting in "+ action + " - "  + cleanupPath + ": " + e, e);
+    }
+  }
+
+  public static void noteAction(String action) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("==============  "+ action +" =============");
+    }
+  }
+
+  /**
+   * downgrade a failure to a message and a warning, then an
+   * exception for the Junit test runner to mark as failed
+   * @param message text message
+   * @param failure what failed
+   * @throws AssumptionViolatedException always
+   */
+  public static void downgrade(String message, Throwable failure) {
+    LOG.warn("Downgrading test " + message, failure);
+    AssumptionViolatedException ave =
+      new AssumptionViolatedException(failure, null);
+    throw ave;
+  }
+
+  /**
+   * report an overridden test as unsupported
+   * @param message message to use in the text
+   * @throws AssumptionViolatedException always
+   */
+  public static void unsupported(String message) {
+    throw new AssumptionViolatedException(message);
+  }
+
+  /**
+   * report a test has been skipped for some reason
+   * @param message message to use in the text
+   * @throws AssumptionViolatedException always
+   */
+  public static void skip(String message) {
+    throw new AssumptionViolatedException(message);
+  }
+
+
+  /**
+   * Make an assertion about the length of a file
+   * @param fs filesystem
+   * @param path path of the file
+   * @param expected expected length
+   * @throws IOException on File IO problems
+   */
+  public static void assertFileHasLength(FileSystem fs, Path path,
+                                         int expected) throws IOException {
+    FileStatus status = fs.getFileStatus(path);
+    assertEquals(
+      "Wrong file length of file " + path + " status: " + status,
+      expected,
+      status.getLen());
+  }
+
+  /**
+   * Assert that a path refers to a directory
+   * @param fs filesystem
+   * @param path path of the directory
+   * @throws IOException on File IO problems
+   */
+  public static void assertIsDirectory(FileSystem fs,
+                                       Path path) throws IOException {
+    FileStatus fileStatus = fs.getFileStatus(path);
+    assertIsDirectory(fileStatus);
+  }
+
+  /**
+   * Assert that a path refers to a directory
+   * @param fileStatus stats to check
+   */
+  public static void assertIsDirectory(FileStatus fileStatus) {
+    assertTrue("Should be a dir -but isn't: " + fileStatus,
+                      fileStatus.isDirectory());
+  }
+
+  /**
+   * Write the text to a file, returning the converted byte array
+   * for use in validating the round trip
+   * @param fs filesystem
+   * @param path path of file
+   * @param text text to write
+   * @param overwrite should the operation overwrite any existing file?
+   * @return the read bytes
+   * @throws IOException on IO problems
+   */
+  public static byte[] writeTextFile(FileSystem fs,
+                                   Path path,
+                                   String text,
+                                   boolean overwrite) throws IOException {
+    FSDataOutputStream stream = fs.create(path, overwrite);
+    byte[] bytes = new byte[0];
+    if (text != null) {
+      bytes = toAsciiByteArray(text);
+      stream.write(bytes);
+    }
+    stream.close();
+    return bytes;
+  }
+
+  /**
+   * Touch a file: fails if it is already there
+   * @param fs filesystem
+   * @param path path
+   * @throws IOException IO problems
+   */
+  public static void touch(FileSystem fs,
+                           Path path) throws IOException {
+    fs.delete(path, true);
+    writeTextFile(fs, path, null, false);
+  }
+
+  public static void assertDeleted(FileSystem fs,
+                                   Path file,
+                                   boolean recursive) throws IOException {
+    assertPathExists(fs, "about to be deleted file", file);
+    boolean deleted = fs.delete(file, recursive);
+    String dir = ls(fs, file.getParent());
+    assertTrue("Delete failed on " + file + ": " + dir, deleted);
+    assertPathDoesNotExist(fs, "Deleted file", file);
+  }
+
+  /**
+   * Read in "length" bytes, convert to an ascii string
+   * @param fs filesystem
+   * @param path path to read
+   * @param length #of bytes to read.
+   * @return the bytes read and converted to a string
+   * @throws IOException
+   */
+  public static String readBytesToString(FileSystem fs,
+                                  Path path,
+                                  int length) throws IOException {
+    FSDataInputStream in = fs.open(path);
+    try {
+      byte[] buf = new byte[length];
+      in.readFully(0, buf);
+      return toChar(buf);
+    } finally {
+      in.close();
+    }
+  }
+
+  public static String getDefaultWorkingDirectory() {
+    return "/user/" + System.getProperty("user.name");
+  }
+
+  public static String ls(FileSystem fileSystem, Path path) throws IOException {
+    return SwiftUtils.ls(fileSystem, path);
+  }
+
+  public static String dumpStats(String pathname, FileStatus[] stats) {
+    return pathname + SwiftUtils.fileStatsToString(stats,"\n");
+  }
+
+  /**
+   /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   * @param fileSystem filesystem to resolve path against
+   * @param filename name of the file
+   * @throws IOException IO problems during file operations
+   */
+  public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+                                                                 IOException {
+    assertPathExists(fileSystem, "Expected file", filename);
+    FileStatus status = fileSystem.getFileStatus(filename);
+    String fileInfo = filename + "  " + status;
+    assertFalse("File claims to be a directory " + fileInfo,
+                status.isDirectory());
+/* disabled for Hadoop v1 compatibility
+    assertFalse("File claims to be a symlink " + fileInfo,
+                       status.isSymlink());
+*/
+  }
+
+  /**
+   * Create a dataset for use in the tests; all data is in the range
+   * base to (base+modulo-1) inclusive
+   * @param len length of data
+   * @param base base of the data
+   * @param modulo the modulo
+   * @return the newly generated dataset
+   */
+  public static byte[] dataset(int len, int base, int modulo) {
+    byte[] dataset = new byte[len];
+    for (int i = 0; i < len; i++) {
+      dataset[i] = (byte) (base + (i % modulo));
+    }
+    return dataset;
+  }
+
+  /**
+   * Assert that a path exists -but make no assertions as to the
+   * type of that entry
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws IOException IO problems
+   */
+  public static void assertPathExists(FileSystem fileSystem, String message,
+                               Path path) throws IOException {
+    if (!fileSystem.exists(path)) {
+      //failure, report it
+      fail(message + ": not found " + path + " in " + path.getParent());
+           ls(fileSystem, path.getParent());
+    }
+  }
+
+  /**
+   * Assert that a path does not exist
+   *
+   * @param fileSystem filesystem to examine
+   * @param message message to include in the assertion failure message
+   * @param path path in the filesystem
+   * @throws IOException IO problems
+   */
+  public static void assertPathDoesNotExist(FileSystem fileSystem,
+                                            String message,
+                                            Path path) throws IOException {
+    try {
+      FileStatus status = fileSystem.getFileStatus(path);
+      fail(message + ": unexpectedly found " + path + " as  " + status);
+    } catch (FileNotFoundException expected) {
+      //this is expected
+
+    }
+  }
+
+
+  /**
+   * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+   * @param fs filesystem
+   * @param dir directory to scan
+   * @param subdir full path to look for
+   * @throws IOException IO probles
+   */
+  public static void assertListStatusFinds(FileSystem fs,
+                                           Path dir,
+                                           Path subdir) throws IOException {
+    FileStatus[] stats = fs.listStatus(dir);
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.toString()).append('\n');
+      if (stat.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+                      + " not found in directory " + dir + ":" + builder,
+                      found);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/main/java/org/apache/hadoop/fs/swift/util/SwiftUtils.java

@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.util;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Various utility classes for SwiftFS support
+ */
+public final class SwiftUtils {
+
+  public static final String READ = "read(buffer, offset, length)";
+
+  /**
+   * Join two (non null) paths, inserting a forward slash between them
+   * if needed
+   *
+   * @param path1 first path
+   * @param path2 second path
+   * @return the combined path
+   */
+  public static String joinPaths(String path1, String path2) {
+    StringBuilder result =
+            new StringBuilder(path1.length() + path2.length() + 1);
+    result.append(path1);
+    boolean insertSlash = true;
+    if (path1.endsWith("/")) {
+      insertSlash = false;
+    } else if (path2.startsWith("/")) {
+      insertSlash = false;
+    }
+    if (insertSlash) {
+      result.append("/");
+    }
+    result.append(path2);
+    return result.toString();
+  }
+
+  /**
+   * This test contains the is-directory logic for Swift, so if
+   * changed there is only one place for it.
+   *
+   * @param fileStatus status to examine
+   * @return true if we consider this status to be representative of a
+   *         directory.
+   */
+  public static boolean isDirectory(FileStatus fileStatus) {
+    return fileStatus.isDirectory() || isFilePretendingToBeDirectory(fileStatus);
+  }
+
+  /**
+   * Test for the entry being a file that is treated as if it is a
+   * directory
+   *
+   * @param fileStatus status
+   * @return true if it meets the rules for being a directory
+   */
+  public static boolean isFilePretendingToBeDirectory(FileStatus fileStatus) {
+    return fileStatus.getLen() == 0;
+  }
+
+  /**
+   * Predicate: Is a swift object referring to the root direcory?
+   * @param swiftObject object to probe
+   * @return true iff the object refers to the root
+   */
+  public static boolean isRootDir(SwiftObjectPath swiftObject) {
+    return swiftObject.objectMatches("") || swiftObject.objectMatches("/");
+  }
+
+  /**
+   * Sprintf() to the log iff the log is at debug level. If the log
+   * is not at debug level, the printf operation is skipped, so
+   * no time is spent generating the string.
+   * @param log log to use
+   * @param text text message
+   * @param args args arguments to the print statement
+   */
+  public static void debug(Log log, String text, Object... args) {
+    if (log.isDebugEnabled()) {
+      log.debug(String.format(text, args));
+    }
+  }
+
+  /**
+   * Log an exception (in text and trace) iff the log is at debug
+   * @param log Log to use
+   * @param text text message
+   * @param ex exception
+   */
+  public static void debugEx(Log log, String text, Exception ex) {
+    if (log.isDebugEnabled()) {
+      log.debug(text + ex, ex);
+    }
+  }
+
+  /**
+   * Sprintf() to the log iff the log is at trace level. If the log
+   * is not at trace level, the printf operation is skipped, so
+   * no time is spent generating the string.
+   * @param log log to use
+   * @param text text message
+   * @param args args arguments to the print statement
+   */
+  public static void trace(Log log, String text, Object... args) {
+    if (log.isTraceEnabled()) {
+      log.trace(String.format(text, args));
+    }
+  }
+
+  /**
+   * Given a partition number, calculate the partition value.
+   * This is used in the SwiftNativeOutputStream, and is placed
+   * here for tests to be able to calculate the filename of
+   * a partition.
+   * @param partNumber part number
+   * @return a string to use as the filename
+   */
+  public static String partitionFilenameFromNumber(int partNumber) {
+    return String.format("%06d", partNumber);
+  }
+
+  /**
+   * List a a path to string
+   * @param fileSystem filesystem
+   * @param path directory
+   * @return a listing of the filestatuses of elements in the directory, one
+   * to a line, precedeed by the full path of the directory
+   * @throws IOException connectivity problems
+   */
+  public static String ls(FileSystem fileSystem, Path path) throws
+                                                            IOException {
+    if (path == null) {
+      //surfaces when someone calls getParent() on something at the top of the path
+      return "/";
+    }
+    FileStatus[] stats;
+    String pathtext = "ls " + path;
+    try {
+      stats = fileSystem.listStatus(path);
+    } catch (FileNotFoundException e) {
+      return pathtext + " -file not found";
+    } catch (IOException e) {
+      return pathtext + " -failed: " + e;
+    }
+    return pathtext + fileStatsToString(stats, "\n");
+  }
+
+  /**
+   * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+   * @param stats array of stats
+   * @param separator separator after every entry
+   * @return a stringified set
+   */
+  public static String fileStatsToString(FileStatus[] stats, String separator) {
+    StringBuilder buf = new StringBuilder(stats.length * 128);
+    for (int i = 0; i < stats.length; i++) {
+      buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+    }
+    return buf.toString();
+  }
+
+  /**
+   * Verify that the basic args to a read operation are valid;
+   * throws an exception if not -with meaningful text includeing
+   * @param buffer destination buffer
+   * @param off offset
+   * @param len number of bytes to read
+   * @throws NullPointerException null buffer
+   * @throws IndexOutOfBoundsException on any invalid range.
+   */
+  public static void validateReadArgs(byte[] buffer, int off, int len) {
+    if (buffer == null) {
+      throw new NullPointerException("Null byte array in"+ READ);
+    }
+    if (off < 0 ) {
+      throw new IndexOutOfBoundsException("Negative buffer offset "
+                                          + off
+                                          + " in " + READ);
+    }
+    if (len < 0 ) {
+      throw new IndexOutOfBoundsException("Negative read length "
+                                          + len
+                                          + " in " + READ);
+    }
+    if (off > buffer.length) {
+      throw new IndexOutOfBoundsException("Buffer offset of "
+                                          + off
+                                          + "beyond buffer size of "
+                                          + buffer.length
+                                          + " in " + READ);
+    }
+  } 
+}

hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm

@@ -0,0 +1,686 @@
+~~ Licensed under the Apache License, Version 2.0 (the "License");
+~~ you may not use this file except in compliance with the License.
+~~ You may obtain a copy of the License at
+~~
+~~   http://www.apache.org/licenses/LICENSE-2.0
+~~
+~~ Unless required by applicable law or agreed to in writing, software
+~~ distributed under the License is distributed on an "AS IS" BASIS,
+~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~~ See the License for the specific language governing permissions and
+~~ limitations under the License. See accompanying LICENSE file.
+
+  ---
+  Hadoop OpenStack Support: Swift Object Store
+  ---
+  ---
+  ${maven.build.timestamp}
+
+%{toc|section=1|fromDepth=0}
+
+Hadoop OpenStack Support: Swift Object Store
+
+* {Introduction}
+
+  {{{http://www.openstack.org/}OpenStack}} is an open source cloud infrastructure
+   which can be accessed
+  from multiple public IaaS providers, and deployed privately. It offers
+  infrastructure services such as VM hosting (Nova), authentication (Keystone)
+  and storage of binary objects (Swift).
+
+  This module enables Apache Hadoop applications -including MapReduce jobs,
+  read and write data to and from instances of the
+   {{{http://www.openstack.org/software/openstack-storage/}OpenStack Swift object store}}.
+
+* Features
+
+    * Read and write of data stored in a Swift object store
+
+    * Support of a pseudo-hierachical file system (directories, subdirectories and
+      files)
+
+    * Standard filesystem operations: <<<create>>>, <<<delete>>>, <<<mkdir>>>,
+     <<<ls>>>, <<<mv>>>, <<<stat>>>.
+
+    * Can act as a source of data in a MapReduce job, or a sink.
+
+    * Support for multiple OpenStack services, and multiple containers from a
+      single service.
+
+    * Supports in-cluster and remote access to Swift data.
+
+    * Supports OpenStack Keystone authentication with password or token.
+
+    * Released under the Apache Software License
+
+    * Tested against the Hadoop 3.x and 1.x branches, against multiple public
+      OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
+
+    * Tested against private OpenStack clusters, including scalability tests of
+      large file uploads.
+
+* Using the Hadoop Swift Filesystem Client
+
+** Concepts: services and containers
+
+  OpenStack swift is an <Object Store>; also known as a <blobstore>. It stores
+  arbitrary binary objects by name in a <container>.
+
+  The Hadoop Swift filesystem library adds another concept, the <service>, which
+  defines which Swift blobstore hosts a container -and how to connect to it.
+
+** Containers and Objects
+
+    * Containers are created by users with accounts on the Swift filestore, and hold
+    <objects>.
+
+    * Objects can be zero bytes long, or they can contain data.
+
+    * Objects in the container can be up to 5GB; there is a special support for
+      larger files than this, which merges multiple objects in to one.
+
+    * Each object is referenced by it's <name>; there is no notion of directories.
+
+    * You can use any characters in an object name that can be 'URL-encoded'; the
+      maximum length of a name is 1034 characters -after URL encoding.
+
+    * Names can have <<</>>> characters in them, which are used to create the illusion of
+      a directory structure. For example <<<dir/dir2/name>>>. Even though this looks
+      like a directory, <it is still just a name>. There is no requirement to have
+      any entries in the container called <<<dir>>> or <<<dir/dir2>>>
+
+    * That said. if the container has zero-byte objects that look like directory
+      names above other objects, they can pretend to be directories. Continuing the
+      example, a 0-byte object called <<<dir>>> would tell clients that it is a
+      directory while <<<dir/dir2>>> or <<<dir/dir2/name>>> were present. This creates an
+      illusion of containers holding a filesystem.
+
+  Client applications talk to Swift over HTTP or HTTPS, reading, writing and
+  deleting objects using standard HTTP operations (GET, PUT and DELETE,
+  respectively). There is also a COPY operation, that creates a new object in the
+  container, with a new name, containing the old data. There is no rename
+  operation itself, objects need to be copied -then the original entry deleted.
+
+** Eventual Consistency
+
+  The Swift Filesystem is *eventually consistent*: an operation on an object may
+  not be immediately visible to that client, or other clients. This is a
+  consequence of the goal of the filesystem: to span a set of machines, across
+  multiple datacenters, in such a way that the data can still be available when
+  many of them fail. (In contrast, the Hadoop HDFS filesystem is *immediately
+  consistent*, but it does not span datacenters.)
+
+  Eventual consistency can cause surprises for client applications that expect
+  immediate consistency: after an object is deleted or overwritten, the object
+  may still be visible -or the old data still retrievable. The Swift Filesystem
+  client for Apache Hadoop attempts to handle this, in conjunction with the
+  MapReduce engine, but there may be still be occasions when eventual consistency
+  causes surprises.
+
+** Non-atomic "directory" operations.
+
+  Hadoop expects some
+  operations to be atomic, especially <<<rename()>>>, which is something
+  the MapReduce layer relies on to commit the output of a job, renaming data
+  from a temp directory to the final path. Because a rename
+  is implemented as a copy of every blob under the directory's path, followed
+  by a delete of the originals, the intermediate state of the operation
+  will be visible to other clients. If two Reducer tasks to rename their temp
+  directory to the final path, both operations may succeed, with the result that
+  output directory contains mixed data. This can happen if MapReduce jobs
+  are being run with <speculation> enabled and Swift used as the direct output
+  of the MR job (it can also happen against Amazon S3).
+
+  Other consequences of the non-atomic operations are:
+
+  1. If a program is looking for the presence of the directory before acting
+  on the data -it may start prematurely. This can be avoided by using
+  other mechanisms to co-ordinate the programs, such as the presence of a file
+  that is written <after> any bulk directory operations.
+
+  2. A <<<rename()>>> or <<<delete()>>> operation may include files added under
+  the source directory tree during the operation, may unintentionally delete
+  it, or delete the 0-byte swift entries that mimic directories and act
+  as parents for the files. Try to avoid doing this.
+
+  The best ways to avoid all these problems is not using Swift as
+  the filesystem between MapReduce jobs or other Hadoop workflows. It
+  can act as a source of data, and a final destination, but it doesn't meet
+  all of Hadoop's expectations of what a filesystem is -it's a <blobstore>.
+
+* Working with Swift Object Stores in Hadoop
+
+  Once installed, the Swift FileSystem client can be used by any Hadoop application
+  to read from or write to data stored in a Swift container.
+
+  Data stored in Swift can be used as the direct input to a MapReduce job
+  -simply use the <<<swift:>>> URL (see below) to declare the source of the data.
+
+  This Swift Filesystem client is designed to work with multiple
+  Swift object stores, both public and private. This allows the client to work
+  with different clusters, reading and writing data to and from either of them.
+
+  It can also work with the same object stores using multiple login details.
+
+  These features are achieved by one basic concept: using a service name in
+  the URI referring to a swift filesystem, and looking up all the connection and
+  login details for that specific service. Different service names can be defined
+  in the Hadoop XML configuration file, so defining different clusters, or
+  providing different login details for the same object store(s).
+
+
+** Swift Filesystem URIs
+
+  Hadoop uses URIs to refer to files within a filesystem. Some common examples
+  are:
+
++--
+    local://etc/hosts
+    hdfs://cluster1/users/example/data/set1
+    hdfs://cluster2.example.org:8020/users/example/data/set1
++--
+
+  The Swift Filesystem Client adds a new URL type <<<swift>>>. In a Swift Filesystem
+  URL, the hostname part of a URL identifies the container and the service to
+  work with; the path the name of the object. Here are some examples
+
++--
+    swift://container.rackspace/my-object.csv
+    swift://data.hpcloud/data/set1
+    swift://dmitry.privatecloud/out/results
++--
+
+  In the last two examples, the paths look like directories: it is not, they are
+  simply the objects named <<<data/set1>>> and <<<out/results>>> respectively.
+
+** Installing
+
+  The <<<hadoop-openstack>>> JAR must be on the classpath of the Hadoop program trying to
+  talk to the Swift service. If installed in the classpath of the Hadoop
+  MapReduce service, then all programs started by the MR engine will pick up the
+  JAR automatically. This is the easiest way to give all Hadoop jobs access to
+  Swift.
+
+  Alternatively, the JAR can be included as one of the JAR files that an
+  application uses. This lets the Hadoop jobs work with a Swift object store even
+  if the Hadoop cluster is not pre-configured for this.
+
+  The library also depends upon the Apache HttpComponents library, which
+  must also be on the classpath.
+
+** Configuring
+
+  To talk to a swift service, the user must must provide:
+
+    [[1]] The URL defining the container and the service.
+
+    [[1]] In the cluster/job configuration, the login details of that service.
+
+  Multiple service definitions can co-exist in the same configuration file: just
+  use different names for them.
+
+*** Example: Rackspace US, in-cluster access using API key
+
+  This service definition is for use in a Hadoop cluster deployed within Rackspace's
+  US infrastructure.
+
++--
+    <property>
+      <name>fs.swift.service.rackspace.auth.url</name>
+      <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
+      <description>Rackspace US (multiregion)</description>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspace.username</name>
+      <value>user4</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspace.region</name>
+      <value>DFW</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspace.apikey</name>
+      <value>fe806aa86dfffe2f6ed8</value>
+    </property>
++--
+
+  Here the API key visible in the account settings API keys page is used to log
+  in. No property for public/private access -the default is to use the private
+  endpoint for Swift operations.
+
+  This configuration also selects one of the regions, DFW, for its data.
+
+  A reference to this service would use the <<<rackspace>>> service name:
+
+---
+    swift://hadoop-container.rackspace/
+---
+
+*** Example: Rackspace UK: remote access with password authentication
+
+  This connects to Rackspace's UK ("LON") datacenter.
+
++--
+    <property>
+      <name>fs.swift.service.rackspaceuk.auth.url</name>
+      <value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
+      <description>Rackspace UK</description>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspaceuk.username</name>
+      <value>user4</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspaceuk.password</name>
+      <value>insert-password-here/value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.rackspace.public</name>
+      <value>true</value>
+    </property>
++--
+
+  This is a public access point connection, using a password over an API key.
+
+  A reference to this service would use the <<<rackspaceuk>>> service name:
+
++--
+    swift://hadoop-container.rackspaceuk/
++--
+
+  Because the public endpoint is used, if this service definition is used within
+  the London datacenter, all accesses will be billed at the public
+  upload/download rates, <irrespective of where the Hadoop cluster is>.
+
+*** Example: HP cloud service definition
+
+  Here is an example that connects to the HP Cloud object store.
+
++--
+    <property>
+      <name>fs.swift.service.hpcloud.auth.url</name>
+      <value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
+      </value>
+      <description>HP Cloud</description>
+    </property>
+
+    <property>
+      <name>fs.swift.service.hpcloud.tenant</name>
+      <value>FE806AA86</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.hpcloud.username</name>
+      <value>FE806AA86DFFFE2F6ED8</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.hpcloud.password</name>
+      <value>secret-password-goes-here</value>
+    </property>
+
+    <property>
+      <name>fs.swift.service.hpcloud.public</name>
+      <value>true</value>
+    </property>
++--
+
+  A reference to this service would use the <<<hpcloud>>> service name:
+
++--
+    swift://hadoop-container.hpcloud/
++--
+
+** General Swift Filesystem configuration options
+
+  Some configuration options apply to the Swift client, independent of
+  the specific Swift filesystem chosen.
+
+*** Blocksize fs.swift.blocksize
+
+  Swift does not break up files into blocks, except in the special case of files
+  over 5GB in length. Accordingly, there isn't a notion of a "block size"
+  to define where the data is kept.
+
+  Hadoop's MapReduce layer depends on files declaring their block size,
+  so that it knows how to partition work. Too small a blocksize means that
+  many mappers work on small pieces of data; too large a block size means
+  that only a few mappers get started.
+
+  The block size value reported by Swift, therefore, controls the basic workload
+  partioning of the MapReduce engine -and can be an important parameter to
+  tune for performance of the cluster.
+
+  The property has a unit of kilobytes; the default value is <<<32*1024>>>: 32 MB
+
++--
+    <property>
+      <name>fs.swift.blocksize</name>
+      <value>32768</value>
+    </property>
++--
+
+  This blocksize has no influence on how files are stored in Swift; it only controls
+  what the reported size of blocks are - a value used in Hadoop MapReduce to
+  divide work.
+
+  Note that the MapReduce engine's split logic can be tuned independently by setting
+  the <<<mapred.min.split.size>>> and <<<mapred.max.split.size>>> properties,
+  which can be done in specific job configurations.
+
++--
+    <property>
+      <name>mapred.min.split.size</name>
+      <value>524288</value>
+    </property>
+
+    <property>
+      <name>mapred.max.split.size</name>
+      <value>1048576</value>
+    </property>
++--
+
+  In an Apache Pig script, these properties would be set as:
+
+---
+    mapred.min.split.size 524288
+    mapred.max.split.size 1048576
+---
+
+*** Partition size fs.swift.partsize
+
+  The Swift filesystem client breaks very large files into partitioned files,
+  uploading each as it progresses, and writing any remaning data and an XML
+  manifest when a partitioned file is closed.
+
+  The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that
+  Swift can support.
+
+  It is possible to set a smaller partition size, in the <<<fs.swift.partsize>>>
+  option. This takes a value in KB.
+
++--
+    <property>
+      <name>fs.swift.partsize</name>
+      <value>1024</value>
+      <description>upload every MB</description>
+    </property>
++--
+
+  When should this value be changed from its default?
+
+  While there is no need to ever change it for basic operation of
+  the Swift filesystem client, it can be tuned
+
+  * If a Swift filesystem is location aware, then breaking a file up into
+  smaller partitions scatters the data round the cluster. For best performance,
+  the property <<<fs.swift.blocksize>>> should be set to a smaller value than the
+  partition size of files.
+
+  * When writing to an unpartitioned file, the entire write is done in the
+  <<<close()>>> operation. When a file is partitioned, the outstanding data to
+  be written whenever the outstanding amount of data is greater than the
+  partition size. This means that data will be written more incrementally
+
+*** Request size fs.swift.requestsize
+
+  The Swift filesystem client reads files in HTTP GET operations, asking for
+  a block of data at a time.
+
+  The default value is 64KB. A larger value may be more efficient over faster
+  networks, as it reduces the overhead of setting up the HTTP operation.
+
+  However, if the file is read with many random accesses, requests for
+  data will be made from different parts of the file -discarding some of the
+  previously requested data. The benefits of larger request sizes may be wasted.
+
+  The property <<<fs.swift.requestsize>>> sets the request size in KB.
+
++--
+    <property>
+      <name>fs.swift.requestsize</name>
+      <value>128</value>
+    </property>
++--
+
+*** Connection timeout fs.swift.connect.timeout
+
+  This sets the timeout in milliseconds to connect to a Swift service.
+
++--
+    <property>
+      <name>fs.swift.connect.timeout</name>
+      <value>15000</value>
+    </property>
++--
+
+  A shorter timeout means that connection failures are raised faster -but
+  may trigger more false alarms. A longer timeout is more resilient to network
+  problems -and may be needed when talking to remote filesystems.
+
+*** Connection timeout fs.swift.socket.timeout
+
+  This sets the timeout in milliseconds to wait for data from a connected socket.
+
++--
+    <property>
+      <name>fs.swift.socket.timeout</name>
+      <value>60000</value>
+    </property>
++--
+
+  A shorter timeout means that connection failures are raised faster -but
+  may trigger more false alarms. A longer timeout is more resilient to network
+  problems -and may be needed when talking to remote filesystems.
+
+*** Connection Retry Count fs.swift.connect.retry.count
+
+  This sets the number of times to try to connect to a service whenever
+  an HTTP request is made.
+
++--
+    <property>
+      <name>fs.swift.connect.retry.count</name>
+      <value>3</value>
+    </property>
++--
+
+  The more retries, the more resilient it is to transient outages -and the
+  less rapid it is at detecting and reporting server connectivity problems.
+
+*** Connection Throttle Delay fs.swift.connect.throttle.delay
+
+  This property adds a delay between bulk file copy and delete operations,
+  to prevent requests being throttled or blocked by the remote service
+
++--
+    <property>
+      <name>fs.swift.connect.throttle.delay</name>
+      <value>0</value>
+    </property>
++--
+
+  It is measured in milliseconds; "0" means do not add any delay.
+
+  Throttling is enabled on the public endpoints of some Swift services.
+  If <<<rename()>>> or <<<delete()>>> operations fail with
+   <<<SwiftThrottledRequestException>>>
+  exceptions, try setting this property.
+
+*** HTTP Proxy
+
+  If the client can only access the Swift filesystem via a web proxy
+  server, the client configuration must specify the proxy via
+  the <<<fs.swift.connect.proxy.host>>> and <<<fs.swift.connect.proxy.port>>>
+  properties.
+
++--
+    <property>
+      <name>fs.swift.proxy.host</name>
+      <value>web-proxy</value>
+    </property>
+
+    <property>
+      <name>fs.swift.proxy.port</name>
+      <value>8088</value>
+    </property>
++--
+
+  If the host is declared, the proxy port must be set to a valid integer value.
+
+
+** Troubleshooting
+
+*** ClassNotFoundException
+
+  The <<<hadoop-openstack>>> JAR -or any dependencies- may not be on your classpath.
+
+  If it is a remote MapReduce job that is failing, make sure that the JAR is
+  installed on the servers in the cluster -or that the job submission process
+  uploads the JAR file to the distributed cache.
+
+*** Failure to Authenticate
+
+  A <<<SwiftAuthenticationFailedException>>> is thrown when the client
+  cannot authenticate with the OpenStack keystone server. This could be
+  because the URL in the service definition is wrong, or because
+  the supplied credentials are invalid.
+
+    [[1]] Check the authentication URL through <<<curl>>> or your browser
+
+    [[1]] Use a Swift client such as CyberDuck to validate your credentials
+
+    [[1]] If you have included a tenant ID, try leaving it out. Similarly,
+    try adding it if you had not included it.
+
+    [[1]] Try switching from API key authentication to password-based authentication,
+    by setting the password.
+
+    [[1]] Change your credentials. As with Amazon AWS clients, some credentials
+    don't seem to like going over the network.
+
+*** Timeout connecting to the Swift Service
+
+  This happens if the client application is running outside an OpenStack cluster,
+  where it does not have access to the private hostname/IP address for filesystem
+  operations. Set the <<<public>>> flag to true -but remember to set it to false
+  for use in-cluster.
+
+** Warnings
+
+    [[1]] Do not share your login details with anyone, which means do not log the
+    details, or check the XML configuration files into any revision control system
+    to which you do not have exclusive access.
+
+    [[1]] Similarly, do not use your real account details in any documentation *or any
+     bug reports submitted online*
+
+    [[1]] Prefer the apikey authentication over passwords as it is easier
+    to revoke a key -and some service providers allow you to set
+    an automatic expiry date on a key when issued.
+
+    [[1]] Do not use the public service endpoint from within a public OpenStack
+    cluster, as it will run up large bills.
+
+    [[1]] Remember: it's not a real filesystem or hierarchical directory structure.
+    Some operations (directory rename and delete) take time and are not atomic or
+    isolated from other operations taking place.
+
+    [[1]] Append is not supported.
+
+    [[1]] Unix-style permissions are not supported. All accounts with write access to
+    a repository have unlimited access; the same goes for those with read access.
+
+    [[1]] In the public clouds, do not make the containers public unless you are happy
+    with anyone reading your data, and are prepared to pay the costs of their
+    downloads.
+
+** Limits
+
+    * Maximum length of an object path: 1024 characters
+
+    * Maximum size of a binary object: no absolute limit. Files > 5GB are
+      partitioned into separate files in the native filesystem, and merged during
+      retrieval. <Warning:> the partitioned/large file support is the
+      most complex part of the Hadoop/Swift FS integration, and, along with
+      authentication, the most troublesome to support.
+
+** Testing the hadoop-openstack module
+
+  The <<<hadoop-openstack>>> can be remotely tested against any public
+  or private cloud infrastructure which supports the OpenStack Keystone
+  authentication mechanism. It can also be tested against private
+  OpenStack clusters. OpenStack Development teams are strongly encouraged to test
+  the Hadoop swift filesystem client against any version of Swift that they
+  are developing or deploying, to stress their cluster and to identify
+  bugs early.
+
+  The module comes with a large suite of JUnit tests -tests that are
+  only executed if the source tree includes credentials to test against a
+  specific cluster.
+
+  After checking out the Hadoop source tree, create the file:
+
++--
+  hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
++--
+
+  Into this file, insert the credentials needed to bond to the test filesystem,
+  as decribed above.
+
+  Next set the property <<<test.fs.swift.name>>> to the URL of a
+  swift container to test against. The tests expect exclusive access
+  to this container -do not keep any other data on it, or expect it
+  to be preserved.
+
++--
+    <property>
+      <name>test.fs.swift.name</name>
+      <value>swift://test.myswift/</value>
+    </property>
++--
+
+  In the base hadoop directory, run:
+
++--
+   mvn clean install -DskipTests
++--
+
+  This builds a set of Hadoop JARs consistent with the <<<hadoop-openstack>>>
+  module that is about to be tested.
+
+  In the <<<hadoop-tools/hadoop-openstack>>> directory run
+
++--
+   mvn test -Dtest=TestSwiftRestClient
++--
+
+  This runs some simple tests which include authenticating
+  against the remote swift service. If these tests fail, so will all
+  the rest. If it does fail: check your authentication.
+
+  Once this test succeeds, you can run the full test suite
+
++--
+    mvn test
++--
+
+  Be advised that these tests can take an hour or more, especially against a
+  remote Swift service -or one that throttles bulk operations.
+
+  Once the <<<auth-keys.xml>>> file is in place, the <<<mvn test>>> runs from
+  the Hadoop source base directory will automatically run these OpenStack tests
+  While this ensures that no regressions have occurred, it can also add significant
+  time to test runs, and may run up bills, depending on who is providing\
+  the Swift storage service. We recommend having a separate source tree
+  set up purely for the Swift tests, and running it manually or by the CI tooling
+  at a lower frequency than normal test runs.
+
+  Finally: Apache Hadoop is an open source project. Contributions of code
+  -including more tests- are very welcome.

hadoop-tools/hadoop-openstack/src/site/site.xml

@@ -0,0 +1,46 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project name="Apache Hadoop ${project.version}">
+
+  <skin>
+    <groupId>org.apache.maven.skins</groupId>
+    <artifactId>maven-stylus-skin</artifactId>
+    <version>1.2</version>
+  </skin>
+
+  <body>
+    <links>
+      <item name="Apache Hadoop" href="http://hadoop.apache.org/"/>
+    </links>
+  </body>
+
+</project>

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/AcceptAllFilter.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+
+/**
+ * A path filter that accepts everything
+ */
+public class AcceptAllFilter implements PathFilter {
+  @Override
+  public boolean accept(Path file) {
+    return true;
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftFileSystemBaseTest.java

@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftOperationFailedException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystemStore;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanupInTeardown;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.getServiceURI;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.noteAction;
+
+/**
+ * This is the base class for most of the Swift tests
+ */
+public class SwiftFileSystemBaseTest extends Assert implements
+                                                    SwiftTestConstants {
+
+  protected static final Log LOG =
+          LogFactory.getLog(SwiftFileSystemBaseTest.class);
+  protected SwiftNativeFileSystem fs;
+  protected static SwiftNativeFileSystem lastFs;
+  protected byte[] data = SwiftTestUtils.dataset(getBlockSize() * 2, 0, 255);
+  private Configuration conf;
+
+  @Before
+  public void setUp() throws Exception {
+    noteAction("setup");
+    final URI uri = getFilesystemURI();
+    conf = createConfiguration();
+
+    fs = createSwiftFS();
+    try {
+      fs.initialize(uri, conf);
+    } catch (IOException e) {
+      //FS init failed, set it to null so that teardown doesn't
+      //attempt to use it
+      fs = null;
+      throw e;
+    }
+    //remember the last FS
+    lastFs = fs;
+    noteAction("setup complete");
+  }
+
+  /**
+   * Configuration generator. May be overridden to inject
+   * some custom options
+   * @return a configuration with which to create FS instances
+   */
+  protected Configuration createConfiguration() {
+    return new Configuration();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    cleanupInTeardown(fs, "/test");
+  }
+
+  @AfterClass
+  public static void classTearDown() throws Exception {
+    if (lastFs  != null) {
+      List<DurationStats> statistics = lastFs.getOperationStatistics();
+      for (DurationStats stat : statistics) {
+        LOG.info(stat.toString());
+      }
+    }
+  }
+
+  /**
+   * Get the configuration used to set up the FS
+   * @return the configuration
+   */
+  public Configuration getConf() {
+    return conf;
+  }
+
+  /**
+   * Describe the test, combining some logging with details
+   * for people reading the code
+   *
+   * @param description test description
+   */
+  protected void describe(String description) {
+    noteAction(description);
+  }
+
+  protected URI getFilesystemURI() throws URISyntaxException, IOException {
+    return getServiceURI(createConfiguration());
+  }
+
+  protected SwiftNativeFileSystem createSwiftFS() throws IOException {
+    SwiftNativeFileSystem swiftNativeFileSystem =
+      new SwiftNativeFileSystem();
+    return swiftNativeFileSystem;
+  }
+
+  protected int getBlockSize() {
+    return 1024;
+  }
+
+  /**
+   * Is rename supported?
+   * @return true
+   */
+  protected boolean renameSupported() {
+    return true;
+  }
+
+  /**
+   * assume in a test that rename is supported;
+   * skip it if not
+   */
+  protected void assumeRenameSupported() {
+    Assume.assumeTrue(renameSupported());
+  }
+
+  /**
+   * Take an unqualified path, and qualify it w.r.t the
+   * current filesystem
+   * @param pathString source path
+   * @return a qualified path instance
+   */
+  protected Path path(String pathString) {
+    return new Path(pathString).makeQualified(fs);
+  }
+
+  /**
+   * Get the filesystem
+   * @return the current FS
+   */
+  public SwiftNativeFileSystem getFs() {
+    return fs;
+  }
+
+  /**
+   * Create a file using the standard {@link #data} bytes.
+   *
+   * @param path path to write
+   * @throws IOException on any problem
+   */
+  protected void createFile(Path path) throws IOException {
+    createFile(path, data);
+  }
+
+  /**
+   * Create a file with the given data.
+   *
+   * @param path       path to write
+   * @param sourceData source dataset
+   * @throws IOException on any problem
+   */
+  protected void createFile(Path path, byte[] sourceData) throws IOException {
+    FSDataOutputStream out = fs.create(path);
+    out.write(sourceData, 0, sourceData.length);
+    out.close();
+  }
+
+  /**
+   * Create and then close a file
+   * @param path path to create
+   * @throws IOException on a failure
+   */
+  protected void createEmptyFile(Path path) throws IOException {
+    FSDataOutputStream out = fs.create(path);
+    out.close();
+  }
+
+  /**
+   * Get the inner store -useful for lower level operations
+   *
+   * @return the store
+   */
+  protected SwiftNativeFileSystemStore getStore() {
+    return fs.getStore();
+  }
+
+  /**
+   * Rename a path
+   * @param src source
+   * @param dst dest
+   * @param renameMustSucceed flag to say "this rename must exist"
+   * @param srcExists add assert that the source exists afterwards
+   * @param dstExists add assert the dest exists afterwards
+   * @throws IOException IO trouble
+   */
+  protected void rename(Path src, Path dst, boolean renameMustSucceed,
+                        boolean srcExists, boolean dstExists) throws IOException {
+    if (renameMustSucceed) {
+      renameToSuccess(src, dst, srcExists, dstExists);
+    } else {
+      renameToFailure(src, dst);
+    }
+  }
+
+  /**
+   * Get a string describing the outcome of a rename, by listing the dest
+   * path and its parent along with some covering text
+   * @param src source patj
+   * @param dst dest path
+   * @return a string for logs and exceptions
+   * @throws IOException IO problems
+   */
+  private String getRenameOutcome(Path src, Path dst) throws IOException {
+    String lsDst = ls(dst);
+    Path parent = dst.getParent();
+    String lsParent = parent != null ? ls(parent) : "";
+    return "  result of " + src + " => " + dst
+            + " - " + lsDst
+            + " \n" + lsParent;
+  }
+
+  /**
+   * Rename, expecting an exception to be thrown
+   *
+   * @param src source
+   * @param dst dest
+   * @throws IOException a failure other than an
+   *                     expected SwiftRenameException or FileNotFoundException
+   */
+  protected void renameToFailure(Path src, Path dst) throws IOException {
+    try {
+      getStore().rename(src, dst);
+      fail("Expected failure renaming " + src + " to " + dst
+              + "- but got success");
+    } catch (SwiftOperationFailedException e) {
+      LOG.debug("Rename failed (expected):" + e);
+    } catch (FileNotFoundException e) {
+      LOG.debug("Rename failed (expected):" + e);
+    }
+  }
+
+  /**
+   * Rename to success
+   *
+   * @param src source
+   * @param dst dest
+   * @param srcExists add assert that the source exists afterwards
+   * @param dstExists add assert the dest exists afterwards
+   * @throws SwiftOperationFailedException operation failure
+   * @throws IOException IO problems
+   */
+  protected void renameToSuccess(Path src, Path dst,
+                                 boolean srcExists, boolean dstExists)
+      throws SwiftOperationFailedException, IOException {
+    getStore().rename(src, dst);
+    String outcome = getRenameOutcome(src, dst);
+    assertEquals("Source " + src + "exists: " + outcome,
+            srcExists, fs.exists(src));
+    assertEquals("Destination " + dstExists + " exists" + outcome,
+            dstExists, fs.exists(dst));
+  }
+
+  /**
+   * List a path in the test FS
+   * @param path path to list
+   * @return the contents of the path/dir
+   * @throws IOException IO problems
+   */
+  protected String ls(Path path) throws IOException {
+    return SwiftTestUtils.ls(fs, path);
+  }
+
+  /**
+   * assert that a path exists
+   * @param message message to use in an assertion
+   * @param path path to probe
+   * @throws IOException IO problems
+   */
+  public void assertExists(String message, Path path) throws IOException {
+    assertPathExists(fs, message, path);
+  }
+
+  /**
+   * assert that a path does not
+   * @param message message to use in an assertion
+   * @param path path to probe
+   * @throws IOException IO problems
+   */
+  public void assertPathDoesNotExist(String message, Path path) throws
+          IOException {
+    SwiftTestUtils.assertPathDoesNotExist(fs, message, path);
+  }
+
+  /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   *
+   * @param filename name of the file
+   * @throws IOException IO problems during file operations
+   */
+  protected void assertIsFile(Path filename) throws IOException {
+    SwiftTestUtils.assertIsFile(fs, filename);
+  }
+
+  /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   *
+   * @throws IOException IO problems during file operations
+   */
+  protected void mkdirs(Path path) throws IOException {
+    assertTrue("Failed to mkdir" + path, fs.mkdirs(path));
+  }
+
+  /**
+   * Assert that a delete succeeded
+   * @param path path to delete
+   * @param recursive recursive flag
+   * @throws IOException IO problems
+   */
+  protected void assertDeleted(Path path, boolean recursive) throws IOException {
+    SwiftTestUtils.assertDeleted(fs, path, recursive);
+  }
+
+  /**
+   * Assert that a value is not equal to the expected value
+   * @param message message if the two values are equal
+   * @param expected expected value
+   * @param actual actual value
+   */
+  protected void assertNotEqual(String message, int expected, int actual) {
+    assertTrue(message,
+               actual != expected);
+  }
+
+  /**
+   * Get the number of partitions written from the Swift Native FS APIs
+   * @param out output stream
+   * @return the number of partitioned files written by the stream
+   */
+  protected int getPartitionsWritten(FSDataOutputStream out) {
+    return SwiftNativeFileSystem.getPartitionsWritten(out);
+  }
+
+  /**
+   * Assert that the no. of partitions written matches expectations
+   * @param action operation (for use in the assertions)
+   * @param out output stream
+   * @param expected expected no. of partitions
+   */
+  protected void assertPartitionsWritten(String action, FSDataOutputStream out,
+                                         long expected) {
+    OutputStream nativeStream = out.getWrappedStream();
+    int written = getPartitionsWritten(out);
+    if(written !=expected) {
+    Assert.fail(action + ": " +
+                TestSwiftFileSystemPartitionedUploads.WRONG_PARTITION_COUNT 
+                + " + expected: " + expected + " actual: " + written
+                + " -- " + nativeStream);
+    }
+  }
+
+  /**
+   * Assert that the result value == -1; which implies
+   * that a read was successful
+   * @param text text to include in a message (usually the operation)
+   * @param result read result to validate
+   */
+  protected void assertMinusOne(String text, int result) {
+    assertEquals(text + " wrong read result " + result, -1, result);
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/SwiftTestConstants.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+/**
+ * Hard coded constants for the test timeouts
+ */
+public interface SwiftTestConstants {
+  /**
+   * Timeout for swift tests: {@value}
+   */
+  int SWIFT_TEST_TIMEOUT = 5 * 60 * 1000;
+
+  /**
+   * Timeout for tests performing bulk operations: {@value}
+   */
+  int SWIFT_BULK_IO_TEST_TIMEOUT = 12 * 60 * 1000;
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestFSMainOperationsSwift.java

@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSMainOperationsBaseTest;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import static org.apache.hadoop.fs.swift.SwiftTestConstants.SWIFT_TEST_TIMEOUT;
+import java.io.IOException;
+import java.net.URI;
+
+public class TestFSMainOperationsSwift extends FSMainOperationsBaseTest {
+
+  @Override
+  @Before
+  public void setUp() throws Exception {
+    Configuration conf = new Configuration();
+    //small blocksize for faster remote tests
+    conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, 2);
+    URI serviceURI = SwiftTestUtils.getServiceURI(conf);
+    fSys = FileSystem.get(serviceURI, conf);
+    super.setUp();
+  }
+
+  private Path wd = null;
+
+  @Override
+  protected FileSystem createFileSystem() throws Exception {
+    return fSys;
+  }
+
+  @Override
+  protected Path getDefaultWorkingDirectory() throws IOException {
+    if (wd == null) {
+      wd = fSys.getWorkingDirectory();
+    }
+    return wd;
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWDAbsolute() throws IOException {
+    Path absoluteDir = getTestRootPath(fSys, "test/existingDir");
+    fSys.mkdirs(absoluteDir);
+    fSys.setWorkingDirectory(absoluteDir);
+    Assert.assertEquals(absoluteDir, fSys.getWorkingDirectory());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testListStatusThrowsExceptionForUnreadableDir() {
+    SwiftTestUtils.skip("unsupported");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testFsStatus() throws Exception {
+    super.testFsStatus();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWorkingDirectory() throws Exception {
+    super.testWorkingDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testMkdirs() throws Exception {
+    super.testMkdirs();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+    super.testMkdirsFailsForSubdirectoryOfExistingFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGetFileStatusThrowsExceptionForNonExistentFile() throws
+                                                                   Exception {
+    super.testGetFileStatusThrowsExceptionForNonExistentFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testListStatusThrowsExceptionForNonExistentFile() throws
+                                                                Exception {
+    super.testListStatusThrowsExceptionForNonExistentFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testListStatus() throws Exception {
+    super.testListStatus();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testListStatusFilterWithNoMatches() throws Exception {
+    super.testListStatusFilterWithNoMatches();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testListStatusFilterWithSomeMatches() throws Exception {
+    super.testListStatusFilterWithSomeMatches();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusNonExistentFile() throws Exception {
+    super.testGlobStatusNonExistentFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusWithNoMatchesInPath() throws Exception {
+    super.testGlobStatusWithNoMatchesInPath();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusSomeMatchesInDirectories() throws Exception {
+    super.testGlobStatusSomeMatchesInDirectories();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusWithMultipleWildCardMatches() throws Exception {
+    super.testGlobStatusWithMultipleWildCardMatches();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusWithMultipleMatchesOfSingleChar() throws Exception {
+    super.testGlobStatusWithMultipleMatchesOfSingleChar();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithEmptyPathResults() throws Exception {
+    super.testGlobStatusFilterWithEmptyPathResults();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithSomePathMatchesAndTrivialFilter() throws
+                                                                        Exception {
+    super.testGlobStatusFilterWithSomePathMatchesAndTrivialFilter();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter() throws
+                                                                                Exception {
+    super.testGlobStatusFilterWithMultipleWildCardMatchesAndTrivialFilter();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter() throws
+                                                                               Exception {
+    super.testGlobStatusFilterWithMultiplePathMatchesAndNonTrivialFilter();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter() throws
+                                                                           Exception {
+    super.testGlobStatusFilterWithNoMatchingPathsAndNonTrivialFilter();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter() throws
+                                                                                 Exception {
+    super.testGlobStatusFilterWithMultiplePathWildcardsAndNonTrivialFilter();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteReadAndDeleteEmptyFile() throws Exception {
+    super.testWriteReadAndDeleteEmptyFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteReadAndDeleteHalfABlock() throws Exception {
+    super.testWriteReadAndDeleteHalfABlock();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteReadAndDeleteOneBlock() throws Exception {
+    super.testWriteReadAndDeleteOneBlock();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
+    super.testWriteReadAndDeleteOneAndAHalfBlocks();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteReadAndDeleteTwoBlocks() throws Exception {
+    super.testWriteReadAndDeleteTwoBlocks();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testOverwrite() throws IOException {
+    super.testOverwrite();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testWriteInNonExistentDirectory() throws IOException {
+    super.testWriteInNonExistentDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testDeleteNonExistentFile() throws IOException {
+    super.testDeleteNonExistentFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testDeleteRecursively() throws IOException {
+    super.testDeleteRecursively();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testDeleteEmptyDirectory() throws IOException {
+    super.testDeleteEmptyDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameNonExistentPath() throws Exception {
+    super.testRenameNonExistentPath();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileToNonExistentDirectory() throws Exception {
+    super.testRenameFileToNonExistentDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileToDestinationWithParentFile() throws Exception {
+    super.testRenameFileToDestinationWithParentFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileToExistingParent() throws Exception {
+    super.testRenameFileToExistingParent();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileToItself() throws Exception {
+    super.testRenameFileToItself();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileAsExistingFile() throws Exception {
+    super.testRenameFileAsExistingFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameFileAsExistingDirectory() throws Exception {
+    super.testRenameFileAsExistingDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryToItself() throws Exception {
+    super.testRenameDirectoryToItself();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryToNonExistentParent() throws Exception {
+    super.testRenameDirectoryToNonExistentParent();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryAsNonExistentDirectory() throws Exception {
+    super.testRenameDirectoryAsNonExistentDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryAsEmptyDirectory() throws Exception {
+    super.testRenameDirectoryAsEmptyDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryAsNonEmptyDirectory() throws Exception {
+    super.testRenameDirectoryAsNonEmptyDirectory();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testRenameDirectoryAsFile() throws Exception {
+    super.testRenameDirectoryAsFile();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testInputStreamClosedTwice() throws IOException {
+    super.testInputStreamClosedTwice();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testOutputStreamClosedTwice() throws IOException {
+    super.testOutputStreamClosedTwice();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testGetWrappedInputStream() throws IOException {
+    super.testGetWrappedInputStream();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  @Override
+  public void testCopyToLocalWithUseRawLocalFileSystemOption() throws
+                                                               Exception {
+    super.testCopyToLocalWithUseRawLocalFileSystemOption();
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestLogResources.java

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Test;
+
+import java.net.URL;
+
+/**
+ * This test just debugs which log resources are being picked up
+ */
+public class TestLogResources implements SwiftTestConstants {
+  protected static final Log LOG =
+    LogFactory.getLog(TestLogResources.class);
+
+  private void printf(String format, Object... args) {
+    String msg = String.format(format, args);
+    System.out.printf(msg + "\n");
+    LOG.info(msg);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testWhichLog4JPropsFile() throws Throwable {
+    locateResource("log4j.properties");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testWhichLog4JXMLFile() throws Throwable {
+    locateResource("log4j.XML");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testCommonsLoggingProps() throws Throwable {
+    locateResource("commons-logging.properties");
+  }
+
+  private void locateResource(String resource) {
+    URL url = this.getClass().getClassLoader().getResource(resource);
+    if (url != null) {
+      printf("resource %s is at %s", resource, url);
+    } else {
+      printf("resource %s is not on the classpath", resource);
+    }
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestReadPastBuffer.java

@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Seek tests verify that
+ * <ol>
+ *   <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
+ *   <li>When you seek past the end of a file, it's an error that should
+ *   raise -what- EOFException?</li>
+ *   <li>when you seek forwards, you get new data</li>
+ *   <li>when you seek backwards, you get the previous data</li>
+ *   <li>That this works for big multi-MB files as well as small ones.</li>
+ * </ol>
+ * These may seem "obvious", but the more the input streams try to be clever
+ * about offsets and buffering, the more likely it is that seek() will start
+ * to get confused.
+ */
+public class TestReadPastBuffer extends SwiftFileSystemBaseTest {
+  protected static final Log LOG =
+    LogFactory.getLog(TestReadPastBuffer.class);
+  public static final int SWIFT_READ_BLOCKSIZE = 4096;
+  public static final int SEEK_FILE_LEN = SWIFT_READ_BLOCKSIZE * 2;
+
+  private Path testPath;
+  private Path readFile;
+  private Path zeroByteFile;
+  private FSDataInputStream instream;
+
+
+  /**
+   * Get a configuration which a small blocksize reported to callers
+   * @return a configuration for this test
+   */
+  @Override
+  public Configuration getConf() {
+    Configuration conf = super.getConf();
+    /*
+     * set to 4KB
+     */
+    conf.setInt(SwiftProtocolConstants.SWIFT_BLOCKSIZE, SWIFT_READ_BLOCKSIZE);
+    return conf;
+  }
+
+  /**
+   * Setup creates dirs under test/hadoop
+   *
+   * @throws Exception
+   */
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    byte[] block = SwiftTestUtils.dataset(SEEK_FILE_LEN, 0, 255);
+
+    //delete the test directory
+    testPath = path("/test");
+    readFile = new Path(testPath, "TestReadPastBuffer.txt");
+    createFile(readFile, block);
+  }
+
+  @After
+  public void cleanFile() {
+    IOUtils.closeStream(instream);
+    instream = null;
+  }
+
+  /**
+   * Create a config with a 1KB request size
+   * @return a config
+   */
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
+    return conf;
+  }
+
+  /**
+   * Seek past the buffer then read
+   * @throws Throwable problems
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekAndReadPastEndOfFile() throws Throwable {
+    instream = fs.open(readFile);
+    assertEquals(0, instream.getPos());
+    //expect that seek to 0 works
+    //go just before the end
+    instream.seek(SEEK_FILE_LEN - 2);
+    assertTrue("Premature EOF", instream.read() != -1);
+    assertTrue("Premature EOF", instream.read() != -1);
+    assertMinusOne("read past end of file", instream.read());
+  }
+
+  /**
+   * Seek past the buffer and attempt a read(buffer)
+   * @throws Throwable failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekBulkReadPastEndOfFile() throws Throwable {
+    instream = fs.open(readFile);
+    assertEquals(0, instream.getPos());
+    //go just before the end
+    instream.seek(SEEK_FILE_LEN - 1);
+    byte[] buffer = new byte[1];
+    int result = instream.read(buffer, 0, 1);
+    //next byte is expected to fail
+    result = instream.read(buffer, 0, 1);
+    assertMinusOne("read past end of file", result);
+    //and this one
+    result = instream.read(buffer, 0, 1);
+    assertMinusOne("read past end of file", result);
+
+    //now do an 0-byte read and expect it to
+    //to be checked first
+    result = instream.read(buffer, 0, 0);
+    assertEquals("EOF checks coming before read range check", 0, result);
+
+  }
+
+
+
+  /**
+   * Read past the buffer size byte by byte and verify that it refreshed
+   * @throws Throwable
+   */
+  @Test
+  public void testReadPastBufferSize() throws Throwable {
+    instream = fs.open(readFile);
+
+    while (instream.read() != -1);
+    //here we have gone past the end of a file and its buffer. Now try again
+    assertMinusOne("reading after the (large) file was read: "+ instream,
+                   instream.read());
+  }
+}
+

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSeek.java

@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConnectionClosedException;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * Seek tests verify that
+ * <ol>
+ *   <li>When you seek on a 0 byte file to byte (0), it's not an error.</li>
+ *   <li>When you seek past the end of a file, it's an error that should
+ *   raise -what- EOFException?</li>
+ *   <li>when you seek forwards, you get new data</li>
+ *   <li>when you seek backwards, you get the previous data</li>
+ *   <li>That this works for big multi-MB files as well as small ones.</li>
+ * </ol>
+ * These may seem "obvious", but the more the input streams try to be clever
+ * about offsets and buffering, the more likely it is that seek() will start
+ * to get confused.
+ */
+public class TestSeek extends SwiftFileSystemBaseTest {
+  protected static final Log LOG =
+    LogFactory.getLog(TestSeek.class);
+  public static final int SMALL_SEEK_FILE_LEN = 256;
+
+  private Path testPath;
+  private Path smallSeekFile;
+  private Path zeroByteFile;
+  private FSDataInputStream instream;
+
+  /**
+   * Setup creates dirs under test/hadoop
+   *
+   * @throws Exception
+   */
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    //delete the test directory
+    testPath = path("/test");
+    smallSeekFile = new Path(testPath, "seekfile.txt");
+    zeroByteFile = new Path(testPath, "zero.txt");
+    byte[] block = SwiftTestUtils.dataset(SMALL_SEEK_FILE_LEN, 0, 255);
+    //this file now has a simple rule: offset => value
+    createFile(smallSeekFile, block);
+    createEmptyFile(zeroByteFile);
+  }
+
+  @After
+  public void cleanFile() {
+    IOUtils.closeStream(instream);
+    instream = null;
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekZeroByteFile() throws Throwable {
+    instream = fs.open(zeroByteFile);
+    assertEquals(0, instream.getPos());
+    //expect initial read to fai;
+    int result = instream.read();
+    assertMinusOne("initial byte read", result);
+    byte[] buffer = new byte[1];
+    //expect that seek to 0 works
+    instream.seek(0);
+    //reread, expect same exception
+    result = instream.read();
+    assertMinusOne("post-seek byte read", result);
+    result = instream.read(buffer, 0, 1);
+    assertMinusOne("post-seek buffer read", result);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testBlockReadZeroByteFile() throws Throwable {
+    instream = fs.open(zeroByteFile);
+    assertEquals(0, instream.getPos());
+    //expect that seek to 0 works
+    byte[] buffer = new byte[1];
+    int result = instream.read(buffer, 0, 1);
+    assertMinusOne("block read zero byte file", result);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekReadClosedFile() throws Throwable {
+    instream = fs.open(smallSeekFile);
+    instream.close();
+    try {
+      instream.seek(0);
+    } catch (SwiftConnectionClosedException e) {
+      //expected a closed file
+    }
+    try {
+      instream.read();
+    } catch (IOException e) {
+      //expected a closed file
+    }
+    try {
+      byte[] buffer = new byte[1];
+      int result = instream.read(buffer, 0, 1);
+    } catch (IOException e) {
+      //expected a closed file
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testNegativeSeek() throws Throwable {
+    instream = fs.open(smallSeekFile);
+    assertEquals(0, instream.getPos());
+    try {
+      instream.seek(-1);
+      long p = instream.getPos();
+      LOG.warn("Seek to -1 returned a position of " + p);
+      int result = instream.read();
+      fail(
+        "expected an exception, got data " + result + " at a position of " + p);
+    } catch (IOException e) {
+      //bad seek -expected
+    }
+    assertEquals(0, instream.getPos());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekFile() throws Throwable {
+    instream = fs.open(smallSeekFile);
+    assertEquals(0, instream.getPos());
+    //expect that seek to 0 works
+    instream.seek(0);
+    int result = instream.read();
+    assertEquals(0, result);
+    assertEquals(1, instream.read());
+    assertEquals(2, instream.getPos());
+    assertEquals(2, instream.read());
+    assertEquals(3, instream.getPos());
+    instream.seek(128);
+    assertEquals(128, instream.getPos());
+    assertEquals(128, instream.read());
+    instream.seek(63);
+    assertEquals(63, instream.read());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekAndReadPastEndOfFile() throws Throwable {
+    instream = fs.open(smallSeekFile);
+    assertEquals(0, instream.getPos());
+    //expect that seek to 0 works
+    //go just before the end
+    instream.seek(SMALL_SEEK_FILE_LEN - 2);
+    assertTrue("Premature EOF", instream.read() != -1);
+    assertTrue("Premature EOF", instream.read() != -1);
+    assertMinusOne("read past end of file", instream.read());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekAndPastEndOfFileThenReseekAndRead() throws Throwable {
+    instream = fs.open(smallSeekFile);
+    //go just before the end. This may or may not fail; it may be delayed until the
+    //read
+    try {
+      instream.seek(SMALL_SEEK_FILE_LEN);
+      //if this doesn't trigger, then read() is expected to fail
+      assertMinusOne("read after seeking past EOF", instream.read());
+    } catch (EOFException expected) {
+      //here an exception was raised in seek
+    }
+    instream.seek(1);
+    assertTrue("Premature EOF", instream.read() != -1);
+  }
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    conf.set(SwiftProtocolConstants.SWIFT_REQUEST_SIZE, "1");
+    return conf;
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSeekBigFile() throws Throwable {
+    Path testSeekFile = new Path(testPath, "bigseekfile.txt");
+    byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
+    createFile(testSeekFile, block);
+    instream = fs.open(testSeekFile);
+    assertEquals(0, instream.getPos());
+    //expect that seek to 0 works
+    instream.seek(0);
+    int result = instream.read();
+    assertEquals(0, result);
+    assertEquals(1, instream.read());
+    assertEquals(2, instream.read());
+
+    //do seek 32KB ahead
+    instream.seek(32768);
+    assertEquals("@32768", block[32768], (byte) instream.read());
+    instream.seek(40000);
+    assertEquals("@40000", block[40000], (byte) instream.read());
+    instream.seek(8191);
+    assertEquals("@8191", block[8191], (byte) instream.read());
+    instream.seek(0);
+    assertEquals("@0", 0, (byte) instream.read());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testPositionedBulkReadDoesntChangePosition() throws Throwable {
+    Path testSeekFile = new Path(testPath, "bigseekfile.txt");
+    byte[] block = SwiftTestUtils.dataset(65536, 0, 255);
+    createFile(testSeekFile, block);
+    instream = fs.open(testSeekFile);
+    instream.seek(39999);
+    assertTrue(-1 != instream.read());
+    assertEquals (40000, instream.getPos());
+
+    byte[] readBuffer = new byte[256];
+    instream.read(128, readBuffer, 0, readBuffer.length);
+    //have gone back
+    assertEquals(40000, instream.getPos());
+    //content is the same too
+    assertEquals("@40000", block[40000], (byte) instream.read());
+    //now verify the picked up data
+    for (int i = 0; i < 256; i++) {
+      assertEquals("@" + i, block[i + 128], readBuffer[i]);
+    }
+  }
+
+  /**
+   * work out the expected byte from a specific offset
+   * @param offset offset in the file
+   * @return the value
+   */
+  int expectedByte(int offset) {
+    return offset & 0xff;
+  }
+}
+

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftConfig.java

@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_LOCATION_AWARE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_TENANT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_BLOCKSIZE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONNECTION_TIMEOUT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PARTITION_SIZE;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_HOST_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PROXY_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_RETRY_COUNT;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PREFIX;
+
+/**
+ * Test the swift service-specific configuration binding features
+ */
+public class TestSwiftConfig extends Assert {
+
+
+  public static final String SERVICE = "openstack";
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testEmptyUrl() throws Exception {
+    final Configuration configuration = new Configuration();
+
+    set(configuration, DOT_TENANT, "tenant");
+    set(configuration, DOT_USERNAME, "username");
+    set(configuration, DOT_PASSWORD, "password");
+    mkInstance(configuration);
+  }
+
+@Test
+  public void testEmptyTenant() throws Exception {
+    final Configuration configuration = new Configuration();
+    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+    set(configuration, DOT_USERNAME, "username");
+    set(configuration, DOT_PASSWORD, "password");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testEmptyUsername() throws Exception {
+    final Configuration configuration = new Configuration();
+    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+    set(configuration, DOT_TENANT, "tenant");
+    set(configuration, DOT_PASSWORD, "password");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testEmptyPassword() throws Exception {
+    final Configuration configuration = new Configuration();
+    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+    set(configuration, DOT_TENANT, "tenant");
+    set(configuration, DOT_USERNAME, "username");
+    mkInstance(configuration);
+  }
+
+  @Test
+  public void testGoodRetryCount() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_RETRY_COUNT, "3");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testBadRetryCount() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_RETRY_COUNT, "three");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testBadConnectTimeout() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_CONNECTION_TIMEOUT, "three");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testZeroBlocksize() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_BLOCKSIZE, "0");
+    mkInstance(configuration);
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testNegativeBlocksize() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_BLOCKSIZE, "-1");
+    mkInstance(configuration);
+  }
+
+  @Test
+  public void testPositiveBlocksize() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    int size = 127;
+    configuration.set(SWIFT_BLOCKSIZE, Integer.toString(size));
+    SwiftRestClient restClient = mkInstance(configuration);
+    assertEquals(size, restClient.getBlocksizeKB());
+  }
+
+  @Test
+  public void testLocationAwareTruePropagates() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    set(configuration, DOT_LOCATION_AWARE, "true");
+    SwiftRestClient restClient = mkInstance(configuration);
+    assertTrue(restClient.isLocationAware());
+  }
+
+  @Test
+  public void testLocationAwareFalsePropagates() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    set(configuration, DOT_LOCATION_AWARE, "false");
+    SwiftRestClient restClient = mkInstance(configuration);
+    assertFalse(restClient.isLocationAware());
+  }
+
+  @Test(expected = org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException.class)
+  public void testNegativePartsize() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    configuration.set(SWIFT_PARTITION_SIZE, "-1");
+    SwiftRestClient restClient = mkInstance(configuration);
+  }
+
+  @Test
+  public void testPositivePartsize() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    int size = 127;
+    configuration.set(SWIFT_PARTITION_SIZE, Integer.toString(size));
+    SwiftRestClient restClient = mkInstance(configuration);
+    assertEquals(size, restClient.getPartSizeKB());
+  }
+
+  @Test
+  public void testProxyData() throws Exception {
+    final Configuration configuration = createCoreConfig();
+    String proxy="web-proxy";
+    int port = 8088;
+    configuration.set(SWIFT_PROXY_HOST_PROPERTY, proxy);
+    configuration.set(SWIFT_PROXY_PORT_PROPERTY, Integer.toString(port));
+    SwiftRestClient restClient = mkInstance(configuration);
+    assertEquals(proxy, restClient.getProxyHost());
+    assertEquals(port, restClient.getProxyPort());
+  }
+
+  private Configuration createCoreConfig() {
+    final Configuration configuration = new Configuration();
+    set(configuration, DOT_AUTH_URL, "http://localhost:8080");
+    set(configuration, DOT_TENANT, "tenant");
+    set(configuration, DOT_USERNAME, "username");
+    set(configuration, DOT_PASSWORD, "password");
+    return configuration;
+  }
+
+  private void set(Configuration configuration, String field, String value) {
+    configuration.set(SWIFT_SERVICE_PREFIX + SERVICE + field, value);
+  }
+
+  private SwiftRestClient mkInstance(Configuration configuration) throws
+          IOException,
+          URISyntaxException {
+    URI uri = new URI("swift://container.openstack/");
+    return SwiftRestClient.getInstance(uri, configuration);
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBasicOps.java

@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftBadRequestException;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertFileHasLength;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertIsDirectory;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
+
+
+/**
+ * Test basic filesystem operations.
+ * Many of these are similar to those in {@link TestSwiftFileSystemContract}
+ * -this is a JUnit4 test suite used to initially test the Swift
+ * component. Once written, there's no reason not to retain these tests.
+ */
+public class TestSwiftFileSystemBasicOps extends SwiftFileSystemBaseTest {
+
+  private static final Log LOG =
+          LogFactory.getLog(TestSwiftFileSystemBasicOps.class);
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLsRoot() throws Throwable {
+    Path path = new Path("/");
+    FileStatus[] statuses = fs.listStatus(path);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testMkDir() throws Throwable {
+    Path path = new Path("/test/MkDir");
+    fs.mkdirs(path);
+    //success then -so try a recursive operation
+    fs.delete(path, true);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteNonexistentFile() throws Throwable {
+    Path path = new Path("/test/DeleteNonexistentFile");
+    assertFalse("delete returned true", fs.delete(path, false));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testPutFile() throws Throwable {
+    Path path = new Path("/test/PutFile");
+    Exception caught = null;
+    writeTextFile(fs, path, "Testing a put to a file", false);
+    assertDeleted(path, false);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testPutGetFile() throws Throwable {
+    Path path = new Path("/test/PutGetFile");
+    try {
+      String text = "Testing a put and get to a file "
+              + System.currentTimeMillis();
+      writeTextFile(fs, path, text, false);
+
+      String result = readBytesToString(fs, path, text.length());
+      assertEquals(text, result);
+    } finally {
+      delete(fs, path);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testPutDeleteFileInSubdir() throws Throwable {
+    Path path =
+            new Path("/test/PutDeleteFileInSubdir/testPutDeleteFileInSubdir");
+    String text = "Testing a put and get to a file in a subdir "
+            + System.currentTimeMillis();
+    writeTextFile(fs, path, text, false);
+    assertDeleted(path, false);
+    //now delete the parent that should have no children
+    assertDeleted(new Path("/test/PutDeleteFileInSubdir"), false);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRecursiveDelete() throws Throwable {
+    Path childpath =
+            new Path("/test/testRecursiveDelete");
+    String text = "Testing a put and get to a file in a subdir "
+            + System.currentTimeMillis();
+    writeTextFile(fs, childpath, text, false);
+    //now delete the parent that should have no children
+    assertDeleted(new Path("/test"), true);
+    assertFalse("child entry still present " + childpath, fs.exists(childpath));
+  }
+
+  private void delete(SwiftNativeFileSystem fs, Path path) {
+    try {
+      if (!fs.delete(path, false)) {
+        LOG.warn("Failed to delete " + path);
+      }
+    } catch (IOException e) {
+      LOG.warn("deleting " + path, e);
+    }
+  }
+
+  private void deleteR(SwiftNativeFileSystem fs, Path path) {
+    try {
+      if (!fs.delete(path, true)) {
+        LOG.warn("Failed to delete " + path);
+      }
+    } catch (IOException e) {
+      LOG.warn("deleting " + path, e);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testOverwrite() throws Throwable {
+    Path path = new Path("/test/Overwrite");
+    try {
+      String text = "Testing a put to a file "
+              + System.currentTimeMillis();
+      writeTextFile(fs, path, text, false);
+      assertFileHasLength(fs, path, text.length());
+      String text2 = "Overwriting a file "
+              + System.currentTimeMillis();
+      writeTextFile(fs, path, text2, true);
+      assertFileHasLength(fs, path, text2.length());
+      String result = readBytesToString(fs, path, text2.length());
+      assertEquals(text2, result);
+    } finally {
+      delete(fs, path);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testOverwriteDirectory() throws Throwable {
+    Path path = new Path("/test/testOverwriteDirectory");
+    try {
+      fs.mkdirs(path.getParent());
+      String text = "Testing a put to a file "
+              + System.currentTimeMillis();
+      writeTextFile(fs, path, text, false);
+      assertFileHasLength(fs, path, text.length());
+    } finally {
+      delete(fs, path);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testFileStatus() throws Throwable {
+    Path path = new Path("/test/FileStatus");
+    try {
+      String text = "Testing File Status "
+              + System.currentTimeMillis();
+      writeTextFile(fs, path, text, false);
+      SwiftTestUtils.assertIsFile(fs, path);
+    } finally {
+      delete(fs, path);
+    }
+  }
+
+  /**
+   * Assert that a newly created directory is a directory
+   *
+   * @throws Throwable if not, or if something else failed
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDirStatus() throws Throwable {
+    Path path = new Path("/test/DirStatus");
+    try {
+      fs.mkdirs(path);
+      assertIsDirectory(fs, path);
+    } finally {
+      delete(fs, path);
+    }
+  }
+
+  /**
+   * Assert that if a directory that has children is deleted, it is still
+   * a directory
+   *
+   * @throws Throwable if not, or if something else failed
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDirStaysADir() throws Throwable {
+    Path path = new Path("/test/dirStaysADir");
+    Path child = new Path(path, "child");
+    try {
+      //create the dir
+      fs.mkdirs(path);
+      //assert the parent has the directory nature
+      assertIsDirectory(fs, path);
+      //create the child dir
+      writeTextFile(fs, child, "child file", true);
+      //assert the parent has the directory nature
+      assertIsDirectory(fs, path);
+      //now rm the child
+      delete(fs, child);
+    } finally {
+      deleteR(fs, path);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testCreateMultilevelDir() throws Throwable {
+    Path base = new Path("/test/CreateMultilevelDir");
+    Path path = new Path(base, "1/2/3");
+    fs.mkdirs(path);
+    assertExists("deep multilevel dir not created", path);
+    fs.delete(base, true);
+    assertPathDoesNotExist("Multilevel delete failed", path);
+    assertPathDoesNotExist("Multilevel delete failed", base);
+
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testCreateDirWithFileParent() throws Throwable {
+    Path path = new Path("/test/CreateDirWithFileParent");
+    Path child = new Path(path, "subdir/child");
+    fs.mkdirs(path.getParent());
+    try {
+      //create the child dir
+      writeTextFile(fs, path, "parent", true);
+      try {
+        fs.mkdirs(child);
+      } catch (SwiftNotDirectoryException expected) {
+        LOG.debug("Expected Exception", expected);
+      }
+    } finally {
+      fs.delete(path, true);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLongObjectNamesForbidden() throws Throwable {
+    StringBuilder buffer = new StringBuilder(1200);
+    buffer.append("/");
+    for (int i = 0; i < (1200 / 4); i++) {
+      buffer.append(String.format("%04x", i));
+    }
+    String pathString = buffer.toString();
+    Path path = new Path(pathString);
+    try {
+      writeTextFile(fs, path, pathString, true);
+      //if we get here, problems.
+      fs.delete(path, false);
+      fail("Managed to create an object with a name of length "
+              + pathString.length());
+    } catch (SwiftBadRequestException e) {
+      //expected
+      //LOG.debug("Caught exception " + e, e);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLsNonExistentFile() throws Exception {
+    try {
+      Path path = new Path("/test/hadoop/file");
+      FileStatus[] statuses = fs.listStatus(path);
+      fail("Should throw FileNotFoundException on " + path
+              + " but got list of length " + statuses.length);
+    } catch (FileNotFoundException fnfe) {
+      // expected
+    }
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlockLocation.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Test block location logic.
+ * The endpoint may or may not be location-aware
+ */
+public class TestSwiftFileSystemBlockLocation extends SwiftFileSystemBaseTest {
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateSingleFileBlocks() throws Throwable {
+    describe("verify that a file returns 1+ blocks");
+    FileStatus fileStatus = createFileAndGetStatus();
+    BlockLocation[] locations =
+      getFs().getFileBlockLocations(fileStatus, 0, 1);
+    assertNotEqual("No block locations supplied for " + fileStatus, 0,
+                   locations.length);
+    for (BlockLocation location : locations) {
+      assertLocationValid(location);
+    }
+  }
+
+  private void assertLocationValid(BlockLocation location) throws
+                                                           IOException {
+    LOG.info(location);
+    String[] hosts = location.getHosts();
+    String[] names = location.getNames();
+    assertNotEqual("No hosts supplied for " + location, 0, hosts.length);
+    //for every host, there's a name.
+    assertEquals("Unequal names and hosts in " + location,
+                 hosts.length, names.length);
+    assertEquals(SwiftProtocolConstants.BLOCK_LOCATION,
+                 location.getNames()[0]);
+    assertEquals(SwiftProtocolConstants.TOPOLOGY_PATH,
+                 location.getTopologyPaths()[0]);
+  }
+
+  private FileStatus createFileAndGetStatus() throws IOException {
+    Path path = path("/test/locatedFile");
+    createFile(path);
+    return fs.getFileStatus(path);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateNullStatus() throws Throwable {
+    describe("verify that a null filestatus maps to a null location array");
+    BlockLocation[] locations =
+      getFs().getFileBlockLocations((FileStatus) null, 0, 1);
+    assertNull(locations);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateNegativeSeek() throws Throwable {
+    describe("verify that a negative offset is illegal");
+    try {
+      BlockLocation[] locations =
+        getFs().getFileBlockLocations(createFileAndGetStatus(),
+                                      -1,
+                                      1);
+      fail("Expected an exception, got " + locations.length + " locations");
+    } catch (IllegalArgumentException e) {
+      //expected
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateNegativeLen() throws Throwable {
+    describe("verify that a negative length is illegal");
+    try {
+      BlockLocation[] locations =
+        getFs().getFileBlockLocations(createFileAndGetStatus(),
+                                      0,
+                                      -1);
+      fail("Expected an exception, got " + locations.length + " locations");
+    } catch (IllegalArgumentException e) {
+      //expected
+    }
+  }
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateOutOfRangeLen() throws Throwable {
+    describe("overshooting the length is legal, as long as the" +
+             " origin location is valid");
+
+    BlockLocation[] locations =
+      getFs().getFileBlockLocations(createFileAndGetStatus(),
+                                    0,
+                                    data.length + 100);
+    assertNotNull(locations);
+    assertTrue(locations.length > 0);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateOutOfRangeSrc() throws Throwable {
+    describe("Seeking out of the file length returns an empty array");
+
+    BlockLocation[] locations =
+      getFs().getFileBlockLocations(createFileAndGetStatus(),
+                                    data.length + 100,
+                                    1);
+    assertEmptyBlockLocations(locations);
+  }
+
+  private void assertEmptyBlockLocations(BlockLocation[] locations) {
+    assertNotNull(locations);
+    if (locations.length!=0) {
+      fail("non empty locations[] with first entry of " + locations[0]);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateDirectory() throws Throwable {
+    describe("verify that locating a directory is an error");
+    createFile(path("/test/filename"));
+    FileStatus status = fs.getFileStatus(path("/test"));
+    LOG.info("Filesystem is " + fs + "; target is " + status);
+    SwiftTestUtils.assertIsDirectory(status);
+    BlockLocation[] locations;
+    locations = getFs().getFileBlockLocations(status,
+                                              0,
+                                              1);
+    assertEmptyBlockLocations(locations);
+  }
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLocateRootDirectory() throws Throwable {
+    describe("verify that locating the root directory is an error");
+    FileStatus status = fs.getFileStatus(path("/"));
+    SwiftTestUtils.assertIsDirectory(status);
+    BlockLocation[] locations;
+    locations = getFs().getFileBlockLocations(status,
+                                              0,
+                                              1);
+    assertEmptyBlockLocations(locations);
+  }
+
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemBlocksize.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+/**
+ * Tests that blocksize is never zero for a file, either in the FS default
+ * or the FileStatus value of a queried file 
+ */
+public class TestSwiftFileSystemBlocksize extends SwiftFileSystemBaseTest {
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDefaultBlocksizeNonZero() throws Throwable {
+    assertTrue("Zero default blocksize", 0L != getFs().getDefaultBlockSize());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDefaultBlocksizeRootPathNonZero() throws Throwable {
+    assertTrue("Zero default blocksize",
+               0L != getFs().getDefaultBlockSize(new Path("/")));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDefaultBlocksizeOtherPathNonZero() throws Throwable {
+    assertTrue("Zero default blocksize",
+               0L != getFs().getDefaultBlockSize(new Path("/test")));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testBlocksizeNonZeroForFile() throws Throwable {
+    Path smallfile = new Path("/test/smallfile");
+    SwiftTestUtils.writeTextFile(fs, smallfile, "blocksize", true);
+    createFile(smallfile);
+    FileStatus status = getFs().getFileStatus(smallfile);
+    assertTrue("Zero blocksize in " + status,
+               status.getBlockSize() != 0L);
+    assertTrue("Zero replication in " + status,
+               status.getReplication() != 0L);
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemConcurrency.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test Swift FS concurrency logic. This isn't a very accurate test,
+ * because it is hard to consistently generate race conditions.
+ * Consider it "best effort"
+ */
+public class TestSwiftFileSystemConcurrency extends SwiftFileSystemBaseTest {
+  protected static final Log LOG =
+    LogFactory.getLog(TestSwiftFileSystemConcurrency.class);
+  private Exception thread1Ex, thread2Ex;
+  public static final String TEST_RACE_CONDITION_ON_DELETE_DIR =
+    "/test/testraceconditionondirdeletetest";
+
+  /**
+   * test on concurrent file system changes
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRaceConditionOnDirDeleteTest() throws Exception {
+    SwiftTestUtils.skip("Skipping unreliable test");
+
+    final String message = "message";
+    final Path fileToRead = new Path(
+      TEST_RACE_CONDITION_ON_DELETE_DIR +"/files/many-files/file");
+    final ExecutorService executorService = Executors.newFixedThreadPool(2);
+    fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR +"/file/test/file1"));
+    fs.create(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR + "/documents/doc1"));
+    fs.create(new Path(
+      TEST_RACE_CONDITION_ON_DELETE_DIR + "/pictures/picture"));
+
+
+    executorService.execute(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          assertDeleted(new Path(TEST_RACE_CONDITION_ON_DELETE_DIR), true);
+        } catch (IOException e) {
+          LOG.warn("deletion thread:" + e, e);
+          thread1Ex = e;
+          throw new RuntimeException(e);
+        }
+      }
+    });
+    executorService.execute(new Runnable() {
+      @Override
+      public void run() {
+        try {
+          final FSDataOutputStream outputStream = fs.create(fileToRead);
+          outputStream.write(message.getBytes());
+          outputStream.close();
+        } catch (IOException e) {
+          LOG.warn("writer thread:" + e, e);
+          thread2Ex = e;
+          throw new RuntimeException(e);
+        }
+      }
+    });
+
+    executorService.awaitTermination(1, TimeUnit.MINUTES);
+    if (thread1Ex != null) {
+      throw thread1Ex;
+    }
+    if (thread2Ex != null) {
+      throw thread2Ex;
+    }
+    try {
+      fs.open(fileToRead);
+      LOG.info("concurrency test failed to trigger a failure");
+    } catch (FileNotFoundException expected) {
+
+    }
+
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemContract.java

@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import junit.framework.AssertionFailedError;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.exceptions.SwiftNotDirectoryException;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+/**
+ * This is the full filesystem contract test -which requires the
+ * Default config set up to point to a filesystem.
+ *
+ * Some of the tests override the base class tests -these
+ * are where SwiftFS does not implement those features, or
+ * when the behavior of SwiftFS does not match the normal
+ * contract -which normally means that directories and equal files
+ * are being treated as equal.
+ */
+public class TestSwiftFileSystemContract
+        extends FileSystemContractBaseTest {
+  private static final Log LOG =
+          LogFactory.getLog(TestSwiftFileSystemContract.class);
+
+  @Override
+  protected void setUp() throws Exception {
+    final URI uri = getFilesystemURI();
+    final Configuration conf = new Configuration();
+    fs = createSwiftFS();
+    try {
+      fs.initialize(uri, conf);
+    } catch (IOException e) {
+      //FS init failed, set it to null so that teardown doesn't
+      //attempt to use it
+      fs = null;
+      throw e;
+    }
+    super.setUp();
+  }
+
+  protected URI getFilesystemURI() throws URISyntaxException, IOException {
+    return SwiftTestUtils.getServiceURI(new Configuration());
+  }
+
+  protected SwiftNativeFileSystem createSwiftFS() throws IOException {
+    SwiftNativeFileSystem swiftNativeFileSystem =
+            new SwiftNativeFileSystem();
+    return swiftNativeFileSystem;
+  }
+
+  @Override
+  public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+    Path testDir = path("/test/hadoop");
+    assertFalse(fs.exists(testDir));
+    assertTrue(fs.mkdirs(testDir));
+    assertTrue(fs.exists(testDir));
+
+    Path filepath = path("/test/hadoop/file");
+    SwiftTestUtils.writeTextFile(fs, filepath, "hello, world", false);
+
+    Path testSubDir = new Path(filepath, "subdir");
+    SwiftTestUtils.assertPathDoesNotExist(fs, "subdir before mkdir", testSubDir);
+
+    try {
+      fs.mkdirs(testSubDir);
+      fail("Should throw IOException.");
+    } catch (SwiftNotDirectoryException e) {
+      // expected
+      assertEquals(filepath,e.getPath());
+    }
+    //now verify that the subdir path does not exist
+    SwiftTestUtils.assertPathDoesNotExist(fs, "subdir after mkdir", testSubDir);
+
+    Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir");
+    try {
+      fs.mkdirs(testDeepSubDir);
+      fail("Should throw IOException.");
+    } catch (SwiftNotDirectoryException e) {
+      // expected
+    }
+    SwiftTestUtils.assertPathDoesNotExist(fs, "testDeepSubDir  after mkdir",
+                                          testDeepSubDir);
+
+  }
+
+  @Override
+  public void testWriteReadAndDeleteEmptyFile() throws Exception {
+    try {
+      super.testWriteReadAndDeleteEmptyFile();
+    } catch (AssertionFailedError e) {
+      SwiftTestUtils.downgrade("empty files get mistaken for directories", e);
+    }
+  }
+
+  @Override
+  public void testMkdirsWithUmask() throws Exception {
+    //unsupported
+  }
+
+  public void testZeroByteFilesAreFiles() throws Exception {
+//    SwiftTestUtils.unsupported("testZeroByteFilesAreFiles");
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDelete.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+/**
+ * Test deletion operations
+ */
+public class TestSwiftFileSystemDelete extends SwiftFileSystemBaseTest {
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteEmptyFile() throws IOException {
+    final Path file = new Path("/test/testDeleteEmptyFile");
+    createEmptyFile(file);
+    SwiftTestUtils.noteAction("about to delete");
+    assertDeleted(file, true);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteEmptyFileTwice() throws IOException {
+    final Path file = new Path("/test/testDeleteEmptyFileTwice");
+    createEmptyFile(file);
+    assertDeleted(file, true);
+    SwiftTestUtils.noteAction("multiple creates, and deletes");
+    assertFalse("Delete returned true", fs.delete(file, false));
+    createEmptyFile(file);
+    assertDeleted(file, true);
+    assertFalse("Delete returned true", fs.delete(file, false));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteNonEmptyFile() throws IOException {
+    final Path file = new Path("/test/testDeleteNonEmptyFile");
+    createFile(file);
+    assertDeleted(file, true);
+  }
+  
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteNonEmptyFileTwice() throws IOException {
+    final Path file = new Path("/test/testDeleteNonEmptyFileTwice");
+    createFile(file);
+    assertDeleted(file, true);
+    assertFalse("Delete returned true", fs.delete(file, false));
+    createFile(file);
+    assertDeleted(file, true);
+    assertFalse("Delete returned true", fs.delete(file, false));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDeleteTestDir() throws IOException {
+    final Path file = new Path("/test/");
+    fs.delete(file, true);
+    assertPathDoesNotExist("Test dir found", file);
+  }
+
+  /**
+   * Test recursive root directory deletion fails if there is an entry underneath
+   * @throws Throwable
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRmRootDirRecursiveIsForbidden() throws Throwable {
+    Path root = path("/");
+    Path testFile = path("/test");
+    createFile(testFile);
+    assertTrue("rm(/) returned false", fs.delete(root, true));
+    assertExists("Root dir is missing", root);
+    assertPathDoesNotExist("test file not deleted", testFile);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemDirectories.java

@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+
+/**
+ * Test swift-specific directory logic.
+ * This class is HDFS-1 compatible; its designed to be subclases by something
+ * with HDFS2 extensions
+ */
+public class TestSwiftFileSystemDirectories extends SwiftFileSystemBaseTest {
+
+  /**
+   * Asserts that a zero byte file has a status of file and not
+   * directory or symlink
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testZeroByteFilesAreDirectories() throws Exception {
+    Path src = path("/test/testZeroByteFilesAreFiles");
+    //create a zero byte file
+    SwiftTestUtils.touch(fs, src);
+    SwiftTestUtils.assertIsDirectory(fs, src);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testNoStatusForMissingDirectories() throws Throwable {
+    Path missing = path("/test/testNoStatusForMissingDirectories");
+    assertPathDoesNotExist("leftover?", missing);
+    try {
+      FileStatus[] statuses = fs.listStatus(missing);
+      //not expected
+      fail("Expected a FileNotFoundException, got the status " + statuses);
+    } catch (FileNotFoundException expected) {
+      //expected
+    }
+  }
+
+  /**
+   * test that a dir off root has a listStatus() call that
+   * works as expected. and that when a child is added. it changes
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDirectoriesOffRootHaveMatchingFileStatus() throws Exception {
+    Path test = path("/test");
+    fs.delete(test, true);
+    mkdirs(test);
+    assertExists("created test directory", test);
+    FileStatus[] statuses = fs.listStatus(test);
+    String statusString = statusToString(test.toString(), statuses);
+    assertEquals("Wrong number of elements in file status " + statusString, 0,
+                 statuses.length);
+
+    Path src = path("/test/file");
+
+    //create a zero byte file
+    SwiftTestUtils.touch(fs, src);
+    //stat it
+    statuses = fs.listStatus(test);
+    statusString = statusToString(test.toString(), statuses);
+    assertEquals("Wrong number of elements in file status " + statusString, 1,
+                 statuses.length);
+    SwiftFileStatus stat = (SwiftFileStatus) statuses[0];
+    assertTrue("isDir(): Not a directory: " + stat, stat.isDir());
+    extraStatusAssertions(stat);
+  }
+
+  /**
+   * test that a dir two levels down has a listStatus() call that
+   * works as expected.
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDirectoriesLowerDownHaveMatchingFileStatus() throws Exception {
+    Path test = path("/test/testDirectoriesLowerDownHaveMatchingFileStatus");
+    fs.delete(test, true);
+    mkdirs(test);
+    assertExists("created test sub directory", test);
+    FileStatus[] statuses = fs.listStatus(test);
+    String statusString = statusToString(test.toString(), statuses);
+    assertEquals("Wrong number of elements in file status " + statusString,0,
+                 statuses.length);
+  }
+
+  private String statusToString(String pathname,
+                                FileStatus[] statuses) {
+    assertNotNull(statuses);
+    return SwiftTestUtils.dumpStats(pathname,statuses);
+  }
+
+  /**
+   * method for subclasses to add extra assertions
+   * @param stat status to look at
+   */
+  protected void extraStatusAssertions(SwiftFileStatus stat) {
+
+  }
+
+  /**
+   * Asserts that a zero byte file has a status of file and not
+   * directory or symlink
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testMultiByteFilesAreFiles() throws Exception {
+    Path src = path("/test/testMultiByteFilesAreFiles");
+    SwiftTestUtils.writeTextFile(fs, src, "testMultiByteFilesAreFiles", false);
+    assertIsFile(src);
+    FileStatus status = fs.getFileStatus(src);
+    assertFalse(status.isDir());
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemExtendedContract.java

@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Locale;
+
+public class TestSwiftFileSystemExtendedContract extends SwiftFileSystemBaseTest {
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testOpenNonExistingFile() throws IOException {
+    final Path p = new Path("/test/testOpenNonExistingFile");
+    //open it as a file, should get FileNotFoundException
+    try {
+      final FSDataInputStream in = fs.open(p);
+      in.close();
+      fail("didn't expect to get here");
+    } catch (FileNotFoundException fnfe) {
+      LOG.debug("Expected: " + fnfe, fnfe);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testFilesystemHasURI() throws Throwable {
+    assertNotNull(fs.getUri());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testCreateFile() throws Exception {
+    final Path f = new Path("/test/testCreateFile");
+    final FSDataOutputStream fsDataOutputStream = fs.create(f);
+    fsDataOutputStream.close();
+    assertExists("created file", f);
+  }
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testWriteReadFile() throws Exception {
+    final Path f = new Path("/test/test");
+    final FSDataOutputStream fsDataOutputStream = fs.create(f);
+    final String message = "Test string";
+    fsDataOutputStream.write(message.getBytes());
+    fsDataOutputStream.close();
+    assertExists("created file", f);
+    FSDataInputStream open = null;
+    try {
+      open = fs.open(f);
+      final byte[] bytes = new byte[512];
+      final int read = open.read(bytes);
+      final byte[] buffer = new byte[read];
+      System.arraycopy(bytes, 0, buffer, 0, read);
+      assertEquals(message, new String(buffer));
+    } finally {
+      fs.delete(f, false);
+      IOUtils.closeStream(open);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testConfDefinesFilesystem() throws Throwable {
+    Configuration conf = new Configuration();
+    SwiftTestUtils.getServiceURI(conf);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testConfIsValid() throws Throwable {
+    Configuration conf = new Configuration();
+    URI fsURI = SwiftTestUtils.getServiceURI(conf);
+    RestClientBindings.bind(fsURI, conf);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testGetSchemeImplemented() throws Throwable {
+    String scheme = fs.getScheme();
+    assertEquals(SwiftNativeFileSystem.SWIFT,scheme);
+  }
+
+  /**
+   * Assert that a filesystem is case sensitive.
+   * This is done by creating a mixed-case filename and asserting that
+   * its lower case version is not there.
+   *
+   * @throws Exception failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testFilesystemIsCaseSensitive() throws Exception {
+    String mixedCaseFilename = "/test/UPPER.TXT";
+    Path upper = path(mixedCaseFilename);
+    Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
+    assertFalse("File exists" + upper, fs.exists(upper));
+    assertFalse("File exists" + lower, fs.exists(lower));
+    FSDataOutputStream out = fs.create(upper);
+    out.writeUTF("UPPER");
+    out.close();
+    FileStatus upperStatus = fs.getFileStatus(upper);
+    assertExists("Original upper case file" + upper, upper);
+    //verify the lower-case version of the filename doesn't exist
+    assertPathDoesNotExist("lower case file", lower);
+    //now overwrite the lower case version of the filename with a
+    //new version.
+    out = fs.create(lower);
+    out.writeUTF("l");
+    out.close();
+    assertExists("lower case file", lower);
+    //verifEy the length of the upper file hasn't changed
+    assertExists("Original upper case file " + upper, upper);
+    FileStatus newStatus = fs.getFileStatus(upper);
+    assertEquals("Expected status:" + upperStatus
+            + " actual status " + newStatus,
+            upperStatus.getLen(),
+            newStatus.getLen());
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemLsOperations.java

@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertListStatusFinds;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.cleanup;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dumpStats;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.touch;
+
+/**
+ * Test the FileSystem#listStatus() operations
+ */
+public class TestSwiftFileSystemLsOperations extends SwiftFileSystemBaseTest {
+
+  private Path[] testDirs;
+
+    /**
+   * Setup creates dirs under test/hadoop
+   *
+   * @throws Exception
+   */
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    //delete the test directory
+    Path test = path("/test");
+    fs.delete(test, true);
+    mkdirs(test);
+  }
+
+  /**
+   * Create subdirectories and files under test/ for those tests
+   * that want them. Doing so adds overhead to setup and teardown,
+   * so should only be done for those tests that need them.
+   * @throws IOException on an IO problem
+   */
+  private void createTestSubdirs() throws IOException {
+    testDirs = new Path[]{
+              path("/test/hadoop/a"),
+              path("/test/hadoop/b"),
+              path("/test/hadoop/c/1"),
+      };
+
+    assertPathDoesNotExist("test directory setup", testDirs[0]);
+    for (Path path : testDirs) {
+      mkdirs(path);
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListLevelTest() throws Exception {
+    createTestSubdirs();
+    FileStatus[] paths = fs.listStatus(path("/test"));
+    assertEquals(dumpStats("/test", paths), 1, paths.length);
+    assertEquals(path("/test/hadoop"), paths[0].getPath());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListLevelTestHadoop() throws Exception {
+    createTestSubdirs();
+    FileStatus[] paths;
+    paths = fs.listStatus(path("/test/hadoop"));
+    String stats = dumpStats("/test/hadoop", paths);
+    assertEquals("Paths.length wrong in " + stats, 3, paths.length);
+    assertEquals("Path element[0] wrong: " + stats, path("/test/hadoop/a"),
+                 paths[0].getPath());
+    assertEquals("Path element[1] wrong: " + stats, path("/test/hadoop/b"),
+                 paths[1].getPath());
+    assertEquals("Path element[2] wrong: " + stats, path("/test/hadoop/c"),
+                 paths[2].getPath());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListStatusEmptyDirectory() throws Exception {
+    createTestSubdirs();
+    FileStatus[] paths;
+    paths = fs.listStatus(path("/test/hadoop/a"));
+    assertEquals(dumpStats("/test/hadoop/a", paths), 0,
+                 paths.length);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListStatusFile() throws Exception {
+    describe("Create a single file under /test;" +
+             " assert that listStatus(/test) finds it");
+    Path file = path("/test/filename");
+    createFile(file);
+    FileStatus[] pathStats = fs.listStatus(file);
+    assertEquals(dumpStats("/test/", pathStats),
+                 1,
+                 pathStats.length);
+    //and assert that the len of that ls'd path is the same as the original
+    FileStatus lsStat = pathStats[0];
+    assertEquals("Wrong file len in listing of " + lsStat,
+      data.length, lsStat.getLen());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListEmptyRoot() throws Throwable {
+    describe("Empty the root dir and verify that an LS / returns {}");
+    cleanup("testListEmptyRoot", fs, "/test");
+    cleanup("testListEmptyRoot", fs, "/user");
+    FileStatus[] fileStatuses = fs.listStatus(path("/"));
+    assertEquals("Non-empty root" + dumpStats("/", fileStatuses),
+                 0,
+                 fileStatuses.length);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListNonEmptyRoot() throws Throwable {
+    Path test = path("/test");
+    touch(fs, test);
+    FileStatus[] fileStatuses = fs.listStatus(path("/"));
+    String stats = dumpStats("/", fileStatuses);
+    assertEquals("Wrong #of root children" + stats, 1, fileStatuses.length);
+    FileStatus status = fileStatuses[0];
+    assertEquals("Wrong path value" + stats,test, status.getPath());
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListStatusRootDir() throws Throwable {
+    Path dir = path("/");
+    Path child = path("/test");
+    touch(fs, child);
+    assertListStatusFinds(fs, dir, child);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListStatusFiltered() throws Throwable {
+    Path dir = path("/");
+    Path child = path("/test");
+    touch(fs, child);
+    FileStatus[] stats = fs.listStatus(dir, new AcceptAllFilter());
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.toString()).append('\n');
+      if (stat.getPath().equals(child)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + child
+                      + " not found in directory " + dir + ":" + builder,
+                      found);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemPartitionedUploads.java

@@ -0,0 +1,442 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.SwiftProtocolConstants;
+import org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Test;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPathExists;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+
+/**
+ * Test partitioned uploads.
+ * This is done by forcing a very small partition size and verifying that it
+ * is picked up.
+ */
+public class TestSwiftFileSystemPartitionedUploads extends
+                                                   SwiftFileSystemBaseTest {
+
+  public static final String WRONG_PARTITION_COUNT =
+    "wrong number of partitions written into ";
+  public static final int PART_SIZE = 1;
+  public static final int PART_SIZE_BYTES = PART_SIZE * 1024;
+  public static final int BLOCK_SIZE = 1024;
+  private URI uri;
+
+  @Override
+  protected Configuration createConfiguration() {
+    Configuration conf = super.createConfiguration();
+    //set the partition size to 1 KB
+    conf.setInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE, PART_SIZE);
+    return conf;
+  }
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testPartitionPropertyPropagatesToConf() throws Throwable {
+    assertEquals(1,
+                 getConf().getInt(SwiftProtocolConstants.SWIFT_PARTITION_SIZE,
+                                  0));
+  }
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testPartionPropertyPropagatesToStore() throws Throwable {
+    assertEquals(1, fs.getStore().getPartsizeKB());
+  }
+
+  /**
+   * tests functionality for big files ( > 5Gb) upload
+   */
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testFilePartUpload() throws Throwable {
+
+    final Path path = new Path("/test/testFilePartUpload");
+
+    int len = 8192;
+    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+    FSDataOutputStream out = fs.create(path,
+                                       false,
+                                       getBufferSize(),
+                                       (short) 1,
+                                       BLOCK_SIZE);
+
+    try {
+      int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+      assertPartitionsWritten("Startup", out, 0);
+      //write 2048
+      int firstWriteLen = 2048;
+      out.write(src, 0, firstWriteLen);
+      //assert
+      long expected = getExpectedPartitionsWritten(firstWriteLen,
+                                                   PART_SIZE_BYTES,
+                                                   false);
+      SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+                       expected);
+      assertPartitionsWritten("First write completed", out, expected);
+      //write the rest
+      int remainder = len - firstWriteLen;
+      SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+      out.write(src, firstWriteLen, remainder);
+      expected =
+        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+      assertPartitionsWritten("Remaining data", out, expected);
+      out.close();
+      expected =
+        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+      assertPartitionsWritten("Stream closed", out, expected);
+
+      Header[] headers = fs.getStore().getObjectHeaders(path, true);
+      for (Header header : headers) {
+        LOG.info(header.toString());
+      }
+
+      byte[] dest = readDataset(fs, path, len);
+      LOG.info("Read dataset from " + path + ": data length =" + len);
+      //compare data
+      SwiftTestUtils.compareByteArrays(src, dest, len);
+      FileStatus status;
+
+      final Path qualifiedPath = path.makeQualified(fs);
+      status = fs.getFileStatus(qualifiedPath);
+      //now see what block location info comes back.
+      //This will vary depending on the Swift version, so the results
+      //aren't checked -merely that the test actually worked
+      BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+      assertNotNull("Null getFileBlockLocations()", locations);
+      assertTrue("empty array returned for getFileBlockLocations()",
+                 locations.length > 0);
+
+      //last bit of test -which seems to play up on partitions, which we download
+      //to a skip
+      try {
+        validatePathLen(path, len);
+      } catch (AssertionError e) {
+        //downgrade to a skip
+        throw new AssumptionViolatedException(e, null);
+      }
+
+    } finally {
+      IOUtils.closeStream(out);
+    }
+  }
+  /**
+   * tests functionality for big files ( > 5Gb) upload
+   */
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testFilePartUploadNoLengthCheck() throws IOException, URISyntaxException {
+
+    final Path path = new Path("/test/testFilePartUploadLengthCheck");
+
+    int len = 8192;
+    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+    FSDataOutputStream out = fs.create(path,
+                                       false,
+                                       getBufferSize(),
+                                       (short) 1,
+                                       BLOCK_SIZE);
+
+    try {
+      int totalPartitionsToWrite = len / PART_SIZE_BYTES;
+      assertPartitionsWritten("Startup", out, 0);
+      //write 2048
+      int firstWriteLen = 2048;
+      out.write(src, 0, firstWriteLen);
+      //assert
+      long expected = getExpectedPartitionsWritten(firstWriteLen,
+                                                   PART_SIZE_BYTES,
+                                                   false);
+      SwiftUtils.debug(LOG, "First write: predict %d partitions written",
+                       expected);
+      assertPartitionsWritten("First write completed", out, expected);
+      //write the rest
+      int remainder = len - firstWriteLen;
+      SwiftUtils.debug(LOG, "remainder: writing: %d bytes", remainder);
+
+      out.write(src, firstWriteLen, remainder);
+      expected =
+        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, false);
+      assertPartitionsWritten("Remaining data", out, expected);
+      out.close();
+      expected =
+        getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+      assertPartitionsWritten("Stream closed", out, expected);
+
+      Header[] headers = fs.getStore().getObjectHeaders(path, true);
+      for (Header header : headers) {
+        LOG.info(header.toString());
+      }
+
+      byte[] dest = readDataset(fs, path, len);
+      LOG.info("Read dataset from " + path + ": data length =" + len);
+      //compare data
+      SwiftTestUtils.compareByteArrays(src, dest, len);
+      FileStatus status = fs.getFileStatus(path);
+
+      //now see what block location info comes back.
+      //This will vary depending on the Swift version, so the results
+      //aren't checked -merely that the test actually worked
+      BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
+      assertNotNull("Null getFileBlockLocations()", locations);
+      assertTrue("empty array returned for getFileBlockLocations()",
+                 locations.length > 0);
+    } finally {
+      IOUtils.closeStream(out);
+    }
+  }
+
+  private FileStatus validatePathLen(Path path, int len) throws IOException {
+    //verify that the length is what was written in a direct status check
+    final Path qualifiedPath = path.makeQualified(fs);
+    FileStatus[] parentDirListing = fs.listStatus(qualifiedPath.getParent());
+    StringBuilder listing = lsToString(parentDirListing);
+    String parentDirLS = listing.toString();
+    FileStatus status = fs.getFileStatus(qualifiedPath);
+    assertEquals("Length of written file " + qualifiedPath
+                 + " from status check " + status
+                 + " in dir " + listing,
+                 len,
+                 status.getLen());
+    String fileInfo = qualifiedPath + "  " + status;
+    assertFalse("File claims to be a directory " + fileInfo,
+                status.isDir());
+
+    FileStatus listedFileStat = resolveChild(parentDirListing, qualifiedPath);
+    assertNotNull("Did not find " + path + " in " + parentDirLS,
+                  listedFileStat);
+    //file is in the parent dir. Now validate it's stats
+    assertEquals("Wrong len for " + path + " in listing " + parentDirLS,
+                 len,
+                 listedFileStat.getLen());
+    listedFileStat.toString();
+    return status;
+  }
+
+  private FileStatus resolveChild(FileStatus[] parentDirListing,
+                                  Path childPath) {
+    FileStatus listedFileStat = null;
+    for (FileStatus stat : parentDirListing) {
+      if (stat.getPath().equals(childPath)) {
+        listedFileStat = stat;
+      }
+    }
+    return listedFileStat;
+  }
+
+  private StringBuilder lsToString(FileStatus[] parentDirListing) {
+    StringBuilder listing = new StringBuilder();
+    for (FileStatus stat : parentDirListing) {
+      listing.append(stat).append("\n");
+    }
+    return listing;
+  }
+
+  /**
+   * Calculate the #of partitions expected from the upload
+   * @param uploaded number of bytes uploaded
+   * @param partSizeBytes the partition size
+   * @param closed whether or not the stream has closed
+   * @return the expected number of partitions, for use in assertions.
+   */
+  private int getExpectedPartitionsWritten(long uploaded,
+                                           int partSizeBytes,
+                                           boolean closed) {
+    //#of partitions in total
+    int partitions = (int) (uploaded / partSizeBytes);
+    //#of bytes past the last partition
+    int remainder = (int) (uploaded % partSizeBytes);
+    if (closed) {
+      //all data is written, so if there was any remainder, it went up
+      //too
+      return partitions + ((remainder > 0) ? 1 : 0);
+    } else {
+      //not closed. All the remainder is buffered,
+      return partitions;
+    }
+  }
+
+  private int getBufferSize() {
+    return fs.getConf().getInt("io.file.buffer.size", 4096);
+  }
+
+  /**
+   * Test sticks up a very large partitioned file and verifies that
+   * it comes back unchanged.
+   * @throws Throwable
+   */
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testManyPartitionedFile() throws Throwable {
+    final Path path = new Path("/test/testManyPartitionedFile");
+
+    int len = PART_SIZE_BYTES * 15;
+    final byte[] src = SwiftTestUtils.dataset(len, 32, 144);
+    FSDataOutputStream out = fs.create(path,
+                                       false,
+                                       getBufferSize(),
+                                       (short) 1,
+                                       BLOCK_SIZE);
+
+    out.write(src, 0, src.length);
+    int expected =
+      getExpectedPartitionsWritten(len, PART_SIZE_BYTES, true);
+    out.close();
+    assertPartitionsWritten("write completed", out, expected);
+    assertEquals("too few bytes written", len,
+                 SwiftNativeFileSystem.getBytesWritten(out));
+    assertEquals("too few bytes uploaded", len,
+                 SwiftNativeFileSystem.getBytesUploaded(out));
+    //now we verify that the data comes back. If it
+    //doesn't, it means that the ordering of the partitions
+    //isn't right
+    byte[] dest = readDataset(fs, path, len);
+    //compare data
+    SwiftTestUtils.compareByteArrays(src, dest, len);
+    //finally, check the data
+    FileStatus[] stats = fs.listStatus(path);
+    assertEquals("wrong entry count in "
+                 + SwiftTestUtils.dumpStats(path.toString(), stats),
+                 expected, stats.length);
+  }
+
+  /**
+   * Test that when a partitioned file is overwritten by a smaller one,
+   * all the old partitioned files go away
+   * @throws Throwable
+   */
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testOverwritePartitionedFile() throws Throwable {
+    final Path path = new Path("/test/testOverwritePartitionedFile");
+
+    final int len1 = 8192;
+    final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+    FSDataOutputStream out = fs.create(path,
+                                       false,
+                                       getBufferSize(),
+                                       (short) 1,
+                                       1024);
+    out.write(src1, 0, len1);
+    out.close();
+    long expected = getExpectedPartitionsWritten(len1,
+                                                 PART_SIZE_BYTES,
+                                                 false);
+    assertPartitionsWritten("initial upload", out, expected);
+    assertExists("Exists", path);
+    FileStatus status = fs.getFileStatus(path);
+    assertEquals("Length", len1, status.getLen());
+    //now write a shorter file with a different dataset
+    final int len2 = 4095;
+    final byte[] src2 = SwiftTestUtils.dataset(len2, 'a', 'z');
+    out = fs.create(path,
+                    true,
+                    getBufferSize(),
+                    (short) 1,
+                    1024);
+    out.write(src2, 0, len2);
+    out.close();
+    status = fs.getFileStatus(path);
+    assertEquals("Length", len2, status.getLen());
+    byte[] dest = readDataset(fs, path, len2);
+    //compare data
+    SwiftTestUtils.compareByteArrays(src2, dest, len2);
+  }
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testDeleteSmallPartitionedFile() throws Throwable {
+    final Path path = new Path("/test/testDeleteSmallPartitionedFile");
+
+    final int len1 = 1024;
+    final byte[] src1 = SwiftTestUtils.dataset(len1, 'A', 'Z');
+    SwiftTestUtils.writeDataset(fs, path, src1, len1, 1024, false);
+    assertExists("Exists", path);
+
+    Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+    Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+    String ls = SwiftTestUtils.ls(fs, path);
+    assertExists("Partition 0001 Exists in " + ls, part_0001);
+    assertPathDoesNotExist("partition 0002 found under " + ls, part_0002);
+    assertExists("Partition 0002 Exists in " + ls, part_0001);
+    fs.delete(path, false);
+    assertPathDoesNotExist("deleted file still there", path);
+    ls = SwiftTestUtils.ls(fs, path);
+    assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+  }
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testDeletePartitionedFile() throws Throwable {
+    final Path path = new Path("/test/testDeletePartitionedFile");
+
+    SwiftTestUtils.writeDataset(fs, path, data, data.length, 1024, false);
+    assertExists("Exists", path);
+
+    Path part_0001 = new Path(path, SwiftUtils.partitionFilenameFromNumber(1));
+    Path part_0002 = new Path(path, SwiftUtils.partitionFilenameFromNumber(2));
+    String ls = SwiftTestUtils.ls(fs, path);
+    assertExists("Partition 0001 Exists in " + ls, part_0001);
+    assertExists("Partition 0002 Exists in " + ls, part_0001);
+    fs.delete(path, false);
+    assertPathDoesNotExist("deleted file still there", path);
+    ls = SwiftTestUtils.ls(fs, path);
+    assertPathDoesNotExist("partition 0001 file still under " + ls, part_0001);
+    assertPathDoesNotExist("partition 0002 file still under " + ls, part_0002);
+  }
+
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testRenamePartitionedFile() throws Throwable {
+    Path src = new Path("/test/testRenamePartitionedFileSrc");
+
+    int len = data.length;
+    SwiftTestUtils.writeDataset(fs, src, data, len, 1024, false);
+    assertExists("Exists", src);
+
+    String partOneName = SwiftUtils.partitionFilenameFromNumber(1);
+    Path srcPart = new Path(src, partOneName);
+    Path dest = new Path("/test/testRenamePartitionedFileDest");
+    Path destPart = new Path(src, partOneName);
+    assertExists("Partition Exists", srcPart);
+    fs.rename(src, dest);
+    assertPathExists(fs, "dest file missing", dest);
+    FileStatus status = fs.getFileStatus(dest);
+    assertEquals("Length of renamed file is wrong", len, status.getLen());
+    byte[] destData = readDataset(fs, dest, len);
+    //compare data
+    SwiftTestUtils.compareByteArrays(data, destData, len);
+    String srcLs = SwiftTestUtils.ls(fs, src);
+    String destLs = SwiftTestUtils.ls(fs, dest);
+
+    assertPathDoesNotExist("deleted file still found in " + srcLs, src);
+
+    assertPathDoesNotExist("partition file still found in " + srcLs, srcPart);
+  }
+
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRead.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeTextFile;
+
+/**
+ * Test filesystem read operations
+ */
+public class TestSwiftFileSystemRead extends SwiftFileSystemBaseTest {
+
+
+  /**
+   * Read past the end of a file: expect the operation to fail
+   * @throws IOException
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testOverRead() throws IOException {
+    final String message = "message";
+    final Path filePath = new Path("/test/file.txt");
+
+    writeTextFile(fs, filePath, message, false);
+
+    try {
+      readBytesToString(fs, filePath, 20);
+      fail("expected an exception");
+    } catch (EOFException e) {
+      //expected
+    }
+  }
+
+  /**
+   * Read and write some JSON
+   * @throws IOException
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRWJson() throws IOException {
+    final String message = "{" +
+                           " 'json': { 'i':43, 'b':true}," +
+                           " 's':'string'" +
+                           "}";
+    final Path filePath = new Path("/test/file.json");
+
+    writeTextFile(fs, filePath, message, false);
+    String readJson = readBytesToString(fs, filePath, message.length());
+    assertEquals(message,readJson);
+    //now find out where it is
+    FileStatus status = fs.getFileStatus(filePath);
+    BlockLocation[] locations = fs.getFileBlockLocations(status, 0, 10);
+  }
+
+  /**
+   * Read and write some XML
+   * @throws IOException
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRWXML() throws IOException {
+    final String message = "<x>" +
+                           " <json i='43' 'b'=true/>" +
+                           " string" +
+                           "</x>";
+    final Path filePath = new Path("/test/file.xml");
+
+    writeTextFile(fs, filePath, message, false);
+    String read = readBytesToString(fs, filePath, message.length());
+    assertEquals(message,read);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftFileSystemRename.java

@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.compareByteArrays;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.dataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readBytesToString;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.readDataset;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.writeDataset;
+
+public class TestSwiftFileSystemRename extends SwiftFileSystemBaseTest {
+
+  /**
+   * Rename a file into a directory
+   *
+   * @throws Exception
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameFileIntoExistingDirectory() throws Exception {
+    assumeRenameSupported();
+
+    Path src = path("/test/olddir/file");
+    createFile(src);
+    Path dst = path("/test/new/newdir");
+    fs.mkdirs(dst);
+    rename(src, dst, true, false, true);
+    Path newFile = path("/test/new/newdir/file");
+    if (!fs.exists(newFile)) {
+      String ls = ls(dst);
+      LOG.info(ls(path("/test/new")));
+      LOG.info(ls(path("/test/hadoop")));
+      fail("did not find " + newFile + " - directory: " + ls);
+    }
+    assertTrue("Destination changed",
+            fs.exists(path("/test/new/newdir/file")));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameFile() throws Exception {
+    assumeRenameSupported();
+
+    final Path old = new Path("/test/alice/file");
+    final Path newPath = new Path("/test/bob/file");
+    fs.mkdirs(newPath.getParent());
+    final FSDataOutputStream fsDataOutputStream = fs.create(old);
+    final byte[] message = "Some data".getBytes();
+    fsDataOutputStream.write(message);
+    fsDataOutputStream.close();
+
+    assertTrue(fs.exists(old));
+    rename(old, newPath, true, false, true);
+
+    final FSDataInputStream bobStream = fs.open(newPath);
+    final byte[] bytes = new byte[512];
+    final int read = bobStream.read(bytes);
+    bobStream.close();
+    final byte[] buffer = new byte[read];
+    System.arraycopy(bytes, 0, buffer, 0, read);
+    assertEquals(new String(message), new String(buffer));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameDirectory() throws Exception {
+    assumeRenameSupported();
+
+    final Path old = new Path("/test/data/logs");
+    final Path newPath = new Path("/test/var/logs");
+    fs.mkdirs(old);
+    fs.mkdirs(newPath.getParent());
+    assertTrue(fs.exists(old));
+    rename(old, newPath, true, false, true);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameTheSameDirectory() throws Exception {
+    assumeRenameSupported();
+
+    final Path old = new Path("/test/usr/data");
+    fs.mkdirs(old);
+    rename(old, old, false, true, true);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameDirectoryIntoExistingDirectory() throws Exception {
+    assumeRenameSupported();
+
+    Path src = path("/test/olddir/dir");
+    fs.mkdirs(src);
+    createFile(path("/test/olddir/dir/file1"));
+    createFile(path("/test/olddir/dir/subdir/file2"));
+
+    Path dst = path("/test/new/newdir");
+    fs.mkdirs(dst);
+    //this renames into a child
+    rename(src, dst, true, false, true);
+    assertExists("new dir", path("/test/new/newdir/dir"));
+    assertExists("Renamed nested file1", path("/test/new/newdir/dir/file1"));
+    assertPathDoesNotExist("Nested file1 should have been deleted",
+            path("/test/olddir/dir/file1"));
+    assertExists("Renamed nested subdir",
+            path("/test/new/newdir/dir/subdir/"));
+    assertExists("file under subdir",
+            path("/test/new/newdir/dir/subdir/file2"));
+
+    assertPathDoesNotExist("Nested /test/hadoop/dir/subdir/file2 still exists",
+            path("/test/olddir/dir/subdir/file2"));
+  }
+
+  /**
+   * trying to rename a directory onto itself should fail,
+   * preserving everything underneath.
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameDirToSelf() throws Throwable {
+    assumeRenameSupported();
+    Path parentdir = path("/test/parentdir");
+    fs.mkdirs(parentdir);
+    Path child = new Path(parentdir, "child");
+    createFile(child);
+
+    rename(parentdir, parentdir, false, true, true);
+    //verify the child is still there
+    assertIsFile(child);
+  }
+
+  /**
+   * Assert that root directory renames are not allowed
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameRootDirForbidden() throws Exception {
+    assumeRenameSupported();
+    rename(path("/"),
+            path("/test/newRootDir"),
+            false, true, false);
+  }
+
+  /**
+   * Assert that renaming a parent directory to be a child
+   * of itself is forbidden
+   *
+   * @throws Exception on failures
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameChildDirForbidden() throws Exception {
+    assumeRenameSupported();
+
+    Path parentdir = path("/test/parentdir");
+    fs.mkdirs(parentdir);
+    Path childFile = new Path(parentdir, "childfile");
+    createFile(childFile);
+    //verify one level down
+    Path childdir = new Path(parentdir, "childdir");
+    rename(parentdir, childdir, false, true, false);
+    //now another level
+    fs.mkdirs(childdir);
+    Path childchilddir = new Path(childdir, "childdir");
+    rename(parentdir, childchilddir, false, true, false);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameFileAndVerifyContents() throws IOException {
+    assumeRenameSupported();
+
+    final Path filePath = new Path("/test/home/user/documents/file.txt");
+    final Path newFilePath = new Path("/test/home/user/files/file.txt");
+    mkdirs(newFilePath.getParent());
+    int len = 1024;
+    byte[] dataset = dataset(len, 'A', 26);
+    writeDataset(fs, filePath, dataset, len, len, false);
+    rename(filePath, newFilePath, true, false, true);
+    byte[] dest = readDataset(fs, newFilePath, len);
+    compareByteArrays(dataset, dest, len);
+    String reread = readBytesToString(fs, newFilePath, 20);
+  }
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testMoveFileUnderParent() throws Throwable {
+    if (!renameSupported()) return;
+    Path filepath = path("test/file");
+    createFile(filepath);
+    //HDFS expects rename src, src -> true
+    rename(filepath, filepath, true, true, true);
+    //verify the file is still there
+    assertIsFile(filepath);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testMoveDirUnderParent() throws Throwable {
+    if (!renameSupported()) {
+      return;
+    }
+    Path testdir = path("test/dir");
+    fs.mkdirs(testdir);
+    Path parent = testdir.getParent();
+    //the outcome here is ambiguous, so is not checked
+    fs.rename(testdir, parent);
+    assertExists("Source directory has been deleted ", testdir);
+  }
+
+  /**
+   * trying to rename a file onto itself should succeed (it's a no-op)
+   */
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameFileToSelf() throws Throwable {
+    if (!renameSupported()) return;
+    Path filepath = path("test/file");
+    createFile(filepath);
+    //HDFS expects rename src, src -> true
+    rename(filepath, filepath, true, true, true);
+    //verify the file is still there
+    assertIsFile(filepath);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenamedConsistence() throws IOException {
+    assumeRenameSupported();
+    describe("verify that overwriting a file with new data doesn't impact" +
+            " the existing content");
+
+    final Path filePath = new Path("/test/home/user/documents/file.txt");
+    final Path newFilePath = new Path("/test/home/user/files/file.txt");
+    mkdirs(newFilePath.getParent());
+    int len = 1024;
+    byte[] dataset = dataset(len, 'A', 26);
+    byte[] dataset2 = dataset(len, 'a', 26);
+    writeDataset(fs, filePath, dataset, len, len, false);
+    rename(filePath, newFilePath, true, false, true);
+    SwiftTestUtils.writeAndRead(fs, filePath, dataset2, len, len, false, true);
+    byte[] dest = readDataset(fs, newFilePath, len);
+    compareByteArrays(dataset, dest, len);
+    String reread = readBytesToString(fs, newFilePath, 20);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRenameMissingFile() throws Throwable {
+    assumeRenameSupported();
+    Path path = path("/test/RenameMissingFile");
+    Path path2 = path("/test/RenameMissingFileDest");
+    mkdirs(path("test"));
+    rename(path, path2, false, false, false);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/TestSwiftObjectPath.java

@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.swift;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.http.RestClientBindings;
+import org.apache.hadoop.fs.swift.http.SwiftRestClient;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftUtils;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Unit tests for SwiftObjectPath class.
+ */
+public class TestSwiftObjectPath implements SwiftTestConstants {
+  private static final Log LOG = LogFactory.getLog(TestSwiftObjectPath.class);
+
+  /**
+   * What an endpoint looks like. This is derived from a (valid)
+   * rackspace endpoint address
+   */
+  private static final String ENDPOINT =
+          "https://storage101.region1.example.org/v1/MossoCloudFS_9fb40cc0-1234-5678-9abc-def000c9a66";
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testParsePath() throws Exception {
+    final String pathString = "/home/user/files/file1";
+    final Path path = new Path(pathString);
+    final URI uri = new URI("http://container.localhost");
+    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+    final SwiftObjectPath actual = new SwiftObjectPath(
+            RestClientBindings.extractContainerName(uri),
+            pathString);
+
+    assertEquals(expected, actual);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testParseUrlPath() throws Exception {
+    final String pathString = "swift://container.service1/home/user/files/file1";
+    final URI uri = new URI(pathString);
+    final Path path = new Path(pathString);
+    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+    final SwiftObjectPath actual = new SwiftObjectPath(
+            RestClientBindings.extractContainerName(uri),
+            "/home/user/files/file1");
+
+    assertEquals(expected, actual);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testParseAuthenticatedUrl() throws Exception {
+    final String pathString = "swift://container.service1/v2/AUTH_00345h34l93459y4/home/tom/documents/finance.docx";
+    final URI uri = new URI(pathString);
+    final Path path = new Path(pathString);
+    final SwiftObjectPath expected = SwiftObjectPath.fromPath(uri, path);
+    final SwiftObjectPath actual = new SwiftObjectPath(
+            RestClientBindings.extractContainerName(uri),
+            "/home/tom/documents/finance.docx");
+
+    assertEquals(expected, actual);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testConvertToPath() throws Throwable {
+    String initialpath = "/dir/file1";
+    Path ipath = new Path(initialpath);
+    SwiftObjectPath objectPath = SwiftObjectPath.fromPath(new URI(initialpath),
+            ipath);
+    URI endpoint = new URI(ENDPOINT);
+    URI uri = SwiftRestClient.pathToURI(objectPath, endpoint);
+    LOG.info("Inital Hadoop Path =" + initialpath);
+    LOG.info("Merged URI=" + uri);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRootDirProbeEmptyPath() throws Throwable {
+    SwiftObjectPath object=new SwiftObjectPath("container","");
+    assertTrue(SwiftUtils.isRootDir(object));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testRootDirProbeRootPath() throws Throwable {
+    SwiftObjectPath object=new SwiftObjectPath("container","/");
+    assertTrue(SwiftUtils.isRootDir(object));
+  }
+
+  private void assertParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+    assertTrue(p1.toString() + " is not a parent of " + p2 ,p1.isEqualToOrParentOf(
+      p2));
+  }
+
+  private void assertNotParentOf(SwiftObjectPath p1, SwiftObjectPath p2) {
+    assertFalse(p1.toString() + " is a parent of " + p2, p1.isEqualToOrParentOf(
+      p2));
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testChildOfProbe() throws Throwable {
+    SwiftObjectPath parent = new SwiftObjectPath("container",
+                                                 "/parent");
+    SwiftObjectPath parent2 = new SwiftObjectPath("container",
+                                                 "/parent2");
+    SwiftObjectPath child = new SwiftObjectPath("container",
+                                                 "/parent/child");
+    SwiftObjectPath sibling = new SwiftObjectPath("container",
+                                                 "/parent/sibling");
+    SwiftObjectPath grandchild = new SwiftObjectPath("container",
+                                                     "/parent/child/grandchild");
+    assertParentOf(parent, child);
+    assertParentOf(parent, grandchild);
+    assertParentOf(child, grandchild);
+    assertParentOf(parent, parent);
+    assertNotParentOf(child, parent);
+    assertParentOf(child, child);
+    assertNotParentOf(parent, parent2);
+    assertNotParentOf(grandchild, parent);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testChildOfRoot() throws Throwable {
+    SwiftObjectPath root = new SwiftObjectPath("container", "/");
+    SwiftObjectPath child = new SwiftObjectPath("container", "child");
+    SwiftObjectPath grandchild = new SwiftObjectPath("container",
+                                                     "/child/grandchild");
+    assertParentOf(root, child);
+    assertParentOf(root, grandchild);
+    assertParentOf(child, grandchild);
+    assertParentOf(root, root);
+    assertNotParentOf(child, root);
+    assertParentOf(child, child);
+    assertNotParentOf(grandchild, root);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestSwiftFileSystemDirectoriesHdfs2.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.swift.TestSwiftFileSystemDirectories;
+import org.apache.hadoop.fs.swift.snative.SwiftFileStatus;
+
+/**
+ * Add some HDFS-2 only assertions to {@link TestSwiftFileSystemDirectories}
+ */
+public class TestSwiftFileSystemDirectoriesHdfs2 extends
+                                                 TestSwiftFileSystemDirectories {
+
+
+  /**
+   * make assertions about fields that only appear in
+   * FileStatus in HDFS2
+   * @param stat status to look at
+   */
+  protected void extraStatusAssertions(SwiftFileStatus stat) {
+    //HDFS2
+    assertTrue("isDirectory(): Not a directory: " + stat, stat.isDirectory());
+    assertFalse("isFile(): declares itself a file: " + stat, stat.isFile());
+    assertFalse("isFile(): declares itself a file: " + stat, stat.isSymlink());
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/hdfs2/TestV2LsOperations.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.hdfs2;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestV2LsOperations extends SwiftFileSystemBaseTest {
+
+  private Path[] testDirs;
+
+  /**
+   * Setup creates dirs under test/hadoop
+   * @throws Exception
+   */
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    //delete the test directory
+    Path test = path("/test");
+    fs.delete(test, true);
+    mkdirs(test);
+  }
+
+  /**
+   * Create subdirectories and files under test/ for those tests
+   * that want them. Doing so adds overhead to setup and teardown,
+   * so should only be done for those tests that need them.
+   * @throws IOException on an IO problem
+   */
+  private void createTestSubdirs() throws IOException {
+    testDirs = new Path[]{
+      path("/test/hadoop/a"),
+      path("/test/hadoop/b"),
+      path("/test/hadoop/c/1"),
+    };
+    assertPathDoesNotExist("test directory setup", testDirs[0]);
+    for (Path path : testDirs) {
+      mkdirs(path);
+    }
+  }
+
+  /**
+   * To get this project to compile under Hadoop 1, this code needs to be
+   * commented out
+   *
+   *
+   * @param fs filesystem
+   * @param dir dir
+   * @param subdir subdir
+   * @param recursive recurse?
+   * @throws IOException IO problems
+   */
+  public static void assertListFilesFinds(FileSystem fs,
+                                          Path dir,
+                                          Path subdir,
+                                          boolean recursive) throws IOException {
+    RemoteIterator<LocatedFileStatus> iterator =
+      fs.listFiles(dir, recursive);
+    boolean found = false;
+    int entries = 0;
+    StringBuilder builder = new StringBuilder();
+    while (iterator.hasNext()) {
+      LocatedFileStatus next = iterator.next();
+      entries++;
+      builder.append(next.toString()).append('\n');
+      if (next.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+               + " not found in directory " + dir + " : "
+               + " entries=" + entries
+               + " content"
+               + builder.toString(),
+               found);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListFilesRootDir() throws Throwable {
+    Path dir = path("/");
+    Path child = new Path(dir, "test");
+    fs.delete(child, true);
+    SwiftTestUtils.writeTextFile(fs, child, "text", false);
+    assertListFilesFinds(fs, dir, child, false);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListFilesSubDir() throws Throwable {
+    createTestSubdirs();
+    Path dir = path("/test");
+    Path child = new Path(dir, "text.txt");
+    SwiftTestUtils.writeTextFile(fs, child, "text", false);
+    assertListFilesFinds(fs, dir, child, false);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testListFilesRecursive() throws Throwable {
+    createTestSubdirs();
+    Path dir = path("/test");
+    Path child = new Path(dir, "hadoop/a/a.txt");
+    SwiftTestUtils.writeTextFile(fs, child, "text", false);
+    assertListFilesFinds(fs, dir, child, true);
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestRestClientBindings.java

@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.exceptions.SwiftConfigurationException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Properties;
+
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_AUTH_URL;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_PASSWORD;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.DOT_USERNAME;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_AUTH_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_CONTAINER_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTPS_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_HTTP_PORT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_PASSWORD_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_REGION_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_SERVICE_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_TENANT_PROPERTY;
+import static org.apache.hadoop.fs.swift.http.SwiftProtocolConstants.SWIFT_USERNAME_PROPERTY;
+import static org.apache.hadoop.fs.swift.util.SwiftTestUtils.assertPropertyEquals;
+
+public class TestRestClientBindings extends Assert
+  implements SwiftTestConstants {
+
+  private static final String SERVICE = "sname";
+  private static final String CONTAINER = "cname";
+  private static final String FS_URI = "swift://"
+          + CONTAINER + "." + SERVICE + "/";
+  private static final String AUTH_URL = "http://localhost:8080/auth";
+  private static final String USER = "user";
+  private static final String PASS = "pass";
+  private static final String TENANT = "tenant";
+  private URI filesysURI;
+  private Configuration conf;
+
+  @Before
+  public void setup() throws URISyntaxException {
+    filesysURI = new URI(FS_URI);
+    conf = new Configuration(true);
+    setInstanceVal(conf, SERVICE, DOT_AUTH_URL, AUTH_URL);
+    setInstanceVal(conf, SERVICE, DOT_USERNAME, USER);
+    setInstanceVal(conf, SERVICE, DOT_PASSWORD, PASS);
+  }
+
+  private void setInstanceVal(Configuration conf,
+                              String host,
+                              String key,
+                              String val) {
+    String instance = RestClientBindings.buildSwiftInstancePrefix(host);
+    String confkey = instance
+            + key;
+    conf.set(confkey, val);
+  }
+
+  public void testPrefixBuilder() throws Throwable {
+    String built = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+    assertEquals("fs.swift.service." + SERVICE, built);
+  }
+
+  public void testBindAgainstConf() throws Exception {
+    Properties props = RestClientBindings.bind(filesysURI, conf);
+    assertPropertyEquals(props, SWIFT_CONTAINER_PROPERTY, CONTAINER);
+    assertPropertyEquals(props, SWIFT_SERVICE_PROPERTY, SERVICE);
+    assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+    assertPropertyEquals(props, SWIFT_AUTH_PROPERTY, AUTH_URL);
+    assertPropertyEquals(props, SWIFT_USERNAME_PROPERTY, USER);
+    assertPropertyEquals(props, SWIFT_PASSWORD_PROPERTY, PASS);
+
+    assertPropertyEquals(props, SWIFT_TENANT_PROPERTY, null);
+    assertPropertyEquals(props, SWIFT_REGION_PROPERTY, null);
+    assertPropertyEquals(props, SWIFT_HTTP_PORT_PROPERTY, null);
+    assertPropertyEquals(props, SWIFT_HTTPS_PORT_PROPERTY, null);
+  }
+
+  public void expectBindingFailure(URI fsURI, Configuration config) {
+    try {
+      Properties binding = RestClientBindings.bind(fsURI, config);
+      //if we get here, binding didn't fail- there is something else.
+      //list the properties but not the values.
+      StringBuilder details = new StringBuilder() ;
+      for (Object key: binding.keySet()) {
+        details.append(key.toString()).append(" ");
+      }
+      fail("Expected a failure, got the binding [ "+ details+"]");
+    } catch (SwiftConfigurationException expected) {
+
+    }
+  }
+
+  public void testBindAgainstConfMissingInstance() throws Exception {
+    Configuration badConf = new Configuration();
+    expectBindingFailure(filesysURI, badConf);
+  }
+
+
+/* Hadoop 2.x+ only, as conf.unset() isn't a v1 feature
+  public void testBindAgainstConfIncompleteInstance() throws Exception {
+    String instance = RestClientBindings.buildSwiftInstancePrefix(SERVICE);
+    conf.unset(instance + DOT_PASSWORD);
+    expectBindingFailure(filesysURI, conf);
+  }
+*/
+
+  @Test(expected = SwiftConfigurationException.class)
+  public void testDottedServiceURL() throws Exception {
+    RestClientBindings.bind(new URI("swift://hadoop.apache.org/"), conf);
+  }
+
+  @Test(expected = SwiftConfigurationException.class)
+  public void testMissingServiceURL() throws Exception {
+    RestClientBindings.bind(new URI("swift:///"), conf);
+  }
+
+  /**
+   * inner test method that expects container extraction to fail
+   * -if not prints a meaningful error message.
+   *
+   * @param hostname hostname to parse
+   */
+  private static void expectExtractContainerFail(String hostname) {
+    try {
+      String container = RestClientBindings.extractContainerName(hostname);
+      fail("Expected an error -got a container of '" + container
+              + "' from " + hostname);
+    } catch (SwiftConfigurationException expected) {
+      //expected
+    }
+  }
+
+  /**
+   * inner test method that expects service extraction to fail
+   * -if not prints a meaningful error message.
+   *
+   * @param hostname hostname to parse
+   */
+  public static void expectExtractServiceFail(String hostname) {
+    try {
+      String service = RestClientBindings.extractServiceName(hostname);
+      fail("Expected an error -got a service of '" + service
+              + "' from " + hostname);
+    } catch (SwiftConfigurationException expected) {
+      //expected
+    }
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testEmptyHostname() throws Throwable {
+    expectExtractContainerFail("");
+    expectExtractServiceFail("");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testDot() throws Throwable {
+    expectExtractContainerFail(".");
+    expectExtractServiceFail(".");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testSimple() throws Throwable {
+    expectExtractContainerFail("simple");
+    expectExtractServiceFail("simple");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testTrailingDot() throws Throwable {
+    expectExtractServiceFail("simple.");
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testLeadingDot() throws Throwable {
+    expectExtractServiceFail(".leading");
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/http/TestSwiftRestClient.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.http;
+
+import org.apache.commons.httpclient.Header;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.SwiftTestConstants;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftObjectPath;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+
+public class TestSwiftRestClient implements SwiftTestConstants {
+  private static final Log LOG =
+          LogFactory.getLog(TestSwiftRestClient.class);
+
+  private Configuration conf;
+  private boolean runTests;
+  private URI serviceURI;
+
+  @Before
+  public void setup() throws IOException {
+    conf = new Configuration();
+    runTests = SwiftTestUtils.hasServiceURI(conf);
+    if (runTests) {
+      serviceURI = SwiftTestUtils.getServiceURI(conf);
+    }
+  }
+
+  protected void assumeEnabled() {
+    Assume.assumeTrue(runTests);
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testCreate() throws Throwable {
+    assumeEnabled();
+    SwiftRestClient client = createClient();
+  }
+
+  private SwiftRestClient createClient() throws IOException {
+    return SwiftRestClient.getInstance(serviceURI, conf);
+  }
+
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testAuthenticate() throws Throwable {
+    assumeEnabled();
+    SwiftRestClient client = createClient();
+    client.authenticate();
+  }
+
+  @Test(timeout = SWIFT_TEST_TIMEOUT)
+  public void testPutAndDelete() throws Throwable {
+    assumeEnabled();
+    SwiftRestClient client = createClient();
+    client.authenticate();
+    Path path = new Path("restTestPutAndDelete");
+    SwiftObjectPath sobject = SwiftObjectPath.fromPath(serviceURI, path);
+    byte[] stuff = new byte[1];
+    stuff[0] = 'a';
+    client.upload(sobject, new ByteArrayInputStream(stuff), stuff.length);
+    //check file exists
+    Duration head = new Duration();
+    Header[] responseHeaders = client.headRequest("expect success",
+                                                  sobject,
+                                                  SwiftRestClient.NEWEST);
+    head.finished();
+    LOG.info("head request duration " + head);
+    for (Header header: responseHeaders) {
+      LOG.info(header.toString());
+    }
+    //delete the file
+    client.delete(sobject);
+    //check file is gone
+    try {
+      Header[] headers = client.headRequest("expect fail",
+                                            sobject,
+                                            SwiftRestClient.NEWEST);
+      Assert.fail("Expected deleted file, but object is still present: "
+                  + sobject);
+    } catch (FileNotFoundException e) {
+      //expected
+    }
+    for (DurationStats stats: client.getOperationStatistics()) {
+      LOG.info(stats);
+    }
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/SwiftScaleTestBase.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.hadoop.fs.swift.SwiftFileSystemBaseTest;
+
+/**
+ * Base class for scale tests; here is where the common scale configuration
+ * keys are defined
+ */
+
+public class SwiftScaleTestBase extends SwiftFileSystemBaseTest {
+
+  public static final String SCALE_TEST = "scale.test.";
+  public static final String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count";
+  public static final long DEFAULT_OPERATION_COUNT = 10;
+
+  protected long getOperationCount() {
+    return getConf().getLong(KEY_OPERATION_COUNT, DEFAULT_OPERATION_COUNT);
+  }
+}

hadoop-tools/hadoop-openstack/src/test/java/org/apache/hadoop/fs/swift/scale/TestWriteManySmallFiles.java

@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.fs.swift.scale;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.swift.util.Duration;
+import org.apache.hadoop.fs.swift.util.DurationStats;
+import org.apache.hadoop.fs.swift.util.SwiftTestUtils;
+import org.junit.Test;
+
+public class TestWriteManySmallFiles extends SwiftScaleTestBase {
+
+  public static final Log LOG = LogFactory.getLog(TestWriteManySmallFiles.class);
+
+  @Test(timeout = SWIFT_BULK_IO_TEST_TIMEOUT)
+  public void testScaledWriteThenRead() throws Throwable {
+    Path dir = new Path("/test/manysmallfiles");
+    Duration rm1 = new Duration();
+    fs.delete(dir, true);
+    rm1.finished();
+    fs.mkdirs(dir);
+    Duration ls1 = new Duration();
+    fs.listStatus(dir);
+    ls1.finished();
+    long count = getOperationCount();
+    SwiftTestUtils.noteAction("Beginning Write of "+ count + " files ");
+    DurationStats writeStats = new DurationStats("write");
+    DurationStats readStats = new DurationStats("read");
+    String format = "%08d";
+    for (long l = 0; l < count; l++) {
+      String name = String.format(format, l);
+      Path p = new Path(dir, "part-" + name);
+      Duration d = new Duration();
+      SwiftTestUtils.writeTextFile(fs, p, name, false);
+      d.finished();
+      writeStats.add(d);
+      Thread.sleep(1000);
+    }
+    //at this point, the directory is full.
+    SwiftTestUtils.noteAction("Beginning ls");
+
+    Duration ls2 = new Duration();
+    FileStatus[] status2 = (FileStatus[]) fs.listStatus(dir);
+    ls2.finished();
+    assertEquals("Not enough entries in the directory", count, status2.length);
+
+    SwiftTestUtils.noteAction("Beginning read");
+
+    for (long l = 0; l < count; l++) {
+      String name = String.format(format, l);
+      Path p = new Path(dir, "part-" + name);
+      Duration d = new Duration();
+      String result = SwiftTestUtils.readBytesToString(fs, p, name.length());
+      assertEquals(name, result);
+      d.finished();
+      readStats.add(d);
+    }
+    //do a recursive delete
+    SwiftTestUtils.noteAction("Beginning delete");
+    Duration rm2 = new Duration();
+    fs.delete(dir, true);
+    rm2.finished();
+    //print the stats
+    LOG.info(String.format("'filesystem','%s'",fs.getUri()));
+    LOG.info(writeStats.toString());
+    LOG.info(readStats.toString());
+    LOG.info(String.format(
+      "'rm1',%d,'ls1',%d",
+      rm1.value(),
+      ls1.value()));
+    LOG.info(String.format(
+      "'rm2',%d,'ls2',%d",
+      rm2.value(),
+      ls2.value()));
+  }
+
+}

hadoop-tools/hadoop-openstack/src/test/resources/core-site.xml

@@ -0,0 +1,51 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~  or more contributor license agreements.  See the NOTICE file
+  ~  distributed with this work for additional information
+  ~  regarding copyright ownership.  The ASF licenses this file
+  ~  to you under the Apache License, Version 2.0 (the
+  ~  "License"); you may not use this file except in compliance
+  ~  with the License.  You may obtain a copy of the License at
+  ~
+  ~       http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~  Unless required by applicable law or agreed to in writing, software
+  ~  distributed under the License is distributed on an "AS IS" BASIS,
+  ~  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~  See the License for the specific language governing permissions and
+  ~  limitations under the License.
+  -->
+
+<!-- Values used when running unit tests.  This is mostly empty, to -->
+<!-- use of the default values, overriding the potentially -->
+<!-- user-editted core-site.xml in the conf/ directory.  -->
+
+<configuration>
+
+
+    <property>
+        <name>hadoop.tmp.dir</name>
+        <value>target/build/test</value>
+        <description>A base for other temporary directories.</description>
+        <final>true</final>
+    </property>
+
+    <!-- Turn security off for tests by default -->
+    <property>
+        <name>hadoop.security.authentication</name>
+        <value>simple</value>
+    </property>
+
+    <!--
+    To run these tests.
+
+    # Create a file auth-keys.xml  - DO NOT ADD TO REVISION CONTROL
+    # add the property test.fs.swift.name to point to a swift filesystem URL
+    # Add the credentials for the service you are testing against
+    -->
+    <include xmlns="http://www.w3.org/2001/XInclude"
+             href="auth-keys.xml"/>
+
+</configuration>

hadoop-tools/hadoop-openstack/src/test/resources/log4j.properties

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c %x - %m%n"
+#log4j.logger.org.apache.hadoop.fs.swift=DEBUG
+
+#crank back on warnings about -1 content length GETs
+log4j.logger.org.apache.commons.httpclient.HttpMethodBase=ERROR

hadoop-tools/hadoop-tools-dist/pom.xml

@@ -77,6 +77,12 @@
       <type>pom</type>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-openstack</artifactId>
+      <scope>compile</scope>
+      <version>${project.version}</version>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-tools/pom.xml

@@ -40,6 +40,7 @@
     <module>hadoop-tools-dist</module>
     <module>hadoop-extras</module>
     <module>hadoop-pipes</module>
+    <module>hadoop-openstack</module>
   </modules>
 
   <build>