|
@@ -24,12 +24,11 @@ import java.net.ConnectException;
|
|
|
import java.net.URI;
|
|
|
import java.security.PrivilegedExceptionAction;
|
|
|
import java.time.Duration;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
|
|
|
-import org.assertj.core.api.Assertions;
|
|
|
-import org.junit.Rule;
|
|
|
-import org.junit.Test;
|
|
|
-import org.junit.rules.TemporaryFolder;
|
|
|
-import org.junit.rules.Timeout;
|
|
|
+import org.junit.jupiter.api.Test;
|
|
|
+import org.junit.jupiter.api.Timeout;
|
|
|
+import org.junit.jupiter.api.io.TempDir;
|
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
import software.amazon.awssdk.core.SdkClient;
|
|
@@ -68,16 +67,22 @@ import static java.lang.String.format;
|
|
|
import static java.util.Objects.requireNonNull;
|
|
|
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
|
|
|
import static org.apache.hadoop.fs.s3a.Constants.*;
|
|
|
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.EU_WEST_1;
|
|
|
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
|
|
|
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
|
|
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
|
|
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
|
|
-import static org.apache.hadoop.fs.s3a.S3ATestConstants.TEST_FS_S3A_NAME;
|
|
|
-import static org.junit.Assert.*;
|
|
|
+import static org.assertj.core.api.Assertions.assertThat;
|
|
|
+import static org.junit.jupiter.api.Assertions.assertEquals;
|
|
|
+import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
|
+import static org.junit.jupiter.api.Assertions.assertNotNull;
|
|
|
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
|
|
+import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
|
+import static org.junit.jupiter.api.Assertions.fail;
|
|
|
|
|
|
/**
|
|
|
* S3A tests for configuration, especially credentials.
|
|
|
*/
|
|
|
+@Timeout(value = S3A_TEST_TIMEOUT, unit = TimeUnit.MILLISECONDS)
|
|
|
public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
private static final String EXAMPLE_ID = "AKASOMEACCESSKEY";
|
|
|
private static final String EXAMPLE_KEY =
|
|
@@ -91,13 +96,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
private static final Logger LOG =
|
|
|
LoggerFactory.getLogger(ITestS3AConfiguration.class);
|
|
|
|
|
|
- @Rule
|
|
|
- public Timeout testTimeout = new Timeout(
|
|
|
- S3ATestConstants.S3A_TEST_TIMEOUT
|
|
|
- );
|
|
|
-
|
|
|
- @Rule
|
|
|
- public final TemporaryFolder tempDir = new TemporaryFolder();
|
|
|
+ @TempDir
|
|
|
+ private java.nio.file.Path tempDir;
|
|
|
|
|
|
/**
|
|
|
* Get the S3 client of the active filesystem.
|
|
@@ -154,8 +154,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
fail("Unexpected endpoint");
|
|
|
}
|
|
|
String region = getS3AInternals().getBucketLocation();
|
|
|
- assertEquals("Endpoint config setting and bucket location differ: ",
|
|
|
- endPointRegion, region);
|
|
|
+ assertEquals(endPointRegion, region,
|
|
|
+ "Endpoint config setting and bucket location differ: ");
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -263,7 +263,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
public void testCredsFromCredentialProvider() throws Exception {
|
|
|
// set up conf to have a cred provider
|
|
|
final Configuration conf = new Configuration();
|
|
|
- final File file = tempDir.newFile("test.jks");
|
|
|
+ final File file = tempDir.resolve("test.jks").toFile();
|
|
|
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
|
|
|
file.toURI());
|
|
|
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
|
@@ -274,8 +274,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
conf.set(Constants.ACCESS_KEY, EXAMPLE_ID + "LJM");
|
|
|
S3xLoginHelper.Login creds =
|
|
|
S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
|
|
|
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
|
|
|
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
|
|
|
+ assertEquals(EXAMPLE_ID, creds.getUser(), "AccessKey incorrect.");
|
|
|
+ assertEquals(EXAMPLE_KEY, creds.getPassword(), "SecretKey incorrect.");
|
|
|
}
|
|
|
|
|
|
void provisionAccessKeys(final Configuration conf) throws Exception {
|
|
@@ -293,7 +293,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
public void testSecretFromCredentialProviderIDFromConfig() throws Exception {
|
|
|
// set up conf to have a cred provider
|
|
|
final Configuration conf = new Configuration();
|
|
|
- final File file = tempDir.newFile("test.jks");
|
|
|
+ final File file = tempDir.resolve("test.jks").toFile();
|
|
|
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
|
|
|
file.toURI());
|
|
|
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
|
@@ -309,15 +309,15 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
conf.set(Constants.ACCESS_KEY, EXAMPLE_ID);
|
|
|
S3xLoginHelper.Login creds =
|
|
|
S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
|
|
|
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
|
|
|
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
|
|
|
+ assertEquals(EXAMPLE_ID, creds.getUser(), "AccessKey incorrect.");
|
|
|
+ assertEquals(EXAMPLE_KEY, creds.getPassword(), "SecretKey incorrect.");
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
public void testIDFromCredentialProviderSecretFromConfig() throws Exception {
|
|
|
// set up conf to have a cred provider
|
|
|
final Configuration conf = new Configuration();
|
|
|
- final File file = tempDir.newFile("test.jks");
|
|
|
+ final File file = tempDir.resolve("test.jks").toFile();
|
|
|
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
|
|
|
file.toURI());
|
|
|
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
|
@@ -333,15 +333,15 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
conf.set(Constants.SECRET_KEY, EXAMPLE_KEY);
|
|
|
S3xLoginHelper.Login creds =
|
|
|
S3AUtils.getAWSAccessKeys(new URI("s3a://foobar"), conf);
|
|
|
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
|
|
|
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
|
|
|
+ assertEquals(EXAMPLE_ID, creds.getUser(), "AccessKey incorrect.");
|
|
|
+ assertEquals(EXAMPLE_KEY, creds.getPassword(), "SecretKey incorrect.");
|
|
|
}
|
|
|
|
|
|
@Test
|
|
|
public void testExcludingS3ACredentialProvider() throws Exception {
|
|
|
// set up conf to have a cred provider
|
|
|
final Configuration conf = new Configuration();
|
|
|
- final File file = tempDir.newFile("test.jks");
|
|
|
+ final File file = tempDir.resolve("test.jks").toFile();
|
|
|
final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider(
|
|
|
file.toURI());
|
|
|
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
|
|
@@ -352,7 +352,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
conf, S3AFileSystem.class);
|
|
|
String newPath = conf.get(
|
|
|
CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH);
|
|
|
- assertFalse("Provider Path incorrect", newPath.contains("s3a://"));
|
|
|
+ assertFalse(newPath.contains("s3a://"), "Provider Path incorrect");
|
|
|
|
|
|
// now let's make sure the new path is created by the S3AFileSystem
|
|
|
// and the integration still works. Let's provision the keys through
|
|
@@ -364,8 +364,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
URI uri2 = new URI("s3a://foobar");
|
|
|
S3xLoginHelper.Login creds =
|
|
|
S3AUtils.getAWSAccessKeys(uri2, conf);
|
|
|
- assertEquals("AccessKey incorrect.", EXAMPLE_ID, creds.getUser());
|
|
|
- assertEquals("SecretKey incorrect.", EXAMPLE_KEY, creds.getPassword());
|
|
|
+ assertEquals(EXAMPLE_ID, creds.getUser(), "AccessKey incorrect.");
|
|
|
+ assertEquals(EXAMPLE_KEY, creds.getPassword(), "SecretKey incorrect.");
|
|
|
|
|
|
}
|
|
|
|
|
@@ -388,8 +388,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
"clientConfiguration");
|
|
|
S3Configuration s3Configuration =
|
|
|
(S3Configuration)clientConfiguration.option(SdkClientOption.SERVICE_CONFIGURATION);
|
|
|
- assertTrue("Expected to find path style access to be switched on!",
|
|
|
- s3Configuration.pathStyleAccessEnabled());
|
|
|
+ assertTrue(s3Configuration.pathStyleAccessEnabled(),
|
|
|
+ "Expected to find path style access to be switched on!");
|
|
|
byte[] file = ContractTestUtils.toAsciiByteArray("test file");
|
|
|
ContractTestUtils.writeAndRead(fs,
|
|
|
createTestPath(new Path("/path/style/access/testFile")),
|
|
@@ -423,7 +423,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
S3Client s3 = getS3Client("User Agent");
|
|
|
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
|
|
|
"clientConfiguration");
|
|
|
- Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
|
|
|
+ assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
|
|
|
.describedAs("User Agent prefix")
|
|
|
.startsWith("Hadoop " + VersionInfo.getVersion());
|
|
|
}
|
|
@@ -439,7 +439,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
S3Client s3 = getS3Client("User agent");
|
|
|
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
|
|
|
"clientConfiguration");
|
|
|
- Assertions.assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
|
|
|
+ assertThat(clientConfiguration.option(SdkClientOption.CLIENT_USER_AGENT))
|
|
|
.describedAs("User Agent prefix")
|
|
|
.startsWith("MyApp, Hadoop " + VersionInfo.getVersion());
|
|
|
}
|
|
@@ -460,7 +460,7 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
}
|
|
|
SdkClientConfiguration clientConfiguration = getField(s3, SdkClientConfiguration.class,
|
|
|
"clientConfiguration");
|
|
|
- Assertions.assertThat(clientConfiguration.option(SdkClientOption.API_CALL_ATTEMPT_TIMEOUT))
|
|
|
+ assertThat(clientConfiguration.option(SdkClientOption.API_CALL_ATTEMPT_TIMEOUT))
|
|
|
.describedAs("Configured " + REQUEST_TIMEOUT +
|
|
|
" is different than what AWS sdk configuration uses internally")
|
|
|
.isEqualTo(timeout);
|
|
@@ -477,12 +477,12 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
getS3AInternals().shareCredentials("testCloseIdempotent");
|
|
|
credentials.close();
|
|
|
fs.close();
|
|
|
- assertTrue("Closing FS didn't close credentials " + credentials,
|
|
|
- credentials.isClosed());
|
|
|
- assertEquals("refcount not zero in " + credentials, 0, credentials.getRefCount());
|
|
|
+ assertTrue(credentials.isClosed(),
|
|
|
+ "Closing FS didn't close credentials " + credentials);
|
|
|
+ assertEquals(0, credentials.getRefCount(), "refcount not zero in " + credentials);
|
|
|
fs.close();
|
|
|
// and the numbers should not change
|
|
|
- assertEquals("refcount not zero in " + credentials, 0, credentials.getRefCount());
|
|
|
+ assertEquals(0, credentials.getRefCount(), "refcount not zero in " + credentials);
|
|
|
}
|
|
|
|
|
|
@Test
|
|
@@ -496,11 +496,11 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
try {
|
|
|
fs = S3ATestUtils.createTestFileSystem(conf);
|
|
|
final Configuration fsConf = fs.getConf();
|
|
|
- Assertions.assertThat(fsConf.get(Constants.BUFFER_DIR))
|
|
|
+ assertThat(fsConf.get(Constants.BUFFER_DIR))
|
|
|
.describedAs("Config option %s", Constants.BUFFER_DIR)
|
|
|
.isEqualTo(blank);
|
|
|
File tmp = createTemporaryFileForWriting();
|
|
|
- assertTrue("not found: " + tmp, tmp.exists());
|
|
|
+ assertTrue(tmp.exists(), "not found: " + tmp);
|
|
|
tmp.delete();
|
|
|
} finally {
|
|
|
removeAllocatorContexts();
|
|
@@ -535,15 +535,15 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
conf.set(format("fs.s3a.bucket.%s.buffer.dir", bucketName), dirs);
|
|
|
fs = S3ATestUtils.createTestFileSystem(conf);
|
|
|
final Configuration fsConf = fs.getConf();
|
|
|
- Assertions.assertThat(fsConf.get(Constants.BUFFER_DIR))
|
|
|
+ assertThat(fsConf.get(Constants.BUFFER_DIR))
|
|
|
.describedAs("Config option %s", Constants.BUFFER_DIR)
|
|
|
.isEqualTo(dirs);
|
|
|
File tmp1 = createTemporaryFileForWriting();
|
|
|
tmp1.delete();
|
|
|
File tmp2 = createTemporaryFileForWriting();
|
|
|
tmp2.delete();
|
|
|
- assertNotEquals("round robin not working",
|
|
|
- tmp1.getParent(), tmp2.getParent());
|
|
|
+ assertNotEquals(tmp1.getParent(), tmp2.getParent(),
|
|
|
+ "round robin not working");
|
|
|
}
|
|
|
|
|
|
@Test
|
|
@@ -559,10 +559,10 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
return S3ATestUtils.createTestFileSystem(conf);
|
|
|
}
|
|
|
});
|
|
|
- assertEquals("username", alice, fs.getUsername());
|
|
|
+ assertEquals(alice, fs.getUsername(), "username");
|
|
|
FileStatus status = fs.getFileStatus(new Path("/"));
|
|
|
- assertEquals("owner in " + status, alice, status.getOwner());
|
|
|
- assertEquals("group in " + status, alice, status.getGroup());
|
|
|
+ assertEquals(alice, status.getOwner(), "owner in " + status);
|
|
|
+ assertEquals(alice, status.getGroup(), "group in " + status);
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -579,13 +579,12 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
private static <T> T getField(Object target, Class<T> fieldType,
|
|
|
String fieldName) throws IllegalAccessException {
|
|
|
Object obj = FieldUtils.readField(target, fieldName, true);
|
|
|
- assertNotNull(format(
|
|
|
+ assertNotNull(obj, format(
|
|
|
"Could not read field named %s in object with class %s.", fieldName,
|
|
|
- target.getClass().getName()), obj);
|
|
|
- assertTrue(format(
|
|
|
+ target.getClass().getName()));
|
|
|
+ assertTrue(fieldType.isAssignableFrom(obj.getClass()), format(
|
|
|
"Unexpected type found for field named %s, expected %s, actual %s.",
|
|
|
- fieldName, fieldType.getName(), obj.getClass().getName()),
|
|
|
- fieldType.isAssignableFrom(obj.getClass()));
|
|
|
+ fieldName, fieldType.getName(), obj.getClass().getName()));
|
|
|
return fieldType.cast(obj);
|
|
|
}
|
|
|
|
|
@@ -600,7 +599,8 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
assertOptionEquals(updated, "fs.s3a.propagation", "propagated");
|
|
|
}
|
|
|
|
|
|
- @Test(timeout = 10_000L)
|
|
|
+ @Test
|
|
|
+ @Timeout(10)
|
|
|
public void testS3SpecificSignerOverride() throws Exception {
|
|
|
Configuration config = new Configuration();
|
|
|
removeBaseAndBucketOverrides(config,
|
|
@@ -633,10 +633,10 @@ public class ITestS3AConfiguration extends AbstractHadoopTestBase {
|
|
|
Invoker.once("head", bucket, () ->
|
|
|
s3Client.headBucket(HeadBucketRequest.builder().bucket(bucket).build())));
|
|
|
|
|
|
- Assertions.assertThat(CustomS3Signer.isS3SignerCalled())
|
|
|
+ assertThat(CustomS3Signer.isS3SignerCalled())
|
|
|
.describedAs("Custom S3 signer not called").isTrue();
|
|
|
|
|
|
- Assertions.assertThat(CustomSTSSigner.isSTSSignerCalled())
|
|
|
+ assertThat(CustomSTSSigner.isSTSSignerCalled())
|
|
|
.describedAs("Custom STS signer not called").isTrue();
|
|
|
}
|
|
|
|