HADOOP-15663. ABFS: Simplify configuration.

Contributed by Da Zhou.

hadoop-common-project/hadoop-common/src/main/resources/core-default.xml

@@ -1618,6 +1618,18 @@
 </property>
 
 <!-- Azure file system properties -->
+<property>
+  <name>fs.AbstractFileSystem.wasb.impl</name>
+  <value>org.apache.hadoop.fs.azure.Wasb</value>
+  <description>AbstractFileSystem implementation class of wasb://</description>
+</property>
+
+<property>
+  <name>fs.AbstractFileSystem.wasbs.impl</name>
+  <value>org.apache.hadoop.fs.azure.Wasbs</value>
+  <description>AbstractFileSystem implementation class of wasbs://</description>
+</property>
+
 <property>
   <name>fs.wasb.impl</name>
   <value>org.apache.hadoop.fs.azure.NativeAzureFileSystem</value>

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java

@@ -173,10 +173,6 @@ public class AbfsConfiguration{
     }
   }
 
-  public boolean isEmulator() {
-    return this.getConfiguration().getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
-  }
-
   public boolean isSecureMode() {
     return isSecure;
   }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java

@@ -77,6 +77,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy;
 import org.apache.hadoop.fs.azurebfs.services.SharedKeyCredentials;
+import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsAction;
@@ -86,6 +87,7 @@ import org.apache.http.client.utils.URIBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_ABFS_ENDPOINT;
 import static org.apache.hadoop.util.Time.now;
 
 /**
@@ -146,7 +148,27 @@ public class AzureBlobFileSystemStore {
 
     final URIBuilder uriBuilder = new URIBuilder();
     uriBuilder.setScheme(scheme);
-    uriBuilder.setHost(hostName);
+
+    // For testing purposes, an IP address and port may be provided to override
+    // the host specified in the FileSystem URI.  Also note that the format of
+    // the Azure Storage Service URI changes from
+    // http[s]://[account][domain-suffix]/[filesystem] to
+    // http[s]://[ip]:[port]/[account]/[filesystem].
+    String endPoint = abfsConfiguration.getConfiguration().get(AZURE_ABFS_ENDPOINT);
+    if (endPoint == null || !endPoint.contains(AbfsHttpConstants.COLON)) {
+      uriBuilder.setHost(hostName);
+      return uriBuilder;
+    }
+
+    // Split ip and port
+    String[] data = endPoint.split(AbfsHttpConstants.COLON);
+    if (data.length != 2) {
+      throw new RuntimeException(String.format("ABFS endpoint is not set correctly : %s, "
+              + "Do not specify scheme when using {IP}:{PORT}", endPoint));
+    }
+    uriBuilder.setHost(data[0].trim());
+    uriBuilder.setPort(Integer.parseInt(data[1].trim()));
+    uriBuilder.setPath("/" + UriUtils.extractAccountNameFromHostName(hostName));
 
     return uriBuilder;
   }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java

@@ -37,9 +37,6 @@ public final class ConfigurationKeys {
   public static final String AZURE_BACKOFF_INTERVAL = "fs.azure.io.retry.backoff.interval";
   public static final String AZURE_MAX_IO_RETRIES = "fs.azure.io.retry.max.retries";
 
-  // Remove this and use common azure storage emulator property for public release.
-  public static final String FS_AZURE_EMULATOR_ENABLED = "fs.azure.abfs.emulator.enabled";
-
   // Read and write buffer sizes defined by the user
   public static final String AZURE_WRITE_BUFFER_SIZE = "fs.azure.write.request.size";
   public static final String AZURE_READ_BUFFER_SIZE = "fs.azure.read.request.size";
@@ -60,6 +57,8 @@ public final class ConfigurationKeys {
   public static final String AZURE_KEY_ACCOUNT_KEYPROVIDER_PREFIX = "fs.azure.account.keyprovider.";
   public static final String AZURE_KEY_ACCOUNT_SHELLKEYPROVIDER_SCRIPT = "fs.azure.shellkeyprovider.script";
 
+  /** End point of ABFS account: {@value}. */
+  public static final String AZURE_ABFS_ENDPOINT = "fs.azure.abfs.endpoint";
   /** Prefix for auth type properties: {@value}. */
   public static final String FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME = "fs.azure.account.auth.type.";
   /** Prefix for oauth token provider type: {@value}. */

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/UriUtils.java

@@ -41,20 +41,21 @@ public final class UriUtils {
   }
 
   /**
-   * Extracts the raw account name from account name.
-   * @param accountName to extract the raw account name.
-   * @return extracted raw account name.
+   * Extracts the account name from the host name.
+   * @param hostName the fully-qualified domain name of the storage service
+   *                 endpoint (e.g. {account}.dfs.core.windows.net.
+   * @return the storage service account name.
    */
-  public static String extractRawAccountFromAccountName(final String accountName) {
-    if (accountName == null || accountName.isEmpty()) {
+  public static String extractAccountNameFromHostName(final String hostName) {
+    if (hostName == null || hostName.isEmpty()) {
       return null;
     }
 
-    if (!containsAbfsUrl(accountName)) {
+    if (!containsAbfsUrl(hostName)) {
       return null;
     }
 
-    String[] splitByDot = accountName.split("\\.");
+    String[] splitByDot = hostName.split("\\.");
     if (splitByDot.length == 0) {
       return null;
     }

hadoop-tools/hadoop-azure/src/site/markdown/testing_azure.md

@@ -90,7 +90,7 @@ For example:
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration>
   <property>
-    <name>fs.azure.test.account.name</name>
+    <name>fs.azure.wasb.account.name</name>
     <value>{ACCOUNTNAME}.blob.core.windows.net</value>
   </property>
   <property>
@@ -126,7 +126,7 @@ Overall, to run all the tests using `mvn test`,  a sample `azure-auth-keys.xml`
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration>
   <property>
-    <name>fs.azure.test.account.name</name>
+    <name>fs.azure.wasb.account.name</name>
     <value>{ACCOUNTNAME}.blob.core.windows.net</value>
   </property>
   <property>
@@ -576,77 +576,172 @@ This will delete the containers; the output log of the test run will
 provide the details and summary of the operation.
 
 
-## Testing ABFS
+## Testing the Azure ABFS Client
 
-The ABFS Connector tests share the same account as the wasb tests; this is
-needed for cross-connector compatibility tests.
-
-This makes for a somewhat complex set of configuration options.
-
-Here are the settings for an account `ACCOUNTNAME`
+Azure Data Lake Storage Gen 2 (ADLS Gen 2) is a set of capabilities dedicated to 
+big data analytics, built on top of Azure Blob Storage. The ABFS and ABFSS 
+schemes target the ADLS Gen 2 REST API, and the WASB and WASBS schemes target
+the Azure Blob Storage REST API.  ADLS Gen 2 offers better performance and
+scalability.  ADLS Gen 2 also offers authentication and authorization compatible 
+with the Hadoop Distributed File System permissions model when hierarchical 
+namespace is enabled for the storage account.  Furthermore, the metadata and data
+produced by ADLS Gen 2 REST API can be consumed by Blob REST API, and vice versa.
+ 
+In order to test ABFS, please add the following configuration to your
+`src/test/resources/azure-auth-keys.xml` file. Note that the ABFS tests include
+compatibility tests which require WASB credentials, in addition to the ABFS
+credentials.
 
 ```xml
-<property>
-  <name>abfs.account.name</name>
-  <value>ACCOUNTNAME</value>
-</property>
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+  <property>
+    <name>fs.azure.abfs.account.name</name>
+    <value>{ACCOUNT_NAME}.dfs.core.windows.net</value>
+  </property>
 
-<property>
-  <name>abfs.account.full.name</name>
-  <value>${abfs.account.name}.dfs.core.windows.net</value>
-</property>
+  <property>
+    <name>fs.azure.account.key.{ACCOUNT_NAME}.dfs.core.windows.net</name>
+    <value>{ACCOUNT_ACCESS_KEY}</value>
+  </property>
 
-<property>
-  <name>abfs.account.key</name>
-  <value>SECRETKEY==</value>
-</property>
+  <property>
+    <name>fs.azure.wasb.account.name</name>
+    <value>{ACCOUNT_NAME}.blob.core.windows.net</value>
+  </property>
+  
+  <property>
+    <name>fs.azure.account.key.{ACCOUNT_NAME}.blob.core.windows.net</name>
+    <value>{ACCOUNT_ACCESS_KEY}</value>
+  </property>
 
-<property>
-  <name>fs.azure.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
-  <value>${abfs.account.key}</value>
-</property>
+  <property>
+    <name>fs.contract.test.fs.abfs</name>
+    <value>abfs://{CONTAINER_NAME}@{ACCOUNT_NAME}.dfs.core.windows.net</value>
+    <description>A file system URI to be used by the contract tests.</description>
+  </property>
 
-<property>
-  <name>fs.azure.account.key.ACCOUNTNAME.blob.core.windows.net</name>
-  <value>${abfs.account.key}</value>
-</property>
+  <property>
+    <name>fs.contract.test.fs.wasb</name>
+    <value>wasb://{CONTAINER_NAME}@{ACCOUNT_NAME}.blob.core.windows.net</value>
+    <description>A file system URI to be used by the contract tests.</description>
+  </property>
+</configuration>
+```
 
-<property>
-  <name>fs.azure.test.account.key.ACCOUNTNAME.dfs.core.windows.net</name>
-  <value>${abfs.account.key}</value>
-</property>
+To run OAuth and ACL test cases you must use a storage account with the
+hierarchical namespace enabled, and set the following configuration settings:
 
+```xml
+<!--=========================== AUTHENTICATION  OPTIONS ===================-->
+<!--ATTENTION:
+      TO RUN ABFS & WASB COMPATIBILITY TESTS, YOU MUST SET AUTH TYPE AS SharedKey.
+      OAUTH IS INTRODUCED TO ABFS ONLY.-->
 <property>
-  <name>fs.azure.test.account.key.ACCOUNTNAME.blob.core.windows.net</name>
-  <value>${abfs.account.key}</value>
+  <name>fs.azure.account.auth.type.{YOUR_ABFS_ACCOUNT_NAME}</name>
+  <value>{AUTH TYPE}</value>
+  <description>The authorization type can be SharedKey, OAuth, or Custom. The 
+  default is SharedKey.</description>
 </property>
 
-<property>
-  <name>fs.azure.account.key.ACCOUNTNAME</name>
-  <value>${abfs.account.key}</value>
-</property>
+<!--=============================   FOR OAUTH   ===========================-->
+<!--IF AUTH TYPE IS SET AS OAUTH, FOLLOW THE STEPS BELOW-->
+<!--NOTICE: AAD client and tenant related properties can be obtained through Azure Portal-->
 
-<property>
-  <name>fs.azure.test.account.key.ACCOUNTNAME</name>
-  <value>${abfs.account.key}</value>
-</property>
+  <!--1. UNCOMMENT BELOW AND CHOOSE YOUR OAUTH PROVIDER TYPE -->
 
-<property>
-  <name>fs.azure.test.account.name</name>
-  <value>${abfs.account.full.name}</value>
-</property>
+  <!--
+  <property>
+    <name>fs.azure.account.oauth.provider.type.{ABFS_ACCOUNT_NAME}</name>
+    <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
+    <description>The full name of token provider class name.</description>
+  </property>
+ -->
 
-<property>
-  <name>fs.contract.test.fs.abfs</name>
-  <value>abfs://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
-  <description>Container for contract tests</description>
-</property>
+  <!--2. UNCOMMENT BELOW AND SET CREDENTIALS ACCORDING TO THE PROVIDER TYPE-->
 
-<property>
-  <name>fs.contract.test.fs.abfss</name>
-  <value>abfss://TESTCONTAINER@ACCOUNTNAME.dfs.core.windows.net</value>
-  <description>Container for contract tests</description>
-</property>
+  <!--2.1. If "ClientCredsTokenProvider" is set as key provider, uncomment below and
+           set auth endpoint, client id and secret below-->
+  <!--
+   <property>
+    <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
+    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+    <description>Token end point, this can be found through Azure portal</description>
+  </property>
 
+   <property>
+     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+     <value>{client id}</value>
+     <description>AAD client id.</description>
+   </property>
+
+   <property>
+     <name>fs.azure.account.oauth2.client.secret.{ABFS_ACCOUNT_NAME}</name>
+     <value>{client secret}</value>
+   </property>
+ -->
+
+  <!--2.2. If "UserPasswordTokenProvider" is set as key provider, uncomment below and
+           set auth endpoint, use name and password-->
+  <!--
+   <property>
+    <name>fs.azure.account.oauth2.client.endpoint.{ABFS_ACCOUNT_NAME}</name>
+    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
+    <description>Token end point, this can be found through Azure portal</description>
+  </property>
 
+   <property>
+     <name>fs.azure.account.oauth2.user.name.{ABFS_ACCOUNT_NAME}</name>
+     <value>{user name}</value>
+   </property>
+
+   <property>
+     <name>fs.azure.account.oauth2.user.password.{ABFS_ACCOUNT_NAME}</name>
+     <value>{user password}</value>
+   </property>
+ -->
+
+  <!--2.3. If "MsiTokenProvider" is set as key provider, uncomment below and
+           set tenantGuid and client id.-->
+  <!--
+   <property>
+     <name>fs.azure.account.oauth2.msi.tenant.{ABFS_ACCOUNT_NAME}</name>
+     <value>{tenantGuid}</value>
+     <description>msi tenantGuid.</description>
+   </property>
+
+   <property>
+     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+     <value>{client id}</value>
+     <description>AAD client id.</description>
+   </property>
+  -->
+
+  <!--2.4. If "RefreshTokenBasedTokenProvider" is set as key provider, uncomment below and
+           set refresh token and client id.-->
+  <!--
+   <property>
+     <name>fs.azure.account.oauth2.refresh.token.{ABFS_ACCOUNT_NAME}</name>
+     <value>{refresh token}</value>
+     <description>refresh token.</description>
+   </property>
+
+   <property>
+     <name>fs.azure.account.oauth2.client.id.{ABFS_ACCOUNT_NAME}</name>
+     <value>{client id}</value>
+     <description>AAD client id.</description>
+   </property>
+  -->
 ```
+
+If running tests against an endpoint that uses the URL format
+http[s]://[ip]:[port]/[account]/[filesystem] instead of
+http[s]://[account][domain-suffix]/[filesystem], please use the following:
+
+```xml
+<property>
+  <name>fs.azure.abfs.endpoint</name>
+  <value>{IP}:{PORT}</value>
+</property>
+```

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/AzureBlobStorageTestAccount.java

@@ -48,6 +48,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
 import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.DEFAULT_STORAGE_EMULATOR_ACCOUNT_NAME;
 import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_LOCAL_SAS_KEY_MODE;
 import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.KEY_USE_SECURE_MODE;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
 
 /**
  * Helper class to create WASB file systems backed by either a mock in-memory
@@ -58,11 +59,14 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
   private static final Logger LOG = LoggerFactory.getLogger(
       AzureBlobStorageTestAccount.class);
 
-  private static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
   private static final String SAS_PROPERTY_NAME = "fs.azure.sas.";
   private static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
-  private static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.test.account.name";
+  public static final String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+  public static final String TEST_ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
+  public static final String WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN = "fs.azure.wasb.account.name";
   public static final String MOCK_ACCOUNT_NAME = "mockAccount.blob.core.windows.net";
+  public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX = ".blob.core.windows.net";
+  public static final String WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX = "\\.blob(\\.preprod)?\\.core\\.windows\\.net";
   public static final String MOCK_CONTAINER_NAME = "mockContainer";
   public static final String WASB_AUTHORITY_DELIMITER = "@";
   public static final String WASB_SCHEME = "wasb";
@@ -379,7 +383,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
         containerName);
     container.create();
 
-    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    String accountName = verifyWasbAccountNameInConfig(conf);
 
     // Ensure that custom throttling is disabled and tolerate concurrent
     // out-of-band appends.
@@ -525,7 +529,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
 
   static CloudStorageAccount createTestAccount(Configuration conf)
       throws URISyntaxException, KeyProviderException {
-    String testAccountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    String testAccountName = verifyWasbAccountNameInConfig(conf);
     if (testAccountName == null) {
       LOG.warn("Skipping live Azure test because of missing test account");
       return null;
@@ -570,16 +574,16 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
     String containerName = useContainerSuffixAsContainerName
         ? containerNameSuffix
         : String.format(
-            "wasbtests-%s-%tQ%s",
+            "wasbtests-%s-%s%s",
             System.getProperty("user.name"),
-            new Date(),
+            UUID.randomUUID().toString(),
             containerNameSuffix);
     container = account.createCloudBlobClient().getContainerReference(
         containerName);
     if (createOptions.contains(CreateOptions.CreateContainer)) {
       container.createIfNotExists();
     }
-    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    String accountName = verifyWasbAccountNameInConfig(conf);
     if (createOptions.contains(CreateOptions.UseSas)) {
       String sas = generateSAS(container,
           createOptions.contains(CreateOptions.Readonly));
@@ -741,7 +745,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
     CloudBlobClient blobClient = account.createCloudBlobClient();
 
     // Capture the account URL and the account name.
-    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    String accountName = verifyWasbAccountNameInConfig(conf);
 
     configureSecureModeTestSettings(conf);
 
@@ -814,7 +818,7 @@ public final class AzureBlobStorageTestAccount implements AutoCloseable,
     CloudBlobClient blobClient = account.createCloudBlobClient();
 
     // Capture the account URL and the account name.
-    String accountName = conf.get(TEST_ACCOUNT_NAME_PROPERTY_NAME);
+    String accountName = verifyWasbAccountNameInConfig(conf);
 
     configureSecureModeTestSettings(conf);
 

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestFileSystemOperationExceptionMessage.java

@@ -29,6 +29,7 @@ import com.microsoft.azure.storage.CloudStorageAccount;
 import org.junit.Test;
 
 import static org.apache.hadoop.fs.azure.AzureNativeFileSystemStore.NO_ACCESS_TO_CONTAINER_MSG;
+import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.verifyWasbAccountNameInConfig;
 
 /**
  * Test for error messages coming from SDK.
@@ -46,7 +47,7 @@ public class ITestFileSystemOperationExceptionMessage
         AzureBlobStorageTestAccount.createTestAccount(conf);
     AzureTestUtils.assume("No test account", account != null);
 
-    String testStorageAccount = conf.get("fs.azure.test.account.name");
+    String testStorageAccount = verifyWasbAccountNameInConfig(conf);
     conf = new Configuration();
     conf.set("fs.AbstractFileSystem.wasb.impl",
         "org.apache.hadoop.fs.azure.Wasb");

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestWasbUriAndConfiguration.java

@@ -499,32 +499,6 @@ public class ITestWasbUriAndConfiguration extends AbstractWasbTestWithTimeout {
     }
   }
 
-  @Test
-  public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception {
-    try {
-      testAccount = AzureBlobStorageTestAccount.createMock();
-      Configuration conf = testAccount.getFileSystem().getConf();
-      String authority = testAccount.getFileSystem().getUri().getAuthority();
-      URI defaultUri = new URI("wasbs", authority, null, null, null);
-      conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString());
-
-      FileSystem fs = FileSystem.get(conf);
-      assertTrue(fs instanceof NativeAzureFileSystem);
-      assertEquals("wasbs", fs.getScheme());
-
-      // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified
-      try{
-        FileContext.getFileContext(conf).getDefaultFileSystem();
-        fail("Should've thrown.");
-      }catch(UnsupportedFileSystemException e){
-      }
-
-    } finally {
-      testAccount.cleanup();
-      FileSystem.closeAll();
-    }
-  }
-
   @Test
   public void testCredentialProviderPathExclusions() throws Exception {
     String providerPath =

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java

@@ -143,12 +143,10 @@ public interface AzureTestConstants {
 
 
 
-  String ACCOUNT_KEY_PROPERTY_NAME
-      = "fs.azure.account.key.";
+  String ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key.";
+  String ACCOUNT_NAME_PROPERTY_NAME = "fs.azure.account.name";
   String SAS_PROPERTY_NAME = "fs.azure.sas.";
   String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
-  String TEST_ACCOUNT_NAME_PROPERTY_NAME
-      = "fs.azure.test.account.name";
   String MOCK_ACCOUNT_NAME
       = "mockAccount.blob.core.windows.net";
   String MOCK_CONTAINER_NAME = "mockContainer";

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java

@@ -36,10 +36,14 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
 
+
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX;
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN;
 import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*;
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getLongGauge;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assume.assumeTrue;
 
 /**
  * Utilities for the Azure tests. Based on {@code S3ATestUtils}, so
@@ -476,4 +480,18 @@ public final class AzureTestUtils extends Assert {
             + KEY_SCALE_TESTS_ENABLED,
         enabled);
   }
+
+  /**
+   * Check the account name for WASB tests is set correctly and return.
+   */
+  public static String verifyWasbAccountNameInConfig(Configuration conf) {
+    String accountName = conf.get(ACCOUNT_NAME_PROPERTY_NAME);
+    if (accountName == null) {
+      accountName = conf.get(WASB_TEST_ACCOUNT_NAME_WITH_DOMAIN);
+    }
+    assumeTrue("Account for WASB is missing or it is not in correct format",
+            accountName != null && !accountName.endsWith(WASB_ACCOUNT_NAME_DOMAIN_SUFFIX_REGEX));
+    return accountName;
+  }
+
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/metrics/TestRollingWindowAverage.java

@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.fs.azure.metrics;
 
-import static org.junit.Assert.*;
-import org.junit.*;
+import static org.junit.Assert.assertEquals;
+import org.junit.Test;
 
 public class TestRollingWindowAverage {
   /**

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java

@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.apache.hadoop.fs.azure.AbstractWasbTestWithTimeout;
 import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
 import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -45,12 +44,12 @@ import org.apache.hadoop.fs.azurebfs.utils.UriUtils;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.io.IOUtils;
 
+import static org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.*;
 import static org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode.FILE_SYSTEM_NOT_FOUND;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.*;
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 import static org.junit.Assume.assumeTrue;
-import static org.hamcrest.CoreMatchers.*;
 
 /**
  * Base for AzureBlobFileSystem Integration tests.
@@ -58,12 +57,12 @@ import static org.hamcrest.CoreMatchers.*;
  * <I>Important: This is for integration tests only.</I>
  */
 public abstract class AbstractAbfsIntegrationTest extends
-    AbstractWasbTestWithTimeout {
+        AbstractAbfsTestWithTimeout {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(AbstractAbfsIntegrationTest.class);
 
-  private boolean isEmulator;
+  private boolean isIPAddress;
   private NativeAzureFileSystem wasb;
   private AzureBlobFileSystem abfs;
   private String abfsScheme;
@@ -75,27 +74,28 @@ public abstract class AbstractAbfsIntegrationTest extends
   private AuthType authType;
 
   protected AbstractAbfsIntegrationTest() {
-    fileSystemName = ABFS_TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
+    fileSystemName = TEST_CONTAINER_PREFIX + UUID.randomUUID().toString();
     configuration = new Configuration();
-    configuration.addResource(ABFS_TEST_RESOURCE_XML);
-    this.accountName = this.configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
+    configuration.addResource(TEST_CONFIGURATION_FILE_NAME);
+
+    this.accountName = this.configuration.get(FS_AZURE_ACCOUNT_NAME);
+    if (accountName == null) {
+      // check if accountName is set using different config key
+      accountName = configuration.get(FS_AZURE_ABFS_ACCOUNT_NAME);
+    }
+    assumeTrue("Not set: " + FS_AZURE_ABFS_ACCOUNT_NAME,
+            accountName != null && !accountName.isEmpty());
+
 
     authType = configuration.getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME
             + accountName, AuthType.SharedKey);
     abfsScheme = authType == AuthType.SharedKey ? FileSystemUriSchemes.ABFS_SCHEME
             : FileSystemUriSchemes.ABFS_SECURE_SCHEME;
 
-    String accountName = configuration.get(FS_AZURE_TEST_ACCOUNT_NAME, "");
-    assumeTrue("Not set: " + FS_AZURE_TEST_ACCOUNT_NAME,
-        !accountName.isEmpty());
-    assertThat("The key in " + FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
-            + " is not bound to an ABFS account",
-        accountName, containsString("dfs.core.windows.net"));
-    String fullKey = FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
-        + accountName;
-
     if (authType == AuthType.SharedKey) {
-      assumeTrue("Not set: " + fullKey, configuration.get(fullKey) != null);
+      String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + accountName;
+      assumeTrue("Not set: " + keyProperty, configuration.get(keyProperty) != null);
+      // Update credentials
     } else {
       String accessTokenProviderKey = FS_AZURE_ACCOUNT_TOKEN_PROVIDER_TYPE_PROPERTY_NAME + accountName;
       assumeTrue("Not set: " + accessTokenProviderKey, configuration.get(accessTokenProviderKey) != null);
@@ -113,7 +113,17 @@ public abstract class AbstractAbfsIntegrationTest extends
     this.testUrl = defaultUri.toString();
     configuration.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString());
     configuration.setBoolean(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, true);
-    this.isEmulator = this.configuration.getBoolean(FS_AZURE_EMULATOR_ENABLED, false);
+    // For testing purposes, an IP address and port may be provided to override
+    // the host specified in the FileSystem URI.  Also note that the format of
+    // the Azure Storage Service URI changes from
+    // http[s]://[account][domain-suffix]/[filesystem] to
+    // http[s]://[ip]:[port]/[account]/[filesystem].
+    String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+    if (endPoint != null && endPoint.contains(":") && endPoint.split(":").length == 2) {
+      this.isIPAddress = true;
+    } else {
+      this.isIPAddress = false;
+    }
   }
 
 
@@ -122,13 +132,22 @@ public abstract class AbstractAbfsIntegrationTest extends
     //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
     createFileSystem();
 
-    if (!isEmulator && authType == AuthType.SharedKey) {
+    if (!isIPAddress && authType == AuthType.SharedKey) {
       final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
       final AzureNativeFileSystemStore azureNativeFileSystemStore =
           new AzureNativeFileSystemStore();
+
+      // update configuration with wasb credentials
+      String accountNameWithoutDomain = accountName.split("\\.")[0];
+      String wasbAccountName = accountNameWithoutDomain + WASB_ACCOUNT_NAME_DOMAIN_SUFFIX;
+      String keyProperty = FS_AZURE_ACCOUNT_KEY_PREFIX + wasbAccountName;
+      if (configuration.get(keyProperty) == null) {
+        configuration.set(keyProperty, getAccountKey());
+      }
+
       azureNativeFileSystemStore.initialize(
           wasbUri,
-          getConfiguration(),
+          configuration,
           new AzureFileSystemInstrumentation(getConfiguration()));
 
       wasb = new NativeAzureFileSystem(azureNativeFileSystemStore);
@@ -201,7 +220,9 @@ public abstract class AbstractAbfsIntegrationTest extends
   }
 
   protected String getHostName() {
-    return configuration.get(FS_AZURE_TEST_HOST_NAME);
+    // READ FROM ENDPOINT, THIS IS CALLED ONLY WHEN TESTING AGAINST DEV-FABRIC
+    String endPoint = configuration.get(AZURE_ABFS_ENDPOINT);
+    return endPoint.split(":")[0];
   }
 
   protected void setTestUrl(String testUrl) {
@@ -220,21 +241,21 @@ public abstract class AbstractAbfsIntegrationTest extends
   }
 
   protected String getAccountName() {
-    return configuration.get(FS_AZURE_TEST_ACCOUNT_NAME);
+    return this.accountName;
   }
 
   protected String getAccountKey() {
     return configuration.get(
-        FS_AZURE_TEST_ACCOUNT_KEY_PREFIX
-            + getAccountName());
+        FS_AZURE_ACCOUNT_KEY_PREFIX
+            + accountName);
   }
 
   protected Configuration getConfiguration() {
     return configuration;
   }
 
-  protected boolean isEmulator() {
-    return isEmulator;
+  protected boolean isIPAddress() {
+    return isIPAddress;
   }
 
   protected AuthType getAuthType() {

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java

@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.azurebfs;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
+
+/**
+ * Base class for any ABFS test with timeouts & named threads.
+ * This class does not attempt to bind to Azure.
+ */
+public class AbstractAbfsTestWithTimeout extends Assert {
+  /**
+   * The name of the current method.
+   */
+  @Rule
+  public TestName methodName = new TestName();
+  /**
+   * Set the timeout for every test.
+   * This is driven by the value returned by {@link #getTestTimeoutMillis()}.
+   */
+  @Rule
+  public Timeout testTimeout = new Timeout(getTestTimeoutMillis());
+
+  /**
+   * Name the junit thread for the class. This will overridden
+   * before the individual test methods are run.
+   */
+  @BeforeClass
+  public static void nameTestThread() {
+    Thread.currentThread().setName("JUnit");
+  }
+
+  /**
+   * Name the thread to the current test method.
+   */
+  @Before
+  public void nameThread() {
+    Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
+  }
+
+  /**
+   * Override point: the test timeout in milliseconds.
+   * @return a timeout in milliseconds
+   */
+  protected int getTestTimeoutMillis() {
+    return TEST_TIMEOUT;
+  }
+}

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java

@@ -69,7 +69,7 @@ public class ITestAzureBlobFileSystemBackCompat extends
 
   private String getBlobConnectionString() {
     String connectionString;
-    if (isEmulator()) {
+    if (isIPAddress()) {
       connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://"
               + this.getHostName() + ":8880/" + this.getAccountName().split("\\.") [0]
               + ";AccountName=" + this.getAccountName().split("\\.")[0]

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java

@@ -50,7 +50,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
       LoggerFactory.getLogger(ITestWasbAbfsCompatibility.class);
 
   public ITestWasbAbfsCompatibility() throws Exception {
-    Assume.assumeFalse("Emulator is not supported", isEmulator());
+    Assume.assumeFalse("Emulator is not supported", isIPAddress());
     Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
   }
 

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsConfigurationFieldsValidation.java

@@ -23,6 +23,7 @@ import java.lang.reflect.Field;
 import org.apache.commons.codec.Charsets;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation;
 import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation;
@@ -98,7 +99,7 @@ public class TestAbfsConfigurationFieldsValidation {
     this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
     this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8);
     Configuration configuration = new Configuration();
-    configuration.addResource("azure-bfs-test.xml");
+    configuration.addResource(TestConfigurationKeys.TEST_CONFIGURATION_FILE_NAME);
     configuration.set(INT_KEY, "1234565");
     configuration.set(LONG_KEY, "4194304");
     configuration.set(STRING_KEY, "stringValue");

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/constants/TestConfigurationKeys.java

@@ -22,10 +22,9 @@ package org.apache.hadoop.fs.azurebfs.constants;
  * Responsible to keep all the Azure Blob File System configurations keys in Hadoop configuration file.
  */
 public final class TestConfigurationKeys {
-  public static final String FS_AZURE_TEST_ACCOUNT_NAME = "fs.azure.test.account.name";
-  public static final String FS_AZURE_TEST_ACCOUNT_KEY_PREFIX = "fs.azure.test.account.key.";
-  public static final String FS_AZURE_TEST_HOST_NAME = "fs.azure.test.host.name";
-  public static final String FS_AZURE_TEST_HOST_PORT = "fs.azure.test.host.port";
+  public static final String FS_AZURE_ACCOUNT_NAME = "fs.azure.account.name";
+  public static final String FS_AZURE_ABFS_ACCOUNT_NAME = "fs.azure.abfs.account.name";
+  public static final String FS_AZURE_ACCOUNT_KEY_PREFIX = "fs.azure.account.key.";
   public static final String FS_AZURE_CONTRACT_TEST_URI = "fs.contract.test.fs.abfs";
 
   public static final String FS_AZURE_BLOB_DATA_CONTRIBUTOR_CLIENT_ID = "fs.azure.account.oauth2.contributor.client.id";
@@ -34,9 +33,9 @@ public final class TestConfigurationKeys {
   public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_ID = "fs.azure.account.oauth2.reader.client.id";
   public static final String FS_AZURE_BLOB_DATA_READER_CLIENT_SECRET = "fs.azure.account.oauth2.reader.client.secret";
 
-  public static final String ABFS_TEST_RESOURCE_XML = "azure-bfs-test.xml";
-
-  public static final String ABFS_TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
+  public static final String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml";
+  public static final String TEST_CONTAINER_PREFIX = "abfs-testcontainer-";
+  public static final int TEST_TIMEOUT = 10 * 60 * 1000;
 
   private TestConfigurationKeys() {}
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ABFSContractTestBinding.java

@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
 import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes;
 import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.junit.Assume;
 
 /**
  * Bind ABFS contract tests to the Azure test setup/teardown.
@@ -42,6 +43,8 @@ public class ABFSContractTestBinding extends AbstractAbfsIntegrationTest {
     if (useExistingFileSystem) {
       Configuration configuration = getConfiguration();
       String testUrl = configuration.get(TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI);
+      Assume.assumeTrue("Contract tests are skipped because of missing config property :"
+      + TestConfigurationKeys.FS_AZURE_CONTRACT_TEST_URI, testUrl != null);
 
       if (getAuthType() != AuthType.SharedKey) {
         testUrl = testUrl.replaceFirst(FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME);

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java

@@ -32,7 +32,6 @@ import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.M
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE;
 
-import static org.junit.Assert.assertEquals;
 
 /**
  * Test configuration validators.

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/CleanUpAbfsTestContainer.java → hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/AbfsTestUtils.java

@@ -22,56 +22,59 @@ import com.microsoft.azure.storage.blob.CloudBlobClient;
 import com.microsoft.azure.storage.blob.CloudBlobContainer;
 
 import org.junit.Assume;
-import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount;
 import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
 
-import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.ABFS_TEST_CONTAINER_PREFIX;
+import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_CONTAINER_PREFIX;
 
-/**
- * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
- * In that case, dev can use this tool to list and delete all test containers.
- * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
- */
-public class CleanUpAbfsTestContainer extends AbstractAbfsIntegrationTest{
+  /*
+  * Some Utils for ABFS tests.
+  * */
+public final class AbfsTestUtils extends AbstractAbfsIntegrationTest{
+    private static final Logger LOG =
+            LoggerFactory.getLogger(AbfsTestUtils.class);
+  /**
+   * If unit tests were interrupted and crushed accidentally, the test containers won't be deleted.
+   * In that case, dev can use this tool to list and delete all test containers.
+   * By default, all test container used in E2E tests sharing same prefix: "abfs-testcontainer-"
+   */
 
-  public CleanUpAbfsTestContainer() {
+  public void checkContainers() throws Throwable {
     Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
-  }
-
-  @Test
-  public void testEnumContainers() throws Throwable {
     int count = 0;
     CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
     CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
     Iterable<CloudBlobContainer> containers
-            = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
+            = blobClient.listContainers(TEST_CONTAINER_PREFIX);
     for (CloudBlobContainer container : containers) {
       count++;
-      System.out.println(String.format("Container %s URI %s",
+      LOG.info("Container {}, URI {}",
               container.getName(),
-              container.getUri()));
+              container.getUri());
     }
-    System.out.println(String.format("Found %d test containers", count));
+    LOG.info("Found {} test containers", count);
   }
 
-  @Test
-  public void testDeleteContainers() throws Throwable {
+
+  public void deleteContainers() throws Throwable {
+    Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
     int count = 0;
     CloudStorageAccount storageAccount = AzureBlobStorageTestAccount.createTestAccount();
     CloudBlobClient blobClient = storageAccount.createCloudBlobClient();
     Iterable<CloudBlobContainer> containers
-            = blobClient.listContainers(ABFS_TEST_CONTAINER_PREFIX);
+            = blobClient.listContainers(TEST_CONTAINER_PREFIX);
     for (CloudBlobContainer container : containers) {
-      System.out.println(String.format("Container %s URI %s",
+      LOG.info("Container {} URI {}",
               container.getName(),
-              container.getUri()));
+              container.getUri());
       if (container.deleteIfExists()) {
         count++;
       }
     }
-    System.out.println(String.format("Deleted %s test containers", count));
+    LOG.info("Deleted {} test containers", count);
   }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java

@@ -38,11 +38,11 @@ public final class TestUriUtils {
 
   @Test
   public void testExtractRawAccountName() throws Exception {
-    Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net"));
-    Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net"));
-    Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
-    Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(""));
-    Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null));
-    Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net"));
+    Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.core.windows.net"));
+    Assert.assertEquals("abfs", UriUtils.extractAccountNameFromHostName("abfs.dfs.preprod.core.windows.net"));
+    Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
+    Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(""));
+    Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName(null));
+    Assert.assertEquals(null, UriUtils.extractAccountNameFromHostName("abfs.dfs.cores.windows.net"));
   }
 }

hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml

@@ -1,188 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
--->
-
-<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
-
-  <!--
-  <property>
-    <name>fs.azure.test.account.name</name>
-    <value>{YOURACCOUNT}</value>
-  </property>
-
-  <property>
-    <name>fs.azure.account.auth.type.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>{AUTH TYPE}</value>
-    <description>The auth type can be : SharedKey, OAuth, Custom. By default "SharedKey" is used.</description>
-  </property>
-
-   <property>
-    <name>fs.contract.test.fs.abfs</name>
-    <value>abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
-    <description>The name of the azure file system for testing.</description>
-  </property>
-
-  <property>
-    <name>fs.contract.test.fs.abfss</name>
-    <value>abfss://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value>
-    <description>The name of the azure file system for testing.</description>
-  </property>
-  -->
-
-  <!--If auth type is "SharedKey", provide SharedKey credentials below -->
-  <!--
-  <property>
-    <name>fs.azure.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>{ACCOUNTKEY}</value>
-  </property>
-
-  <property>
-    <name>fs.azure.test.account.key.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>{ACCOUNTKEY}</value>
-  </property>
-  -->
-
-  <!--If auth type is "OAuth", set below properties, AAD client and tenant related properties can be obtained through Azure Portal-->
-  <!--
-  <property>
-    <name>fs.azure.account.oauth.provider.type.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>org.apache.hadoop.fs.azurebfs.oauth2.{Token Provider Class name}</value>
-    <description>The full name of token provider class name.</description>
-  </property>
- -->
-
-  <!--If "ClientCredsTokenProvider" is set as key provider, set auth endpoint, client id and secret below-->
-  <!--
-   <property>
-    <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>Token end point, this can be found through Azure portal</description>
-  </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{client id}</value>
-     <description>AAD client id.</description>
-   </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.client.secret.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{client secret}</value>
-   </property>
- -->
-
-  <!--If "UserPasswordTokenProvider" is set as key provider, set auth endpoint, use name and password below-->
-  <!--
-   <property>
-    <name>fs.azure.account.oauth2.client.endpoint.{YOURACCOUNT}.dfs.core.windows.net</name>
-    <value>https://login.microsoftonline.com/{TENANTID}/oauth2/token</value>
-    <description>Token end point, this can be found through Azure portal</description>
-  </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.user.name.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{user name}</value>
-   </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.user.password.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{user password}</value>
-   </property>
- -->
-
-  <!--If "MsiTokenProvider" is set as key provider, set tenantGuid and client id below-->
-  <!--
-   <property>
-     <name>fs.azure.account.oauth2.msi.tenant.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{tenantGuid}</value>
-     <description>msi tenantGuid.</description>
-   </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{client id}</value>
-     <description>AAD client id.</description>
-   </property>
-  -->
-
-  <!--If "RefreshTokenBasedTokenProvider" is set as key provider, set refresh token and client id below-->
-  <!--
-   <property>
-     <name>fs.azure.account.oauth2.refresh.token.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{refresh token}</value>
-     <description>refresh token.</description>
-   </property>
-
-   <property>
-     <name>fs.azure.account.oauth2.client.id.{YOURACCOUNT}.dfs.core.windows.net</name>
-     <value>{client id}</value>
-     <description>AAD client id.</description>
-   </property>
-  -->
-
-  <!--Below four configure properties are provided for Blob Data Contributor a
-  nd Blob Data Reader OAuth access test "ITestAzureBlobFileSystemOauth",
-  using"ClientCredsTokenProvider" as key
-  provider. These are for test only.-->
-
-  <!--
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.id</name>
-    <value>{ID of client which has Data Contributor role}</value>
-  </property>
-
-  <property>
-    <name>fs.azure.account.oauth2.contributor.client.secret</name>
-    <value>{secret of client which has Data Contributor role}</value>
-  </property>
-
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.id</name>
-    <value>{ID of client which has Data Reader role}</value>
-  </property>
-
-  <property>
-    <name>fs.azure.account.oauth2.reader.client.secret</name>
-    <value>{Secret of client which has Data Reader role}</value>
-  </property>
-  -->
-
-
-
-  <!-- Save above configuration properties in a separate file named -->
-  <!-- azure-bfs-auth-keys.xml in the same directory as this file. -->
-  <!-- DO NOT ADD azure-bfs-auth-keys.xml TO REVISION CONTROL.  The keys to your -->
-  <!-- Azure Storage account are a secret and must not be shared. -->
-
-  <include xmlns="http://www.w3.org/2001/XInclude" href="azure-bfs-auth-keys.xml">
-    <fallback />
-  </include>
-
-  <!--<property>-->
-  <!--<name>fs.azure.test.host.name</name>-->
-  <!--<value>{HOSTNAME}</value>-->
-  <!--</property>-->
-
-  <!--<property>-->
-  <!--<name>fs.azure.test.host.port</name>-->
-  <!--<value>{PORT}</value>-->
-  <!--</property>-->
-
-  <!--<property>-->
-  <!--<name>fs.azure.abfs.emulator.enabled</name>-->
-  <!--<value>false</value>-->
-  <!--</property>-->
-
-</configuration>

hadoop-tools/hadoop-azure/src/test/resources/azure-test.xml

@@ -16,49 +16,39 @@
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <configuration xmlns:xi="http://www.w3.org/2001/XInclude">
 
-  <!-- For tests against live azure, provide the following account information -->
+  <!--====================  WASB CONFIGURATION ====================-->
+  <!-- For tests against azure-emulator -->
+  <property>
+    <name>fs.azure.test.emulator</name>
+    <value>false</value>
+  </property>
 
-  <!--
-    <property>
-      <name>fs.azure.test.account.name</name>
-      <value>{ACCOUNTNAME}.blob.core.windows.net</value>
-    </property>
-    <property>
-      <name>fs.azure.account.key.{ACCOUNTNAME}.blob.core.windows.net</name>
-      <value>{ACCOUNTKEY}</value>
-    </property>
-  -->
+  <property>
+    <name>fs.azure.secure.mode</name>
+    <value>false</value>
+  </property>
 
-  <!-- uncomment to test in Azure secure mode -->
-  <!--
-    <property>
-      <name>fs.azure.secure.mode</name>
-      <value>true</value>
-    </property>
-  -->
+  <!--====================  ABFS CONFIGURATION ====================-->
+  <!-- SEE relevant section in "site/markdown/testing_azure.md"-->
 
+  <!--========== CONFIGURATION SHARED BY WASB AND ABFS ==========-->
   <property>
     <name>fs.azure.user.agent.prefix</name>
     <value>MSFT</value>
   </property>
 
-  <!-- Save the above configuration properties in a separate file named -->
-  <!-- azure-auth-keys.xml in the same directory as this file. -->
-  <!-- DO NOT ADD azure-auth-keys.xml TO REVISION CONTROL.  The keys to your -->
-  <!-- Azure Storage account are a secret and must not be shared. -->
+  <!--====================== IMPORTANT!! ========================-->
+  <!--WARNING!!
+      PLEASE CREATE A FILE NAMED "azure-auth-keys.xml"
+      AND SEE "site/markdown/testing_azure.md", THEN
+      STORE THE CONFIGURATION PROPERTIES WITHIN IT.
+      TO PREVENT ACCIDENTAL LEAKS OF YOUR STORAGE ACCOUNT CREDENTIALS,
+      THIS FILE IS LISTED IN .gitignore TO PREVENT YOU FROM INCLUDING
+      IT IN PATCHES OR COMMITS. -->
+  <!--=============================================================-->
+
   <include xmlns="http://www.w3.org/2001/XInclude" href="azure-auth-keys.xml">
     <fallback />
   </include>
 
-  <!-- For tests against azure-emulator -->
-  <!--
-  <property>
-    <name>fs.azure.test.emulator</name>
-    <value>true</value>
-  </property>
-  -->
-  <property>
-    <name>fs.AbstractFileSystem.wasb.impl</name>
-    <value>org.apache.hadoop.fs.azure.Wasb</value>
-  </property>
 </configuration>