Integration of TOS: Add switch to unit tests that depending on tos.

hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/TosFS.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.tosfs.object.tos.TOS;
+import org.apache.hadoop.util.Preconditions;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class TosFS extends RawFS {
+  public TosFS(URI uri, Configuration conf) throws IOException, URISyntaxException {
+    super(verifyURI(uri, conf), conf);
+  }
+
+  private static URI verifyURI(URI uri, Configuration conf) {
+    Preconditions.checkNotNull(uri);
+
+    String scheme = uri.getScheme();
+    if (scheme == null || scheme.isEmpty()) {
+      scheme = FileSystem.getDefaultUri(conf).getScheme();
+    }
+    Preconditions.checkArgument(scheme.equals(TOS.TOS_SCHEME),
+        "Unsupported scheme %s, expected scheme is %s.", scheme, TOS.TOS_SCHEME);
+
+    return uri;
+  }
+}

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestEnv.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs;
+
+import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+
+public class TestEnv {
+  public static final String ENV_TOS_UNIT_TEST_ENABLED = "TOS_UNIT_TEST_ENABLED";
+  private static final boolean TOS_TEST_ENABLED;
+
+  static {
+    TOS_TEST_ENABLED = ParseUtils.envAsBoolean(ENV_TOS_UNIT_TEST_ENABLED, false);
+  }
+
+  public static boolean checkTestEnabled() {
+    return TOS_TEST_ENABLED;
+  }
+}

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosChecksum.java

@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.tosfs.util.TempFiles;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
 import org.junit.After;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -52,6 +53,7 @@ public class TestTosChecksum {
 
   @Parameterized.Parameters(name = "checksumType = {0}, conf = {1}, uri = {2}, objectStorage = {3}")
   public static Iterable<Object[]> createStorage() throws URISyntaxException {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
     return createTestObjectStorage(FILE_STORE_ROOT);
   }
 

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/TestTosFileSystem.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.fs.tosfs;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -30,6 +32,12 @@ import java.net.URISyntaxException;
 import static org.junit.Assert.assertThrows;
 
 public class TestTosFileSystem {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Test
   public void testUriVerification() throws URISyntaxException, IOException {
     Configuration conf = new Configuration(false);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/CommitterTestBase.java

@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
@@ -33,6 +34,7 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.FileNotFoundException;
@@ -76,6 +78,11 @@ public abstract class CommitterTestBase {
     IOUtils.closeStream(fs);
   }
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @AfterClass
   public static void afterClass() {
     List<String> committerThreads = Thread.getAllStackTraces().keySet()

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/MRJobTestBase.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.examples.terasort.TeraSort;
 import org.apache.hadoop.examples.terasort.TeraSortConfigKeys;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
@@ -37,6 +38,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
@@ -65,6 +67,8 @@ public abstract class MRJobTestBase {
 
   @BeforeClass
   public static void beforeClass() throws IOException {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+
     conf.setBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, false);
     conf.setBoolean(YarnConfiguration.NM_DISK_HEALTH_CHECK_ENABLE, false);
     conf.setInt(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 100);
@@ -91,16 +95,16 @@ public abstract class MRJobTestBase {
 
   @AfterClass
   public static void afterClass() throws IOException {
+    if (!TestEnv.checkTestEnabled()) {
+      return;
+    }
+
     fs.delete(testDataPath, true);
     if (yarnCluster != null) {
       yarnCluster.stop();
     }
   }
 
-  @Before
-  public void before() throws IOException {
-  }
-
   @After
   public void after() throws IOException {
   }

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/TestCommitter.java

@@ -18,12 +18,13 @@ package org.apache.hadoop.fs.tosfs.commit;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.tosfs.util.ParseUtils;
+import org.apache.hadoop.fs.tosfs.util.TestUtility;
 
 public class TestCommitter extends CommitterTestBase {
   @Override
   protected Configuration newConf() {
     Configuration conf = new Configuration();
-    conf.set("fs.defaultFS", String.format("tos://%s", ParseUtils.envAsString("TOS_BUCKET", false)));
+    conf.set("fs.defaultFS", String.format("tos://%s", TestUtility.bucket()));
     return conf;
   }
 }

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/commit/mapred/CommitterTestBase.java

@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.commit.CommitUtils;
 import org.apache.hadoop.fs.tosfs.commit.Pending;
 import org.apache.hadoop.fs.tosfs.commit.PendingSet;
@@ -37,6 +38,7 @@ import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.FileNotFoundException;
@@ -75,8 +77,17 @@ public abstract class CommitterTestBase {
     IOUtils.closeStream(fs);
   }
 
+  @BeforeClass
+  public static void beforeClass() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @AfterClass
   public static void afterClass() {
+    if (!TestEnv.checkTestEnabled()) {
+      return;
+    }
+
     List<String> committerThreads = Thread.getAllStackTraces().keySet()
         .stream()
         .map(Thread::getName)

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestChecksum.java

@@ -26,9 +26,12 @@ import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.FileNotFoundException;
@@ -38,6 +41,11 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
 
 public class TestChecksum extends AbstractFSContractTestBase {
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestCreate.java

@@ -21,8 +21,17 @@ package org.apache.hadoop.fs.tosfs.contract;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 
 public class TestCreate extends AbstractContractCreateTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDelete.java

@@ -23,6 +23,9 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -31,6 +34,12 @@ import static org.apache.hadoop.fs.tosfs.object.ObjectTestUtils.assertDirExist;
 import static org.apache.hadoop.fs.tosfs.object.ObjectTestUtils.assertObjectNotExist;
 
 public class TestDelete extends AbstractContractDeleteTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestDistCp.java

@@ -20,10 +20,19 @@ package org.apache.hadoop.fs.tosfs.contract;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class TestDistCp extends AbstractContractDistCpTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestGetFileStatus.java

@@ -25,11 +25,14 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.tosfs.RawFileStatus;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.conf.TosKeys;
 import org.apache.hadoop.fs.tosfs.object.Constants;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
 import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -57,6 +60,11 @@ public class TestGetFileStatus extends AbstractContractGetFileStatusTest {
     this.getFileStatusEnabled = getFileStatusEnabled;
   }
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     conf.setBoolean(TosKeys.FS_TOS_GET_FILE_STATUS_ENABLED, getFileStatusEnabled);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestMkdir.java

@@ -21,8 +21,17 @@ package org.apache.hadoop.fs.tosfs.contract;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 
 public class TestMkdir extends AbstractContractMkdirTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestOpen.java

@@ -25,8 +25,11 @@ import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.tosfs.RawFileStatus;
 import org.apache.hadoop.fs.tosfs.RawFileSystem;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.object.exceptions.ChecksumMismatchException;
 import org.apache.hadoop.fs.tosfs.util.Range;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -35,6 +38,12 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
 
 public class TestOpen extends AbstractContractOpenTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRename.java

@@ -23,7 +23,10 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -35,6 +38,11 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
 
 public class TestRename extends AbstractContractRenameTest {
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     // Add follow two keys into hadoop configuration.

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestRootDir.java

@@ -19,8 +19,17 @@ package org.apache.hadoop.fs.tosfs.contract;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 
 public class TestRootDir extends AbstractContractRootDirectoryTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestSeek.java

@@ -19,8 +19,17 @@ package org.apache.hadoop.fs.tosfs.contract;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 
 public class TestSeek extends AbstractContractSeekTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestUnbuffer.java

@@ -19,8 +19,17 @@ package org.apache.hadoop.fs.tosfs.contract;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.contract.AbstractContractUnbufferTest;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 
 public class TestUnbuffer extends AbstractContractUnbufferTest {
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TestXAttr.java

@@ -22,7 +22,10 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.common.Bytes;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.FileNotFoundException;
@@ -35,6 +38,12 @@ import java.util.Map;
 public class TestXAttr extends AbstractFSContractTestBase {
   private static final String XATTR_NAME = "xAttrName";
   private static final byte[] XATTR_VALUE = "xAttrValue".getBytes();
+
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   protected AbstractFSContract createContract(Configuration conf) {
     return new TosContract(conf);

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/contract/TosContract.java

@@ -20,8 +20,12 @@ import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -40,6 +44,11 @@ public class TosContract extends AbstractBondedFSContract {
     testDir = "/test-" + UUIDUtils.random();
   }
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   public String getScheme() {
     return "tos";

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestDirectoryStorage.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.tosfs.object;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.TosKeys;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
@@ -27,6 +28,7 @@ import org.assertj.core.api.Assertions;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -42,6 +44,11 @@ public class TestDirectoryStorage {
             scheme(), TestUtility.bucket(), conf);
   }
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @After
   public void tearDown() {
     CommonUtils.runQuietly(() -> storage.deleteAll(""));

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java

@@ -22,6 +22,7 @@ import com.volcengine.tos.TosServerException;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.TosKeys;
 import org.apache.hadoop.fs.tosfs.object.exceptions.InvalidObjectKeyException;
 import org.apache.hadoop.fs.tosfs.object.exceptions.NotAppendableException;
@@ -75,6 +76,7 @@ public class TestObjectStorage {
 
   @Parameterized.Parameters(name = "ObjectStorage = {0}")
   public static List<ObjectStorage> createStorage() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
     return TestUtility.createTestObjectStorage(FILE_STORE_ROOT);
   }
 

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestDelegationClientBuilder.java

@@ -33,6 +33,7 @@ import com.volcengine.tos.model.object.ListObjectsV2Input;
 import com.volcengine.tos.model.object.ListObjectsV2Output;
 import com.volcengine.tos.model.object.PutObjectInput;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.common.Tasks;
 import org.apache.hadoop.fs.tosfs.common.ThreadPools;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
@@ -44,7 +45,9 @@ import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
 import org.junit.After;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -88,6 +91,11 @@ public class TestDelegationClientBuilder {
   // Maximum retry times of the tos http client.
   public static final String MAX_RETRY_COUNT_KEY = "fs.tos.http.maxRetryCount";
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Before
   public void setUp() {
     TOSV2 tosSdkClientV2 =

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSObjectStorage.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.tosfs.object.tos;
 
 import com.volcengine.tos.internal.model.CRC64Checksum;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.common.Bytes;
 import org.apache.hadoop.fs.tosfs.conf.TosKeys;
 import org.apache.hadoop.fs.tosfs.object.ChecksumType;
@@ -68,6 +69,8 @@ public class TestTOSObjectStorage {
 
   @Parameterized.Parameters(name = "ObjectStorage = {0}, Checksum = {1}, ChecksumType = {2}")
   public static Iterable<Object[]> collections() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+
     List<Object[]> values = new ArrayList<>();
 
     Configuration conf = new Configuration();

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/tos/TestTOSRetryPolicy.java

@@ -26,6 +26,7 @@ import com.volcengine.tos.model.RequestInfo;
 import com.volcengine.tos.model.object.PutObjectOutput;
 import com.volcengine.tos.model.object.UploadPartV2Output;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.conf.TosKeys;
 import org.apache.hadoop.fs.tosfs.object.InputStreamProvider;
@@ -34,7 +35,9 @@ import org.apache.hadoop.fs.tosfs.object.Part;
 import org.apache.hadoop.fs.tosfs.object.tos.auth.SimpleCredentialsProvider;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.junit.After;
+import org.junit.Assume;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 import java.io.IOException;
@@ -61,6 +64,11 @@ public class TestTOSRetryPolicy {
   private TOSV2 tosClient;
   private DelegationClient client;
 
+  @BeforeClass
+  public static void before() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Before
   public void setUp() {
     client = createRetryableDelegationClient();

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.tosfs.ops;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.common.ThreadPools;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
@@ -26,6 +27,7 @@ import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
 import org.apache.hadoop.util.Lists;
 import org.junit.AfterClass;
+import org.junit.Assume;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -61,11 +63,16 @@ public class TestDefaultFsOps extends TestBaseFsOps {
 
   @BeforeClass
   public static void beforeClass() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
     threadPool = ThreadPools.newWorkerPool("TestDefaultFsHelper-pool");
   }
 
   @AfterClass
   public static void afterClass() {
+    if (!TestEnv.checkTestEnabled()) {
+      return;
+    }
+
     if (!threadPool.isShutdown()) {
       threadPool.shutdown();
     }

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDirectoryFsOps.java

@@ -19,11 +19,13 @@
 package org.apache.hadoop.fs.tosfs.ops;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.object.DirectoryStorage;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
 import org.junit.Assume;
+import org.junit.BeforeClass;
 
 import static org.apache.hadoop.fs.tosfs.object.tos.TOS.TOS_SCHEME;
 
@@ -37,6 +39,11 @@ public class TestDirectoryFsOps extends TestBaseFsOps {
     this.fsOps = new DirectoryFsOps((DirectoryStorage) storage, this::toFileStatus);
   }
 
+  @BeforeClass
+  public static void beforeClass() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
+  }
+
   @Override
   public void testRenameDir() {
     // Will remove this test case once test environment support

hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestRenameOp.java

@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.tosfs.ops;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.TestEnv;
 import org.apache.hadoop.fs.tosfs.common.ThreadPools;
 import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
@@ -62,6 +63,7 @@ public class TestRenameOp extends TestBaseOps {
 
   @Parameterized.Parameters
   public static List<ObjectStorage> createStorage() {
+    Assume.assumeTrue(TestEnv.checkTestEnabled());
     return TestUtility.createTestObjectStorage(FILE_STORE_ROOT);
   }