ソースを参照

Integration of TOS: Fix checkstyle.

lijinglun 5 ヶ月 前
コミット
0c0981ef87
25 ファイル変更328 行追加36 行削除
  1. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java
  2. 3 3
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java
  3. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java
  4. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java
  5. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java
  6. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java
  7. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java
  8. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java
  9. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java
  10. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java
  11. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java
  12. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java
  13. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java
  14. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java
  15. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java
  16. 3 3
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java
  17. 4 3
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java
  18. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java
  19. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java
  20. 126 0
      hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java
  21. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java
  22. 2 2
      hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java
  23. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java
  24. 1 1
      hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java
  25. 165 0
      hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFSUtils.java

@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.tosfs;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 import java.util.Objects;
 

+ 3 - 3
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/RawFileSystem.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.tosfs;
 
-import com.google.common.collect.Iterators;
 import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -61,9 +60,10 @@ import org.apache.hadoop.fs.tosfs.util.FuseUtils;
 import org.apache.hadoop.fs.tosfs.util.Range;
 import org.apache.hadoop.fs.tosfs.util.RemoteIterators;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Iterators;
 import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitContext.java

@@ -17,7 +17,7 @@
 package org.apache.hadoop.fs.tosfs.commit;
 
 import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
 
 import java.util.List;
 

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/CommitUtils.java

@@ -31,8 +31,8 @@ import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
 import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Committer.java

@@ -36,8 +36,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/MagicOutputStream.java

@@ -25,8 +25,8 @@ import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
 import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
 import org.apache.hadoop.fs.tosfs.object.Part;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 import java.util.Arrays;

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/Pending.java

@@ -21,8 +21,8 @@ import org.apache.hadoop.fs.tosfs.object.Part;
 import org.apache.hadoop.fs.tosfs.util.JsonCodec;
 import org.apache.hadoop.fs.tosfs.util.Serializer;
 import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.IOException;
 import java.util.List;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/PendingSet.java

@@ -21,8 +21,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.tosfs.util.JsonCodec;
 import org.apache.hadoop.fs.tosfs.util.Serializer;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/SuccessData.java

@@ -21,7 +21,7 @@ import org.apache.hadoop.fs.tosfs.util.Serializer;
 import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
 import org.apache.hadoop.thirdparty.com.google.common.base.Throwables;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
 import org.apache.hadoop.util.StringUtils;
 
 import java.io.IOException;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/commit/mapred/Committer.java

@@ -25,7 +25,7 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.JobContext;
 import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.mapred.TaskAttemptContext;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/conf/ConfKeys.java

@@ -18,7 +18,7 @@
 
 package org.apache.hadoop.fs.tosfs.conf;
 
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 
 public class ConfKeys {
 

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectMultiRangeInputStream.java

@@ -25,8 +25,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.fs.tosfs.util.FSUtils;
 import org.apache.hadoop.fs.tosfs.util.Range;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
 
 import java.io.EOFException;
 import java.io.IOException;

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectOutputStream.java

@@ -23,8 +23,8 @@ import org.apache.hadoop.fs.tosfs.object.staging.FileStagingPart;
 import org.apache.hadoop.fs.tosfs.object.staging.StagingPart;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectRangeInputStream.java

@@ -23,9 +23,9 @@ import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.tosfs.util.FSUtils;
 import org.apache.hadoop.fs.tosfs.util.Range;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
 import org.apache.hadoop.thirdparty.com.google.common.io.ByteStreams;
 import org.apache.hadoop.thirdparty.com.google.common.primitives.Ints;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/staging/FileStagingPart.java

@@ -18,7 +18,7 @@ package org.apache.hadoop.fs.tosfs.object.staging;
 
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 3 - 3
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DefaultFsOps.java

@@ -30,15 +30,15 @@ import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
 import org.apache.hadoop.fs.tosfs.object.request.ListObjectsRequest;
 import org.apache.hadoop.fs.tosfs.object.response.ListObjectsResponse;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Function;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.fs.tosfs.util.Iterables;
+import org.apache.hadoop.util.Lists;
 
 import java.io.IOException;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
 import static org.apache.hadoop.fs.tosfs.object.ObjectUtils.SLASH;

+ 4 - 3
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/DirectoryFsOps.java

@@ -24,10 +24,10 @@ import org.apache.hadoop.fs.tosfs.RawFileStatus;
 import org.apache.hadoop.fs.tosfs.object.DirectoryStorage;
 import org.apache.hadoop.fs.tosfs.object.ObjectInfo;
 import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Function;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hadoop.fs.tosfs.util.Iterables;
 
 import java.io.IOException;
+import java.util.function.Function;
 import java.util.function.Predicate;
 
 /**
@@ -86,7 +86,8 @@ public class DirectoryFsOps implements FsOps {
   @Override
   public Iterable<RawFileStatus> listDir(Path dir, boolean recursive, Predicate<String> postFilter) {
     String key = ObjectUtils.pathToKey(dir, true);
-    Iterable<ObjectInfo> objs = Iterables.filter(storage.listDir(key, recursive), obj -> postFilter.test(obj.key()));
+    Iterable<ObjectInfo> objs =
+        Iterables.filter(storage.listDir(key, recursive), obj -> postFilter.test(obj.key()));
     return Iterables.transform(objs, objMapper);
   }
 

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/ops/RenameOp.java

@@ -28,8 +28,8 @@ import org.apache.hadoop.fs.tosfs.object.ObjectStorage;
 import org.apache.hadoop.fs.tosfs.object.ObjectUtils;
 import org.apache.hadoop.fs.tosfs.object.Part;
 import org.apache.hadoop.fs.tosfs.util.CommonUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/FSUtils.java

@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.tosfs.util;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Preconditions;
 import org.apache.hadoop.util.ReflectionUtils;
 
 import java.io.IOException;

+ 126 - 0
hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Iterables.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.util.Preconditions;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class Iterables {
+
+  public static <F, T> Iterable<T> transform(final Iterable<F> fromIterable,
+      final Function<? super F, ? extends T> function) {
+    Preconditions.checkNotNull(fromIterable);
+    Preconditions.checkNotNull(function);
+    return () -> new Iterator<T>() {
+      private Iterator<F> iterator = fromIterable.iterator();
+
+      @Override
+      public boolean hasNext() {
+        return iterator.hasNext();
+      }
+
+      @Override
+      public T next() {
+        return function.apply(iterator.next());
+      }
+    };
+  }
+
+  public static <T> Iterable<T> filter(final Iterable<T> unfiltered,
+      final Predicate<? super T> predicate) {
+    Preconditions.checkNotNull(unfiltered);
+    Preconditions.checkNotNull(predicate);
+    return () -> new Iterator<T>() {
+      private Iterator<T> iterator = unfiltered.iterator();
+      private boolean advance = true;
+      private T value;
+
+      @Override
+      public boolean hasNext() {
+        if (!advance) {
+          return true;
+        }
+
+        while (iterator.hasNext()) {
+          value = iterator.next();
+          if (predicate.test(value)) {
+            advance = false;
+            return true;
+          }
+        }
+
+        return false;
+      }
+
+      @Override
+      public T next() {
+        if (hasNext()) {
+          advance = true;
+          return value;
+        }
+        throw new NoSuchElementException("No more items in iterator.");
+      }
+    };
+  }
+
+  public static <T extends @Nullable Object> Iterable<T> concat(
+      Iterable<? extends Iterable<? extends T>> inputs) {
+    return () -> new ConcatenatedIterator<>(inputs.iterator());
+  }
+
+  private static class ConcatenatedIterator<T> implements Iterator<T> {
+    // Iterators is the iterator of iterables.
+    private final Iterator<? extends Iterable<? extends T>> iterators;
+    private Iterator<? extends T> curIter;
+
+    ConcatenatedIterator(Iterator<? extends Iterable<? extends T>> iterators) {
+      Preconditions.checkNotNull(iterators, "Iterators should not be null.");
+      this.iterators = iterators;
+    }
+
+    @Override
+    public boolean hasNext() {
+      while (curIter == null || !curIter.hasNext()) {
+        if (curIter != null) {
+          curIter = null;
+        }
+
+        if (!iterators.hasNext()) {
+          return false;
+        }
+
+        curIter = iterators.next().iterator();
+      }
+      return true;
+    }
+
+    @Override
+    public T next() {
+      if (hasNext()) {
+        return curIter.next();
+      }
+      throw new NoSuchElementException("No more elements");
+    }
+  }
+}

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectOutputStream.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.fs.tosfs.util.FSUtils;
 import org.apache.hadoop.fs.tosfs.util.TempFiles;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;

+ 2 - 2
hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/object/TestObjectStorage.java

@@ -31,9 +31,9 @@ import org.apache.hadoop.fs.tosfs.util.CommonUtils;
 import org.apache.hadoop.fs.tosfs.util.TempFiles;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
 import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
+import org.apache.hadoop.util.Lists;
+import org.apache.hadoop.util.Preconditions;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/ops/TestDefaultFsOps.java

@@ -24,7 +24,7 @@ import org.apache.hadoop.fs.tosfs.conf.ConfKeys;
 import org.apache.hadoop.fs.tosfs.object.ObjectStorageFactory;
 import org.apache.hadoop.fs.tosfs.util.TestUtility;
 import org.apache.hadoop.fs.tosfs.util.UUIDUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.runner.RunWith;

+ 1 - 1
hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TempFiles.java

@@ -19,7 +19,7 @@
 package org.apache.hadoop.fs.tosfs.util;
 
 import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.thirdparty.com.google.common.collect.Lists;
+import org.apache.hadoop.util.Lists;
 
 import java.io.Closeable;
 import java.io.File;

+ 165 - 0
hadoop-cloud-storage-project/hadoop-tos/src/test/java/org/apache/hadoop/fs/tosfs/util/TestIterables.java

@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+public class TestIterables {
+
+  @Test
+  public void testTransform() {
+    List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+    Function<Integer, Integer> transform = i -> i + 10;
+    Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+    for (int i = 0; i < 5; i++) {
+      assertTrue(iter.hasNext());
+      int value = iter.next();
+      assertEquals(10 + i + 1, value);
+    }
+    assertFalse(iter.hasNext());
+  }
+
+  @Test
+  public void testTransformEmptyIterable() {
+    List<Integer> list = Arrays.asList();
+    Function<Integer, Integer> transform = i -> i + 10;
+    Iterator<Integer> iter = Iterables.transform(list, transform).iterator();
+
+    assertFalse(iter.hasNext());
+  }
+
+  @Test
+  public void testFilter() {
+    // Filter odd elements.
+    List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
+    Predicate<Integer> filter = i -> (i % 2) == 0;
+    Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+    for (int i = 0; i < 2; i++) {
+      assertTrue(iter.hasNext());
+      int value = iter.next();
+      assertEquals((i + 1) * 2, value);
+    }
+    assertFalse(iter.hasNext());
+
+    // Ignore all elements.
+    filter = i -> false;
+    iter = Iterables.filter(list, filter).iterator();
+    assertFalse(iter.hasNext());
+  }
+
+  @Test
+  public void testFilterEmptyIterable() {
+    List<Integer> list = Arrays.asList();
+    Predicate<Integer> filter = i -> (i % 2) == 0;
+    Iterator<Integer> iter = Iterables.filter(list, filter).iterator();
+
+    assertFalse(iter.hasNext());
+  }
+
+  // Full iterators.
+  @Test
+  public void testConcatFullIterators() {
+    List<Integer> expectedList = new ArrayList<>();
+    List<Iterable<Integer>> iterList = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      List<Integer> list = new ArrayList<>();
+      for (int j = 0; j < 10; j++) {
+        list.add(i * 10 + j);
+        expectedList.add(i * 10 + j);
+      }
+      iterList.add(list);
+    }
+
+    verifyConcat(expectedList.iterator(), iterList);
+  }
+
+  // Empty iterators.
+  @Test
+  public void testConcatEmptyIterators() {
+    List<Integer> expectedList = new ArrayList<>();
+    List<Iterable<Integer>> iterList = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      iterList.add(Collections.emptyList());
+    }
+
+    verifyConcat(expectedList.iterator(), iterList);
+  }
+
+  // Mix full and empty iterators.
+  @Test
+  public void testConcatMixFullAndEmptyIterators() {
+    List<Integer> expectedList = new ArrayList<>();
+    List<Iterable<Integer>> iterList = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      List<Integer> list = new ArrayList<>();
+      for (int j = 0; j < 10; j++) {
+        list.add(i * 10 + j);
+        expectedList.add(i * 10 + j);
+      }
+      iterList.add(list);
+      iterList.add(Collections.emptyList());
+      iterList.add(Collections.emptyList());
+    }
+
+    verifyConcat(expectedList.iterator(), iterList);
+  }
+
+  // Invalid iterators.
+  @Test
+  public void testConcatNullMetaIterator() {
+    assertThrows("Expect null verification error.", NullPointerException.class,
+        () -> verifyConcat(Collections.emptyIterator(), null));
+  }
+
+  // Concat null iterators.
+  @Test
+  public void testConcatNullElementIterators() {
+    List<Iterable<Integer>> list = new ArrayList<>();
+    for (int i = 0; i < 3; i++) {
+      list.add(() -> null);
+    }
+    verifyConcat(Collections.emptyIterator(), list);
+  }
+
+  private <T> void verifyConcat(Iterator<T> expectedValues, Iterable<Iterable<T>> metaIter) {
+    Iterator<T> iter = Iterables.concat(metaIter).iterator();
+    while (expectedValues.hasNext()) {
+      assertTrue(iter.hasNext());
+      T v1 = expectedValues.next();
+      T v2 = iter.next();
+      assertEquals(v1, v2);
+    }
+    assertFalse(iter.hasNext());
+  }
+}