Integration of TOS: Add ObjectUtils UUIDUtils and Range.

hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/object/ObjectUtils.java

@@ -1,2 +1,91 @@
-package org.apache.hadoop.fs.tosfs.util;public class ObjectUtils {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.object;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.tosfs.util.Range;
+import org.apache.hadoop.thirdparty.com.google.common.base.Joiner;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Lists;
+
+import java.util.List;
+
+public class ObjectUtils {
+  public static final String SLASH = "/";
+
+  private ObjectUtils() {
+  }
+
+  public static Path keyToPath(String key) {
+    return new Path(SLASH + key);
+  }
+
+  public static String path(String key) {
+    return key.startsWith(SLASH) ? key : SLASH + key;
+  }
+
+  public static String pathToKey(Path p) {
+    return pathToKey(p, false);
+  }
+
+  public static String pathToKey(Path p, Boolean isDir) {
+    Preconditions.checkArgument(p != null, "Null path");
+    if (p.toUri().getScheme() != null && p.toUri().getPath().isEmpty()) {
+      return "";
+    }
+    String key = p.toUri().getPath().substring(1);
+    if (isDir && !key.isEmpty()) {
+      return key.endsWith(SLASH) ? key : key + SLASH;
+    }
+    return key;
+  }
+
+  public static void deleteAllObjects(ObjectStorage storage, Iterable<ObjectInfo> objects, int batchSize) {
+    List<String> keysToDelete = Lists.newArrayList();
+    for (ObjectInfo obj : objects) {
+      keysToDelete.add(obj.key());
+
+      if (keysToDelete.size() == batchSize) {
+        batchDelete(storage, keysToDelete);
+        keysToDelete.clear();
+      }
+    }
+
+    if (!keysToDelete.isEmpty()) {
+      batchDelete(storage, keysToDelete);
+    }
+  }
+
+  private static void batchDelete(ObjectStorage storage, List<String> keys) {
+    List<String> failedKeys = storage.batchDelete(keys);
+    if (!failedKeys.isEmpty()) {
+      throw new RuntimeException(String.format("Failed to delete %s objects, detail: %s",
+          failedKeys.size(), Joiner.on(",").join(failedKeys)));
+    }
+  }
+
+  public static Range calculateRange(final long offset, final long limit, final long objSize) {
+    Preconditions.checkArgument(offset >= 0, String.format("offset is a negative number: %s", offset));
+    Preconditions.checkArgument(offset <= objSize,
+        String.format("offset: %s is bigger than object size: %s", offset, objSize));
+    long len = limit < 0 ? objSize - offset : Math.min(objSize - offset, limit);
+    return Range.of(offset, len);
+  }
 }
+

hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/Range.java

@@ -1,2 +1,122 @@
-package org.apache.hadoop.fs.tosfs.util;public class Range {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import org.apache.hadoop.thirdparty.com.google.common.base.MoreObjects;
+import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
+import org.apache.hadoop.util.Lists;
+
+import java.util.List;
+import java.util.Objects;
+
+public class Range {
+  private final long off;
+  private final long len;
+
+  private Range(long off, long len) {
+    this.off = off;
+    this.len = len;
+  }
+
+  public static Range of(long off, long len) {
+    return new Range(off, len);
+  }
+
+  public long off() {
+    return off;
+  }
+
+  public long len() {
+    return len;
+  }
+
+  public long end() {
+    return off + len;
+  }
+
+  public boolean include(long pos) {
+    return pos >= off && pos < off + len;
+  }
+
+  public boolean include(Range r) {
+    return off() <= r.off() && r.end() <= end();
+  }
+
+  public boolean overlap(Range r) {
+    return r.off() < end() && off() < r.end();
+  }
+
+  public boolean overlap(long off1, long len1) {
+    return off1 < end() && off() < (off1 + len1);
+  }
+
+  public Range alignTo(long fixedSize) {
+    // Get the aligned start.
+    long alignedStart = off - off % fixedSize;
+    // Get the aligned end.
+    long mod = end() % fixedSize;
+    long alignedEnd = mod == 0 ? end() : (end() - mod + fixedSize);
+    return Range.of(alignedStart, alignedEnd - alignedStart);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(off, len);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    } else if (!(o instanceof Range)) {
+      return false;
+    }
+
+    Range that = (Range) o;
+    return Objects.equals(off, that.off)
+        && Objects.equals(len, that.len);
+  }
+
+  public String toString() {
+    return MoreObjects.toStringHelper(this)
+        .add("offset", off)
+        .add("length", len)
+        .toString();
+  }
+
+  public static List<Range> split(long totalSize, long width) {
+    Preconditions.checkArgument(totalSize >= 0, "Size %s must be >= 0", totalSize);
+    Preconditions.checkArgument(width > 0, "Width %s must be positive", width);
+
+    long remain = totalSize % width;
+    long rangeNum = totalSize / width;
+
+    List<Range> ranges = Lists.newArrayListWithCapacity((int) rangeNum + (remain == 0 ? 0 : 1));
+    for (int i = 0; i < rangeNum; i++) {
+      ranges.add(Range.of(i * width, width));
+    }
+
+    if (remain > 0) {
+      ranges.add(Range.of(totalSize - remain, remain));
+    }
+
+    return ranges;
+  }
 }
+

hadoop-cloud-storage-project/hadoop-tos/src/main/java/org/apache/hadoop/fs/tosfs/util/UUIDUtils.java

@@ -1,2 +1,31 @@
-package org.apache.hadoop.fs.tosfs.util;public class UUIDUtils {
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.tosfs.util;
+
+import java.util.UUID;
+
+public class UUIDUtils {
+  private UUIDUtils() {
+  }
+
+  public static String random() {
+    UUID uuid = UUID.randomUUID();
+    return uuid.toString().replace("-", "");
+  }
 }