HDDS-936. Need a tool to map containers to ozone objects. Contributed by Sarun Singla

hadoop-ozone/tools/pom.xml

@@ -78,6 +78,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <version>2.15.0</version>
       <scope>test</scope>
     </dependency>
+      <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-ozone-ozone-manager</artifactId>
+          <version>0.4.0-SNAPSHOT</version>
+      </dependency>
   </dependencies>
   <build>
     <plugins>

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/BlockIdDetails.java

@@ -0,0 +1,66 @@
+package org.apache.hadoop.ozone.fsck;
+
+import java.util.Objects;
+
+/**
+ * Getter and Setter for BlockDetails.
+ */
+
+public class BlockIdDetails {
+
+  private String bucketName;
+  private String blockVol;
+  private  String keyName;
+
+  public String getBucketName() {
+    return bucketName;
+  }
+
+  public void setBucketName(String bucketName) {
+    this.bucketName = bucketName;
+  }
+
+  public String getBlockVol() {
+    return blockVol;
+  }
+
+  public void setBlockVol(String blockVol) {
+    this.blockVol = blockVol;
+  }
+
+  public String getKeyName() {
+    return keyName;
+  }
+
+  public void setKeyName(String keyName) {
+    this.keyName = keyName;
+  }
+
+  @Override
+  public String toString() {
+    return "BlockIdDetails{" +
+        "bucketName='" + bucketName + '\'' +
+        ", blockVol='" + blockVol + '\'' +
+        ", keyName='" + keyName + '\'' +
+        '}';
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+    BlockIdDetails that = (BlockIdDetails) o;
+    return Objects.equals(bucketName, that.bucketName) &&
+        Objects.equals(blockVol, that.blockVol) &&
+        Objects.equals(keyName, that.keyName);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(bucketName, blockVol, keyName);
+  }
+}

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/ContainerMapper.java

@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.fsck;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
+import org.apache.hadoop.utils.db.Table;
+import org.apache.hadoop.utils.db.TableIterator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+
+
+/**
+ * Generates Container Id to Blocks and BlockDetails mapping.
+ */
+
+public class ContainerMapper {
+
+
+  private static Table getMetaTable(OzoneConfiguration configuration)
+      throws IOException {
+    OmMetadataManagerImpl metadataManager =
+        new OmMetadataManagerImpl(configuration);
+    return metadataManager.getKeyTable();
+  }
+
+  public static void main(String[] args) throws IOException {
+    String path = args[0];
+    if (path == null) {
+      throw new IOException("Path cannot be null");
+    }
+
+    OzoneConfiguration configuration = new OzoneConfiguration();
+    configuration.set(OZONE_OM_DB_DIRS, path);
+
+    ContainerMapper containerMapper = new ContainerMapper();
+    Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
+        containerMapper.parseOmDB(configuration);
+
+
+    ObjectMapper mapper = new ObjectMapper();
+    System.out.println(mapper.writeValueAsString(dataMap));
+
+  }
+
+  /**
+   * Generates Container Id to Blocks and BlockDetails mapping.
+   * @param configuration @{@link OzoneConfiguration}
+   * @return Map<Long, List<Map<Long, @BlockDetails>>>
+   *   Map of ContainerId -> (Block, Block info)
+   * @throws IOException
+   */
+  public Map<Long, List<Map<Long, BlockIdDetails>>>
+      parseOmDB(OzoneConfiguration configuration) throws IOException {
+    String path = configuration.get(OZONE_OM_DB_DIRS);
+    if (path == null || path.isEmpty()) {
+      throw new IOException(OZONE_OM_DB_DIRS + "should be set ");
+    } else {
+      Table keyTable = getMetaTable(configuration);
+      Map<Long, List<Map<Long, BlockIdDetails>>> dataMap = new HashMap<>();
+
+      if (keyTable != null) {
+        try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+                 keyValueTableIterator = keyTable.iterator()) {
+          while (keyValueTableIterator.hasNext()) {
+            Table.KeyValue<String, OmKeyInfo> keyValue =
+                keyValueTableIterator.next();
+            OmKeyInfo omKeyInfo = keyValue.getValue();
+            byte[] value = omKeyInfo.getProtobuf().toByteArray();
+            OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(
+                OzoneManagerProtocolProtos.KeyInfo.parseFrom(value));
+            for (OmKeyLocationInfoGroup keyLocationInfoGroup : keyInfo
+                .getKeyLocationVersions()) {
+              List<OmKeyLocationInfo> keyLocationInfo = keyLocationInfoGroup
+                  .getLocationList();
+              for (OmKeyLocationInfo keyLocation : keyLocationInfo) {
+                BlockIdDetails blockIdDetails = new BlockIdDetails();
+                Map<Long, BlockIdDetails> innerMap = new HashMap<>();
+
+                long containerID = keyLocation.getBlockID().getContainerID();
+                long blockID = keyLocation.getBlockID().getLocalID();
+                blockIdDetails.setBucketName(keyInfo.getBucketName());
+                blockIdDetails.setBlockVol(keyInfo.getVolumeName());
+                blockIdDetails.setKeyName(keyInfo.getKeyName());
+
+                List<Map<Long, BlockIdDetails>> innerList = new ArrayList<>();
+                innerMap.put(blockID, blockIdDetails);
+
+                if (dataMap.containsKey(containerID)) {
+                  innerList = dataMap.get(containerID);
+                }
+
+                innerList.add(innerMap);
+                dataMap.put(containerID, innerList);
+              }
+            }
+          }
+        }
+      }
+
+      return dataMap;
+
+    }
+  }
+}
+
+

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/fsck/package-info.java

@@ -0,0 +1,44 @@
+/**
+ * Package info.
+ * <p>
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * fsck tool.
+ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * fsck tool.
+ */
+
+
+package org.apache.hadoop.ozone.fsck;

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/TestContainerMapper.java

@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.fsck;
+
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.*;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.ratis.util.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_DB_DIRS;
+import static org.junit.Assert.*;
+
+/**
+ * Test cases for ContainerMapper.
+ */
+
+public class TestContainerMapper {
+
+  private static MiniOzoneCluster cluster = null;
+  private static OzoneClient ozClient = null;
+  private static ObjectStore store = null;
+  private static OzoneManager ozoneManager;
+  private static StorageContainerLocationProtocolClientSideTranslatorPB
+      storageContainerLocationClient;
+  private static final String SCM_ID = UUID.randomUUID().toString();
+  private static String volName = UUID.randomUUID().toString();
+  private static String bucketName = UUID.randomUUID().toString();
+  private static OzoneConfiguration conf;
+  private static List<String> keyList = new ArrayList<>();
+  private static String dbPath;
+
+
+  @BeforeClass
+  public static void init() throws Exception {
+    conf = new OzoneConfiguration();
+    dbPath = GenericTestUtils.getRandomizedTempPath();
+    conf.set(OZONE_OM_DB_DIRS, dbPath);
+    conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, "100MB");
+    cluster = MiniOzoneCluster.newBuilder(conf)
+        .setNumDatanodes(1)
+        .setScmId(SCM_ID)
+        .build();
+    cluster.waitForClusterToBeReady();
+    ozClient = OzoneClientFactory.getRpcClient(conf);
+    store = ozClient.getObjectStore();
+    storageContainerLocationClient =
+        cluster.getStorageContainerLocationClient();
+    ozoneManager = cluster.getOzoneManager();
+    store.createVolume(volName);
+    OzoneVolume volume = store.getVolume(volName);
+    volume.createBucket(bucketName);
+    OzoneBucket bucket = volume.getBucket(bucketName);
+    byte[] data = generateData(10 * 1024 * 1024, (byte)98);
+
+    for (int i = 0; i < 20; i++) {
+      String key = UUID.randomUUID().toString();
+      keyList.add(key);
+      OzoneOutputStream out = bucket.createKey(key, data.length,
+          ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
+          new HashMap<String, String>());
+      out.write(data, 0, data.length);
+      out.close();
+    }
+    cluster.stop();
+  }
+
+  @Test
+  public void testContainerMapper() throws Exception {
+    ContainerMapper containerMapper = new ContainerMapper();
+    Map<Long, List<Map<Long, BlockIdDetails>>> dataMap =
+        containerMapper.parseOmDB(conf);
+    // As we have created 20 keys with 10 MB size, and each
+    // container max size is 100 MB, it should create 3 containers because
+    // containers are closing before reaching the threshold
+    assertEquals(3, dataMap.size());
+  }
+
+  private static byte[] generateData(int size, byte val) {
+    byte[] chars = new byte[size];
+    Arrays.fill(chars, val);
+    return chars;
+  }
+
+  @AfterClass
+  public static void shutdown() throws IOException {
+    cluster.shutdown();
+    FileUtils.deleteFully(new File(dbPath));
+  }
+}

hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/fsck/package-info.java

@@ -0,0 +1,44 @@
+/**
+ * Package info.
+ * <p>
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * <p>
+ * fsck tool.
+ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * fsck tool.
+ */
+
+
+package org.apache.hadoop.ozone.fsck;