HADOOP-829. Within HDFS, clearly separate three different representations for datanodes. Contributed by Dhruba.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@488415 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -114,6 +114,11 @@ Trunk (unreleased changes)
 32. HADOOP-811.  Add a utility, MultithreadedMapRunner.
     (Alejandro Abdelnur via cutting)
 
+33. HADOOP-829.  Within HDFS, clearly separate three different
+    representations for datanodes: one for RPCs, one for
+    namenode-internal use, and one for namespace persistence.
+    (Dhruba Borthakur via cutting) 
+
 
 Release 0.9.2 - 2006-12-15
 

src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java

@@ -24,6 +24,11 @@ import java.util.*;
  * such as available storage capacity, last update time, etc.,
  * and maintains a set of blocks stored on the datanode. 
  *
+ * This data structure is a data structure that is internal
+ * to the namenode. It is *not* sent over-the-wire to the Client
+ * or the Datnodes. Neither is it stored persistently in the
+ * fsImage.
+
  * @author Mike Cafarella
  * @author Konstantin Shvachko
  **************************************************/

src/java/org/apache/hadoop/dfs/DatanodeID.java

@@ -68,7 +68,14 @@ public class DatanodeID implements WritableComparable {
   public int getInfoPort() {
     return infoPort;
   }
-  
+
+  /**
+   * @sets data storage ID.
+   */
+  void setStorageID(String storageID) {
+    this.storageID = storageID;
+  }
+
   /**
    * @return hostname and no :portNumber.
    */

src/java/org/apache/hadoop/dfs/DatanodeInfo.java

@@ -30,6 +30,8 @@ import org.apache.hadoop.io.WritableFactory;
 
 /** 
  * DatanodeInfo represents the status of a DataNode.
+ * This object is used for communication in the
+ * Datanode Protocol and the Client Protocol.
  *
  * @author Mike Cafarella
  * @author Konstantin Shvachko
@@ -72,6 +74,26 @@ public class DatanodeInfo extends DatanodeID {
   /** number of active connections */
   public int getXceiverCount() { return xceiverCount; }
 
+  /** Sets raw capacity. */
+  void setCapacity(long capacity) { 
+    this.capacity = capacity; 
+  }
+
+  /** Sets raw free space. */
+  void setRemaining(long remaining) { 
+    this.remaining = remaining; 
+  }
+
+  /** Sets time when this information was accurate. */
+  void setLastUpdate(long lastUpdate) { 
+    this.lastUpdate = lastUpdate; 
+  }
+
+  /** Sets number of active connections */
+  void setXceiverCount(int xceiverCount) { 
+    this.xceiverCount = xceiverCount; 
+  }
+
   /** A formatted string for reporting the status of the DataNode. */
   public String getDatanodeReport() {
     StringBuffer buffer = new StringBuffer();

src/java/org/apache/hadoop/dfs/FSImage.java

@@ -25,6 +25,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Random;
@@ -33,6 +35,12 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.dfs.FSDirectory.INode;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.io.UTF8;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableFactories;
+import org.apache.hadoop.io.WritableFactory;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.UTF8;
 
 /**
  * FSImage handles checkpointing and logging of the namespace edits.
@@ -318,8 +326,10 @@ class FSImage {
     Map datanodeMap = FSNamesystem.getFSNamesystem().datanodeMap;
     int size = datanodeMap.size();
     out.writeInt( size );
-    for( Iterator it = datanodeMap.values().iterator(); it.hasNext(); )
-      ((DatanodeDescriptor)it.next()).write( out );
+    for( Iterator it = datanodeMap.values().iterator(); it.hasNext(); ) {
+      DatanodeImage nodeImage = new DatanodeImage((DatanodeDescriptor) it.next());
+      nodeImage.write( out );
+    }
   }
 
   void loadDatanodes( int version, DataInputStream in ) throws IOException {
@@ -328,9 +338,75 @@ class FSImage {
     FSNamesystem fsNamesys = FSNamesystem.getFSNamesystem();
     int size = in.readInt();
     for( int i = 0; i < size; i++ ) {
-      DatanodeDescriptor node = new DatanodeDescriptor();
-      node.readFields(in);
-      fsNamesys.unprotectedAddDatanode( node );
+      DatanodeImage nodeImage = new DatanodeImage();
+      nodeImage.readFields(in);
+      fsNamesys.unprotectedAddDatanode(nodeImage.getDatanodeDescriptor());
+    }
+  }
+
+  class DatanodeImage implements WritableComparable {
+
+    /**************************************************
+     * DatanodeImage is used to store persistent information
+     * about datanodes into the fsImage.
+     **************************************************/
+    DatanodeDescriptor              node;
+
+    DatanodeImage() {
+      node = new DatanodeDescriptor();
+    }
+
+    DatanodeImage(DatanodeDescriptor from) {
+      node = from;
+    }
+
+    /** 
+     * Returns the underlying Datanode Descriptor
+     */
+    DatanodeDescriptor getDatanodeDescriptor() { 
+      return node; 
+    }
+
+    public int compareTo(Object o) {
+      return node.compareTo(o);
+    }
+
+    /////////////////////////////////////////////////
+    // Writable
+    /////////////////////////////////////////////////
+    /**
+     * Public method that serializes the information about a
+     * Datanode to be stored in the fsImage.
+     */
+    public void write(DataOutput out) throws IOException {
+      DatanodeID id = new DatanodeID(node.getName(), node.getStorageID(),
+                                     node.getInfoPort());
+      id.write(out);
+      out.writeLong(node.getCapacity());
+      out.writeLong(node.getRemaining());
+      out.writeLong(node.getLastUpdate());
+      out.writeInt(node.getXceiverCount());
+    }
+
+    /**
+     * Public method that reads a serialized Datanode
+     * from the fsImage.
+     */
+    public void readFields(DataInput in) throws IOException {
+      DatanodeID id = new DatanodeID();
+      id.readFields(in);
+      long capacity = in.readLong();
+      long remaining = in.readLong();
+      long lastUpdate = in.readLong();
+      int xceiverCount = in.readInt();
+
+      // update the DatanodeDescriptor with the data we read in
+      node.updateRegInfo(id);
+      node.setStorageID(id.getStorageID());
+      node.setCapacity(capacity);
+      node.setRemaining(remaining);
+      node.setLastUpdate(lastUpdate);
+      node.setXceiverCount(xceiverCount);
     }
   }
 }