HADOOP-3329. DatanodeDescriptor objects should not be stored in the

fsimage. (dhruba)



git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@654941 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -44,6 +44,9 @@ Trunk (unreleased changes)
     HADOOP-3226. Run combiners multiple times over map outputs as they
     are merged in both the map and the reduce tasks. (cdouglas via omalley)
 
+    HADOOP-3329.  DatanodeDescriptor objects should not be stored in the
+    fsimage. (dhruba)
+
   NEW FEATURES
 
     HADOOP-3074. Provides a UrlStreamHandler for DFS and other FS,

src/java/org/apache/hadoop/dfs/DatanodeDescriptor.java

@@ -374,23 +374,6 @@ public class DatanodeDescriptor extends DatanodeInfo {
   }
 
   /** Serialization for FSEditLog */
-  //TODO: remove this method in HADOOP-3329
-  void write2FSEditLog(DataOutput out) throws IOException {
-    UTF8.writeString(out, name);
-    UTF8.writeString(out, storageID);
-    out.writeShort(infoPort);
-    out.writeLong(capacity);
-    out.writeLong(dfsUsed);
-    out.writeLong(remaining);
-    out.writeLong(lastUpdate);
-    out.writeInt(xceiverCount);
-    Text.writeString(out, location);
-    Text.writeString(out, hostName == null? "": hostName);
-    WritableUtils.writeEnum(out, getAdminState());
-  }
-  
-  /** Serialization for FSEditLog */
-  //TODO: remove this method in HADOOP-3329
   void readFieldsFromFSEditLog(DataInput in) throws IOException {
     this.name = UTF8.readString(in);
     this.storageID = UTF8.readString(in);

src/java/org/apache/hadoop/dfs/FSConstants.java

@@ -190,7 +190,7 @@ public interface FSConstants {
   // Version is reflected in the data storage file.
   // Versions are negative.
   // Decrement LAYOUT_VERSION to define a new version.
-  public static final int LAYOUT_VERSION = -13;
+  public static final int LAYOUT_VERSION = -14;
   // Current version: 
-  // Fix bug introduced by OPEN, CLOSE and GENSTAMP transactions for supporting appends
+  // Remove storing locations of last block of a file in fsimage
 }

src/java/org/apache/hadoop/dfs/FSDirectory.java

@@ -309,9 +309,6 @@ class FSDirectory implements FSConstants {
       fileNode.removeBlock(block);
       namesystem.blocksMap.removeINode(block);
 
-      // Remove the block locations for the last block.
-      fileNode.setLastBlockLocations(new DatanodeDescriptor[0]);
-
       // write modified block locations to log
       fsImage.getEditLog().logOpenFile(path, fileNode);
       NameNode.stateChangeLog.debug("DIR* FSDirectory.addFile: "

src/java/org/apache/hadoop/dfs/FSEditLog.java

@@ -384,7 +384,6 @@ class FSEditLog {
     INode old = null;
     String clientName = null;
     String clientMachine = null;
-    DatanodeDescriptor lastLocations[] = null;
     String path = null;
     int numOpAdd = 0, numOpClose = 0, numOpDelete = 0,
         numOpRename = 0, numOpSetRepl = 0, numOpMkDir = 0,
@@ -470,15 +469,15 @@ class FSEditLog {
             }
 
             // clientname, clientMachine and block locations of last block.
-            lastLocations = null;
             if (opcode == OP_ADD && logVersion <= -12) {
               clientName = FSImage.readString(in);
               clientMachine = FSImage.readString(in);
-              lastLocations = readDatanodeDescriptorArray(in);
+              if (-13 <= logVersion) {
+                readDatanodeDescriptorArray(in);
+              }
             } else {
               clientName = "";
               clientMachine = "";
-              lastLocations = EMPTY_ARRAY_DN_DESCRIPTORS;
             }
   
             // The open lease transaction re-creates a file if necessary.
@@ -512,8 +511,7 @@ class FSEditLog {
                                         node.getPermissionStatus(),
                                         clientName, 
                                         clientMachine, 
-                                        null,
-                                        lastLocations);
+                                        null);
               fsDir.replaceNode(path, node, cons);
               fsNamesys.leaseManager.addLease(path, clientName);
             } else if (opcode == OP_CLOSE) {
@@ -800,7 +798,6 @@ class FSEditLog {
    */
   void logOpenFile(String path, INodeFileUnderConstruction newNode) 
                    throws IOException {
-    final DatanodeDescriptor[] locations = newNode.getLastBlockLocations();
 
     UTF8 nameReplicationPair[] = new UTF8[] { 
       new UTF8(path), 
@@ -812,13 +809,7 @@ class FSEditLog {
             new ArrayWritable(Block.class, newNode.getBlocks()),
             newNode.getPermissionStatus(),
             new UTF8(newNode.getClientName()),
-            new UTF8(newNode.getClientMachine()),
-            new Writable() {
-              public void readFields(DataInput in) {}
-              public void write(DataOutput out) throws IOException {
-                writeDatanodeDescriptorArray(out, locations);
-              }
-    });
+            new UTF8(newNode.getClientMachine()));
   }
 
   /** 
@@ -1013,19 +1004,6 @@ class FSEditLog {
   }
 
   /** This method is defined for compatibility reason. */
-  //TODO: remove this class in HADOOP-3329
-  static private void writeDatanodeDescriptorArray(DataOutput out,
-      DatanodeDescriptor[] locations) throws IOException {
-    out.writeInt(locations.length);                 // write values
-    for (int i = 0; i < locations.length; i++) {
-      locations[i].write2FSEditLog(out);
-    }
-  }
-
-  /** This method is defined for compatibility reason. */
-  private static final DatanodeDescriptor[] EMPTY_ARRAY_DN_DESCRIPTORS
-                                                  = new DatanodeDescriptor[0];
-  //TODO: remove this class in HADOOP-3329
   static private DatanodeDescriptor[] readDatanodeDescriptorArray(DataInput in
       ) throws IOException {
     DatanodeDescriptor[] locations = new DatanodeDescriptor[in.readInt()];

src/java/org/apache/hadoop/dfs/FSImage.java

@@ -1035,6 +1035,7 @@ class FSImage extends Storage {
     String clientName = readString(in);
     String clientMachine = readString(in);
 
+    // These locations are not used at all
     int numLocs = in.readInt();
     DatanodeDescriptor[] locations = new DatanodeDescriptor[numLocs];
     for (int i = 0; i < numLocs; i++) {
@@ -1050,9 +1051,7 @@ class FSImage extends Storage {
                                           perm,
                                           clientName,
                                           clientMachine,
-                                          null,
-                                          locations);
-
+                                          null);
   }
 
   // Helper function that writes an INodeUnderConstruction
@@ -1075,11 +1074,7 @@ class FSImage extends Storage {
     writeString(cons.getClientName(), out);
     writeString(cons.getClientMachine(), out);
 
-    int numLocs = cons.getLastBlockLocations().length;
-    out.writeInt(numLocs);
-    for (int i = 0; i < numLocs; i++) {
-      cons.getLastBlockLocations()[i].write(out);
-    }
+    out.writeInt(0); //  do not store locations of last block
   }
 
   /**

src/java/org/apache/hadoop/dfs/FSNamesystem.java

@@ -1015,7 +1015,6 @@ class FSNamesystem implements FSConstants, FSNamesystemMBean {
 
       // allocate new block record block locations in INode.
       newBlock = allocateBlock(src, pendingFile);
-      pendingFile.setLastBlockLocations(targets);
     }
         
     // Create next block

src/java/org/apache/hadoop/dfs/INode.java

@@ -781,7 +781,6 @@ class INodeFileUnderConstruction extends INodeFile {
   protected StringBytesWritable clientName;         // lease holder
   protected StringBytesWritable clientMachine;
   protected DatanodeDescriptor clientNode; // if client is a cluster node too.
-  protected DatanodeDescriptor[] targets;  // locations for last block
 
   INodeFileUnderConstruction() {
     clientName = null;
@@ -802,7 +801,6 @@ class INodeFileUnderConstruction extends INodeFile {
     this.clientName = new StringBytesWritable(clientName);
     this.clientMachine = new StringBytesWritable(clientMachine);
     this.clientNode = clientNode;
-    this.targets = new DatanodeDescriptor[0];
   }
 
   INodeFileUnderConstruction(byte[] name,
@@ -813,8 +811,7 @@ class INodeFileUnderConstruction extends INodeFile {
                              PermissionStatus perm,
                              String clientName,
                              String clientMachine,
-                             DatanodeDescriptor clientNode,
-                             DatanodeDescriptor[] targets) 
+                             DatanodeDescriptor clientNode)
                              throws IOException {
     super(perm, blocks, blockReplication, modificationTime, 
           preferredBlockSize);
@@ -822,7 +819,6 @@ class INodeFileUnderConstruction extends INodeFile {
     this.clientName = new StringBytesWritable(clientName);
     this.clientMachine = new StringBytesWritable(clientMachine);
     this.clientNode = clientNode;
-    this.targets = targets;
   }
 
   String getClientName() throws IOException {
@@ -837,14 +833,6 @@ class INodeFileUnderConstruction extends INodeFile {
     return clientNode;
   }
 
-  void setLastBlockLocations(DatanodeDescriptor[] targets) {
-    this.targets = targets;
-  }
-
-  DatanodeDescriptor[] getLastBlockLocations() {
-    return this.targets;
-  }
-
   /**
    * Is this inode being constructed?
    */