HADOOP-4174. Move fs image/edit log methods from ClientProtocol to NamenodeProtocol. (shv via szetszwo)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696039 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -320,6 +320,9 @@ Trunk (unreleased changes)
     HADOOP-4151. Add a byte-comparable interface that both Text and 
     BytesWritable implement. (cdouglas via omalley)
 
+    HADOOP-4174. Move fs image/edit log methods from ClientProtocol to
+    NamenodeProtocol. (shv via szetszwo)
+
   OPTIMIZATIONS
 
     HADOOP-3556. Removed lock contention in MD5Hash by changing the 

src/hdfs/org/apache/hadoop/hdfs/protocol/BlocksWithLocations.java

@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
-
-/** A class to implement an array of BlockLocations
- *  It provide efficient customized serialization/deserialization methods
- *  in stead of using the default array (de)serialization provided by RPC
- */
-public class BlocksWithLocations implements Writable {
-
-  /**
-   * A class to keep track of a block and its locations
-   */
-  public static class BlockWithLocations  implements Writable {
-    Block block;
-    String datanodeIDs[];
-    
-    /** default constructor */
-    public BlockWithLocations() {
-      block = new Block();
-      datanodeIDs = null;
-    }
-    
-    /** constructor */
-    public BlockWithLocations(Block b, String[] datanodes) {
-      block = b;
-      datanodeIDs = datanodes;
-    }
-    
-    /** get the block */
-    public Block getBlock() {
-      return block;
-    }
-    
-    /** get the block's locations */
-    public String[] getDatanodes() {
-      return datanodeIDs;
-    }
-    
-    /** deserialization method */
-    public void readFields(DataInput in) throws IOException {
-      block.readFields(in);
-      int len = WritableUtils.readVInt(in); // variable length integer
-      datanodeIDs = new String[len];
-      for(int i=0; i<len; i++) {
-        datanodeIDs[i] = Text.readString(in);
-      }
-    }
-    
-    /** serialization method */
-    public void write(DataOutput out) throws IOException {
-      block.write(out);
-      WritableUtils.writeVInt(out, datanodeIDs.length); // variable length int
-      for(String id:datanodeIDs) {
-        Text.writeString(out, id);
-      }
-    }
-  }
-
-  private BlockWithLocations[] blocks;
-
-  /** default constructor */
-  BlocksWithLocations() {
-  }
-
-  /** Constructor with one parameter */
-  public BlocksWithLocations( BlockWithLocations[] blocks ) {
-    this.blocks = blocks;
-  }
-
-  /** getter */
-  public BlockWithLocations[] getBlocks() {
-    return blocks;
-  }
-
-  /** serialization method */
-  public void write( DataOutput out ) throws IOException {
-    WritableUtils.writeVInt(out, blocks.length);
-    for(int i=0; i<blocks.length; i++) {
-      blocks[i].write(out);
-    }
-  }
-
-  /** deserialization method */
-  public void readFields(DataInput in) throws IOException {
-    int len = WritableUtils.readVInt(in);
-    blocks = new BlockWithLocations[len];
-    for(int i=0; i<len; i++) {
-      blocks[i] = new BlockWithLocations();
-      blocks[i].readFields(in);
-    }
-  }
-}

src/hdfs/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -22,7 +22,6 @@ import java.io.*;
 import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.hdfs.protocol.FSConstants.UpgradeAction;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
-import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
 import org.apache.hadoop.fs.permission.*;
 import org.apache.hadoop.fs.ContentSummary;
 
@@ -39,9 +38,10 @@ public interface ClientProtocol extends VersionedProtocol {
    * Compared to the previous version the following changes have been introduced:
    * (Only the latest change is reflected.
    * The log of historical changes can be retrieved from the svn).
-   * 37 : Added setTimes
+   * 38 : getEditLogSize(), rollEditLog(), rollFSImage() 
+   * moved to NamenodeProtocol.
    */
-  public static final long versionID = 37L;
+  public static final long versionID = 38L;
   
   ///////////////////////////////////////
   // File contents
@@ -365,30 +365,6 @@ public interface ClientProtocol extends VersionedProtocol {
    */
   public void refreshNodes() throws IOException;
 
-
-  /**
-   * Get the size of the current edit log (in bytes).
-   * @return The number of bytes in the current edit log.
-   * @throws IOException
-   */
-  public long getEditLogSize() throws IOException;
-
-  /**
-   * Closes the current edit log and opens a new one. The 
-   * call fails if the file system is in SafeMode.
-   * @throws IOException
-   * @return a unique token to identify this transaction.
-   */
-  public CheckpointSignature rollEditLog() throws IOException;
-
-  /**
-   * Rolls the fsImage log. It removes the old fsImage, copies the
-   * new image to fsImage, removes the old edits and renames edits.new 
-   * to edits. The call fails if any of the four files are missing.
-   * @throws IOException
-   */
-  public void rollFsImage() throws IOException;
-
   /**
    * Finalize previous upgrade.
    * Remove file system state saved during the upgrade.

src/hdfs/org/apache/hadoop/hdfs/server/balancer/Balancer.java

@@ -54,10 +54,10 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.server.common.Util;
 import org.apache.hadoop.hdfs.DFSClient;
 import org.apache.hadoop.hdfs.protocol.*;
-import org.apache.hadoop.hdfs.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FsShell;
 import org.apache.hadoop.fs.Path;

src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -26,7 +26,6 @@ import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.hdfs.server.namenode.BlocksMap.BlockInfo;
-import org.apache.hadoop.hdfs.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMBean;
 import org.apache.hadoop.hdfs.server.namenode.metrics.FSNamesystemMetrics;
 import org.apache.hadoop.security.UnixUserGroupInformation;
@@ -40,6 +39,8 @@ import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.net.ScriptBasedMapping;
 import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
 import org.apache.hadoop.hdfs.server.protocol.DisallowedDatanodeException;

src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -28,7 +28,7 @@ import org.apache.hadoop.hdfs.protocol.*;
 import org.apache.hadoop.hdfs.server.common.IncorrectVersionException;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
 import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
-import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;

src/hdfs/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
 import org.apache.commons.logging.*;
 
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
 import org.apache.hadoop.ipc.*;
@@ -51,7 +51,7 @@ import org.apache.hadoop.metrics.jvm.JvmMetrics;
  * primary NameNode.
  *
  **********************************************************/
-public class SecondaryNameNode implements FSConstants, Runnable {
+public class SecondaryNameNode implements Runnable {
     
   public static final Log LOG = 
     LogFactory.getLog(SecondaryNameNode.class.getName());
@@ -59,7 +59,7 @@ public class SecondaryNameNode implements FSConstants, Runnable {
   private String fsName;
   private CheckpointStorage checkpointImage;
 
-  private ClientProtocol namenode;
+  private NamenodeProtocol namenode;
   private Configuration conf;
   private InetSocketAddress nameNodeAddr;
   private boolean shouldRun;
@@ -131,8 +131,8 @@ public class SecondaryNameNode implements FSConstants, Runnable {
 
     this.conf = conf;
     this.namenode =
-        (ClientProtocol) RPC.waitForProxy(ClientProtocol.class,
-            ClientProtocol.versionID, nameNodeAddr, conf);
+        (NamenodeProtocol) RPC.waitForProxy(NamenodeProtocol.class,
+            NamenodeProtocol.versionID, nameNodeAddr, conf);
 
     // initialize checkpoint directories
     fsName = getInfoServer();
@@ -513,7 +513,7 @@ public class SecondaryNameNode implements FSConstants, Runnable {
               "cannot access checkpoint directory.");
         StorageState curState;
         try {
-          curState = sd.analyzeStorage(StartupOption.REGULAR);
+          curState = sd.analyzeStorage(FSConstants.StartupOption.REGULAR);
           // sd is locked but not opened
           switch(curState) {
           case NON_EXISTENT:

src/hdfs/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java

@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.protocol;
 
 import java.io.IOException;
 
-import org.apache.hadoop.hdfs.protocol.BlocksWithLocations;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
 import org.apache.hadoop.ipc.VersionedProtocol;
 
 /*****************************************************************************
@@ -30,9 +30,9 @@ import org.apache.hadoop.ipc.VersionedProtocol;
  *****************************************************************************/
 public interface NamenodeProtocol extends VersionedProtocol {
   /**
-   * 1: changed the serialization in DatanodeInfo
+   * 2: Added getEditLogSize(), rollEditLog(), rollFSImage().
    */
-  public static final long versionID = 1L;
+  public static final long versionID = 2L;
 
   /** Get a list of blocks belonged to <code>datanode</code>
     * whose total size is equal to <code>size</code>
@@ -45,4 +45,26 @@ public interface NamenodeProtocol extends VersionedProtocol {
   public BlocksWithLocations getBlocks(DatanodeInfo datanode, long size)
   throws IOException;
 
+  /**
+   * Get the size of the current edit log (in bytes).
+   * @return The number of bytes in the current edit log.
+   * @throws IOException
+   */
+  public long getEditLogSize() throws IOException;
+
+  /**
+   * Closes the current edit log and opens a new one. The 
+   * call fails if the file system is in SafeMode.
+   * @throws IOException
+   * @return a unique token to identify this transaction.
+   */
+  public CheckpointSignature rollEditLog() throws IOException;
+
+  /**
+   * Rolls the fsImage log. It removes the old fsImage, copies the
+   * new image to fsImage, removes the old edits and renames edits.new 
+   * to edits. The call fails if any of the four files are missing.
+   * @throws IOException
+   */
+  public void rollFsImage() throws IOException;
 }

src/test/org/apache/hadoop/hdfs/TestGetBlocks.java

@@ -25,9 +25,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;