HADOOP-3720. Re-read the config file when dfsadmin -refreshNodes is invoked

so dfs.hosts and dfs.hosts.exclude are observed. Contributed by lohit vijayarenu.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@677438 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -120,6 +120,10 @@ Trunk (unreleased changes)
     HADOOP-3721. Refactor CompositeRecordReader and related mapred.join classes
     to make them clearer. (cdouglas)
 
+    HADOOP-3720. Re-read the config file when dfsadmin -refreshNodes is invoked
+    so dfs.hosts and dfs.hosts.exclude are observed. (lohit vijayarenu via
+    cdouglas)
+
 Release 0.18.0 - Unreleased
 
   INCOMPATIBLE CHANGES

src/core/org/apache/hadoop/util/HostsFileReader.java

@@ -63,7 +63,7 @@ public class HostsFileReader {
     }  
   }
 
-  public void refresh() throws IOException {
+  public synchronized void refresh() throws IOException {
     includes.clear();
     excludes.clear();
     
@@ -83,4 +83,18 @@ public class HostsFileReader {
     return excludes;
   }
 
+  public synchronized void setIncludesFile(String includesFile) {
+    this.includesFile = includesFile;
+  }
+  
+  public synchronized void setExcludesFile(String excludesFile) {
+    this.excludesFile = excludesFile;
+  }
+
+  public synchronized void updateFileNames(String includesFile, 
+                                           String excludesFile) 
+                                           throws IOException {
+    setIncludesFile(includesFile);
+    setExcludesFile(excludesFile);
+  }
 }

src/docs/src/documentation/content/xdocs/hdfs_design.xml

@@ -313,7 +313,7 @@
             <td> Generate a list of DataNodes </td> <td> <code>bin/hadoop dfsadmin -report</code> </td>
           </tr>
           <tr>
-            <td> Decommission DataNode <code>datanodename</code> </td><td> <code>bin/hadoop dfsadmin -decommission datanodename</code> </td>
+            <td> Recommission or decommission DataNode(s) </td><td> <code>bin/hadoop dfsadmin -refreshNodes</code> </td>
           </tr>
         </table>
       </section>

src/docs/src/documentation/content/xdocs/hdfs_user_guide.xml

@@ -190,6 +190,17 @@
    		<code>-finalizeUpgrade</code>
    		: removes previous backup of the cluster made during last upgrade.
    	</li>
+    <li>
+      <code>-refreshNodes</code>
+      : Updates the set of hosts allowed to connect to namenode.
+      Re-reads the config file to update values defined by dfs.hosts and 
+      dfs.host.exclude and reads the entires (hostnames) in those files. 
+      Each entry not defined in dfs.hosts but in dfs.hosts.exclude 
+      is decommissioned. Each entry defined in dfs.hosts and also in 
+      dfs.host.exclude is stopped from decommissioning if it has aleady 
+      been marked for decommission. Entires not present in both the lists 
+      are decommissioned. 
+    </li>
    	</ul>
    	<p>
    	  For command usage, see <a href="commands_manual.html#dfsadmin">dfsadmin command</a>.

src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -3480,6 +3480,7 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
   }
 
   /**
+   * Rereads the config to get hosts and exclude list file names.
    * Rereads the files to update the hosts and exclude lists.  It
    * checks if any of the hosts have changed states:
    * 1. Added to hosts  --> no further work needed here.
@@ -3487,8 +3488,14 @@ public class FSNamesystem implements FSConstants, FSNamesystemMBean {
    * 3. Added to exclude --> start decommission.
    * 4. Removed from exclude --> stop decommission.
    */
-  void refreshNodes() throws IOException {
+  public void refreshNodes(Configuration conf) throws IOException {
     checkSuperuserPrivilege();
+    // Reread the config to get dfs.hosts and dfs.hosts.exclude filenames.
+    // Update the file names and refresh internal includes and excludes list
+    if (conf == null)
+      conf = new Configuration();
+    hostsReader.updateFileNames(conf.get("dfs.hosts",""), 
+                                conf.get("dfs.hosts.exclude", ""));
     hostsReader.refresh();
     synchronized (this) {
       for (Iterator<DatanodeDescriptor> it = datanodeMap.values().iterator();

src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -521,10 +521,11 @@ public class NameNode implements ClientProtocol, DatanodeProtocol,
 
   /*
    * Refresh the list of datanodes that the namenode should allow to  
-   * connect.  Uses the files list in the configuration to update the list. 
+   * connect.  Re-reads conf by creating new Configuration object and 
+   * uses the files list in the configuration to update the list. 
    */
   public void refreshNodes() throws IOException {
-    namesystem.refreshNodes();
+    namesystem.refreshNodes(new Configuration());
   }
 
   /**

src/hdfs/org/apache/hadoop/hdfs/tools/DFSAdmin.java

@@ -305,9 +305,16 @@ public class DFSAdmin extends FsShell {
       "\t\tcondition.  Safe mode can also be entered manually, but then\n" +
       "\t\tit can only be turned off manually as well.\n";
 
-    String refreshNodes = "-refreshNodes: \tRe-read the hosts and exclude files to update the set\n" +
-      "\t\tof Datanodes that are allowed to connect to the Namenode\n" +
-      "\t\tand those that should be decommissioned of recommissioned.\n";
+    String refreshNodes = "-refreshNodes: \tUpdates the set of hosts allowed " +
+                          "to connect to namenode.\n\n" +
+      "\t\tRe-reads the config file to update values defined by \n" +
+      "\t\tdfs.hosts and dfs.host.exclude and reads the \n" +
+      "\t\tentires (hostnames) in those files.\n\n" +
+      "\t\tEach entry not defined in dfs.hosts but in \n" + 
+      "\t\tdfs.hosts.exclude is decommissioned. Each entry defined \n" +
+      "\t\tin dfs.hosts and also in dfs.host.exclude is stopped from \n" +
+      "\t\tdecommissioning if it has aleady been marked for decommission.\n" + 
+      "\t\tEntires not present in both the lists are decommissioned.\n";
 
     String finalizeUpgrade = "-finalizeUpgrade: Finalize upgrade of DFS.\n" +
       "\t\tDatanodes delete their previous version working directories,\n" +

src/test/org/apache/hadoop/hdfs/TestDecommission.java

@@ -28,6 +28,7 @@ import java.lang.InterruptedException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -170,7 +171,9 @@ public class TestDecommission extends TestCase {
   /*
    * decommission one random node.
    */
-  private String decommissionNode(DFSClient client, 
+  private String decommissionNode(NameNode namenode,
+                                  Configuration conf,
+                                  DFSClient client, 
                                   FileSystem filesys,
                                   FileSystem localFileSys)
     throws IOException {
@@ -195,7 +198,7 @@ public class TestDecommission extends TestCase {
     ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes);
     nodes.add(nodename);
     writeConfigFile(localFileSys, excludeFile, nodes);
-    dfs.refreshNodes();
+    namenode.namesystem.refreshNodes(conf);
     return nodename;
   }
 
@@ -303,7 +306,8 @@ public class TestDecommission extends TestCase {
                            replicas + " replicas.");
         checkFile(fileSys, file1, replicas);
         printFileLocations(fileSys, file1);
-        String downnode = decommissionNode(client, fileSys, localFileSys);
+        String downnode = decommissionNode(cluster.getNameNode(), conf,
+                                           client, fileSys, localFileSys);
         decommissionedNodes.add(downnode);
         waitNodeState(fileSys, downnode, NodeState.DECOMMISSIONED);
         checkFile(fileSys, file1, replicas, downnode);