HDFS-7564. NFS gateway dynamically reload UID/GID mapping file /etc/nfs.map. Contributed by Yongjun Zhang

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java

@@ -75,6 +75,10 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
 
   private final File staticMappingFile;
   private StaticMapping staticMapping = null;
+  // Last time the static map was modified, measured time difference in
+  // milliseconds since midnight, January 1, 1970 UTC
+  private long lastModificationTimeStaticMap = 0;
+  
   private boolean constructFullMapAtInit = false;
 
   // Used for parsing the static mapping file.
@@ -88,7 +92,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   // Maps for id to name map. Guarded by this object monitor lock
   private BiMap<Integer, String> uidNameMap = HashBiMap.create();
   private BiMap<Integer, String> gidNameMap = HashBiMap.create();
-
   private long lastUpdateTime = 0; // Last time maps were updated
 
   /*
@@ -118,7 +121,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
         conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
             IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
     staticMappingFile = new File(staticFilePath);
-
+    updateStaticMapping();
     updateMaps();
   }
 
@@ -290,20 +293,42 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     return true;
   }
 
-  private synchronized void initStaticMapping() throws IOException {
-    staticMapping = new StaticMapping(
-        new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
+  private synchronized void updateStaticMapping() throws IOException {
+    final boolean init = (staticMapping == null);
+    //
+    // if the static mapping file
+    //   - was modified after last update, load the map again;
+    //   - did not exist but was added since last update, load the map;
+    //   - existed before but deleted since last update, clear the map
+    //
     if (staticMappingFile.exists()) {
-      LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
-      staticMapping = parseStaticMap(staticMappingFile);
+      // check modification time, reload the file if the last modification
+      // time changed since prior load.
+      long lmTime = staticMappingFile.lastModified();
+      if (lmTime != lastModificationTimeStaticMap) {
+        LOG.info(init? "Using " : "Reloading " + "'" + staticMappingFile
+            + "' for static UID/GID mapping...");
+        lastModificationTimeStaticMap = lmTime;
+        staticMapping = parseStaticMap(staticMappingFile);        
+      }
     } else {
-      LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
-          + "' does not exist.");
+      if (init) {
+        staticMapping = new StaticMapping(new HashMap<Integer, Integer>(),
+            new HashMap<Integer, Integer>());
+      }
+      if (lastModificationTimeStaticMap != 0 || init) {
+        // print the following log at initialization or when the static
+        // mapping file was deleted after prior load
+        LOG.info("Not doing static UID/GID mapping because '"
+            + staticMappingFile + "' does not exist.");
+      }
+      lastModificationTimeStaticMap = 0;
+      staticMapping.clear();
     }
-  }  
+  }
 
   /*
-   * Reset the maps to empty.
+   * Refresh static map, and reset the other maps to empty.
    * For testing code, a full map may be re-constructed here when the object
    * was created with constructFullMapAtInit being set to true.
    */
@@ -314,15 +339,16 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
 
     if (constructFullMapAtInit) {
       loadFullMaps();
+      // set constructFullMapAtInit to false to allow testing code to
+      // do incremental update to maps after initial construction
+      constructFullMapAtInit = false;
     } else {
+      updateStaticMapping();
       clearNameMaps();
     }
   }
   
   synchronized private void loadFullUserMap() throws IOException {
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
     BiMap<Integer, String> uMap = HashBiMap.create();
     if (OS.startsWith("Mac")) {
       updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
@@ -336,9 +362,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   }
 
   synchronized private void loadFullGroupMap() throws IOException {
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
     BiMap<Integer, String> gMap = HashBiMap.create();
 
     if (OS.startsWith("Mac")) {
@@ -353,7 +376,6 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
   }
 
   synchronized private void loadFullMaps() throws IOException {
-    initStaticMapping();
     loadFullUserMap();
     loadFullGroupMap();
   }
@@ -443,9 +465,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     }
 
     boolean updated = false;
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
+    updateStaticMapping();
 
     if (OS.startsWith("Linux")) {
       if (isGrp) {
@@ -481,9 +501,7 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
     }
     
     boolean updated = false;
-    if (staticMapping == null) {
-      initStaticMapping();
-    }
+    updateStaticMapping();
 
     if (OS.startsWith("Linux")) {
       if (isGrp) {
@@ -547,6 +565,15 @@ public class ShellBasedIdMapping implements IdMappingServiceProvider {
       this.uidMapping = new PassThroughMap<Integer>(uidMapping);
       this.gidMapping = new PassThroughMap<Integer>(gidMapping);
     }
+
+    public void clear() {
+      uidMapping.clear();
+      gidMapping.clear();
+    }
+
+    public boolean isNonEmpty() {
+      return uidMapping.size() > 0 || gidMapping.size() > 0;
+    }
   }
   
   static StaticMapping parseStaticMap(File staticMapFile)

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java

@@ -41,6 +41,13 @@ public class TestShellBasedIdMapping {
   private static final Map<Integer, Integer> EMPTY_PASS_THROUGH_MAP =
       new PassThroughMap<Integer>();
   
+  private void createStaticMapFile(final File smapFile, final String smapStr)
+      throws IOException {
+    OutputStream out = new FileOutputStream(smapFile);
+    out.write(smapStr.getBytes());
+    out.close();
+  }
+
   @Test
   public void testStaticMapParsing() throws IOException {
     File tempStaticMapFile = File.createTempFile("nfs-", ".map");
@@ -58,11 +65,10 @@ public class TestShellBasedIdMapping {
         "gid 12 202\n" +
         "uid 4294967294 123\n" +
         "gid 4294967295 321";
-    OutputStream out = new FileOutputStream(tempStaticMapFile);
-    out.write(staticMapFileContents.getBytes());
-    out.close();
-    StaticMapping parsedMap = ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);
-    
+    createStaticMapFile(tempStaticMapFile, staticMapFileContents);
+
+    StaticMapping parsedMap =
+        ShellBasedIdMapping.parseStaticMap(tempStaticMapFile);    
     assertEquals(10, (int)parsedMap.uidMapping.get(100));
     assertEquals(11, (int)parsedMap.uidMapping.get(201));
     assertEquals(12, (int)parsedMap.uidMapping.get(301));
@@ -120,6 +126,78 @@ public class TestShellBasedIdMapping {
     assertEquals(498, (int)gMap.inverse().get("mapred2"));
   }
 
+  // Test staticMap refreshing
+  @Test
+  public void testStaticMapUpdate() throws IOException {
+    File tempStaticMapFile = File.createTempFile("nfs-", ".map");
+    tempStaticMapFile.delete();
+    Configuration conf = new Configuration();
+    conf.setLong(IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY, 1000);    
+    conf.set(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
+        tempStaticMapFile.getPath());
+
+    ShellBasedIdMapping refIdMapping =
+        new ShellBasedIdMapping(conf, true);
+    ShellBasedIdMapping incrIdMapping = new ShellBasedIdMapping(conf);
+
+    BiMap<Integer, String> uidNameMap = refIdMapping.getUidNameMap();
+    BiMap<Integer, String> gidNameMap = refIdMapping.getGidNameMap();
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getUid()
+    incrIdMapping.clearNameMaps();
+    uidNameMap = refIdMapping.getUidNameMap();
+    {
+      BiMap.Entry<Integer, String> me = uidNameMap.entrySet().iterator().next();
+      Integer id = me.getKey();
+      String name = me.getValue();
+
+      // The static map is empty, so the id found for "name" would be
+      // the same as "id"
+      Integer nid = incrIdMapping.getUid(name);
+      assertEquals(id, nid);
+      
+      // Clear map and update staticMap file
+      incrIdMapping.clearNameMaps();
+      Integer rid = id + 10000;
+      String smapStr = "uid " + rid + " " + id;
+      createStaticMapFile(tempStaticMapFile, smapStr);
+
+      // Now the id found for "name" should be the id specified by
+      // the staticMap
+      nid = incrIdMapping.getUid(name);
+      assertEquals(rid, nid);
+    }
+
+    // Force empty map, to see effect of incremental map update of calling
+    // getGid()
+    incrIdMapping.clearNameMaps();
+    gidNameMap = refIdMapping.getGidNameMap();
+    {
+      BiMap.Entry<Integer, String> me = gidNameMap.entrySet().iterator().next();
+      Integer id = me.getKey();
+      String name = me.getValue();
+
+      // The static map is empty, so the id found for "name" would be
+      // the same as "id"
+      Integer nid = incrIdMapping.getGid(name);
+      assertEquals(id, nid);
+
+      // Clear map and update staticMap file
+      incrIdMapping.clearNameMaps();
+      Integer rid = id + 10000;
+      String smapStr = "gid " + rid + " " + id;
+      // Sleep a bit to avoid that two changes have the same modification time
+      try {Thread.sleep(1000);} catch (InterruptedException e) {}
+      createStaticMapFile(tempStaticMapFile, smapStr);
+
+      // Now the id found for "name" should be the id specified by
+      // the staticMap
+      nid = incrIdMapping.getGid(name);
+      assertEquals(rid, nid);
+    }
+  }
+
   @Test
   public void testDuplicates() throws IOException {
     assumeTrue(!Shell.WINDOWS);

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
 
     HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
 
+    HDFS-7564. NFS gateway dynamically reload UID/GID mapping file /etc/nfs.map
+    (Yongjun Zhang via brandonli)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES