YARN-4768. getAvailablePhysicalMemorySize can be inaccurate on linux. Contributed by Nathan Roberts.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SysInfoLinux.java

@@ -53,7 +53,7 @@ public class SysInfoLinux extends SysInfo {
    */
   private static final String PROCFS_MEMFILE = "/proc/meminfo";
   private static final Pattern PROCFS_MEMFILE_FORMAT =
-      Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB");
+      Pattern.compile("^([a-zA-Z_()]*):[ \t]*([0-9]*)[ \t]*(kB)?");
 
   // We need the values for the following keys in meminfo
   private static final String MEMTOTAL_STRING = "MemTotal";
@@ -61,6 +61,12 @@ public class SysInfoLinux extends SysInfo {
   private static final String MEMFREE_STRING = "MemFree";
   private static final String SWAPFREE_STRING = "SwapFree";
   private static final String INACTIVE_STRING = "Inactive";
+  private static final String INACTIVEFILE_STRING = "Inactive(file)";
+  private static final String HARDWARECORRUPTED_STRING = "HardwareCorrupted";
+  private static final String HUGEPAGESTOTAL_STRING = "HugePages_Total";
+  private static final String HUGEPAGESIZE_STRING = "Hugepagesize";
+
+
 
   /**
    * Patterns for parsing /proc/cpuinfo.
@@ -122,7 +128,13 @@ public class SysInfoLinux extends SysInfo {
   private long swapSize = 0;
   private long ramSizeFree = 0;  // free ram space on the machine (kB)
   private long swapSizeFree = 0; // free swap space on the machine (kB)
-  private long inactiveSize = 0; // inactive cache memory (kB)
+  private long inactiveSize = 0; // inactive memory (kB)
+  private long inactiveFileSize = -1; // inactive cache memory, -1 if not there
+  private long hardwareCorruptSize = 0; // RAM corrupt and not available
+  private long hugePagesTotal = 0; // # of hugepages reserved
+  private long hugePageSize = 0; // # size of each hugepage
+
+
   /* number of logical processors on the system. */
   private int numProcessors = 0;
   /* number of physical cores on the system. */
@@ -245,6 +257,14 @@ public class SysInfoLinux extends SysInfo {
             swapSizeFree = Long.parseLong(mat.group(2));
           } else if (mat.group(1).equals(INACTIVE_STRING)) {
             inactiveSize = Long.parseLong(mat.group(2));
+          } else if (mat.group(1).equals(INACTIVEFILE_STRING)) {
+            inactiveFileSize = Long.parseLong(mat.group(2));
+          } else if (mat.group(1).equals(HARDWARECORRUPTED_STRING)) {
+            hardwareCorruptSize = Long.parseLong(mat.group(2));
+          } else if (mat.group(1).equals(HUGEPAGESTOTAL_STRING)) {
+            hugePagesTotal = Long.parseLong(mat.group(2));
+          } else if (mat.group(1).equals(HUGEPAGESIZE_STRING)) {
+            hugePageSize = Long.parseLong(mat.group(2));
           }
         }
         str = in.readLine();
@@ -554,28 +574,31 @@ public class SysInfoLinux extends SysInfo {
   @Override
   public long getPhysicalMemorySize() {
     readProcMemInfoFile();
-    return ramSize * 1024;
+    return (ramSize
+            - hardwareCorruptSize
+            - (hugePagesTotal * hugePageSize)) * 1024;
   }
 
   /** {@inheritDoc} */
   @Override
   public long getVirtualMemorySize() {
-    readProcMemInfoFile();
-    return (ramSize + swapSize) * 1024;
+    return getPhysicalMemorySize() + (swapSize * 1024);
   }
 
   /** {@inheritDoc} */
   @Override
   public long getAvailablePhysicalMemorySize() {
     readProcMemInfoFile(true);
-    return (ramSizeFree + inactiveSize) * 1024;
+    long inactive = inactiveFileSize != -1
+        ? inactiveFileSize
+        : inactiveSize;
+    return (ramSizeFree + inactive) * 1024;
   }
 
   /** {@inheritDoc} */
   @Override
   public long getAvailableVirtualMemorySize() {
-    readProcMemInfoFile(true);
-    return (ramSizeFree + swapSizeFree + inactiveSize) * 1024;
+    return getAvailablePhysicalMemorySize() + (swapSizeFree * 1024);
   }
 
   /** {@inheritDoc} */

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestSysInfoLinux.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 
 /**
  * A JUnit test to test {@link SysInfoLinux}
@@ -110,11 +111,56 @@ public class TestSysInfoLinux {
     "VmallocTotal: 34359738367 kB\n" +
     "VmallocUsed:      1632 kB\n" +
     "VmallocChunk: 34359736375 kB\n" +
-    "HugePages_Total:     0\n" +
+    "HugePages_Total:     %d\n" +
     "HugePages_Free:      0\n" +
     "HugePages_Rsvd:      0\n" +
     "Hugepagesize:     2048 kB";
 
+  static final String MEMINFO_FORMAT_2 =
+    "MemTotal:       %d kB\n" +
+    "MemFree:        %d kB\n" +
+    "Buffers:          129976 kB\n" +
+    "Cached:         32317676 kB\n" +
+    "SwapCached:            0 kB\n" +
+    "Active:         88938588 kB\n" +
+    "Inactive:       %d kB\n" +
+    "Active(anon):   77502200 kB\n" +
+    "Inactive(anon):  6385336 kB\n" +
+    "Active(file):   11436388 kB\n" +
+    "Inactive(file): %d kB\n" +
+    "Unevictable:           0 kB\n" +
+    "Mlocked:               0 kB\n" +
+    "SwapTotal:      %d kB\n" +
+    "SwapFree:       %d kB\n" +
+    "Dirty:            575864 kB\n" +
+    "Writeback:            16 kB\n" +
+    "AnonPages:      83886180 kB\n" +
+    "Mapped:           108640 kB\n" +
+    "Shmem:              1880 kB\n" +
+    "Slab:            2413448 kB\n" +
+    "SReclaimable:    2194488 kB\n" +
+    "SUnreclaim:       218960 kB\n" +
+    "KernelStack:       31496 kB\n" +
+    "PageTables:       195176 kB\n" +
+    "NFS_Unstable:          0 kB\n" +
+    "Bounce:                0 kB\n" +
+    "WritebackTmp:          0 kB\n" +
+    "CommitLimit:    97683468 kB\n" +
+    "Committed_AS:   94553560 kB\n" +
+    "VmallocTotal:   34359738367 kB\n" +
+    "VmallocUsed:      498580 kB\n" +
+    "VmallocChunk:   34256922296 kB\n" +
+    "HardwareCorrupted: %d kB\n" +
+    "AnonHugePages:         0 kB\n" +
+    "HugePages_Total:       %d\n" +
+    "HugePages_Free:        0\n" +
+    "HugePages_Rsvd:        0\n" +
+    "HugePages_Surp:        0\n" +
+    "Hugepagesize:       2048 kB\n" +
+    "DirectMap4k:        4096 kB\n" +
+    "DirectMap2M:     2027520 kB\n" +
+    "DirectMap1G:    132120576 kB\n";
+
   static final String CPUINFO_FORMAT =
     "processor : %s\n" +
     "vendor_id : AuthenticAMD\n" +
@@ -285,19 +331,57 @@ public class TestSysInfoLinux {
     long inactive = 567732L;
     long swapTotal = 2096472L;
     long swapFree = 1818480L;
+    int nrHugePages = 10;
     File tempFile = new File(FAKE_MEMFILE);
     tempFile.deleteOnExit();
     FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
     fWriter.write(String.format(MEMINFO_FORMAT,
-      memTotal, memFree, inactive, swapTotal, swapFree));
+      memTotal, memFree, inactive, swapTotal, swapFree, nrHugePages));
 
     fWriter.close();
     assertEquals(plugin.getAvailablePhysicalMemorySize(),
                  1024L * (memFree + inactive));
     assertEquals(plugin.getAvailableVirtualMemorySize(),
                  1024L * (memFree + inactive + swapFree));
-    assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
-    assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
+    assertEquals(plugin.getPhysicalMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048)));
+    assertEquals(plugin.getVirtualMemorySize(),
+        1024L * (memTotal - (nrHugePages * 2048) + swapTotal));
+  }
+
+  /**
+   * Test parsing /proc/meminfo with Inactive(file) present
+   * @throws IOException
+   */
+  @Test
+  public void parsingProcMemFile2() throws IOException {
+    long memTotal = 131403836L;
+    long memFree = 11257036L;
+    long inactive = 27396032L;
+    long inactiveFile = 21010696L;
+    long swapTotal = 31981552L;
+    long swapFree = 1818480L;
+    long hardwareCorrupt = 31960904L;
+    int nrHugePages = 10;
+    File tempFile = new File(FAKE_MEMFILE);
+    tempFile.deleteOnExit();
+    FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
+    fWriter.write(String.format(MEMINFO_FORMAT_2,
+      memTotal, memFree, inactive, inactiveFile, swapTotal, swapFree,
+      hardwareCorrupt, nrHugePages));
+
+    fWriter.close();
+    assertEquals(plugin.getAvailablePhysicalMemorySize(),
+                 1024L * (memFree + inactiveFile));
+    assertFalse(plugin.getAvailablePhysicalMemorySize() ==
+                 1024L * (memFree + inactive));
+    assertEquals(plugin.getAvailableVirtualMemorySize(),
+                 1024L * (memFree + inactiveFile + swapFree));
+    assertEquals(plugin.getPhysicalMemorySize(),
+                 1024L * (memTotal - hardwareCorrupt - (nrHugePages * 2048)));
+    assertEquals(plugin.getVirtualMemorySize(),
+                 1024L * (memTotal - hardwareCorrupt -
+                          (nrHugePages * 2048) + swapTotal));
   }
 
   @Test