Large page use crashes the JVM on some Linux systems

[B. Blaser]

Hi Claes,

On 24 April 2018 at 21:39, Claes Redestad <claes.redestad at oracle.com> wrote:
> The root issue here could very well be that the SHM sanity test is
> insufficient. Adding the same test as we already do for TLBFS seems like the
> wrong approach.
>
> I'm not the most knowledgeable about SHM, though, in fact not knowledgeable
> at all, so let's try and get you subscribed to hotspot-dev and spark a
> discussion on the list.
>
> /Claes

As suggested, I've added a SHM sanity test (derived from the TLBFS
one) which ensure that the allocated memory is using huge pages in
'/proc/self/maps'.
Does the following patch look better (tier1 seems OK)?

Thanks,
Bernard

diff --git a/src/hotspot/os/linux/os_linux.cpp
b/src/hotspot/os/linux/os_linux.cpp
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -3317,6 +3317,28 @@
   return result;
 }

+static bool check_hugepage(void* p) {
+  bool result = false;
+  FILE *fp = fopen("/proc/self/maps", "r");
+  if (fp) {
+    while (!feof(fp)) {
+      char chars[257];
+      long x = 0;
+      if (fgets(chars, sizeof(chars), fp)) {
+        if (sscanf(chars, "%lx-%*x", &x) == 1
+            && x == (long)p) {
+          if (strstr (chars, "hugepage")) {
+            result = true;
+            break;
+          }
+        }
+      }
+    }
+    fclose(fp);
+  }
+  return result;
+}
+
 bool os::Linux::hugetlbfs_sanity_check(bool warn, size_t page_size) {
   bool result = false;
   void *p = mmap(NULL, page_size, PROT_READ|PROT_WRITE,
@@ -3325,23 +3347,7 @@

   if (p != MAP_FAILED) {
     // We don't know if this really is a huge page or not.
-    FILE *fp = fopen("/proc/self/maps", "r");
-    if (fp) {
-      while (!feof(fp)) {
-        char chars[257];
-        long x = 0;
-        if (fgets(chars, sizeof(chars), fp)) {
-          if (sscanf(chars, "%lx-%*x", &x) == 1
-              && x == (long)p) {
-            if (strstr (chars, "hugepage")) {
-              result = true;
-              break;
-            }
-          }
-        }
-      }
-      fclose(fp);
-    }
+    result = check_hugepage(p);
     munmap(p, page_size);
   }

@@ -3352,6 +3358,27 @@
   return result;
 }

+bool os::Linux::shm_sanity_check(bool warn, size_t page_size) {
+  bool result = false;
+  int shmid = shmget(IPC_PRIVATE, page_size,
SHM_HUGETLB|IPC_CREAT|SHM_R|SHM_W);
+  if (shmid != -1) {
+    void* p = shmat(shmid, NULL, 0);
+    shmctl(shmid, IPC_RMID, NULL);
+
+    if (p != (void*) -1) {
+      // We don't know if this really is a huge page or not.
+      result = check_hugepage(p);
+      shmdt(p);
+    }
+  }
+
+  if (warn && !result) {
+    warning("SHM is not supported by the operating system.");
+  }
+
+  return result;
+}
+
 // Set the coredump_filter bits to include largepages in core dump (bit 6)
 //
 // From the coredump_filter documentation:
@@ -3506,6 +3533,14 @@
     UseHugeTLBFS = false;
   }

+  if (UseSHM) {
+    bool warn_on_failure = !FLAG_IS_DEFAULT(UseSHM);
+    if (shm_sanity_check(warn_on_failure, page_size)) {
+      return true;
+    }
+    UseSHM = false;
+  }
+
   return UseSHM;
 }

diff --git a/src/hotspot/os/linux/os_linux.hpp
b/src/hotspot/os/linux/os_linux.hpp
--- a/src/hotspot/os/linux/os_linux.hpp
+++ b/src/hotspot/os/linux/os_linux.hpp
@@ -99,6 +99,7 @@
   static bool setup_large_page_type(size_t page_size);
   static bool transparent_huge_pages_sanity_check(bool warn, size_t
pages_size);
   static bool hugetlbfs_sanity_check(bool warn, size_t page_size);
+  static bool shm_sanity_check(bool warn, size_t page_size);

   static char* reserve_memory_special_shm(size_t bytes, size_t
alignment, char* req_addr, bool exec);
   static char* reserve_memory_special_huge_tlbfs(size_t bytes, size_t
alignment, char* req_addr, bool exec);
diff --git a/test/hotspot/jtreg/gc/g1/TestLargePageUseForAuxMemory.java
b/test/hotspot/jtreg/gc/g1/TestLargePageUseForAuxMemory.java
--- a/test/hotspot/jtreg/gc/g1/TestLargePageUseForAuxMemory.java
+++ b/test/hotspot/jtreg/gc/g1/TestLargePageUseForAuxMemory.java
@@ -60,7 +60,8 @@
         }

         long size = parseMemoryString(pageSizeStr);
-        if (size != expectedSize) {
+        // Small pages may be used if no large pages are available.
+        if (size != expectedSize && size != smallPageSize) {
             output.reportDiagnosticSummary();
             throw new RuntimeException("Match from '" + pattern + "'
got " + size + " expected: " + expectedSize);
         }