Check core dumping is enabled when set CrashOnOutOfMemoryError

[KUBOTA Yuji]

Hi everyone.

When set -XX:+CrashOnOutOfMemoryError, I think we should know
whether core dumping has been enabled or not before launching
process to prevent lack of information for after-the-fact analysis.

So I create a patch to check the core dumping, please review it.
I do not have an account of openjdk, so I need sponsor.

I send to serviceability-dev because JDK-8138745 [1] has been
reviewed here. If unfit, please let me know.

[1]:Implement ExitOnOutOfMemory and CrashOnOutOfMemory in HotSpot
     https://bugs.openjdk.java.net/browse/JDK-8138745

diff --git a/src/share/vm/runtime/commandLineFlagConstraintsRuntime.cpp
b/src/share/vm/runtime/commandLineFlagConstraintsRuntime.cpp
--- a/src/share/vm/runtime/commandLineFlagConstraintsRuntime.cpp
+++ b/src/share/vm/runtime/commandLineFlagConstraintsRuntime.cpp
@@ -119,6 +119,17 @@
   }
 }

+Flag::Error CrashOnOutOfMemoryErrorFunc(bool value, bool verbose) {
+  char buffer[O_BUFLEN];
+  os::check_dump_limit(buffer, sizeof(buffer));
+  if (!VMError::coredump_status) {
+    CommandLineError::print(verbose,
+                            "%s\n", VMError::coredump_message);
+    return Flag::VIOLATES_CONSTRAINT;
+  }
+  return Flag::SUCCESS;
+}
+
 Flag::Error PerfDataSamplingIntervalFunc(intx value, bool verbose) {
   if ((value % PeriodicTask::interval_gran != 0)) {
     CommandLineError::print(verbose,
diff --git a/src/share/vm/runtime/commandLineFlagConstraintsRuntime.hpp
b/src/share/vm/runtime/commandLineFlagConstraintsRuntime.hpp
--- a/src/share/vm/runtime/commandLineFlagConstraintsRuntime.hpp
+++ b/src/share/vm/runtime/commandLineFlagConstraintsRuntime.hpp
@@ -43,6 +43,8 @@
 Flag::Error BiasedLockingBulkRevokeThresholdFunc(intx value, bool verbose);
 Flag::Error BiasedLockingDecayTimeFunc(intx value, bool verbose);

+Flag::Error CrashOnOutOfMemoryErrorFunc(bool value, bool verbose);
+
 Flag::Error PerfDataSamplingIntervalFunc(intx value, bool verbose);

 #endif /* SHARE_VM_RUNTIME_COMMANDLINEFLAGCONSTRAINTSRUNTIME_HPP */
diff --git a/src/share/vm/runtime/globals.hpp b/src/share/vm/runtime/globals.hpp
--- a/src/share/vm/runtime/globals.hpp
+++ b/src/share/vm/runtime/globals.hpp
@@ -1401,6 +1401,7 @@
   product(bool, CrashOnOutOfMemoryError, false,                             \
           "JVM aborts, producing an error log and core/mini dump, on the "  \
           "first occurrence of an out-of-memory error")                     \
+          constraint(CrashOnOutOfMemoryErrorFunc,AfterErgo)                 \
                                                                             \
   /* tracing */                                                             \
                                                                             \
diff --git a/src/share/vm/utilities/vmError.hpp
b/src/share/vm/utilities/vmError.hpp
--- a/src/share/vm/utilities/vmError.hpp
+++ b/src/share/vm/utilities/vmError.hpp
@@ -65,14 +65,6 @@
   // so use thread id instead of Thread* to identify thread.
   static volatile intptr_t first_error_tid;

-  // Core dump status, false if we have been unable to write a
core/minidump for some reason
-  static bool coredump_status;
-
-  // When coredump_status is set to true this will contain the
name/path to the core/minidump,
-  // if coredump_status if false, this will (hopefully) contain a
useful error explaining why
-  // no core/minidump has been written to disk
-  static char coredump_message[O_BUFLEN];
-

   // set signal handlers on Solaris/Linux or the default exception filter
   // on Windows, to handle recursive crashes.
@@ -111,6 +103,14 @@
   // Record status of core/minidump
   static void record_coredump_status(const char* message, bool status);

+  // Core dump status, false if we have been unable to write a
core/minidump for some reason
+  static bool coredump_status;
+
+  // When coredump_status is set to true this will contain the
name/path to the core/minidump,
+  // if coredump_status if false, this will (hopefully) contain a
useful error explaining why
+  // no core/minidump has been written to disk
+  static char coredump_message[O_BUFLEN];
+
   // support for VM.info diagnostic command
   static void print_vm_info(outputStream* st);


Thanks,
Yuji