JVM crashes in JIT compiled code without hs_err_pid file

[Alexandr Miloslavskiy]

Hi Thomas,

 > someone in the process may have established a competing signal 
handler via AddVectoredExceptionHandler

I tested it and you were right, if 'AddVectoredExceptionHandler()' is 
present then it's called instead of 'RtlAddFunctionTable()'.
See attached code for a demo.

I'll try to figure if this was the case for the customer.

 > Possibility 2: registering code cache did not work

Sounds unlikely, because in the dump, I can see that it's registered 
(when a dump is created, it also collects all function tables). Also, in 
such case, application wouldn't run longer than a second or two before 
crashing.
-------------- next part --------------
#include <windows.h>
#include <stdio.h>
#include <stdint.h>

#pragma comment(linker, "/SUBSYSTEM:console /ENTRY:mainCRTStartup")

LONG FunctionableHandler(IN PEXCEPTION_RECORD ExceptionRecord, IN ULONG64 EstablisherFrame, IN OUT PCONTEXT ContextRecord, IN OUT PDISPATCHER_CONTEXT DispatcherContext)
{
	printf(
		"FunctionableHandler: exception %08X at %p\n",
		ExceptionRecord->ExceptionCode,
		ExceptionRecord->ExceptionAddress
	);

	TerminateProcess(GetCurrentProcess(), 0);
	return 0;
}

LONG VectoredExceptionHandler(_EXCEPTION_POINTERS *ExceptionInfo)
{
	printf(
		"VectoredExceptionHandler: exception %08X at %p\n",
		ExceptionInfo->ExceptionRecord->ExceptionCode,
		ExceptionInfo->ExceptionRecord->ExceptionAddress
	);

	TerminateProcess(GetCurrentProcess(), 0);
	return 0;
}

// Same as in OpenJDK
typedef unsigned char UBYTE; 
typedef struct _UNWIND_INFO_EH_ONLY {
	UBYTE Version : 3;
	UBYTE Flags : 5;
	UBYTE SizeOfProlog;
	UBYTE CountOfCodes;
	UBYTE FrameRegister : 4;
	UBYTE FrameOffset : 4;
	union {
		OPTIONAL ULONG ExceptionHandler;
		OPTIONAL ULONG FunctionEntry;
	};
	OPTIONAL ULONG ExceptionData[1];
} UNWIND_INFO_EH_ONLY, *PUNWIND_INFO_EH_ONLY;

struct CodePageContents
{
	UNWIND_INFO_EH_ONLY	unwind;
	RUNTIME_FUNCTION	runtimeFunction;
	uint8_t				exceptionHandlerCode[16];
	uint8_t				jitCode[64];
};

typedef void (*PlainFunction)();

PlainFunction createJitCode()
{
	const uint32_t codePageSize = 0x1000;
	CodePageContents* memoryPage = (CodePageContents*)VirtualAlloc(0, codePageSize, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
	
	// Set up exception handling
	{
		UNWIND_INFO_EH_ONLY* punwind = &memoryPage->unwind;
		punwind->Version			= 1;
		punwind->Flags				= UNW_FLAG_EHANDLER;
		punwind->SizeOfProlog		= 0;
		punwind->CountOfCodes		= 0;
		punwind->FrameRegister		= 0;
		punwind->FrameOffset		= 0;
		punwind->ExceptionHandler	= (ULONG)((char*)memoryPage->exceptionHandlerCode - (char*)memoryPage);
		punwind->ExceptionData[0]	= 0;

		RUNTIME_FUNCTION* prt = &memoryPage->runtimeFunction;
		prt->BeginAddress			= 0;
		prt->EndAddress				= codePageSize;
		prt->UnwindData				= (DWORD)((char*)&memoryPage->unwind - (char*)memoryPage);

		RtlAddFunctionTable(prt, 1, (DWORD64)memoryPage);

		uint8_t* code = memoryPage->exceptionHandlerCode;
		
		// mov rax, FunctionableHandler
		code[0] = 0x48;
		code[1] = 0xb8;
		code += 2;
		*(uint64_t*)code = (uint64_t)FunctionableHandler;
		code += 8;

		// jmp rax
		code[0] = 0xff;
		code[1] = 0xe0;
	}

	// "JIT" code
	{
		uint8_t* code = memoryPage->jitCode;

		// xor eax, eax
		code[0] = 0x31;
		code[1] = 0xc0;
		code += 2;

		// mov eax,DWORD PTR [eax] 
		code[0] = 0x67;
		code[1] = 0x8b;
		code[2] = 0x00;
		code += 3;
	}

	return (PlainFunction)&memoryPage->jitCode;
}

int main()
{
	PlainFunction function = createJitCode();

	// Comment/uncomment as needed
	AddVectoredExceptionHandler(TRUE, VectoredExceptionHandler);

	// Run "jit" code and crash
	function();

	return 0;
}