[aarch64-port-dev ] Add missing instruction synchronization barriers and cache flushes
Andrew Haley
aph at redhat.com
Wed Sep 3 14:55:38 UTC 2014
Some more concurrency fixes. In particular, we need an ISB whenever
the code cache may have been changed during a safepoint.
Andrew.
-------------- next part --------------
# HG changeset patch
# User aph
# Date 1409591418 14400
# Mon Sep 01 13:10:18 2014 -0400
# Node ID 32fae3443576ac6b4b5ac0770c0829ce6c08764e
# Parent 24958c74f6112b0625a9c659e5af58bd02e5cda0
Add missing instruction synchronization barriers and cache flushes.
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -518,6 +518,7 @@
__ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
__ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
__ blrt(rscratch1, 1, 0, 1);
+ __ maybe_isb();
__ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
__ mov(rscratch1, r0);
__ pop(0x3, sp); // r0 & r1
@@ -2919,6 +2920,7 @@
if (info != NULL) {
add_call_info_here(info);
}
+ __ maybe_isb();
}
void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/c1_Runtime1_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -80,6 +80,7 @@
pop(r0, sp);
#endif
reset_last_Java_frame(true, true);
+ maybe_isb();
// check for pending exceptions
{ Label L;
@@ -569,6 +570,7 @@
}
#endif
__ reset_last_Java_frame(true, false);
+ __ maybe_isb();
// check for pending exceptions
{ Label L;
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/jniFastGetField_aarch64.cpp
--- a/src/cpu/aarch64/vm/jniFastGetField_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/jniFastGetField_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -127,10 +127,15 @@
case T_DOUBLE: slow_case_addr = jni_GetDoubleField_addr(); break;
default: ShouldNotReachHere();
}
- // tail call
- __ lea(rscratch1, ExternalAddress(slow_case_addr));
- __ br(rscratch1);
+ {
+ __ enter();
+ __ lea(rscratch1, ExternalAddress(slow_case_addr));
+ __ blr(rscratch1);
+ __ maybe_isb();
+ __ leave();
+ __ ret(lr);
+ }
__ flush ();
return fast_entry;
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/macroAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -1197,6 +1197,7 @@
bind(*retaddr);
ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
+ maybe_isb();
}
void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/macroAssembler_aarch64.hpp
--- a/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.hpp Mon Sep 01 13:10:18 2014 -0400
@@ -1102,6 +1102,9 @@
Register tmp1, Register tmp2,
Register tmp3, Register tmp4,
int int_cnt1, Register result);
+
+ // ISB may be needed because of a safepoint
+ void maybe_isb() { isb(); }
};
// Used by aarch64.ad to control code generation
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/nativeInst_aarch64.cpp
--- a/src/cpu/aarch64/vm/nativeInst_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/nativeInst_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -37,11 +37,6 @@
#include "c1/c1_Runtime1.hpp"
#endif
-void NativeInstruction::wrote(int offset) {
- // FIXME: Native needs ISB here
-; }
-
-
void NativeCall::verify() { ; }
address NativeCall::destination() const {
@@ -51,10 +46,13 @@
// Inserts a native call instruction at a given pc
void NativeCall::insert(address code_pos, address entry) { Unimplemented(); }
+//-------------------------------------------------------------------
+
void NativeMovConstReg::verify() {
// make sure code pattern is actually mov reg64, imm64 instructions
}
+
intptr_t NativeMovConstReg::data() const {
// das(uint64_t(instruction_address()),2);
address addr = MacroAssembler::pd_call_destination(instruction_address());
@@ -71,6 +69,7 @@
*(intptr_t*)addr = x;
} else {
MacroAssembler::pd_patch_instruction(instruction_address(), (address)x);
+ ICache::invalidate_range(instruction_address(), instruction_size);
}
};
@@ -102,6 +101,7 @@
*(long*)addr = x;
} else {
MacroAssembler::pd_patch_instruction(pc, (address)intptr_t(x));
+ ICache::invalidate_range(instruction_address(), instruction_size);
}
}
@@ -138,8 +138,11 @@
dest = instruction_address();
MacroAssembler::pd_patch_instruction(instruction_address(), dest);
+ ICache::invalidate_range(instruction_address(), instruction_size);
};
+//-------------------------------------------------------------------
+
bool NativeInstruction::is_safepoint_poll() {
// a safepoint_poll is implemented in two steps as either
//
@@ -189,7 +192,9 @@
return Instruction_aarch64::extract(int_at(0), 30, 23) == 0b11100101;
}
-// MT safe inserting of a jump over an unknown instruction sequence (used by nmethod::makeZombie)
+//-------------------------------------------------------------------
+
+// MT safe inserting of a jump over a jump or a nop (used by nmethod::makeZombie)
void NativeJump::patch_verified_entry(address entry, address verified_entry, address dest) {
ptrdiff_t disp = dest - verified_entry;
@@ -203,23 +208,22 @@
ICache::invalidate_range(verified_entry, instruction_size);
}
-
void NativeGeneralJump::verify() { }
-
void NativeGeneralJump::insert_unconditional(address code_pos, address entry) {
+ NativeGeneralJump* n_jump = (NativeGeneralJump*)code_pos;
ptrdiff_t disp = entry - code_pos;
guarantee(disp < 1 << 27 && disp > - (1 << 27), "branch overflow");
unsigned int insn = (0b000101 << 26) | ((disp >> 2) & 0x3ffffff);
-
*(unsigned int*)code_pos = insn;
ICache::invalidate_range(code_pos, instruction_size);
}
// MT-safe patching of a long jump instruction.
void NativeGeneralJump::replace_mt_safe(address instr_addr, address code_buffer) {
- assert(nativeInstruction_at(instr_addr)->is_jump_or_nop(),
+ NativeGeneralJump* n_jump = (NativeGeneralJump*)instr_addr;
+ assert(n_jump->is_jump_or_nop(),
"Aarch64 cannot replace non-jump with jump");
uint32_t instr = *(uint32_t*)code_buffer;
*(uint32_t*)instr_addr = instr;
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/nativeInst_aarch64.hpp
--- a/src/cpu/aarch64/vm/nativeInst_aarch64.hpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/nativeInst_aarch64.hpp Mon Sep 01 13:10:18 2014 -0400
@@ -80,14 +80,10 @@
oop oop_at (int offset) const { return *(oop*) addr_at(offset); }
- void set_char_at(int offset, char c) { *addr_at(offset) = (u_char)c; wrote(offset); }
- void set_int_at(int offset, jint i) { *(jint*)addr_at(offset) = i; wrote(offset); }
- void set_ptr_at (int offset, intptr_t ptr) { *(intptr_t*) addr_at(offset) = ptr; wrote(offset); }
- void set_oop_at (int offset, oop o) { *(oop*) addr_at(offset) = o; wrote(offset); }
-
- // This doesn't really do anything on AArch64, but it is the place where
- // cache invalidation belongs, generically:
- void wrote(int offset);
+ void set_char_at(int offset, char c) { *addr_at(offset) = (u_char)c; }
+ void set_int_at(int offset, jint i) { *(jint*)addr_at(offset) = i; }
+ void set_ptr_at (int offset, intptr_t ptr) { *(intptr_t*) addr_at(offset) = ptr; }
+ void set_oop_at (int offset, oop o) { *(oop*) addr_at(offset) = o; }
public:
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp
--- a/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/sharedRuntime_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -320,6 +320,7 @@
__ mov(c_rarg1, lr);
__ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
__ blrt(rscratch1, 2, 0, 0);
+ __ maybe_isb();
__ pop_CPU_state();
// restore sp
@@ -1174,7 +1175,7 @@
__ lea(rscratch1, RuntimeAddress(dest));
__ mov(rscratch2, (gpargs << 6) | (fpargs << 2) | type);
__ blrt(rscratch1, rscratch2);
- // __ blrt(rscratch1, gpargs, fpargs, type);
+ __ maybe_isb();
}
}
@@ -1977,6 +1978,7 @@
__ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition)));
}
__ blrt(rscratch1, 1, 0, 1);
+ __ maybe_isb();
// Restore any method result value
restore_native_result(masm, ret_type, stack_slots);
@@ -2823,6 +2825,8 @@
__ reset_last_Java_frame(false, true);
+ __ maybe_isb();
+
__ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
__ cbz(rscratch1, noException);
@@ -2892,6 +2896,8 @@
oop_maps->add_gc_map( __ offset() - start, map);
+ __ maybe_isb();
+
// r0 contains the address we are going to jump to assuming no exception got installed
// clear last_Java_sp
@@ -3014,6 +3020,7 @@
__ mov(c_rarg0, rthread);
__ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
__ blrt(rscratch1, 1, 0, MacroAssembler::ret_type_integral);
+ __ maybe_isb();
// Set an oopmap for the call site. This oopmap will only be used if we
// are unwinding the stack. Hence, all locations will be dead.
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/stubGenerator_aarch64.cpp
--- a/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/stubGenerator_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -2232,6 +2232,7 @@
oop_maps->add_gc_map(the_pc - start, map);
__ reset_last_Java_frame(true, true);
+ __ maybe_isb();
__ leave();
diff -r 24958c74f611 -r 32fae3443576 src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp
--- a/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp Mon Sep 01 07:02:47 2014 -0400
+++ b/src/cpu/aarch64/vm/templateInterpreter_aarch64.cpp Mon Sep 01 13:10:18 2014 -0400
@@ -1036,6 +1036,7 @@
// Call the native method.
__ blrt(r10, rscratch1);
+ __ maybe_isb();
__ get_method(rmethod);
// result potentially in r0 or v0
@@ -1093,6 +1094,7 @@
__ mov(c_rarg0, rthread);
__ mov(rscratch2, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
__ blrt(rscratch2, 1, 0, 0);
+ __ maybe_isb();
__ get_method(rmethod);
__ reinit_heapbase();
__ bind(Continue);
More information about the aarch64-port-dev
mailing list