[aarch64-port-dev ] openjdk atomics
Andrew Haley
aph at redhat.com
Thu Feb 13 10:49:28 PST 2014
Here attached.
I have tested this, but it's goint go need someone to review it properly.
Andrew.
# HG changeset patch
# User aph
# Date 1392317141 0
# Thu Feb 13 18:45:41 2014 +0000
# Node ID 5614eacca5c6687ac6d80cdf3d16c2ea2af37118
# Parent a69fcb4e97f6098c2f0b712a9306664a0fe74368
Change all load/store exclusive operations as recommended by ARM.
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/aarch64.ad
--- a/src/cpu/aarch64/vm/aarch64.ad Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/aarch64.ad Thu Feb 13 18:45:41 2014 +0000
@@ -2409,18 +2409,20 @@
}
}
Label retry_load, done;
+ __ membar(__ AnyAny);
__ bind(retry_load);
- __ ldaxr(rscratch1, addr_reg);
+ __ ldxr(rscratch1, addr_reg);
__ cmp(rscratch1, old_reg);
__ br(Assembler::NE, done);
// if we store+flush with no intervening write tmp will be zero
- __ stlxr(rscratch1, new_reg, addr_reg);
+ __ stxr(rscratch1, new_reg, addr_reg);
__ cmpw(rscratch1, zr); // cannot use cbzw as must set flag
__ br(Assembler::EQ, done);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
__ b(retry_load);
__ bind(done);
+ __ membar(__ AnyAny);
%}
enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
@@ -2454,17 +2456,19 @@
}
Label retry_load, done;
__ bind(retry_load);
- __ ldaxrw(rscratch1, addr_reg);
+ __ membar(__ AnyAny);
+ __ ldxrw(rscratch1, addr_reg);
__ cmpw(rscratch1, old_reg);
__ br(Assembler::NE, done);
// if we store+flush with no intervening write tmp wil be zero
- __ stlxrw(rscratch1, new_reg, addr_reg);
+ __ stxrw(rscratch1, new_reg, addr_reg);
__ cmpw(rscratch1, zr); // cannot use cbzw as must set flag
__ br(Assembler::EQ, done);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
__ b(retry_load);
__ bind(done);
+ __ membar(__ AnyAny);
%}
// auxiliary used for CompareAndSwapX to set result register
@@ -2976,9 +2980,6 @@
__ br(target_reg);
%}
- // TODO
- // reimplement this with custom cmpxchgptr code
- // which avoids some of the unnecessary branching
enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
MacroAssembler _masm(&cbuf);
Register oop = as_Register($object$$reg);
@@ -3023,13 +3024,30 @@
// Initialize the box. (Must happen before we update the object mark!)
__ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- // Compare object markOop with mark and if equal exchange scratch1 with object markOop.
- __ cmpxchgptr(/*compare_value=*/disp_hdr,
- /*exchange_value=*/box,
- /*where=*/oop,
- /*result=*/tmp,
- cont,
- /*fail*/NULL);
+ // Compare object markOop with mark and if equal exchange scratch1
+ // with object markOop.
+ // Note that this is simply a CAS: it does not generate any
+ // barriers. These are separately generated by
+ // membar_acquire_lock().
+ {
+ Label retry_load;
+ __ bind(retry_load);
+ __ ldxr(tmp, oop);
+ __ cmp(tmp, disp_hdr);
+ __ br(Assembler::NE, cas_failed);
+ __ stxr(tmp, box, oop);
+ __ cbzw(tmp, cont);
+ __ b(retry_load);
+ }
+
+ // Formerly:
+ // __ cmpxchgptr(/*oldv=*/disp_hdr,
+ // /*newv=*/box,
+ // /*addr=*/oop,
+ // /*tmp=*/tmp,
+ // cont,
+ // /*fail*/NULL);
+
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
// If the compare-and-exchange succeeded, then we found an unlocked
@@ -3059,14 +3077,26 @@
// Try to CAS m->owner from NULL to current thread.
__ addw(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
__ mov(disp_hdr, zr);
- Label next;
- __ cmpxchgptr(/*compare_value=*/disp_hdr,
- /*exchange_value=*/rthread,
- /*where=*/tmp,
- /*result=*/rscratch1,
- /*succeed*/next,
- /*fail*/NULL);
- __ bind(next);
+
+ {
+ Label retry_load, fail;
+ __ bind(retry_load);
+ __ ldxr(rscratch1, tmp);
+ __ cmp(disp_hdr, rscratch1);
+ __ br(Assembler::NE, fail);
+ __ stxr(rscratch1, rthread, tmp);
+ __ cbnzw(rscratch1, retry_load);
+ __ bind(fail);
+ }
+
+ // Label next;
+ // __ cmpxchgptr(/*oldv=*/disp_hdr,
+ // /*newv=*/rthread,
+ // /*addr=*/tmp,
+ // /*tmp=*/rscratch1,
+ // /*succeed*/next,
+ // /*fail*/NULL);
+ // __ bind(next);
// store a non-null value into the box.
__ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
@@ -3129,25 +3159,30 @@
// Handle existing monitor.
if ((EmitSync & 0x02) == 0) {
__ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
- // we can use AArch64's bit test and branch here but
- // markoopDesc does not define a bit index just the bit value
- // so assert in case the bit pos changes
-# define __monitor_value_log2 1
- assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
- __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
-# undef __monitor_value_log2
+ __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
}
// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markOop of the
// object.
- __ cmpxchgptr(/*compare_value=*/box,
- /*exchange_value=*/disp_hdr,
- /*where=*/oop,
- /*result=*/tmp,
- cont,
- /*cas_failed*/NULL);
+ {
+ Label retry_load;
+ __ bind(retry_load);
+ __ ldxr(rscratch1, oop);
+ __ cmp(box, tmp);
+ __ br(Assembler::NE, cas_failed);
+ __ stxr(rscratch1, disp_hdr, oop);
+ __ cbzw(rscratch1, cont);
+ __ b(retry_load);
+ }
+
+ // __ cmpxchgptr(/*compare_value=*/box,
+ // /*exchange_value=*/disp_hdr,
+ // /*where=*/oop,
+ // /*result=*/tmp,
+ // cont,
+ // /*cas_failed*/NULL);
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
__ bind(cas_failed);
@@ -5842,11 +5877,6 @@
// ============================================================================
// MemBar Instruction
-// TODO
-
-// check that we are using the right type of barrier instruction and
-// the correct
-
instruct membar_acquire()
%{
match(MemBarAcquire);
@@ -5928,7 +5958,7 @@
ins_encode %{
__ block_comment("MEMBAR-release-lock");
- __ membar(Assembler::Membar_mask_bits(Assembler::StoreStore|Assembler::LoadStore));
+ __ membar(Assembler::AnyAny);
%}
ins_pipe(pipe_class_memory);
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/assembler_aarch64.hpp
--- a/src/cpu/aarch64/vm/assembler_aarch64.hpp Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/assembler_aarch64.hpp Thu Feb 13 18:45:41 2014 +0000
@@ -1004,8 +1004,12 @@
StoreStore = ST,
LoadStore = LD,
LoadLoad = LD,
- StoreLoad = SY,
- AnyAny = SY
+ // We can use ISH for a full barrier because the ARM ARM says
+ // "This architecture assumes that all Processing Elements that
+ // use the same operating system or hypervisor are in the same
+ // Inner Shareable shareability domain."
+ StoreLoad = ISH,
+ AnyAny = ISH
};
void membar(Membar_mask_bits order_constraint) {
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp Thu Feb 13 18:45:41 2014 +0000
@@ -1580,33 +1580,38 @@
// flush and load exclusive from the memory location
// and fail if it is not what we expect
__ bind(retry_load);
- __ ldaxrw(rscratch1, addr);
+
+ __ membar(__ AnyAny);
+ __ ldxrw(rscratch1, addr);
__ cmpw(rscratch1, cmpval);
__ cset(rscratch1, Assembler::NE);
__ br(Assembler::NE, nope);
// if we store+flush with no intervening write rscratch1 wil be zero
- __ stlxrw(rscratch1, newval, addr);
+ __ stxrw(rscratch1, newval, addr);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
__ cbnzw(rscratch1, retry_load);
__ bind(nope);
+ __ membar(__ AnyAny);
}
void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
Label retry_load, nope;
// flush and load exclusive from the memory location
// and fail if it is not what we expect
+ __ membar(__ AnyAny);
__ bind(retry_load);
- __ ldaxr(rscratch1, addr);
+ __ ldxr(rscratch1, addr);
__ cmp(rscratch1, cmpval);
__ cset(rscratch1, Assembler::NE);
__ br(Assembler::NE, nope);
// if we store+flush with no intervening write rscratch1 wil be zero
- __ stlxr(rscratch1, newval, addr);
+ __ stxr(rscratch1, newval, addr);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
__ cbnz(rscratch1, retry_load);
__ bind(nope);
+ __ membar(__ AnyAny);
}
@@ -3080,23 +3085,23 @@
case T_INT:
lda = &MacroAssembler::ldaxrw;
add = &MacroAssembler::addw;
- stl = &MacroAssembler::stlxrw;
+ stl = &MacroAssembler::stxrw;
break;
case T_LONG:
lda = &MacroAssembler::ldaxr;
add = &MacroAssembler::add;
- stl = &MacroAssembler::stlxr;
+ stl = &MacroAssembler::stxr;
break;
case T_OBJECT:
case T_ARRAY:
if (UseCompressedOops) {
lda = &MacroAssembler::ldaxrw;
add = &MacroAssembler::addw;
- stl = &MacroAssembler::stlxrw;
+ stl = &MacroAssembler::stxrw;
} else {
lda = &MacroAssembler::ldaxr;
add = &MacroAssembler::add;
- stl = &MacroAssembler::stlxr;
+ stl = &MacroAssembler::stxr;
}
break;
default:
@@ -3150,7 +3155,7 @@
default:
ShouldNotReachHere();
}
- asm("nop");
+ __ membar(__ AnyAny);
}
#undef __
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/macroAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp Thu Feb 13 18:45:41 2014 +0000
@@ -1408,10 +1408,10 @@
Label retry_load;
bind(retry_load);
// flush and load exclusive from the memory location
- ldaxrw(tmp, counter_addr);
+ ldxrw(tmp, counter_addr);
addw(tmp, tmp, 1);
// if we store+flush with no intervening write tmp wil be zero
- stlxrw(tmp, tmp, counter_addr);
+ stxrw(tmp, tmp, counter_addr);
cbnzw(tmp, retry_load);
}
@@ -1841,17 +1841,20 @@
bind(retry_load);
// flush and load exclusive from the memory location
// and fail if it is not what we expect
- ldaxr(tmp, addr);
+ membar(AnyAny);
+ ldxr(tmp, addr);
cmp(tmp, oldv);
br(Assembler::NE, nope);
// if we store+flush with no intervening write tmp wil be zero
- stlxr(tmp, newv, addr);
+ stxr(tmp, newv, addr);
+ membar(AnyAny);
cbzw(tmp, succeed);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
b(retry_load);
// if the memory word differs we return it in oldv and signal a fail
bind(nope);
+ membar(AnyAny);
mov(oldv, tmp);
if (fail)
b(*fail);
@@ -1868,17 +1871,20 @@
bind(retry_load);
// flush and load exclusive from the memory location
// and fail if it is not what we expect
- ldaxrw(tmp, addr);
+ membar(AnyAny);
+ ldxrw(tmp, addr);
cmp(tmp, oldv);
br(Assembler::NE, nope);
// if we store+flush with no intervening write tmp wil be zero
- stlxrw(tmp, newv, addr);
+ stxrw(tmp, newv, addr);
+ membar(AnyAny);
cbzw(tmp, succeed);
// retry so we only ever return after a load fails to compare
// ensures we don't return a stale value after a failed write.
b(retry_load);
// if the memory word differs we return it in oldv and signal a fail
bind(nope);
+ membar(AnyAny);
mov(oldv, tmp);
if (fail)
b(*fail);
More information about the aarch64-port-dev
mailing list