[aarch64-port-dev ] openjdk atomics

Andrew Haley aph at redhat.com
Thu Feb 13 10:49:28 PST 2014


Here attached.

I have tested this, but it's goint go need someone to review it properly.

Andrew.



# HG changeset patch
# User aph
# Date 1392317141 0
#      Thu Feb 13 18:45:41 2014 +0000
# Node ID 5614eacca5c6687ac6d80cdf3d16c2ea2af37118
# Parent  a69fcb4e97f6098c2f0b712a9306664a0fe74368
Change all load/store exclusive operations as recommended by ARM.

diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/aarch64.ad
--- a/src/cpu/aarch64/vm/aarch64.ad	Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/aarch64.ad	Thu Feb 13 18:45:41 2014 +0000
@@ -2409,18 +2409,20 @@
       }
     }
     Label retry_load, done;
+    __ membar(__ AnyAny);
     __ bind(retry_load);
-    __ ldaxr(rscratch1, addr_reg);
+    __ ldxr(rscratch1, addr_reg);
     __ cmp(rscratch1, old_reg);
     __ br(Assembler::NE, done);
     // if we store+flush with no intervening write tmp will be zero
-    __ stlxr(rscratch1, new_reg, addr_reg);
+    __ stxr(rscratch1, new_reg, addr_reg);
     __ cmpw(rscratch1, zr);       // cannot use cbzw as must set flag
     __ br(Assembler::EQ, done);
     // retry so we only ever return after a load fails to compare
     // ensures we don't return a stale value after a failed write.
     __ b(retry_load);
     __ bind(done);
+    __ membar(__ AnyAny);
   %}

   enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
@@ -2454,17 +2456,19 @@
     }
     Label retry_load, done;
     __ bind(retry_load);
-    __ ldaxrw(rscratch1, addr_reg);
+    __ membar(__ AnyAny);
+    __ ldxrw(rscratch1, addr_reg);
     __ cmpw(rscratch1, old_reg);
     __ br(Assembler::NE, done);
     // if we store+flush with no intervening write tmp wil be zero
-    __ stlxrw(rscratch1, new_reg, addr_reg);
+    __ stxrw(rscratch1, new_reg, addr_reg);
     __ cmpw(rscratch1, zr);       // cannot use cbzw as must set flag
     __ br(Assembler::EQ, done);
     // retry so we only ever return after a load fails to compare
     // ensures we don't return a stale value after a failed write.
     __ b(retry_load);
     __ bind(done);
+    __ membar(__ AnyAny);
   %}

   // auxiliary used for CompareAndSwapX to set result register
@@ -2976,9 +2980,6 @@
     __ br(target_reg);
   %}

-  // TODO
-  // reimplement this with custom cmpxchgptr code
-  // which avoids some of the unnecessary branching
   enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
     MacroAssembler _masm(&cbuf);
     Register oop = as_Register($object$$reg);
@@ -3023,13 +3024,30 @@
     // Initialize the box. (Must happen before we update the object mark!)
     __ str(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));

-    // Compare object markOop with mark and if equal exchange scratch1 with object markOop.
-    __ cmpxchgptr(/*compare_value=*/disp_hdr,
-                  /*exchange_value=*/box,
-                  /*where=*/oop,
-                  /*result=*/tmp,
-                  cont,
-                  /*fail*/NULL);
+    // Compare object markOop with mark and if equal exchange scratch1
+    // with object markOop.
+    // Note that this is simply a CAS: it does not generate any
+    // barriers.  These are separately generated by
+    // membar_acquire_lock().
+    {
+      Label retry_load;
+      __ bind(retry_load);
+      __ ldxr(tmp, oop);
+      __ cmp(tmp, disp_hdr);
+      __ br(Assembler::NE, cas_failed);
+      __ stxr(tmp, box, oop);
+      __ cbzw(tmp, cont);
+      __ b(retry_load);
+    }
+
+    // Formerly:
+    // __ cmpxchgptr(/*oldv=*/disp_hdr,
+    //               /*newv=*/box,
+    //               /*addr=*/oop,
+    //               /*tmp=*/tmp,
+    //               cont,
+    //               /*fail*/NULL);
+
     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");

     // If the compare-and-exchange succeeded, then we found an unlocked
@@ -3059,14 +3077,26 @@
       // Try to CAS m->owner from NULL to current thread.
       __ addw(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
       __ mov(disp_hdr, zr);
-      Label next;
-      __ cmpxchgptr(/*compare_value=*/disp_hdr,
-                    /*exchange_value=*/rthread,
-                    /*where=*/tmp,
-                    /*result=*/rscratch1,
-                    /*succeed*/next,
-                    /*fail*/NULL);
-      __ bind(next);
+
+      {
+	Label retry_load, fail;
+	__ bind(retry_load);
+	__ ldxr(rscratch1, tmp);
+	__ cmp(disp_hdr, rscratch1);
+	__ br(Assembler::NE, fail);
+	__ stxr(rscratch1, rthread, tmp);
+	__ cbnzw(rscratch1, retry_load);
+	__ bind(fail);
+      }
+
+      // Label next;
+      // __ cmpxchgptr(/*oldv=*/disp_hdr,
+      //               /*newv=*/rthread,
+      //               /*addr=*/tmp,
+      //               /*tmp=*/rscratch1,
+      //               /*succeed*/next,
+      //               /*fail*/NULL);
+      // __ bind(next);

       // store a non-null value into the box.
       __ str(box, Address(box, BasicLock::displaced_header_offset_in_bytes()));
@@ -3129,25 +3159,30 @@
     // Handle existing monitor.
     if ((EmitSync & 0x02) == 0) {
       __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
-      // we can use AArch64's bit test and branch here but
-      // markoopDesc does not define a bit index just the bit value
-      // so assert in case the bit pos changes
-#     define __monitor_value_log2 1
-      assert(markOopDesc::monitor_value == (1 << __monitor_value_log2), "incorrect bit position");
-      __ tbnz(disp_hdr, __monitor_value_log2, object_has_monitor);
-#     undef __monitor_value_log2
+      __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
     }

     // Check if it is still a light weight lock, this is is true if we
     // see the stack address of the basicLock in the markOop of the
     // object.

-    __ cmpxchgptr(/*compare_value=*/box,
-                  /*exchange_value=*/disp_hdr,
-                  /*where=*/oop,
-                  /*result=*/tmp,
-                  cont,
-                  /*cas_failed*/NULL);
+      {
+	Label retry_load;
+	__ bind(retry_load);
+	__ ldxr(rscratch1, oop);
+	__ cmp(box, tmp);
+	__ br(Assembler::NE, cas_failed);
+	__ stxr(rscratch1, disp_hdr, oop);
+	__ cbzw(rscratch1, cont);
+	__ b(retry_load);
+      }
+
+    // __ cmpxchgptr(/*compare_value=*/box,
+    //               /*exchange_value=*/disp_hdr,
+    //               /*where=*/oop,
+    //               /*result=*/tmp,
+    //               cont,
+    //               /*cas_failed*/NULL);
     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");

     __ bind(cas_failed);
@@ -5842,11 +5877,6 @@
 // ============================================================================
 // MemBar Instruction

-// TODO
-
-// check that we are using the right type of barrier instruction and
-// the correct
-
 instruct membar_acquire()
 %{
   match(MemBarAcquire);
@@ -5928,7 +5958,7 @@

   ins_encode %{
     __ block_comment("MEMBAR-release-lock");
-    __ membar(Assembler::Membar_mask_bits(Assembler::StoreStore|Assembler::LoadStore));
+    __ membar(Assembler::AnyAny);
   %}

   ins_pipe(pipe_class_memory);
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/assembler_aarch64.hpp
--- a/src/cpu/aarch64/vm/assembler_aarch64.hpp	Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/assembler_aarch64.hpp	Thu Feb 13 18:45:41 2014 +0000
@@ -1004,8 +1004,12 @@
     StoreStore = ST,
     LoadStore  = LD,
     LoadLoad   = LD,
-    StoreLoad  = SY,
-    AnyAny     = SY
+    // We can use ISH for a full barrier because the ARM ARM says
+    // "This architecture assumes that all Processing Elements that
+    // use the same operating system or hypervisor are in the same
+    // Inner Shareable shareability domain."
+    StoreLoad  = ISH,
+    AnyAny     = ISH
   };

   void membar(Membar_mask_bits order_constraint) {
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp	Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/c1_LIRAssembler_aarch64.cpp	Thu Feb 13 18:45:41 2014 +0000
@@ -1580,33 +1580,38 @@
   // flush and load exclusive from the memory location
   // and fail if it is not what we expect
   __ bind(retry_load);
-  __ ldaxrw(rscratch1, addr);
+
+  __ membar(__ AnyAny);
+  __ ldxrw(rscratch1, addr);
   __ cmpw(rscratch1, cmpval);
   __ cset(rscratch1, Assembler::NE);
   __ br(Assembler::NE, nope);
   // if we store+flush with no intervening write rscratch1 wil be zero
-  __ stlxrw(rscratch1, newval, addr);
+  __ stxrw(rscratch1, newval, addr);
   // retry so we only ever return after a load fails to compare
   // ensures we don't return a stale value after a failed write.
   __ cbnzw(rscratch1, retry_load);
   __ bind(nope);
+  __ membar(__ AnyAny);
 }

 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
   Label retry_load, nope;
   // flush and load exclusive from the memory location
   // and fail if it is not what we expect
+  __ membar(__ AnyAny);
   __ bind(retry_load);
-  __ ldaxr(rscratch1, addr);
+  __ ldxr(rscratch1, addr);
   __ cmp(rscratch1, cmpval);
   __ cset(rscratch1, Assembler::NE);
   __ br(Assembler::NE, nope);
   // if we store+flush with no intervening write rscratch1 wil be zero
-  __ stlxr(rscratch1, newval, addr);
+  __ stxr(rscratch1, newval, addr);
   // retry so we only ever return after a load fails to compare
   // ensures we don't return a stale value after a failed write.
   __ cbnz(rscratch1, retry_load);
   __ bind(nope);
+  __ membar(__ AnyAny);
 }


@@ -3080,23 +3085,23 @@
   case T_INT:
     lda = &MacroAssembler::ldaxrw;
     add = &MacroAssembler::addw;
-    stl = &MacroAssembler::stlxrw;
+    stl = &MacroAssembler::stxrw;
     break;
   case T_LONG:
     lda = &MacroAssembler::ldaxr;
     add = &MacroAssembler::add;
-    stl = &MacroAssembler::stlxr;
+    stl = &MacroAssembler::stxr;
     break;
   case T_OBJECT:
   case T_ARRAY:
     if (UseCompressedOops) {
       lda = &MacroAssembler::ldaxrw;
       add = &MacroAssembler::addw;
-      stl = &MacroAssembler::stlxrw;
+      stl = &MacroAssembler::stxrw;
     } else {
       lda = &MacroAssembler::ldaxr;
       add = &MacroAssembler::add;
-      stl = &MacroAssembler::stlxr;
+      stl = &MacroAssembler::stxr;
     }
     break;
   default:
@@ -3150,7 +3155,7 @@
   default:
     ShouldNotReachHere();
   }
-  asm("nop");
+  __ membar(__ AnyAny);
 }

 #undef __
diff -r a69fcb4e97f6 -r 5614eacca5c6 src/cpu/aarch64/vm/macroAssembler_aarch64.cpp
--- a/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp	Tue Feb 11 15:36:54 2014 +0000
+++ b/src/cpu/aarch64/vm/macroAssembler_aarch64.cpp	Thu Feb 13 18:45:41 2014 +0000
@@ -1408,10 +1408,10 @@
   Label retry_load;
   bind(retry_load);
   // flush and load exclusive from the memory location
-  ldaxrw(tmp, counter_addr);
+  ldxrw(tmp, counter_addr);
   addw(tmp, tmp, 1);
   // if we store+flush with no intervening write tmp wil be zero
-  stlxrw(tmp, tmp, counter_addr);
+  stxrw(tmp, tmp, counter_addr);
   cbnzw(tmp, retry_load);
 }

@@ -1841,17 +1841,20 @@
   bind(retry_load);
   // flush and load exclusive from the memory location
   // and fail if it is not what we expect
-  ldaxr(tmp, addr);
+  membar(AnyAny);
+  ldxr(tmp, addr);
   cmp(tmp, oldv);
   br(Assembler::NE, nope);
   // if we store+flush with no intervening write tmp wil be zero
-  stlxr(tmp, newv, addr);
+  stxr(tmp, newv, addr);
+  membar(AnyAny);
   cbzw(tmp, succeed);
   // retry so we only ever return after a load fails to compare
   // ensures we don't return a stale value after a failed write.
   b(retry_load);
   // if the memory word differs we return it in oldv and signal a fail
   bind(nope);
+  membar(AnyAny);
   mov(oldv, tmp);
   if (fail)
     b(*fail);
@@ -1868,17 +1871,20 @@
   bind(retry_load);
   // flush and load exclusive from the memory location
   // and fail if it is not what we expect
-  ldaxrw(tmp, addr);
+  membar(AnyAny);
+  ldxrw(tmp, addr);
   cmp(tmp, oldv);
   br(Assembler::NE, nope);
   // if we store+flush with no intervening write tmp wil be zero
-  stlxrw(tmp, newv, addr);
+  stxrw(tmp, newv, addr);
+  membar(AnyAny);
   cbzw(tmp, succeed);
   // retry so we only ever return after a load fails to compare
   // ensures we don't return a stale value after a failed write.
   b(retry_load);
   // if the memory word differs we return it in oldv and signal a fail
   bind(nope);
+  membar(AnyAny);
   mov(oldv, tmp);
   if (fail)
     b(*fail);


More information about the aarch64-port-dev mailing list