Skip to content

Commit 6313223

Browse files
luhenryluhenry
committedDec 19, 2023
8315856: RISC-V: Use Zacas extension for cmpxchg
Reviewed-by: rehn, fyang
1 parent 3bc5679 commit 6313223

5 files changed

+185
-29
lines changed
 

‎src/hotspot/cpu/riscv/assembler_riscv.hpp

+2
Original file line numberDiff line numberDiff line change
@@ -758,6 +758,8 @@ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
758758
INSN(amomax_d , 0b0101111, 0b011, 0b10100);
759759
INSN(amominu_d, 0b0101111, 0b011, 0b11000);
760760
INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
761+
INSN(amocas_w, 0b0101111, 0b010, 0b00101);
762+
INSN(amocas_d, 0b0101111, 0b011, 0b00101);
761763
#undef INSN
762764

763765
enum operand_size { int8, int16, int32, uint32, int64 };

‎src/hotspot/cpu/riscv/globals_riscv.hpp

+1
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
105105
product(bool, UseZba, false, "Use Zba instructions") \
106106
product(bool, UseZbb, false, "Use Zbb instructions") \
107107
product(bool, UseZbs, false, "Use Zbs instructions") \
108+
product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
108109
product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \
109110
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
110111
product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions") \

‎src/hotspot/cpu/riscv/macroAssembler_riscv.cpp

+168-29
Original file line numberDiff line numberDiff line change
@@ -2732,20 +2732,29 @@ void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Reg
27322732
// oldv holds comparison value
27332733
// newv holds value to write in exchange
27342734
// addr identifies memory word to compare against/update
2735-
Label retry_load, nope;
2736-
bind(retry_load);
2737-
// Load reserved from the memory location
2738-
load_reserved(tmp, addr, int64, Assembler::aqrl);
2739-
// Fail and exit if it is not what we expect
2740-
bne(tmp, oldv, nope);
2741-
// If the store conditional succeeds, tmp will be zero
2742-
store_conditional(tmp, newv, addr, int64, Assembler::rl);
2743-
beqz(tmp, succeed);
2744-
// Retry only when the store conditional failed
2745-
j(retry_load);
2746-
2747-
bind(nope);
2735+
if (UseZacas) {
2736+
mv(tmp, oldv);
2737+
atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl);
2738+
beq(tmp, oldv, succeed);
2739+
} else {
2740+
Label retry_load, nope;
2741+
bind(retry_load);
2742+
// Load reserved from the memory location
2743+
load_reserved(tmp, addr, int64, Assembler::aqrl);
2744+
// Fail and exit if it is not what we expect
2745+
bne(tmp, oldv, nope);
2746+
// If the store conditional succeeds, tmp will be zero
2747+
store_conditional(tmp, newv, addr, int64, Assembler::rl);
2748+
beqz(tmp, succeed);
2749+
// Retry only when the store conditional failed
2750+
j(retry_load);
2751+
2752+
bind(nope);
2753+
}
2754+
2755+
// neither amocas nor lr/sc have an implied barrier in the failing case
27482756
membar(AnyAny);
2757+
27492758
mv(oldv, tmp);
27502759
if (fail != nullptr) {
27512760
j(*fail);
@@ -2819,7 +2828,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
28192828
}
28202829
sll(mask, mask, shift);
28212830

2822-
xori(not_mask, mask, -1);
2831+
notr(not_mask, mask);
28232832

28242833
sll(expected, expected, shift);
28252834
andr(expected, expected, mask);
@@ -2829,7 +2838,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
28292838
}
28302839

28312840
// cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps.
2832-
// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w,
2841+
// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w or amocas.w,
28332842
// which are forced to work with 4-byte aligned address.
28342843
void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
28352844
Register new_val,
@@ -2844,14 +2853,29 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
28442853
Label retry, fail, done;
28452854

28462855
bind(retry);
2847-
lr_w(old, aligned_addr, acquire);
2848-
andr(tmp, old, mask);
2849-
bne(tmp, expected, fail);
28502856

2851-
andr(tmp, old, not_mask);
2852-
orr(tmp, tmp, new_val);
2853-
sc_w(tmp, tmp, aligned_addr, release);
2854-
bnez(tmp, retry);
2857+
if (UseZacas) {
2858+
lw(old, aligned_addr);
2859+
2860+
// if old & mask != expected
2861+
andr(tmp, old, mask);
2862+
bne(tmp, expected, fail);
2863+
2864+
andr(tmp, old, not_mask);
2865+
orr(tmp, tmp, new_val);
2866+
2867+
atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release);
2868+
bne(tmp, old, retry);
2869+
} else {
2870+
lr_w(old, aligned_addr, acquire);
2871+
andr(tmp, old, mask);
2872+
bne(tmp, expected, fail);
2873+
2874+
andr(tmp, old, not_mask);
2875+
orr(tmp, tmp, new_val);
2876+
sc_w(tmp, tmp, aligned_addr, release);
2877+
bnez(tmp, retry);
2878+
}
28552879

28562880
if (result_as_bool) {
28572881
mv(result, 1);
@@ -2891,14 +2915,28 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
28912915

28922916
Label fail, done;
28932917

2894-
lr_w(old, aligned_addr, acquire);
2895-
andr(tmp, old, mask);
2896-
bne(tmp, expected, fail);
2918+
if (UseZacas) {
2919+
lw(old, aligned_addr);
28972920

2898-
andr(tmp, old, not_mask);
2899-
orr(tmp, tmp, new_val);
2900-
sc_w(tmp, tmp, aligned_addr, release);
2901-
bnez(tmp, fail);
2921+
// if old & mask != expected
2922+
andr(tmp, old, mask);
2923+
bne(tmp, expected, fail);
2924+
2925+
andr(tmp, old, not_mask);
2926+
orr(tmp, tmp, new_val);
2927+
2928+
atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release);
2929+
bne(tmp, old, fail);
2930+
} else {
2931+
lr_w(old, aligned_addr, acquire);
2932+
andr(tmp, old, mask);
2933+
bne(tmp, expected, fail);
2934+
2935+
andr(tmp, old, not_mask);
2936+
orr(tmp, tmp, new_val);
2937+
sc_w(tmp, tmp, aligned_addr, release);
2938+
bnez(tmp, fail);
2939+
}
29022940

29032941
// Success
29042942
mv(result, 1);
@@ -2921,6 +2959,19 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
29212959
assert_different_registers(expected, t0);
29222960
assert_different_registers(new_val, t0);
29232961

2962+
if (UseZacas) {
2963+
if (result_as_bool) {
2964+
mv(t0, expected);
2965+
atomic_cas(t0, new_val, addr, size, acquire, release);
2966+
xorr(t0, t0, expected);
2967+
seqz(result, t0);
2968+
} else {
2969+
mv(result, expected);
2970+
atomic_cas(result, new_val, addr, size, acquire, release);
2971+
}
2972+
return;
2973+
}
2974+
29242975
Label retry_load, done, ne_done;
29252976
bind(retry_load);
29262977
load_reserved(t0, addr, size, acquire);
@@ -2952,6 +3003,11 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
29523003
enum operand_size size,
29533004
Assembler::Aqrl acquire, Assembler::Aqrl release,
29543005
Register result) {
3006+
if (UseZacas) {
3007+
cmpxchg(addr, expected, new_val, size, acquire, release, result, true);
3008+
return;
3009+
}
3010+
29553011
assert_different_registers(addr, t0);
29563012
assert_different_registers(expected, t0);
29573013
assert_different_registers(new_val, t0);
@@ -3018,6 +3074,89 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)
30183074

30193075
#undef ATOMIC_XCHGU
30203076

3077+
#define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE) \
3078+
void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
3079+
assert(UseZacas, "invariant"); \
3080+
prev = prev->is_valid() ? prev : zr; \
3081+
AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \
3082+
return; \
3083+
}
3084+
3085+
ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed)
3086+
ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed)
3087+
ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl)
3088+
ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl)
3089+
ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl)
3090+
ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl)
3091+
3092+
#undef ATOMIC_CAS
3093+
3094+
#define ATOMIC_CASU(OP1, OP2) \
3095+
void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \
3096+
atomic_##OP2(prev, newv, addr); \
3097+
zero_extend(prev, prev, 32); \
3098+
return; \
3099+
}
3100+
3101+
ATOMIC_CASU(caswu, casw)
3102+
ATOMIC_CASU(caslwu, caslw)
3103+
ATOMIC_CASU(casalwu, casalw)
3104+
3105+
#undef ATOMIC_CASU
3106+
3107+
void MacroAssembler::atomic_cas(
3108+
Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
3109+
switch (size) {
3110+
case int64:
3111+
switch ((Assembler::Aqrl)(acquire | release)) {
3112+
case Assembler::relaxed:
3113+
atomic_cas(prev, newv, addr);
3114+
break;
3115+
case Assembler::rl:
3116+
atomic_casl(prev, newv, addr);
3117+
break;
3118+
case Assembler::aqrl:
3119+
atomic_casal(prev, newv, addr);
3120+
break;
3121+
default:
3122+
ShouldNotReachHere();
3123+
}
3124+
break;
3125+
case int32:
3126+
switch ((Assembler::Aqrl)(acquire | release)) {
3127+
case Assembler::relaxed:
3128+
atomic_casw(prev, newv, addr);
3129+
break;
3130+
case Assembler::rl:
3131+
atomic_caslw(prev, newv, addr);
3132+
break;
3133+
case Assembler::aqrl:
3134+
atomic_casalw(prev, newv, addr);
3135+
break;
3136+
default:
3137+
ShouldNotReachHere();
3138+
}
3139+
break;
3140+
case uint32:
3141+
switch ((Assembler::Aqrl)(acquire | release)) {
3142+
case Assembler::relaxed:
3143+
atomic_caswu(prev, newv, addr);
3144+
break;
3145+
case Assembler::rl:
3146+
atomic_caslwu(prev, newv, addr);
3147+
break;
3148+
case Assembler::aqrl:
3149+
atomic_casalwu(prev, newv, addr);
3150+
break;
3151+
default:
3152+
ShouldNotReachHere();
3153+
}
3154+
break;
3155+
default:
3156+
ShouldNotReachHere();
3157+
}
3158+
}
3159+
30213160
void MacroAssembler::far_jump(const Address &entry, Register tmp) {
30223161
assert(ReservedCodeCacheSize < 4*G, "branch out of range");
30233162
assert(CodeCache::find_blob(entry.target()) != nullptr,

‎src/hotspot/cpu/riscv/macroAssembler_riscv.hpp

+13
Original file line numberDiff line numberDiff line change
@@ -1063,6 +1063,19 @@ class MacroAssembler: public Assembler {
10631063
void atomic_xchgwu(Register prev, Register newv, Register addr);
10641064
void atomic_xchgalwu(Register prev, Register newv, Register addr);
10651065

1066+
void atomic_cas(Register prev, Register newv, Register addr);
1067+
void atomic_casw(Register prev, Register newv, Register addr);
1068+
void atomic_casl(Register prev, Register newv, Register addr);
1069+
void atomic_caslw(Register prev, Register newv, Register addr);
1070+
void atomic_casal(Register prev, Register newv, Register addr);
1071+
void atomic_casalw(Register prev, Register newv, Register addr);
1072+
void atomic_caswu(Register prev, Register newv, Register addr);
1073+
void atomic_caslwu(Register prev, Register newv, Register addr);
1074+
void atomic_casalwu(Register prev, Register newv, Register addr);
1075+
1076+
void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
1077+
Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
1078+
10661079
// Emit a far call/jump. Only invalidates the tmp register which
10671080
// is used to keep the entry address for jalr.
10681081
// The address must be inside the code cache.

‎src/hotspot/cpu/riscv/vm_version_riscv.hpp

+1
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ class VM_Version : public Abstract_VM_Version {
142142
decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
143143
decl(ext_Ztso , "Ztso" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso)) \
144144
decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
145+
decl(ext_Zacas , "Zacas" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \
145146
decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
146147
decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
147148
decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \

0 commit comments

Comments
 (0)
Failed to load comments.