Skip to content

Commit 4fc6d41

Browse files
mikabl-armmikabl-arm
committedNov 5, 2024
8341194: [REDO] Implement C2 VectorizedHashCode on AArch64
Reviewed-by: aph, adinn
·
jdk-25+13jdk-24+23
1 parent abf2dc7 commit 4fc6d41

12 files changed

+1346
-580
lines changed
 

‎src/hotspot/cpu/aarch64/aarch64.ad

+44
Original file line numberDiff line numberDiff line change
@@ -5028,6 +5028,24 @@ operand vRegD_V7()
50285028
interface(REG_INTER);
50295029
%}
50305030

5031+
operand vRegD_V12()
5032+
%{
5033+
constraint(ALLOC_IN_RC(v12_reg));
5034+
match(RegD);
5035+
op_cost(0);
5036+
format %{ %}
5037+
interface(REG_INTER);
5038+
%}
5039+
5040+
operand vRegD_V13()
5041+
%{
5042+
constraint(ALLOC_IN_RC(v13_reg));
5043+
match(RegD);
5044+
op_cost(0);
5045+
format %{ %}
5046+
interface(REG_INTER);
5047+
%}
5048+
50315049
operand pReg()
50325050
%{
50335051
constraint(ALLOC_IN_RC(pr_reg));
@@ -16770,6 +16788,32 @@ instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
1677016788
ins_pipe(pipe_class_memory);
1677116789
%}
1677216790

16791+
instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16792+
vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16793+
vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16794+
vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16795+
%{
16796+
match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16797+
effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16798+
TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16799+
16800+
format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16801+
ins_encode %{
16802+
address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16803+
$vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16804+
$vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16805+
$vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16806+
$vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16807+
$vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16808+
(BasicType)$basic_type$$constant);
16809+
if (tpc == nullptr) {
16810+
ciEnv::current()->record_failure("CodeCache is full");
16811+
return;
16812+
}
16813+
%}
16814+
ins_pipe(pipe_class_memory);
16815+
%}
16816+
1677316817
instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
1677416818
%{
1677516819
match(Set result (CountPositives ary1 len));

‎src/hotspot/cpu/aarch64/assembler_aarch64.hpp

+67-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2014, 2024, Red Hat Inc. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -287,6 +287,11 @@ class Instruction_aarch64 {
287287
f(r->raw_encoding(), lsb + 4, lsb);
288288
}
289289

290+
//<0-15>reg: As `rf(FloatRegister)`, but only the lower 16 FloatRegisters are allowed.
291+
void lrf(FloatRegister r, int lsb) {
292+
f(r->raw_encoding(), lsb + 3, lsb);
293+
}
294+
290295
void prf(PRegister r, int lsb) {
291296
f(r->raw_encoding(), lsb + 3, lsb);
292297
}
@@ -765,6 +770,7 @@ class Assembler : public AbstractAssembler {
765770
#define f current_insn.f
766771
#define sf current_insn.sf
767772
#define rf current_insn.rf
773+
#define lrf current_insn.lrf
768774
#define srf current_insn.srf
769775
#define zrf current_insn.zrf
770776
#define prf current_insn.prf
@@ -1590,6 +1596,16 @@ class Assembler : public AbstractAssembler {
15901596

15911597
#undef INSN
15921598

1599+
// Load/store a register, but with a BasicType parameter. Loaded signed integer values are
1600+
// extended to 64 bits.
1601+
void load(Register Rt, const Address &adr, BasicType bt) {
1602+
int op = (is_signed_subword_type(bt) || bt == T_INT) ? 0b10 : 0b01;
1603+
ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), op);
1604+
}
1605+
void store(Register Rt, const Address &adr, BasicType bt) {
1606+
ld_st2(Rt, adr, exact_log2(type2aelembytes(bt)), 0b00);
1607+
}
1608+
15931609
/* SIMD extensions
15941610
*
15951611
* We just use FloatRegister in the following. They are exactly the same
@@ -2587,6 +2603,7 @@ template<typename R, typename... Rx>
25872603
INSN(addpv, 0, 0b101111, true); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S, T2D
25882604
INSN(smullv, 0, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
25892605
INSN(umullv, 1, 0b110000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
2606+
INSN(smlalv, 0, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
25902607
INSN(umlalv, 1, 0b100000, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
25912608
INSN(maxv, 0, 0b011001, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
25922609
INSN(minv, 0, 0b011011, false); // accepted arrangements: T8B, T16B, T4H, T8H, T2S, T4S
@@ -2860,6 +2877,28 @@ template<typename R, typename... Rx>
28602877
// FMULX - Vector - Scalar
28612878
INSN(fmulxvs, 1, 0b1001);
28622879

2880+
#undef INSN
2881+
2882+
#define INSN(NAME, op1, op2) \
2883+
void NAME(FloatRegister Vd, SIMD_Arrangement T, FloatRegister Vn, FloatRegister Vm, int index) { \
2884+
starti; \
2885+
assert(T == T4H || T == T8H || T == T2S || T == T4S, "invalid arrangement"); \
2886+
assert(index >= 0 && \
2887+
((T == T2S && index <= 1) || (T != T2S && index <= 3) || (T == T8H && index <= 7)), \
2888+
"invalid index"); \
2889+
assert((T != T4H && T != T8H) || Vm->encoding() < 16, "invalid source SIMD&FP register"); \
2890+
f(0, 31), f((int)T & 1, 30), f(op1, 29), f(0b01111, 28, 24); \
2891+
if (T == T4H || T == T8H) { \
2892+
f(0b01, 23, 22), f(index & 0b11, 21, 20), lrf(Vm, 16), f(index >> 2 & 1, 11); \
2893+
} else { \
2894+
f(0b10, 23, 22), f(index & 1, 21), rf(Vm, 16), f(index >> 1, 11); \
2895+
} \
2896+
f(op2, 15, 12), f(0, 10), rf(Vn, 5), rf(Vd, 0); \
2897+
}
2898+
2899+
// MUL - Vector - Scalar
2900+
INSN(mulvs, 0, 0b1000);
2901+
28632902
#undef INSN
28642903

28652904
// Floating-point Reciprocal Estimate
@@ -3023,6 +3062,33 @@ template<typename R, typename... Rx>
30233062
umov(Xd, Vn, T, index);
30243063
}
30253064

3065+
protected:
3066+
void _xaddwv(bool is_unsigned, FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta,
3067+
FloatRegister Vm, SIMD_Arrangement Tb) {
3068+
starti;
3069+
assert((Tb >> 1) + 1 == (Ta >> 1), "Incompatible arrangement");
3070+
f(0, 31), f((int)Tb & 1, 30), f(is_unsigned ? 1 : 0, 29), f(0b01110, 28, 24);
3071+
f((int)(Ta >> 1) - 1, 23, 22), f(1, 21), rf(Vm, 16), f(0b000100, 15, 10), rf(Vn, 5), rf(Vd, 0);
3072+
}
3073+
3074+
public:
3075+
#define INSN(NAME, assertion, is_unsigned) \
3076+
void NAME(FloatRegister Vd, FloatRegister Vn, SIMD_Arrangement Ta, FloatRegister Vm, \
3077+
SIMD_Arrangement Tb) { \
3078+
assert((assertion), "invalid arrangement"); \
3079+
_xaddwv(is_unsigned, Vd, Vn, Ta, Vm, Tb); \
3080+
}
3081+
3082+
public:
3083+
3084+
INSN(uaddwv, Tb == T8B || Tb == T4H || Tb == T2S, /*is_unsigned*/true)
3085+
INSN(uaddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/true)
3086+
INSN(saddwv, Tb == T8B || Tb == T4H || Tb == T2S, /*is_unsigned*/false)
3087+
INSN(saddwv2, Tb == T16B || Tb == T8H || Tb == T4S, /*is_unsigned*/false)
3088+
3089+
#undef INSN
3090+
3091+
30263092
private:
30273093
void _pmull(FloatRegister Vd, SIMD_Arrangement Ta, FloatRegister Vn, FloatRegister Vm, SIMD_Arrangement Tb) {
30283094
starti;

‎src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

+96
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "opto/subnode.hpp"
3434
#include "runtime/stubRoutines.hpp"
3535
#include "utilities/globalDefinitions.hpp"
36+
#include "utilities/powerOfTwo.hpp"
3637

3738
#ifdef PRODUCT
3839
#define BLOCK_COMMENT(str) /* nothing */
@@ -46,6 +47,101 @@
4647

4748
typedef void (MacroAssembler::* chr_insn)(Register Rt, const Address &adr);
4849

50+
// jdk.internal.util.ArraysSupport.vectorizedHashCode
51+
address C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register result,
52+
FloatRegister vdata0, FloatRegister vdata1,
53+
FloatRegister vdata2, FloatRegister vdata3,
54+
FloatRegister vmul0, FloatRegister vmul1,
55+
FloatRegister vmul2, FloatRegister vmul3,
56+
FloatRegister vpow, FloatRegister vpowm,
57+
BasicType eltype) {
58+
ARRAYS_HASHCODE_REGISTERS;
59+
60+
Register tmp1 = rscratch1, tmp2 = rscratch2;
61+
62+
Label TAIL, STUB_SWITCH, STUB_SWITCH_OUT, LOOP, BR_BASE, LARGE, DONE;
63+
64+
// Vectorization factor. Number of array elements loaded to one SIMD&FP registers by the stubs. We
65+
// use 8H load arrangements for chars and shorts and 8B for booleans and bytes. It's possible to
66+
// use 4H for chars and shorts instead, but using 8H gives better performance.
67+
const size_t vf = eltype == T_BOOLEAN || eltype == T_BYTE ? 8
68+
: eltype == T_CHAR || eltype == T_SHORT ? 8
69+
: eltype == T_INT ? 4
70+
: 0;
71+
guarantee(vf, "unsupported eltype");
72+
73+
// Unroll factor for the scalar loop below. The value is chosen based on performance analysis.
74+
const size_t unroll_factor = 4;
75+
76+
switch (eltype) {
77+
case T_BOOLEAN:
78+
BLOCK_COMMENT("arrays_hashcode(unsigned byte) {");
79+
break;
80+
case T_CHAR:
81+
BLOCK_COMMENT("arrays_hashcode(char) {");
82+
break;
83+
case T_BYTE:
84+
BLOCK_COMMENT("arrays_hashcode(byte) {");
85+
break;
86+
case T_SHORT:
87+
BLOCK_COMMENT("arrays_hashcode(short) {");
88+
break;
89+
case T_INT:
90+
BLOCK_COMMENT("arrays_hashcode(int) {");
91+
break;
92+
default:
93+
ShouldNotReachHere();
94+
}
95+
96+
// large_arrays_hashcode(T_INT) performs worse than the scalar loop below when the Neon loop
97+
// implemented by the stub executes just once. Call the stub only if at least two iterations will
98+
// be executed.
99+
const size_t large_threshold = eltype == T_INT ? vf * 2 : vf;
100+
cmpw(cnt, large_threshold);
101+
br(Assembler::HS, LARGE);
102+
103+
bind(TAIL);
104+
105+
// The andr performs cnt % uf where uf = unroll_factor. The subtract shifted by 3 offsets past
106+
// uf - (cnt % uf) pairs of load + madd insns i.e. it only executes cnt % uf load + madd pairs.
107+
// Iteration eats up the remainder, uf elements at a time.
108+
assert(is_power_of_2(unroll_factor), "can't use this value to calculate the jump target PC");
109+
andr(tmp2, cnt, unroll_factor - 1);
110+
adr(tmp1, BR_BASE);
111+
sub(tmp1, tmp1, tmp2, ext::sxtw, 3);
112+
movw(tmp2, 0x1f);
113+
br(tmp1);
114+
115+
bind(LOOP);
116+
for (size_t i = 0; i < unroll_factor; ++i) {
117+
load(tmp1, Address(post(ary, type2aelembytes(eltype))), eltype);
118+
maddw(result, result, tmp2, tmp1);
119+
}
120+
bind(BR_BASE);
121+
subsw(cnt, cnt, unroll_factor);
122+
br(Assembler::HS, LOOP);
123+
124+
b(DONE);
125+
126+
bind(LARGE);
127+
128+
RuntimeAddress stub = RuntimeAddress(StubRoutines::aarch64::large_arrays_hashcode(eltype));
129+
assert(stub.target() != nullptr, "array_hashcode stub has not been generated");
130+
address tpc = trampoline_call(stub);
131+
if (tpc == nullptr) {
132+
DEBUG_ONLY(reset_labels(TAIL, BR_BASE));
133+
postcond(pc() == badAddress);
134+
return nullptr;
135+
}
136+
137+
bind(DONE);
138+
139+
BLOCK_COMMENT("} // arrays_hashcode");
140+
141+
postcond(pc() != badAddress);
142+
return pc();
143+
}
144+
49145
void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register tmpReg,
50146
Register tmp2Reg, Register tmp3Reg) {
51147
Register oop = objectReg;

‎src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp

+7
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,13 @@
3535
enum shift_kind kind = Assembler::LSL, unsigned shift = 0);
3636

3737
public:
38+
// jdk.internal.util.ArraysSupport.vectorizedHashCode
39+
address arrays_hashcode(Register ary, Register cnt, Register result, FloatRegister vdata0,
40+
FloatRegister vdata1, FloatRegister vdata2, FloatRegister vdata3,
41+
FloatRegister vmul0, FloatRegister vmul1, FloatRegister vmul2,
42+
FloatRegister vmul3, FloatRegister vpow, FloatRegister vpowm,
43+
BasicType eltype);
44+
3845
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
3946
void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
4047
void fast_unlock(Register object, Register box, Register tmp, Register tmp2);

‎src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

+18
Original file line numberDiff line numberDiff line change
@@ -1439,6 +1439,24 @@ class MacroAssembler: public Assembler {
14391439
address arrays_equals(Register a1, Register a2, Register result, Register cnt1,
14401440
Register tmp1, Register tmp2, Register tmp3, int elem_size);
14411441

1442+
// Ensure that the inline code and the stub use the same registers.
1443+
#define ARRAYS_HASHCODE_REGISTERS \
1444+
do { \
1445+
assert(result == r0 && \
1446+
ary == r1 && \
1447+
cnt == r2 && \
1448+
vdata0 == v3 && \
1449+
vdata1 == v2 && \
1450+
vdata2 == v1 && \
1451+
vdata3 == v0 && \
1452+
vmul0 == v4 && \
1453+
vmul1 == v5 && \
1454+
vmul2 == v6 && \
1455+
vmul3 == v7 && \
1456+
vpow == v12 && \
1457+
vpowm == v13, "registers must match aarch64.ad"); \
1458+
} while (0)
1459+
14421460
void string_equals(Register a1, Register a2, Register result, Register cnt1);
14431461

14441462
void fill_words(Register base, Register cnt, Register value);

1 commit comments

Comments
 (1)

openjdk-notifier[bot] commented on Nov 5, 2024

@openjdk-notifier[bot]
openjdk-notifier[bot]
on Nov 5, 2024

Review

Issues

Please sign in to comment.