Skip to content

Commit 336bbbe

Browse files
rkennkeRealFYangtstuefe
committedFeb 23, 2024
8139457: Relax alignment of array elements
Co-authored-by: Fei Yang <fyang@openjdk.org> Co-authored-by: Thomas Stuefe <stuefe@openjdk.org> Reviewed-by: stuefe, stefank, shade, coleenp, kdnilsen, aboldtch
·
jdk-25+17jdk-23-ga
1 parent cb809f8 commit 336bbbe

33 files changed

+560
-171
lines changed
 

‎src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -1209,7 +1209,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
12091209
len,
12101210
tmp1,
12111211
tmp2,
1212-
arrayOopDesc::header_size(op->type()),
1212+
arrayOopDesc::base_offset_in_bytes(op->type()),
12131213
array_element_size(op->type()),
12141214
op->klass()->as_register(),
12151215
*op->stub()->entry());

‎src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp

+12-3
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
188188

189189
if (len->is_valid()) {
190190
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
191+
int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
192+
if (!is_aligned(base_offset, BytesPerWord)) {
193+
assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
194+
// Clear gap/first 4 bytes following the length field.
195+
strw(zr, Address(obj, base_offset));
196+
}
191197
} else if (UseCompressedClassPointers) {
192198
store_klass_gap(obj, zr);
193199
}
@@ -266,7 +272,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
266272

267273
verify_oop(obj);
268274
}
269-
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
275+
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
270276
assert_different_registers(obj, len, t1, t2, klass);
271277

272278
// determine alignment mask
@@ -279,16 +285,19 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
279285

280286
const Register arr_size = t2; // okay to be the same
281287
// align object end
282-
mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
288+
mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
283289
add(arr_size, arr_size, len, ext::uxtw, f);
284290
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
285291

286292
try_allocate(obj, arr_size, 0, t1, t2, slow_case);
287293

288294
initialize_header(obj, klass, len, t1, t2);
289295

296+
// Align-up to word boundary, because we clear the 4 bytes potentially
297+
// following the length field in initialize_header().
298+
int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
290299
// clear rest of allocated space
291-
initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
300+
initialize_body(obj, arr_size, base_offset, t1, t2);
292301
if (Compilation::current()->bailed_out()) {
293302
return;
294303
}

‎src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -100,7 +100,7 @@ using MacroAssembler::null_check;
100100
// header_size: size of object header in words
101101
// f : element scale factor
102102
// slow_case : exit to slow case implementation if fast allocation fails
103-
void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, int f, Register klass, Label& slow_case);
103+
void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
104104

105105
int rsp_offset() const { return _rsp_offset; }
106106
void set_rsp_offset(int n) { _rsp_offset = n; }

‎src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -968,7 +968,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
968968
op->tmp1()->as_register(),
969969
op->tmp2()->as_register(),
970970
op->tmp3()->as_register(),
971-
arrayOopDesc::header_size(op->type()),
971+
arrayOopDesc::base_offset_in_bytes(op->type()),
972972
type2aelembytes(op->type()),
973973
op->klass()->as_register(),
974974
*op->stub()->entry());

‎src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -2298,7 +2298,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
22982298
op->tmp1()->as_register(),
22992299
op->tmp2()->as_register(),
23002300
op->tmp3()->as_register(),
2301-
arrayOopDesc::header_size(op->type()),
2301+
arrayOopDesc::base_offset_in_bytes(op->type()),
23022302
type2aelembytes(op->type()),
23032303
op->klass()->as_register(),
23042304
*op->stub()->entry());

‎src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp

+15-5
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2012, 2018 SAP SE. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -310,7 +310,7 @@ void C1_MacroAssembler::allocate_array(
310310
Register t1, // temp register
311311
Register t2, // temp register
312312
Register t3, // temp register
313-
int hdr_size, // object header size in words
313+
int base_offset_in_bytes, // elements offset in bytes
314314
int elt_size, // element size in bytes
315315
Register klass, // object klass
316316
Label& slow_case // continuation point if fast allocation fails
@@ -342,7 +342,7 @@ void C1_MacroAssembler::allocate_array(
342342
sldi(t1, len, log2_elt_size);
343343
arr_len_in_bytes = t1;
344344
}
345-
addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
345+
addi(arr_size, arr_len_in_bytes, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
346346
clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes); // Align array size.
347347

348348
// Allocate space & initialize header.
@@ -352,8 +352,18 @@ void C1_MacroAssembler::allocate_array(
352352
// Initialize body.
353353
const Register base = t2;
354354
const Register index = t3;
355-
addi(base, obj, hdr_size * wordSize); // compute address of first element
356-
addi(index, arr_size, -(hdr_size * wordSize)); // compute index = number of bytes to clear
355+
addi(base, obj, base_offset_in_bytes); // compute address of first element
356+
addi(index, arr_size, -(base_offset_in_bytes)); // compute index = number of bytes to clear
357+
358+
// Zero first 4 bytes, if start offset is not word aligned.
359+
if (!is_aligned(base_offset_in_bytes, BytesPerWord)) {
360+
assert(is_aligned(base_offset_in_bytes, BytesPerInt), "must be 4-byte aligned");
361+
li(t1, 0);
362+
stw(t1, 0, base);
363+
addi(base, base, BytesPerInt);
364+
// Note: initialize_body will align index down, no need to correct it here.
365+
}
366+
357367
initialize_body(base, index);
358368

359369
if (CURRENT_ENV->dtrace_alloc_probes()) {

‎src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2012, 2015 SAP SE. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -80,7 +80,7 @@
8080
Register t1, // temp register
8181
Register t2, // temp register
8282
Register t3, // temp register
83-
int hdr_size, // object header size in words
83+
int base_offset_in_bytes, // elements offset in bytes
8484
int elt_size, // element size in bytes
8585
Register klass, // object klass
8686
Label& slow_case // continuation point if fast allocation fails

‎src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
44
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
55
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -1020,7 +1020,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
10201020
len,
10211021
tmp1,
10221022
tmp2,
1023-
arrayOopDesc::header_size(op->type()),
1023+
arrayOopDesc::base_offset_in_bytes(op->type()),
10241024
array_element_size(op->type()),
10251025
op->klass()->as_register(),
10261026
*op->stub()->entry());

‎src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp

+15-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
44
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
55
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -280,7 +280,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
280280
verify_oop(obj);
281281
}
282282

283-
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) {
283+
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
284284
assert_different_registers(obj, len, tmp1, tmp2, klass);
285285

286286
// determine alignment mask
@@ -292,17 +292,28 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
292292

293293
const Register arr_size = tmp2; // okay to be the same
294294
// align object end
295-
mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
295+
mv(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
296296
shadd(arr_size, len, arr_size, t0, f);
297297
andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
298298

299299
try_allocate(obj, arr_size, 0, tmp1, tmp2, slow_case);
300300

301301
initialize_header(obj, klass, len, tmp1, tmp2);
302302

303+
// Clear leading 4 bytes, if necessary.
304+
// TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
305+
// for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
306+
int base_offset = base_offset_in_bytes;
307+
if (!is_aligned(base_offset, BytesPerWord)) {
308+
assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
309+
sw(zr, Address(obj, base_offset));
310+
base_offset += BytesPerInt;
311+
}
312+
assert(is_aligned(base_offset, BytesPerWord), "must be word-aligned");
313+
303314
// clear rest of allocated space
304315
const Register len_zero = len;
305-
initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
316+
initialize_body(obj, arr_size, base_offset, len_zero);
306317

307318
membar(MacroAssembler::StoreStore);
308319

‎src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
44
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
55
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -101,7 +101,7 @@ using MacroAssembler::null_check;
101101
// header_size: size of object header in words
102102
// f : element scale factor
103103
// slow_case : exit to slow case implementation if fast allocation fails
104-
void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case);
104+
void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
105105

106106
int rsp_offset() const { return _rsp_offset; }
107107

‎src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -2382,7 +2382,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
23822382
op->len()->as_register(),
23832383
op->tmp1()->as_register(),
23842384
op->tmp2()->as_register(),
2385-
arrayOopDesc::header_size(op->type()),
2385+
arrayOopDesc::base_offset_in_bytes(op->type()),
23862386
type2aelembytes(op->type()),
23872387
op->klass()->as_register(),
23882388
*op->stub()->entry());

‎src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -271,7 +271,7 @@ void C1_MacroAssembler::allocate_array(
271271
Register len, // array length
272272
Register t1, // temp register
273273
Register t2, // temp register
274-
int hdr_size, // object header size in words
274+
int base_offset_in_bytes, // elements offset in bytes
275275
int elt_size, // element size in bytes
276276
Register klass, // object klass
277277
Label& slow_case // Continuation point if fast allocation fails.
@@ -297,8 +297,8 @@ void C1_MacroAssembler::allocate_array(
297297
case 8: z_sllg(arr_size, len, 3); break;
298298
default: ShouldNotReachHere();
299299
}
300-
add2reg(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
301-
z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
300+
add2reg(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
301+
z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
302302

303303
try_allocate(obj, arr_size, 0, t1, slow_case);
304304

@@ -308,9 +308,9 @@ void C1_MacroAssembler::allocate_array(
308308
Label done;
309309
Register object_fields = t1;
310310
Register Rzero = Z_R1_scratch;
311-
z_aghi(arr_size, -(hdr_size * BytesPerWord));
311+
z_aghi(arr_size, -base_offset_in_bytes);
312312
z_bre(done); // Jump if size of fields is zero.
313-
z_la(object_fields, hdr_size * BytesPerWord, obj);
313+
z_la(object_fields, base_offset_in_bytes, obj);
314314
z_xgr(Rzero, Rzero);
315315
initialize_body(object_fields, arr_size, Rzero);
316316
bind(done);

‎src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
33
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
44
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
55
*
@@ -86,7 +86,7 @@
8686
Register len, // array length
8787
Register t1, // temp register
8888
Register t2, // temp register
89-
int hdr_size, // object header size in words
89+
int base_offset_in_bytes, // elements offset in bytes
9090
int elt_size, // element size in bytes
9191
Register klass, // object klass
9292
Label& slow_case // Continuation point if fast allocation fails.

‎src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -1618,7 +1618,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
16181618
len,
16191619
tmp1,
16201620
tmp2,
1621-
arrayOopDesc::header_size(op->type()),
1621+
arrayOopDesc::base_offset_in_bytes(op->type()),
16221622
array_element_size(op->type()),
16231623
op->klass()->as_register(),
16241624
*op->stub()->entry());

‎src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp

+15-3
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,15 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
186186

187187
if (len->is_valid()) {
188188
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
189+
#ifdef _LP64
190+
int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
191+
if (!is_aligned(base_offset, BytesPerWord)) {
192+
assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
193+
// Clear gap/first 4 bytes following the length field.
194+
xorl(t1, t1);
195+
movl(Address(obj, base_offset), t1);
196+
}
197+
#endif
189198
}
190199
#ifdef _LP64
191200
else if (UseCompressedClassPointers) {
@@ -269,7 +278,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
269278
verify_oop(obj);
270279
}
271280

272-
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) {
281+
void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
273282
assert(obj == rax, "obj must be in rax, for cmpxchg");
274283
assert_different_registers(obj, len, t1, t2, klass);
275284

@@ -282,7 +291,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
282291

283292
const Register arr_size = t2; // okay to be the same
284293
// align object end
285-
movptr(arr_size, header_size * BytesPerWord + MinObjAlignmentInBytesMask);
294+
movptr(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask);
286295
lea(arr_size, Address(arr_size, len, f));
287296
andptr(arr_size, ~MinObjAlignmentInBytesMask);
288297

@@ -292,7 +301,10 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
292301

293302
// clear rest of allocated space
294303
const Register len_zero = len;
295-
initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
304+
// Align-up to word boundary, because we clear the 4 bytes potentially
305+
// following the length field in initialize_header().
306+
int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
307+
initialize_body(obj, arr_size, base_offset, len_zero);
296308

297309
if (CURRENT_ENV->dtrace_alloc_probes()) {
298310
assert(obj == rax, "must be");

‎src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@
8989
// header_size: size of object header in words
9090
// f : element scale factor
9191
// slow_case : exit to slow case implementation if fast allocation fails
92-
void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case);
92+
void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case);
9393

9494
int rsp_offset() const { return _rsp_offset; }
9595
void set_rsp_offset(int n) { _rsp_offset = n; }

0 commit comments

Comments
 (0)
Please sign in to comment.