If the target architecture is riscv64, the path should
-be debian-ports instead of debian.
To create an Ubuntu-based chroot:
sudo debootstrap \
--arch=arm64 \
diff --git a/doc/building.md b/doc/building.md
index 611cfe15915..abbd935f652 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -1316,9 +1316,6 @@ For example, cross-compiling to AArch64 from x86_64 could be done like this:
https://httpredir.debian.org/debian/
```
- If the target architecture is `riscv64`, the path should be `debian-ports`
- instead of `debian`.
-
* To create an Ubuntu-based chroot:
```
diff --git a/make/CompileDemos.gmk b/make/CompileDemos.gmk
index 4d8a9598da7..dfe30c70173 100644
--- a/make/CompileDemos.gmk
+++ b/make/CompileDemos.gmk
@@ -58,7 +58,7 @@ DEMO_MANIFEST := $(SUPPORT_OUTPUTDIR)/demos/java-main-manifest.mf
# This rule will be depended on due to the MANIFEST line in SetupBuildDemo
# and SetupBuildJvmtiDemo.
$(eval $(call SetupTextFileProcessing, BUILD_JAVA_MANIFEST, \
- SOURCE_FILES := $(TOPDIR)/make/data/mainmanifest/manifest.mf, \
+ SOURCE_FILES := $(TOPDIR)/make/data/mainmanifest/manifest.mf.template, \
OUTPUT_FILE := $(DEMO_MANIFEST), \
REPLACEMENTS := \
@@VERSION_SPECIFICATION@@ => $(VERSION_SPECIFICATION) ; \
diff --git a/make/JrtfsJar.gmk b/make/JrtfsJar.gmk
index 50ae82ca565..b0b7ed6ce08 100644
--- a/make/JrtfsJar.gmk
+++ b/make/JrtfsJar.gmk
@@ -33,7 +33,7 @@ include TextFileProcessing.gmk
# This rule will be depended on due to the MANIFEST line
$(eval $(call SetupTextFileProcessing, BUILD_JAVA_MANIFEST, \
- SOURCE_FILES := $(TOPDIR)/make/data/mainmanifest/manifest.mf, \
+ SOURCE_FILES := $(TOPDIR)/make/data/mainmanifest/manifest.mf.template, \
OUTPUT_FILE := $(SUPPORT_OUTPUTDIR)/java-main-manifest.mf, \
REPLACEMENTS := \
@@VERSION_SPECIFICATION@@ => $(VERSION_SPECIFICATION) ; \
diff --git a/make/MacBundles.gmk b/make/MacBundles.gmk
index c1c51570a1d..39da6c9cdb3 100644
--- a/make/MacBundles.gmk
+++ b/make/MacBundles.gmk
@@ -69,7 +69,7 @@ ifeq ($(call isTargetOs, macosx), true)
))
$(eval $(call SetupTextFileProcessing, BUILD_JDK_PLIST, \
- SOURCE_FILES := $(MACOSX_PLIST_SRC)/JDK-Info.plist, \
+ SOURCE_FILES := $(MACOSX_PLIST_SRC)/JDK-Info.plist.template, \
OUTPUT_FILE := $(JDK_MACOSX_CONTENTS_DIR)/Info.plist, \
REPLACEMENTS := \
@@ID@@ => $(MACOSX_BUNDLE_ID_BASE).jdk ; \
@@ -82,7 +82,7 @@ ifeq ($(call isTargetOs, macosx), true)
))
$(eval $(call SetupTextFileProcessing, BUILD_JRE_PLIST, \
- SOURCE_FILES := $(MACOSX_PLIST_SRC)/JRE-Info.plist, \
+ SOURCE_FILES := $(MACOSX_PLIST_SRC)/JRE-Info.plist.template, \
OUTPUT_FILE := $(JRE_MACOSX_CONTENTS_DIR)/Info.plist, \
REPLACEMENTS := \
@@ID@@ => $(MACOSX_BUNDLE_ID_BASE).jre ; \
diff --git a/make/Main.gmk b/make/Main.gmk
index a50d3c50278..0014c97cc0b 100644
--- a/make/Main.gmk
+++ b/make/Main.gmk
@@ -744,9 +744,16 @@ endif
$(eval $(call SetupTarget, build-test-lib, \
MAKEFILE := test/BuildTestLib, \
+ TARGET := build-test-lib, \
DEPS := exploded-image, \
))
+$(eval $(call SetupTarget, test-image-lib, \
+ MAKEFILE := test/BuildTestLib, \
+ TARGET := test-image-lib, \
+ DEPS := build-test-lib, \
+))
+
ifeq ($(BUILD_FAILURE_HANDLER), true)
# Builds the failure handler jtreg extension
$(eval $(call SetupTarget, build-test-failure-handler, \
@@ -781,7 +788,7 @@ endif
$(eval $(call SetupTarget, build-microbenchmark, \
MAKEFILE := test/BuildMicrobenchmark, \
- DEPS := interim-langtools exploded-image, \
+ DEPS := interim-langtools exploded-image build-test-lib, \
))
################################################################################
@@ -1264,7 +1271,7 @@ all-docs-bundles: docs-jdk-bundles docs-javase-bundles docs-reference-bundles
# This target builds the test image
test-image: prepare-test-image test-image-jdk-jtreg-native \
test-image-demos-jdk test-image-libtest-jtreg-native \
- test-image-lib-native
+ test-image-lib test-image-lib-native
ifneq ($(JVM_TEST_IMAGE_TARGETS), )
# If JVM_TEST_IMAGE_TARGETS is externally defined, use it instead of the
diff --git a/make/autoconf/Makefile.in b/make/autoconf/Makefile.template
similarity index 100%
rename from make/autoconf/Makefile.in
rename to make/autoconf/Makefile.template
diff --git a/make/autoconf/basic.m4 b/make/autoconf/basic.m4
index af963359583..5a1fbc07952 100644
--- a/make/autoconf/basic.m4
+++ b/make/autoconf/basic.m4
@@ -448,17 +448,17 @@ AC_DEFUN_ONCE([BASIC_SETUP_OUTPUT_DIR],
AC_SUBST(CONFIGURESUPPORT_OUTPUTDIR)
# The spec.gmk file contains all variables for the make system.
- AC_CONFIG_FILES([$OUTPUTDIR/spec.gmk:$AUTOCONF_DIR/spec.gmk.in])
+ AC_CONFIG_FILES([$OUTPUTDIR/spec.gmk:$AUTOCONF_DIR/spec.gmk.template])
# The bootcycle-spec.gmk file contains support for boot cycle builds.
- AC_CONFIG_FILES([$OUTPUTDIR/bootcycle-spec.gmk:$AUTOCONF_DIR/bootcycle-spec.gmk.in])
+ AC_CONFIG_FILES([$OUTPUTDIR/bootcycle-spec.gmk:$AUTOCONF_DIR/bootcycle-spec.gmk.template])
# The buildjdk-spec.gmk file contains support for building a buildjdk when cross compiling.
- AC_CONFIG_FILES([$OUTPUTDIR/buildjdk-spec.gmk:$AUTOCONF_DIR/buildjdk-spec.gmk.in])
+ AC_CONFIG_FILES([$OUTPUTDIR/buildjdk-spec.gmk:$AUTOCONF_DIR/buildjdk-spec.gmk.template])
# The compare.sh is used to compare the build output to other builds.
- AC_CONFIG_FILES([$OUTPUTDIR/compare.sh:$AUTOCONF_DIR/compare.sh.in])
+ AC_CONFIG_FILES([$OUTPUTDIR/compare.sh:$AUTOCONF_DIR/compare.sh.template])
# The generated Makefile knows where the spec.gmk is and where the source is.
# You can run make from the OUTPUTDIR, or from the top-level Makefile
# which will look for generated configurations
- AC_CONFIG_FILES([$OUTPUTDIR/Makefile:$AUTOCONF_DIR/Makefile.in])
+ AC_CONFIG_FILES([$OUTPUTDIR/Makefile:$AUTOCONF_DIR/Makefile.template])
])
###############################################################################
diff --git a/make/autoconf/bootcycle-spec.gmk.in b/make/autoconf/bootcycle-spec.gmk.template
similarity index 100%
rename from make/autoconf/bootcycle-spec.gmk.in
rename to make/autoconf/bootcycle-spec.gmk.template
diff --git a/make/autoconf/buildjdk-spec.gmk.in b/make/autoconf/buildjdk-spec.gmk.template
similarity index 100%
rename from make/autoconf/buildjdk-spec.gmk.in
rename to make/autoconf/buildjdk-spec.gmk.template
diff --git a/make/autoconf/compare.sh.in b/make/autoconf/compare.sh.template
similarity index 100%
rename from make/autoconf/compare.sh.in
rename to make/autoconf/compare.sh.template
diff --git a/make/autoconf/jdk-version.m4 b/make/autoconf/jdk-version.m4
index 6a7662556fd..7c9ecad7779 100644
--- a/make/autoconf/jdk-version.m4
+++ b/make/autoconf/jdk-version.m4
@@ -110,6 +110,15 @@ AC_DEFUN_ONCE([JDKVER_SETUP_JDK_VERSION_NUMBERS],
CHECK_VALUE: [UTIL_CHECK_STRING_NON_EMPTY_PRINTABLE])
AC_SUBST(COMPANY_NAME)
+ # Set the JDK RC Company name
+ # Otherwise uses the value set for "vendor-name".
+ UTIL_ARG_WITH(NAME: jdk-rc-company-name, TYPE: string,
+ DEFAULT: $COMPANY_NAME,
+ DESC: [Set JDK RC company name. This is used for CompanyName properties of MS Windows binaries.],
+ DEFAULT_DESC: [from branding.conf],
+ CHECK_VALUE: [UTIL_CHECK_STRING_NON_EMPTY_PRINTABLE])
+ AC_SUBST(JDK_RC_COMPANY_NAME)
+
# The vendor URL, if any
# Only set VENDOR_URL if '--with-vendor-url' was used and is not empty.
# Otherwise we will use the value from "branding.conf" included above.
diff --git a/make/autoconf/spec.gmk.in b/make/autoconf/spec.gmk.template
similarity index 99%
rename from make/autoconf/spec.gmk.in
rename to make/autoconf/spec.gmk.template
index 8a1eb8ed27f..b78d3da6b32 100644
--- a/make/autoconf/spec.gmk.in
+++ b/make/autoconf/spec.gmk.template
@@ -191,6 +191,7 @@ PRODUCT_NAME := @PRODUCT_NAME@
PRODUCT_SUFFIX := @PRODUCT_SUFFIX@
JDK_RC_PLATFORM_NAME := @JDK_RC_PLATFORM_NAME@
JDK_RC_NAME := @JDK_RC_NAME@
+JDK_RC_COMPANY_NAME := @JDK_RC_COMPANY_NAME@
COMPANY_NAME := @COMPANY_NAME@
HOTSPOT_VM_DISTRO := @HOTSPOT_VM_DISTRO@
MACOSX_BUNDLE_NAME_BASE := @MACOSX_BUNDLE_NAME_BASE@
diff --git a/make/common/JdkNativeCompilation.gmk b/make/common/JdkNativeCompilation.gmk
index 6a963ac2c49..1a1333cf517 100644
--- a/make/common/JdkNativeCompilation.gmk
+++ b/make/common/JdkNativeCompilation.gmk
@@ -98,7 +98,7 @@ GLOBAL_VERSION_INFO_RESOURCE := $(TOPDIR)/src/java.base/windows/native/common/ve
JDK_RCFLAGS=$(RCFLAGS) \
-D"JDK_VERSION_STRING=$(VERSION_STRING)" \
- -D"JDK_COMPANY=$(COMPANY_NAME)" \
+ -D"JDK_COMPANY=$(JDK_RC_COMPANY_NAME)" \
-D"JDK_VER=$(VERSION_NUMBER_FOUR_POSITIONS)" \
-D"JDK_COPYRIGHT=Copyright \xA9 $(COPYRIGHT_YEAR)" \
-D"JDK_NAME=$(JDK_RC_NAME) $(VERSION_SHORT)" \
diff --git a/make/common/modules/LauncherCommon.gmk b/make/common/modules/LauncherCommon.gmk
index 21d2b9a9300..8041e9fe681 100644
--- a/make/common/modules/LauncherCommon.gmk
+++ b/make/common/modules/LauncherCommon.gmk
@@ -112,7 +112,7 @@ define SetupBuildLauncherBody
$1_PLIST_FILE := $$(SUPPORT_OUTPUTDIR)/native/$$(MODULE)/$1/Info.plist
$$(eval $$(call SetupTextFileProcessing, BUILD_PLIST_$1, \
- SOURCE_FILES := $(TOPDIR)/make/data/bundle/cmdline-Info.plist, \
+ SOURCE_FILES := $(TOPDIR)/make/data/bundle/cmdline-Info.plist.template, \
OUTPUT_FILE := $$($1_PLIST_FILE), \
REPLACEMENTS := \
@@ID@@ => $(MACOSX_BUNDLE_ID_BASE).$1 ; \
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index d497f5d82d3..3bef7eb025f 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -1206,7 +1206,7 @@ var getJibProfilesDependencies = function (input, common) {
jcov: {
organization: common.organization,
- revision: "3.0-15-jdk-asm+1.0",
+ revision: "3.0-16-jdk-asm+1.0",
ext: "zip",
environment_name: "JCOV_HOME",
},
diff --git a/make/data/bundle/JDK-Info.plist b/make/data/bundle/JDK-Info.plist.template
similarity index 100%
rename from make/data/bundle/JDK-Info.plist
rename to make/data/bundle/JDK-Info.plist.template
diff --git a/make/data/bundle/JRE-Info.plist b/make/data/bundle/JRE-Info.plist.template
similarity index 100%
rename from make/data/bundle/JRE-Info.plist
rename to make/data/bundle/JRE-Info.plist.template
diff --git a/make/data/bundle/cmdline-Info.plist b/make/data/bundle/cmdline-Info.plist.template
similarity index 100%
rename from make/data/bundle/cmdline-Info.plist
rename to make/data/bundle/cmdline-Info.plist.template
diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix
index b0c868d0451..bdc10add4f2 100644
--- a/make/data/hotspot-symbols/symbols-unix
+++ b/make/data/hotspot-symbols/symbols-unix
@@ -228,6 +228,7 @@ JVM_VirtualThreadEnd
JVM_VirtualThreadMount
JVM_VirtualThreadUnmount
JVM_VirtualThreadHideFrames
+JVM_VirtualThreadDisableSuspend
# Scoped values
JVM_EnsureMaterializedForStackWalk_func
diff --git a/make/data/mainmanifest/manifest.mf b/make/data/mainmanifest/manifest.mf.template
similarity index 100%
rename from make/data/mainmanifest/manifest.mf
rename to make/data/mainmanifest/manifest.mf.template
diff --git a/make/hotspot/gensrc/GenerateSources.gmk b/make/hotspot/gensrc/GenerateSources.gmk
index 9b42493f628..c92408b7d54 100644
--- a/make/hotspot/gensrc/GenerateSources.gmk
+++ b/make/hotspot/gensrc/GenerateSources.gmk
@@ -48,7 +48,7 @@ $(eval $(call IncludeCustomExtension, hotspot/gensrc/GenerateSources.gmk))
# Setup the hotspot launcher script for developer use
$(eval $(call SetupTextFileProcessing, CREATE_HOTSPOT_LAUNCHER, \
- SOURCE_FILES := $(TOPDIR)/make/scripts/hotspot.sh, \
+ SOURCE_FILES := $(TOPDIR)/make/scripts/hotspot.sh.template, \
OUTPUT_FILE := $(JVM_OUTPUTDIR)/hotspot, \
REPLACEMENTS := \
@@LIBARCH@@ => $(OPENJDK_TARGET_CPU_LEGACY_LIB) ; \
diff --git a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
index b6298c00c9a..442a245f92f 100644
--- a/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
+++ b/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1289,25 +1289,58 @@ private static Map coverageLevelsMap() throws Exception {
*/
private static void generateTZDBShortNamesMap() throws IOException {
Files.walk(Path.of(tzDataDir), 1, FileVisitOption.FOLLOW_LINKS)
- .filter(p -> p.toFile().isFile())
+ .filter(p -> p.toFile().isFile() && !p.endsWith("jdk11_backward"))
.forEach(p -> {
try {
String zone = null;
String rule = null;
String format = null;
+ boolean inVanguard = false;
+ boolean inRearguard = false;
for (var line : Files.readAllLines(p)) {
- if (line.contains("#STDOFF")) continue;
+ // Interpret the line in rearguard mode so that STD/DST
+ // correctly handles negative DST cases, such as "GMT/IST"
+ // vs. "IST/GMT" case for Europe/Dublin
+ if (inVanguard) {
+ if (line.startsWith("# Rearguard")) {
+ inVanguard = false;
+ inRearguard = true;
+ }
+ continue;
+ } else if (line.startsWith("# Vanguard")) {
+ inVanguard = true;
+ continue;
+ }
+ if (inRearguard) {
+ if (line.startsWith("# End of rearguard")) {
+ inRearguard = false;
+ continue;
+ } else {
+ if (line.startsWith("#\t")) {
+ line = line.substring(1); // omit #
+ }
+ }
+ }
+ if (line.isBlank() || line.matches("^[ \t]*#.*")) {
+ // ignore blank/comment lines
+ continue;
+ }
+ // remove comments in-line
line = line.replaceAll("[ \t]*#.*", "");
// Zone line
if (line.startsWith("Zone")) {
+ if (zone != null) {
+ tzdbShortNamesMap.put(zone, format + NBSP + rule);
+ }
var zl = line.split("[ \t]+", -1);
zone = zl[1];
rule = zl[3];
format = zl[4];
} else {
if (zone != null) {
- if (line.isBlank()) {
+ if (line.startsWith("Rule") ||
+ line.startsWith("Link")) {
tzdbShortNamesMap.put(zone, format + NBSP + rule);
zone = null;
rule = null;
diff --git a/make/jdk/src/classes/build/tools/generatecurrencydata/GenerateCurrencyData.java b/make/jdk/src/classes/build/tools/generatecurrencydata/GenerateCurrencyData.java
index 561edbef034..9655e08016c 100644
--- a/make/jdk/src/classes/build/tools/generatecurrencydata/GenerateCurrencyData.java
+++ b/make/jdk/src/classes/build/tools/generatecurrencydata/GenerateCurrencyData.java
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
import java.io.FileOutputStream;
import java.io.InputStream;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
@@ -339,9 +340,15 @@ private static void buildOtherTables() {
validCurrencyCodes.substring(i * 7 + 3, i * 7 + 6));
checkCurrencyCode(currencyCode);
int tableEntry = mainTable[(currencyCode.charAt(0) - 'A') * A_TO_Z + (currencyCode.charAt(1) - 'A')];
- if (tableEntry == INVALID_COUNTRY_ENTRY ||
- (tableEntry & SPECIAL_CASE_COUNTRY_MASK) != 0 ||
- (tableEntry & SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK) != (currencyCode.charAt(2) - 'A')) {
+
+ // Do not allow a future currency to be classified as an otherCurrency,
+ // otherwise it will leak out into Currency:getAvailableCurrencies
+ boolean futureCurrency = Arrays.asList(specialCaseNewCurrencies).contains(currencyCode);
+ boolean simpleCurrency = (tableEntry & SIMPLE_CASE_COUNTRY_FINAL_CHAR_MASK) == (currencyCode.charAt(2) - 'A');
+
+ // If neither a simple currency, or one defined in the future
+ // then the current currency is applicable to be added to the otherTable
+ if (!futureCurrency && !simpleCurrency) {
if (otherCurrenciesCount == maxOtherCurrencies) {
throw new RuntimeException("too many other currencies");
}
diff --git a/make/modules/java.base/Lib.gmk b/make/modules/java.base/Lib.gmk
index 30261bab6e1..924cb8aae26 100644
--- a/make/modules/java.base/Lib.gmk
+++ b/make/modules/java.base/Lib.gmk
@@ -245,7 +245,7 @@ ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2)
TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
OPTIMIZATION := HIGH, \
CFLAGS := $(CFLAGS_JDKLIB), \
- CXXFLAGS := $(CXXFLAGS_JDKLIB), \
+ CXXFLAGS := $(CXXFLAGS_JDKLIB) -std=c++17, \
LDFLAGS := $(LDFLAGS_JDKLIB) \
$(call SET_SHARED_LIBRARY_ORIGIN), \
LIBS := $(LIBCXX), \
diff --git a/make/modules/java.instrument/Lib.gmk b/make/modules/java.instrument/Lib.gmk
index ae985b3158f..94246226644 100644
--- a/make/modules/java.instrument/Lib.gmk
+++ b/make/modules/java.instrument/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -47,7 +47,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBINSTRUMENT, \
LDFLAGS_macosx := -L$(call FindLibDirForModule, java.base), \
LDFLAGS_aix := -L$(SUPPORT_OUTPUTDIR)/native/java.base, \
LIBS := $(JDKLIB_LIBS), \
- LIBS_unix := -ljava -ljvm $(LIBZ_LIBS), \
+ LIBS_unix := $(LIBZ_LIBS), \
LIBS_linux := -ljli $(LIBDL), \
LIBS_aix := -liconv -ljli_static $(LIBDL), \
LIBS_macosx := -ljli -liconv -framework Cocoa -framework Security \
diff --git a/make/scripts/hotspot.sh b/make/scripts/hotspot.sh.template
similarity index 100%
rename from make/scripts/hotspot.sh
rename to make/scripts/hotspot.sh.template
diff --git a/make/test/BuildMicrobenchmark.gmk b/make/test/BuildMicrobenchmark.gmk
index 1d4a9c63508..7a5a6f7f5b9 100644
--- a/make/test/BuildMicrobenchmark.gmk
+++ b/make/test/BuildMicrobenchmark.gmk
@@ -53,11 +53,10 @@ JMH_UNPACKED_DIR := $(MICROBENCHMARK_OUTPUT)/jmh_jars
JMH_UNPACKED_JARS_DONE := $(JMH_UNPACKED_DIR)/_unpacked.marker
# External dependencies
-JMH_COMPILE_JARS := $(JMH_CORE_JAR) $(JMH_GENERATOR_JAR)
+WHITEBOX_JAR := $(SUPPORT_OUTPUTDIR)/test/lib/wb.jar
+JMH_COMPILE_JARS := $(JMH_CORE_JAR) $(JMH_GENERATOR_JAR) $(WHITEBOX_JAR)
JMH_RUNTIME_JARS := $(JMH_CORE_JAR) $(JMH_COMMONS_MATH_JAR) $(JMH_JOPT_SIMPLE_JAR)
-MICROBENCHMARK_CLASSPATH := $(call PathList, $(JMH_COMPILE_JARS))
-
# Native dependencies
MICROBENCHMARK_NATIVE_SRC_DIRS := $(MICROBENCHMARK_SRC)
MICROBENCHMARK_NATIVE_OUTPUT := $(MICROBENCHMARK_OUTPUT)/native
@@ -92,25 +91,28 @@ $(eval $(call SetupJavaCompilation, BUILD_INDIFY, \
$(eval $(call SetupJavaCompilation, BUILD_JDK_MICROBENCHMARK, \
TARGET_RELEASE := $(TARGET_RELEASE_NEWJDK_UPGRADED), \
SMALL_JAVA := false, \
- CLASSPATH := $(MICROBENCHMARK_CLASSPATH), \
- DISABLED_WARNINGS := restricted this-escape processing rawtypes unchecked cast serial preview deprecation, \
+ CLASSPATH := $(JMH_COMPILE_JARS), \
+ DISABLED_WARNINGS := restricted this-escape processing rawtypes cast \
+ serial preview unchecked deprecation, \
SRC := $(MICROBENCHMARK_SRC), \
BIN := $(MICROBENCHMARK_CLASSES), \
- JAVAC_FLAGS := --add-exports java.base/sun.security.util=ALL-UNNAMED \
- --add-exports java.base/sun.invoke.util=ALL-UNNAMED \
+ JAVAC_FLAGS := \
--add-exports java.base/jdk.internal.classfile.impl=ALL-UNNAMED \
+ --add-exports java.base/jdk.internal.event=ALL-UNNAMED \
+ --add-exports java.base/jdk.internal.foreign=ALL-UNNAMED \
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
- --add-exports java.base/jdk.internal.org.objectweb.asm=ALL-UNNAMED \
--add-exports java.base/jdk.internal.org.objectweb.asm.tree=ALL-UNNAMED \
+ --add-exports java.base/jdk.internal.org.objectweb.asm=ALL-UNNAMED \
--add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
- --add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
- --add-exports java.base/jdk.internal.event=ALL-UNNAMED \
- --add-exports java.base/jdk.internal.foreign=ALL-UNNAMED \
+ --add-exports java.base/sun.invoke.util=ALL-UNNAMED \
+ --add-exports java.base/sun.security.util=ALL-UNNAMED \
--enable-preview \
-processor org.openjdk.jmh.generators.BenchmarkProcessor, \
- JAVA_FLAGS := --add-modules jdk.unsupported --limit-modules java.management \
+ JAVA_FLAGS := \
--add-exports java.base/jdk.internal.vm=ALL-UNNAMED \
- --enable-preview, \
+ --add-modules jdk.unsupported \
+ --enable-preview \
+ --limit-modules java.management, \
))
$(BUILD_JDK_MICROBENCHMARK): $(JMH_COMPILE_JARS)
diff --git a/make/test/BuildTestLib.gmk b/make/test/BuildTestLib.gmk
index 72f777ec6c1..4445e483ac2 100644
--- a/make/test/BuildTestLib.gmk
+++ b/make/test/BuildTestLib.gmk
@@ -23,12 +23,22 @@
# questions.
#
+################################################################################
+# This file builds the Java components of testlib.
+# It also covers the test-image part, where the built files are copied to the
+# test image.
+################################################################################
+
default: all
include $(SPEC)
include MakeBase.gmk
include JavaCompilation.gmk
+################################################################################
+# Targets for building the test lib jars
+################################################################################
+
TARGETS :=
TEST_LIB_SOURCE_DIR := $(TOPDIR)/test/lib
@@ -48,23 +58,42 @@ TARGETS += $(BUILD_WB_JAR)
$(eval $(call SetupJavaCompilation, BUILD_TEST_LIB_JAR, \
TARGET_RELEASE := $(TARGET_RELEASE_NEWJDK_UPGRADED), \
SRC := $(TEST_LIB_SOURCE_DIR), \
- EXCLUDES := jdk/test/lib/containers jdk/test/lib/security, \
+ EXCLUDES := jdk/test/lib/containers jdk/test/lib/security org, \
BIN := $(TEST_LIB_SUPPORT)/test-lib_classes, \
HEADERS := $(TEST_LIB_SUPPORT)/test-lib_headers, \
JAR := $(TEST_LIB_SUPPORT)/test-lib.jar, \
- DISABLED_WARNINGS := try deprecation rawtypes unchecked serial cast removal preview, \
+ DISABLED_WARNINGS := try deprecation rawtypes unchecked serial cast removal preview varargs, \
JAVAC_FLAGS := --add-exports java.base/sun.security.util=ALL-UNNAMED \
--add-exports java.base/jdk.internal.classfile=ALL-UNNAMED \
--add-exports java.base/jdk.internal.classfile.attribute=ALL-UNNAMED \
--add-exports java.base/jdk.internal.classfile.constantpool=ALL-UNNAMED \
--add-exports java.base/jdk.internal.module=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+ --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED \
--enable-preview, \
))
TARGETS += $(BUILD_TEST_LIB_JAR)
-##########################################################################################
+build-test-lib: $(TARGETS)
+
+################################################################################
+# Targets for building test-image.
+################################################################################
+
+# Copy the jars to the test image.
+$(eval $(call SetupCopyFiles, COPY_LIBTEST_JARS, \
+ DEST := $(TEST_IMAGE_DIR)/lib-test, \
+ FILES := $(BUILD_WB_JAR_JAR) $(BUILD_TEST_LIB_JAR_JAR), \
+))
+#
+
+test-image-lib: $(COPY_LIBTEST_JARS)
-all: $(TARGETS)
+all: build-test-lib
-.PHONY: default all
+.PHONY: default all build-test-lib test-image-lib
diff --git a/make/test/BuildTestLibNative.gmk b/make/test/BuildTestLibNative.gmk
index 00c7607913b..455936d163f 100644
--- a/make/test/BuildTestLibNative.gmk
+++ b/make/test/BuildTestLibNative.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -45,6 +45,10 @@ BUILD_LIBTEST_OUTPUT_DIR := $(OUTPUTDIR)/support/test/lib/native
BUILD_LIBTEST_IMAGE_DIR := $(TEST_IMAGE_DIR)/lib
+ifeq ($(call isTargetOs, windows), false)
+ BUILD_LIBTEST_LIBRARIES_EXCLUDE += libFileUtils.c
+endif
+
# This evaluation is expensive and should only be done if this target was
# explicitly called.
ifneq ($(filter build-test-lib-native, $(MAKECMDGOALS)), )
diff --git a/make/test/JtregNativeHotspot.gmk b/make/test/JtregNativeHotspot.gmk
index d8bf596d38a..0a7b4cdd73c 100644
--- a/make/test/JtregNativeHotspot.gmk
+++ b/make/test/JtregNativeHotspot.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -858,6 +858,11 @@ else
exeinvoke.c exestack-gap.c exestack-tls.c libAsyncGetCallTraceTest.cpp
endif
+ifeq ($(call And, $(call isTargetOs, linux) $(call isTargetCpu, aarch64)), false)
+ BUILD_HOTSPOT_JTREG_EXCLUDE += libTestSVEWithJNI.c
+endif
+
+
BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm
ifeq ($(call isTargetOs, windows), true)
diff --git a/src/demo/share/jfc/J2Ddemo/java2d/Intro.java b/src/demo/share/jfc/J2Ddemo/java2d/Intro.java
index 046768e3a6a..a4fd2e5b753 100644
--- a/src/demo/share/jfc/J2Ddemo/java2d/Intro.java
+++ b/src/demo/share/jfc/J2Ddemo/java2d/Intro.java
@@ -1,6 +1,6 @@
/*
*
- * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2024, Oracle and/or its affiliates. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -1716,7 +1716,7 @@ public Contributors(int beg, int end, Surface surf) {
this.beginning = beg;
this.ending = end;
fm = surf.getMetrics(font);
- java.util.Arrays.sort(members);
+ Arrays.sort(members);
cast.add("CONTRIBUTORS");
cast.add(" ");
cast.addAll(Arrays.asList(members));
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index ceb9cdf6f54..d89ef4c8124 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
@@ -8269,6 +8269,24 @@ instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
ins_pipe(pipe_class_default);
%}
+// ============================================================================
+// VerifyVectorAlignment Instruction
+
+instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
+ match(Set addr (VerifyVectorAlignment addr mask));
+ effect(KILL cr);
+ format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
+ ins_encode %{
+ Label Lskip;
+ // check if masked bits of addr are zero
+ __ tst($addr$$Register, $mask$$constant);
+ __ br(Assembler::EQ, Lskip);
+ __ stop("verify_vector_alignment found a misaligned vector memory access");
+ __ bind(Lskip);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// ============================================================================
// MemBar Instruction
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index 5a69934abd5..01ae45c3107 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -284,7 +284,8 @@ void LIR_Assembler::osr_entry() {
__ bind(L);
}
#endif
- __ ldp(r19, r20, Address(OSR_buf, slot_offset));
+ __ ldr(r19, Address(OSR_buf, slot_offset));
+ __ ldr(r20, Address(OSR_buf, slot_offset + BytesPerWord));
__ str(r19, frame_map()->address_for_monitor_lock(i));
__ str(r20, frame_map()->address_for_monitor_object(i));
}
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index d1001687bd5..c58ff8828bc 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -28,6 +28,7 @@
#include "code/compiledIC.hpp"
#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
@@ -90,9 +91,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.hpp
index e0f04e3cf7b..8a08c2ebf14 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.hpp
@@ -159,8 +159,6 @@
static void verify_deopt_original_pc( CompiledMethod* nm, intptr_t* unextended_sp);
#endif
- const ImmutableOopMap* get_oop_map() const;
-
public:
// Constructors
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
index 70d67b4aae2..7c5bc8b1f71 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -198,7 +198,7 @@ inline bool frame::equal(frame other) const {
&& unextended_sp() == other.unextended_sp()
&& fp() == other.fp()
&& pc() == other.pc();
- assert(!ret || ret && cb() == other.cb() && _deopt_state == other._deopt_state, "inconsistent construction");
+ assert(!ret || (cb() == other.cb() && _deopt_state == other._deopt_state), "inconsistent construction");
return ret;
}
@@ -362,20 +362,6 @@ inline int frame::sender_sp_ret_address_offset() {
return frame::sender_sp_offset - frame::return_addr_offset;
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
//------------------------------------------------------------------------------
// frame::sender
inline frame frame::sender(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
index 2293d70c8da..63f6c9491c8 100644
--- a/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globalDefinitions_aarch64.hpp
@@ -42,8 +42,12 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
// and Operational Models for ARMv8"
#define CPU_MULTI_COPY_ATOMIC
+// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
// According to the ARMv8 ARM, "Concurrent modification and execution
// of instructions can lead to the resulting instruction performing
// any behavior that can be achieved by executing any sequence of
diff --git a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
index aa22459d217..08bff22d7d0 100644
--- a/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/matcher_aarch64.hpp
@@ -193,4 +193,9 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ return false;
+ }
+
#endif // CPU_AARCH64_MATCHER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
index 27bf35e12c8..a12caa3daee 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.cpp
@@ -560,18 +560,23 @@ static bool is_movk_to_zr(uint32_t insn) {
}
#endif
-void NativePostCallNop::patch(jint diff) {
+bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+ if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
+ return false; // cannot encode
+ }
+ uint32_t data = ((uint32_t)oopmap_slot << 24) | cb_offset;
#ifdef ASSERT
- assert(diff != 0, "must be");
+ assert(data != 0, "must be");
uint32_t insn1 = uint_at(4);
uint32_t insn2 = uint_at(8);
assert (is_movk_to_zr(insn1) && is_movk_to_zr(insn2), "must be");
#endif
- uint32_t lo = diff & 0xffff;
- uint32_t hi = (uint32_t)diff >> 16;
+ uint32_t lo = data & 0xffff;
+ uint32_t hi = data >> 16;
Instruction_aarch64::patch(addr_at(4), 20, 5, lo);
Instruction_aarch64::patch(addr_at(8), 20, 5, hi);
+ return true; // successfully encoded
}
void NativeDeoptInstruction::verify() {
diff --git a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
index 1740fde772f..c0be84d3f5c 100644
--- a/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/nativeInst_aarch64.hpp
@@ -691,16 +691,20 @@ class NativePostCallNop: public NativeInstruction {
return (insns & 0xffe0001fffffffff) == 0xf280001fd503201f;
}
- jint displacement() const {
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
uint64_t movk_insns = *(uint64_t*)addr_at(4);
uint32_t lo = (movk_insns >> 5) & 0xffff;
uint32_t hi = (movk_insns >> (5 + 32)) & 0xffff;
- uint32_t result = (hi << 16) | lo;
-
- return (jint)result;
+ uint32_t data = (hi << 16) | lo;
+ if (data == 0) {
+ return false; // no information encoded
+ }
+ cb_offset = (data & 0xffffff);
+ oopmap_slot = (data >> 24) & 0xff;
+ return true; // decoding succeeded
}
- void patch(jint diff);
+ bool patch(int32_t oopmap_slot, int32_t cb_offset);
void make_deopt();
};
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index 5ab36f92e19..4f7618adcf0 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -311,7 +311,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
uint int_args = 0;
uint fp_args = 0;
- uint stk_args = 0; // inc by 2 each time
+ uint stk_args = 0;
for (int i = 0; i < total_args_passed; i++) {
switch (sig_bt[i]) {
@@ -323,8 +323,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_VOID:
@@ -341,6 +342,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -349,8 +351,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_DOUBLE:
@@ -358,6 +361,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -368,7 +372,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
}
}
- return align_up(stk_args, 2);
+ return stk_args;
}
diff --git a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
index 68c66a32794..5c37401e867 100644
--- a/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/templateTable_aarch64.cpp
@@ -3890,11 +3890,9 @@ void TemplateTable::_new() {
// get InstanceKlass
__ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
- // make sure klass is initialized & doesn't have finalizer
- // make sure klass is fully initialized
- __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
- __ cmp(rscratch1, (u1)InstanceKlass::fully_initialized);
- __ br(Assembler::NE, slow_case);
+ // make sure klass is initialized
+ assert(VM_Version::supports_fast_class_init_checks(), "Optimization requires support for fast class initialization checks");
+ __ clinit_barrier(r4, rscratch1, nullptr /*L_fast_path*/, &slow_case);
__ allocate_instance(r4, r0, r3, r1, true, slow_case);
__ b(done);
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 8b67a207cf9..4b2e5cc5a4d 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -165,6 +165,7 @@ enum Ampere_CPU_Model {
static int dcache_line_size() { return _dcache_line_size; }
static int get_initial_sve_vector_length() { return _initial_sve_vector_length; };
+ // Aarch64 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index c4197235987..e31ad91613a 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -9193,7 +9193,7 @@ instruct clear_array(iRegX cnt, iRegP base, iRegI temp, iRegX zero, Universe dum
ins_encode %{
__ mov($zero$$Register, 0);
__ mov($temp$$Register, $cnt$$Register);
- Label(loop);
+ Label loop;
__ bind(loop);
__ subs($temp$$Register, $temp$$Register, 4);
__ str($zero$$Register, Address($base$$Register, $temp$$Register), ge);
diff --git a/src/hotspot/cpu/arm/assembler_arm.hpp b/src/hotspot/cpu/arm/assembler_arm.hpp
index b1b4199e88d..6a3092f7b3b 100644
--- a/src/hotspot/cpu/arm/assembler_arm.hpp
+++ b/src/hotspot/cpu/arm/assembler_arm.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -175,7 +175,7 @@ class Address {
if (_index == noreg) {
assert(-256 < _disp && _disp < 256, "encoding constraint");
return _mode | up(_disp) << 23 | 1 << 22 | _base->encoding() << 16 |
- (abs(_disp) & 0xf0) << 4 | abs(_disp) & 0x0f;
+ (abs(_disp) & 0xf0) << 4 | (abs(_disp) & 0x0f);
} else {
assert(_index != PC && (_mode == basic_offset || _index != _base), "unpredictable instruction");
assert(_disp == 0 && _shift == lsl && _shift_imm == 0, "encoding constraint");
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index 72f58275990..f2f11c52772 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2644,8 +2644,8 @@ void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* arg
void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
- assert(src->is_double_cpu() && dest->is_address() ||
- src->is_address() && dest->is_double_cpu(),
+ assert((src->is_double_cpu() && dest->is_address()) ||
+ (src->is_address() && dest->is_double_cpu()),
"Simple move_op is called for all other cases");
int null_check_offset;
diff --git a/src/hotspot/cpu/arm/compiledIC_arm.cpp b/src/hotspot/cpu/arm/compiledIC_arm.cpp
index 89eda97d7e2..2d4187b7d6c 100644
--- a/src/hotspot/cpu/arm/compiledIC_arm.cpp
+++ b/src/hotspot/cpu/arm/compiledIC_arm.cpp
@@ -28,6 +28,7 @@
#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/nmethod.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
@@ -105,9 +106,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/arm/frame_arm.hpp b/src/hotspot/cpu/arm/frame_arm.hpp
index 7ef941ad0ff..56f8fc9932e 100644
--- a/src/hotspot/cpu/arm/frame_arm.hpp
+++ b/src/hotspot/cpu/arm/frame_arm.hpp
@@ -99,8 +99,6 @@
}
#endif
- const ImmutableOopMap* get_oop_map() const;
-
public:
// Constructors
diff --git a/src/hotspot/cpu/arm/frame_arm.inline.hpp b/src/hotspot/cpu/arm/frame_arm.inline.hpp
index 9191d03baf0..8a08c0d0e9c 100644
--- a/src/hotspot/cpu/arm/frame_arm.inline.hpp
+++ b/src/hotspot/cpu/arm/frame_arm.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -93,7 +93,7 @@ inline bool frame::equal(frame other) const {
&& unextended_sp() == other.unextended_sp()
&& fp() == other.fp()
&& pc() == other.pc();
- assert(!ret || ret && cb() == other.cb() && _deopt_state == other._deopt_state, "inconsistent construction");
+ assert(!ret || (cb() == other.cb() && _deopt_state == other._deopt_state), "inconsistent construction");
return ret;
}
@@ -218,20 +218,6 @@ inline int frame::frame_size() const {
return sender_sp() - sp();
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
inline int frame::compiled_frame_stack_argsize() const {
Unimplemented();
return 0;
diff --git a/src/hotspot/cpu/arm/globalDefinitions_arm.hpp b/src/hotspot/cpu/arm/globalDefinitions_arm.hpp
index 2041cf9e17e..5b68dbc9dbd 100644
--- a/src/hotspot/cpu/arm/globalDefinitions_arm.hpp
+++ b/src/hotspot/cpu/arm/globalDefinitions_arm.hpp
@@ -49,8 +49,12 @@ const bool HaveVFP = true;
// arm32 is not specified as multi-copy-atomic
// So we must not #define CPU_MULTI_COPY_ATOMIC
+// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
#define STUBROUTINES_MD_HPP "stubRoutines_arm.hpp"
#define INTERP_MASM_MD_HPP "interp_masm_arm.hpp"
#define TEMPLATETABLE_MD_HPP "templateTable_arm.hpp"
diff --git a/src/hotspot/cpu/arm/interp_masm_arm.cpp b/src/hotspot/cpu/arm/interp_masm_arm.cpp
index 481c3d09e58..635acd781f9 100644
--- a/src/hotspot/cpu/arm/interp_masm_arm.cpp
+++ b/src/hotspot/cpu/arm/interp_masm_arm.cpp
@@ -303,15 +303,19 @@ void InterpreterMacroAssembler::load_field_entry(Register cache, Register index,
}
void InterpreterMacroAssembler::load_method_entry(Register cache, Register index, int bcp_offset) {
+ assert_different_registers(cache, index);
+
// Get index out of bytecode pointer
get_index_at_bcp(index, bcp_offset, cache /* as tmp */, sizeof(u2));
+
+ // sizeof(ResolvedMethodEntry) is not a power of 2 on Arm, so can't use shift
mov(cache, sizeof(ResolvedMethodEntry));
mul(index, index, cache); // Scale the index to be the entry index * sizeof(ResolvedMethodEntry)
// load constant pool cache pointer
ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
// Get address of method entries array
- ldr(cache, Address(cache, ConstantPoolCache::method_entries_offset()));
+ ldr(cache, Address(cache, in_bytes(ConstantPoolCache::method_entries_offset())));
add(cache, cache, Array::base_offset_in_bytes());
add(cache, cache, index);
}
diff --git a/src/hotspot/cpu/arm/matcher_arm.hpp b/src/hotspot/cpu/arm/matcher_arm.hpp
index 1e8f7683e76..eb26cbcbd7a 100644
--- a/src/hotspot/cpu/arm/matcher_arm.hpp
+++ b/src/hotspot/cpu/arm/matcher_arm.hpp
@@ -186,4 +186,9 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ return false;
+ }
+
#endif // CPU_ARM_MATCHER_ARM_HPP
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
index e5e5e89c200..23ee01d3352 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -162,7 +162,7 @@ void NativeMovConstReg::set_data(intptr_t x, address pc) {
unsigned int hi = (unsigned int)(x >> 16);
this->set_encoding((this->encoding() & 0xfff0f000) | (lo & 0xf000) << 4 | (lo & 0xfff));
next->set_encoding((next->encoding() & 0xfff0f000) | (hi & 0xf000) << 4 | (hi & 0xfff));
- } else if (oop_addr == nullptr & metadata_addr == nullptr) {
+ } else if (oop_addr == nullptr && metadata_addr == nullptr) {
// A static ldr_literal (without oop or metadata relocation)
assert(is_ldr_literal(), "must be");
int offset = ldr_offset();
@@ -341,10 +341,6 @@ void NativePostCallNop::make_deopt() {
NativeDeoptInstruction::insert(addr_at(0));
}
-void NativePostCallNop::patch(jint diff) {
- // unsupported for now
-}
-
void NativeDeoptInstruction::verify() {
}
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
index 804292c24b0..7006d770981 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -396,7 +396,7 @@ class NativeMovConstReg: public NativeInstruction {
inline NativeMovConstReg* nativeMovConstReg_at(address address) {
NativeInstruction* ni = nativeInstruction_at(address);
assert(ni->is_ldr_literal() || ni->is_pc_rel() ||
- ni->is_movw() && VM_Version::supports_movw(), "must be");
+ (ni->is_movw() && VM_Version::supports_movw()), "must be");
return (NativeMovConstReg*)address;
}
@@ -438,8 +438,8 @@ inline NativeCall* nativeCall_before(address return_address) {
class NativePostCallNop: public NativeInstruction {
public:
bool check() const { return is_nop(); }
- int displacement() const { return 0; }
- void patch(jint diff);
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; }
void make_deopt();
};
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index 9a65f9f09fe..716c7b7575e 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -441,7 +441,6 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
}
}
- if (slot & 1) slot++;
return slot;
}
diff --git a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
index ba9a3fd7a9b..efaf78ee568 100644
--- a/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
+++ b/src/hotspot/cpu/arm/templateInterpreterGenerator_arm.cpp
@@ -370,17 +370,16 @@ address TemplateInterpreterGenerator::generate_return_entry_for(TosState state,
if (index_size == sizeof(u4)) {
__ load_resolved_indy_entry(Rcache, Rindex);
__ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
- __ check_stack_top();
- __ add(Rstack_top, Rstack_top, AsmOperand(Rcache, lsl, Interpreter::logStackElementSize));
} else {
// Pop N words from the stack
assert(index_size == sizeof(u2), "Can only be u2");
__ load_method_entry(Rcache, Rindex);
- __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedIndyEntry::num_parameters_offset())));
- __ check_stack_top();
- __ add(Rstack_top, Rstack_top, AsmOperand(Rcache, lsl, Interpreter::logStackElementSize));
+ __ ldrh(Rcache, Address(Rcache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
}
+ __ check_stack_top();
+ __ add(Rstack_top, Rstack_top, AsmOperand(Rcache, lsl, Interpreter::logStackElementSize));
+
__ convert_retval_to_tos(state);
__ check_and_handle_popframe();
diff --git a/src/hotspot/cpu/arm/templateTable_arm.cpp b/src/hotspot/cpu/arm/templateTable_arm.cpp
index e478f08c977..55a0120e7c7 100644
--- a/src/hotspot/cpu/arm/templateTable_arm.cpp
+++ b/src/hotspot/cpu/arm/templateTable_arm.cpp
@@ -3666,15 +3666,15 @@ void TemplateTable::prepare_invoke(Register Rcache, Register recv) {
// load receiver if needed (after extra argument is pushed so parameter size is correct)
if (load_receiver) {
__ ldrh(recv, Address(Rcache, in_bytes(ResolvedMethodEntry::num_parameters_offset())));
- Address recv_addr = __ receiver_argument_address(Rstack_top, Rtemp, recv);
- __ ldr(recv, recv_addr);
+ __ add(recv, Rstack_top, AsmOperand(recv, lsl, Interpreter::logStackElementSize));
+ __ ldr(recv, Address(recv, -Interpreter::stackElementSize));
__ verify_oop(recv);
}
// load return address
{ const address table = (address) Interpreter::invoke_return_entry_table_for(code);
- __ mov_slow(Rtemp, table);
- __ ldr(LR, Address::indexed_ptr(Rtemp, ret_type));
+ __ mov_slow(LR, table);
+ __ ldr(LR, Address::indexed_ptr(LR, ret_type));
}
}
@@ -3744,10 +3744,13 @@ void TemplateTable::invokevirtual(int byte_no) {
void TemplateTable::invokespecial(int byte_no) {
transition(vtos, vtos);
assert(byte_no == f1_byte, "use this argument");
+
const Register Rrecv = R2_tmp;
- load_resolved_method_entry_special_or_static(R2_tmp, // ResolvedMethodEntry*
+ const Register Rflags = R3_tmp;
+
+ load_resolved_method_entry_special_or_static(Rrecv, // ResolvedMethodEntry*
Rmethod, // Method*
- R3_tmp); // Flags
+ Rflags); // Flags
prepare_invoke(Rrecv, Rrecv);
__ verify_oop(Rrecv);
__ null_check(Rrecv, Rtemp);
@@ -3760,12 +3763,16 @@ void TemplateTable::invokespecial(int byte_no) {
void TemplateTable::invokestatic(int byte_no) {
transition(vtos, vtos);
assert(byte_no == f1_byte, "use this argument");
- load_resolved_method_entry_special_or_static(R2_tmp, // ResolvedMethodEntry*
+
+ const Register Rrecv = R2_tmp;
+ const Register Rflags = R3_tmp;
+
+ load_resolved_method_entry_special_or_static(Rrecv, // ResolvedMethodEntry*
Rmethod, // Method*
- R3_tmp); // Flags
- prepare_invoke(R2_tmp, R2_tmp);
+ Rflags); // Flags
+ prepare_invoke(Rrecv, Rrecv);
// do the call
- __ profile_call(R2_tmp);
+ __ profile_call(Rrecv);
__ jump_from_interpreted(Rmethod);
}
@@ -3788,10 +3795,10 @@ void TemplateTable::invokeinterface(int byte_no) {
const Register Rflags = R3_tmp;
const Register Rklass = R2_tmp; // Note! Same register with Rrecv
- load_resolved_method_entry_interface(R2_tmp, // ResolvedMethodEntry*
- R1_tmp, // Klass*
+ load_resolved_method_entry_interface(Rrecv, // ResolvedMethodEntry*
+ Rinterf, // Klass*
Rmethod, // Method* or itable/vtable index
- R3_tmp); // Flags
+ Rflags); // Flags
prepare_invoke(Rrecv, Rrecv);
// First check for Object case, then private interface method,
diff --git a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
index 2e5548afcb1..54f9cfa9367 100644
--- a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
+++ b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
@@ -167,9 +167,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/ppc/frame_ppc.hpp b/src/hotspot/cpu/ppc/frame_ppc.hpp
index e2e2b9d015d..d2c9927f119 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.hpp
@@ -400,8 +400,6 @@
public:
- const ImmutableOopMap* get_oop_map() const;
-
// Constructors
inline frame(intptr_t* sp, intptr_t* fp, address pc);
inline frame(intptr_t* sp, address pc, intptr_t* unextended_sp = nullptr, intptr_t* fp = nullptr, CodeBlob* cb = nullptr);
diff --git a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
index e7dca45b3c3..220b9c3241e 100644
--- a/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/frame_ppc.inline.hpp
@@ -78,8 +78,8 @@ inline void frame::setup() {
// Continuation frames on the java heap are not aligned.
// When thawing interpreted frames the sp can be unaligned (see new_stack_frame()).
assert(_on_heap ||
- (is_aligned(_sp, alignment_in_bytes) || is_interpreted_frame()) &&
- (is_aligned(_fp, alignment_in_bytes) || !is_fully_initialized()),
+ ((is_aligned(_sp, alignment_in_bytes) || is_interpreted_frame()) &&
+ (is_aligned(_fp, alignment_in_bytes) || !is_fully_initialized())),
"invalid alignment sp:" PTR_FORMAT " unextended_sp:" PTR_FORMAT " fp:" PTR_FORMAT, p2i(_sp), p2i(_unextended_sp), p2i(_fp));
}
@@ -361,20 +361,6 @@ inline void frame::set_saved_oop_result(RegisterMap* map, oop obj) {
*result_adr = obj;
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
inline int frame::compiled_frame_stack_argsize() const {
assert(cb()->is_compiled(), "");
return (cb()->as_compiled_method()->method()->num_stack_arg_slots() * VMRegImpl::stack_slot_size) >> LogBytesPerWord;
diff --git a/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp b/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
index bced453d784..30670d053fa 100644
--- a/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globalDefinitions_ppc.hpp
@@ -48,9 +48,12 @@ const bool CCallingConventionRequiresIntsAsLongs = true;
// PPC64 is not specified as multi-copy-atomic
// So we must not #define CPU_MULTI_COPY_ATOMIC
-// The expected size in bytes of a cache line, used to pad data structures.
+// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 128
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
#define SUPPORT_RESERVED_STACK_AREA
// If UseSIGTRAP is active, we only use the poll bit and no polling page.
diff --git a/src/hotspot/cpu/ppc/matcher_ppc.hpp b/src/hotspot/cpu/ppc/matcher_ppc.hpp
index 44d1a3cd305..b195ba4eeb2 100644
--- a/src/hotspot/cpu/ppc/matcher_ppc.hpp
+++ b/src/hotspot/cpu/ppc/matcher_ppc.hpp
@@ -195,4 +195,9 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ return false;
+ }
+
#endif // CPU_PPC_MATCHER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
index f60f3f147ae..06754ccfdd7 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.cpp
@@ -429,10 +429,6 @@ void NativePostCallNop::make_deopt() {
NativeDeoptInstruction::insert(addr_at(0));
}
-void NativePostCallNop::patch(jint diff) {
- // unsupported for now
-}
-
void NativeDeoptInstruction::verify() {
}
diff --git a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
index 378f4ea928a..ec6f5a90a72 100644
--- a/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
+++ b/src/hotspot/cpu/ppc/nativeInst_ppc.hpp
@@ -508,8 +508,8 @@ class NativeMovRegMem: public NativeInstruction {
class NativePostCallNop: public NativeInstruction {
public:
bool check() const { return is_nop(); }
- int displacement() const { return 0; }
- void patch(jint diff);
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset) { return false; }
void make_deopt();
};
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index 824996168a9..3382df355da 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -734,7 +734,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
ShouldNotReachHere();
}
}
- return align_up(stk, 2);
+ return stk;
}
#if defined(COMPILER1) || defined(COMPILER2)
diff --git a/src/hotspot/cpu/ppc/vm_version_ppc.hpp b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
index b1168ded456..a5831ef1590 100644
--- a/src/hotspot/cpu/ppc/vm_version_ppc.hpp
+++ b/src/hotspot/cpu/ppc/vm_version_ppc.hpp
@@ -91,7 +91,7 @@ class VM_Version: public Abstract_VM_Version {
// Override Abstract_VM_Version implementation
static void print_platform_virtualization_info(outputStream*);
- // PPC64 supports fast class initialization checks for static methods.
+ // PPC64 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
diff --git a/src/hotspot/cpu/riscv/assembler_riscv.hpp b/src/hotspot/cpu/riscv/assembler_riscv.hpp
index 922e412356d..f44840d9f8c 100644
--- a/src/hotspot/cpu/riscv/assembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/assembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -506,7 +506,7 @@ class Assembler : public AbstractAssembler {
INSN(sllw, 0b0111011, 0b001, 0b0000000);
INSN(sraw, 0b0111011, 0b101, 0b0100000);
INSN(srlw, 0b0111011, 0b101, 0b0000000);
- INSN(mul, 0b0110011, 0b000, 0b0000001);
+ INSN(_mul, 0b0110011, 0b000, 0b0000001);
INSN(mulh, 0b0110011, 0b001, 0b0000001);
INSN(mulhsu,0b0110011, 0b010, 0b0000001);
INSN(mulhu, 0b0110011, 0b011, 0b0000001);
@@ -537,9 +537,9 @@ class Assembler : public AbstractAssembler {
}
INSN(lb, 0b0000011, 0b000);
- INSN(lbu, 0b0000011, 0b100);
- INSN(lh, 0b0000011, 0b001);
- INSN(lhu, 0b0000011, 0b101);
+ INSN(_lbu, 0b0000011, 0b100);
+ INSN(_lh, 0b0000011, 0b001);
+ INSN(_lhu, 0b0000011, 0b101);
INSN(_lw, 0b0000011, 0b010);
INSN(lwu, 0b0000011, 0b110);
INSN(_ld, 0b0000011, 0b011);
@@ -609,8 +609,8 @@ class Assembler : public AbstractAssembler {
emit(insn); \
} \
- INSN(sb, Register, 0b0100011, 0b000);
- INSN(sh, Register, 0b0100011, 0b001);
+ INSN(_sb, Register, 0b0100011, 0b000);
+ INSN(_sh, Register, 0b0100011, 0b001);
INSN(_sw, Register, 0b0100011, 0b010);
INSN(_sd, Register, 0b0100011, 0b011);
INSN(fsw, FloatRegister, 0b0100111, 0b010);
@@ -758,6 +758,8 @@ enum Aqrl {relaxed = 0b00, rl = 0b01, aq = 0b10, aqrl = 0b11};
INSN(amomax_d , 0b0101111, 0b011, 0b10100);
INSN(amominu_d, 0b0101111, 0b011, 0b11000);
INSN(amomaxu_d, 0b0101111, 0b011, 0b11100);
+ INSN(amocas_w, 0b0101111, 0b010, 0b00101);
+ INSN(amocas_d, 0b0101111, 0b011, 0b00101);
#undef INSN
enum operand_size { int8, int16, int32, uint32, int64 };
@@ -1154,10 +1156,8 @@ static Assembler::SEW elemtype_to_sew(BasicType etype) {
}
#define patch_vtype(hsb, lsb, vlmul, vsew, vta, vma, vill) \
- if (vill == 1) { \
- guarantee((vlmul | vsew | vta | vma == 0), \
- "the other bits in vtype shall be zero"); \
- } \
+ /* If vill then other bits of vtype must be zero. */ \
+ guarantee(!vill, "vill not supported"); \
patch((address)&insn, lsb + 2, lsb, vlmul); \
patch((address)&insn, lsb + 5, lsb + 3, vsew); \
patch((address)&insn, lsb + 6, vta); \
@@ -1332,6 +1332,7 @@ enum VectorMask {
INSN(vsll_vi, 0b1010111, 0b011, 0b100101);
// Vector Slide Instructions
+ INSN(vslideup_vi, 0b1010111, 0b011, 0b001110);
INSN(vslidedown_vi, 0b1010111, 0b011, 0b001111);
#undef INSN
@@ -1687,7 +1688,6 @@ enum VectorMask {
INSN(vmv_v_x, 0b1010111, 0b100, v0, 0b1, 0b010111);
#undef INSN
-#undef patch_VArith
#define INSN(NAME, op, funct13, funct6) \
void NAME(VectorRegister Vd, VectorMask vm = unmasked) { \
@@ -1729,14 +1729,29 @@ enum Nf {
patch_reg((address)&insn, 15, Rs1); \
emit(insn)
-#define INSN(NAME, op, lumop, vm, mop, nf) \
- void NAME(VectorRegister Vd, Register Rs1, uint32_t width = 0, bool mew = false) { \
+#define INSN(NAME, op, width, lumop, vm, mop, mew, nf) \
+ void NAME(VectorRegister Vd, Register Rs1) { \
guarantee(is_uimm3(width), "width is invalid"); \
patch_VLdSt(op, Vd, width, Rs1, lumop, vm, mop, mew, nf); \
}
// Vector Load/Store Instructions
- INSN(vl1re8_v, 0b0000111, 0b01000, 0b1, 0b00, g1);
+ INSN(vl1re8_v, 0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1);
+ INSN(vl1re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g1);
+ INSN(vl1re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g1);
+ INSN(vl1re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g1);
+ INSN(vl2re8_v, 0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g2);
+ INSN(vl2re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g2);
+ INSN(vl2re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g2);
+ INSN(vl2re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g2);
+ INSN(vl4re8_v, 0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g4);
+ INSN(vl4re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g4);
+ INSN(vl4re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g4);
+ INSN(vl4re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g4);
+ INSN(vl8re8_v, 0b0000111, 0b000, 0b01000, 0b1, 0b00, 0b0, g8);
+ INSN(vl8re16_v, 0b0000111, 0b101, 0b01000, 0b1, 0b00, 0b0, g8);
+ INSN(vl8re32_v, 0b0000111, 0b110, 0b01000, 0b1, 0b00, 0b0, g8);
+ INSN(vl8re64_v, 0b0000111, 0b111, 0b01000, 0b1, 0b00, 0b0, g8);
#undef INSN
@@ -1747,6 +1762,9 @@ enum Nf {
// Vector Load/Store Instructions
INSN(vs1r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g1);
+ INSN(vs2r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g2);
+ INSN(vs4r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g4);
+ INSN(vs8r_v, 0b0100111, 0b000, 0b01000, 0b1, 0b00, 0b0, g8);
#undef INSN
@@ -1792,9 +1810,11 @@ enum Nf {
}
// Vector unordered indexed load instructions
+ INSN( vluxei8_v, 0b0000111, 0b000, 0b01, 0b0);
INSN(vluxei32_v, 0b0000111, 0b110, 0b01, 0b0);
// Vector unordered indexed store instructions
+ INSN( vsuxei8_v, 0b0100111, 0b000, 0b01, 0b0);
INSN(vsuxei32_v, 0b0100111, 0b110, 0b01, 0b0);
#undef INSN
@@ -1818,6 +1838,55 @@ enum Nf {
#undef INSN
#undef patch_VLdSt
+// ====================================
+// RISC-V Vector Crypto Extension
+// ====================================
+
+#define INSN(NAME, op, funct3, funct6) \
+ void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1, VectorMask vm = unmasked) { \
+ patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6); \
+ }
+
+ // Vector Bit-manipulation used in Cryptography (Zvkb) Extension
+ INSN(vandn_vv, 0b1010111, 0b000, 0b000001);
+ INSN(vandn_vx, 0b1010111, 0b100, 0b000001);
+ INSN(vandn_vi, 0b1010111, 0b011, 0b000001);
+ INSN(vclmul_vv, 0b1010111, 0b010, 0b001100);
+ INSN(vclmul_vx, 0b1010111, 0b110, 0b001100);
+ INSN(vclmulh_vv, 0b1010111, 0b010, 0b001101);
+ INSN(vclmulh_vx, 0b1010111, 0b110, 0b001101);
+ INSN(vror_vv, 0b1010111, 0b000, 0b010100);
+ INSN(vror_vx, 0b1010111, 0b100, 0b010100);
+ INSN(vrol_vv, 0b1010111, 0b000, 0b010101);
+ INSN(vrol_vx, 0b1010111, 0b100, 0b010101);
+
+#undef INSN
+
+#define INSN(NAME, op, funct3, Vs1, funct6) \
+ void NAME(VectorRegister Vd, VectorRegister Vs2, VectorMask vm = unmasked) { \
+ patch_VArith(op, Vd, funct3, Vs1, Vs2, vm, funct6); \
+ }
+
+ // Vector Bit-manipulation used in Cryptography (Zvkb) Extension
+ INSN(vbrev8_v, 0b1010111, 0b010, 0b01000, 0b010010);
+ INSN(vrev8_v, 0b1010111, 0b010, 0b01001, 0b010010);
+
+#undef INSN
+
+#define INSN(NAME, op, funct3, vm, funct6) \
+ void NAME(VectorRegister Vd, VectorRegister Vs2, VectorRegister Vs1) { \
+ patch_VArith(op, Vd, funct3, Vs1->raw_encoding(), Vs2, vm, funct6); \
+ }
+
+ // Vector SHA-2 Secure Hash (Zvknh[ab]) Extension
+ INSN(vsha2ms_vv, 0b1110111, 0b010, 0b1, 0b101101);
+ INSN(vsha2ch_vv, 0b1110111, 0b010, 0b1, 0b101110);
+ INSN(vsha2cl_vv, 0b1110111, 0b010, 0b1, 0b101111);
+
+#undef INSN
+
+#undef patch_VArith
+
// ====================================
// RISC-V Bit-Manipulation Extension
// Currently only support Zba, Zbb and Zbs bitmanip extensions.
@@ -1867,9 +1936,9 @@ enum Nf {
}
INSN(rev8, 0b0010011, 0b101, 0b011010111000);
- INSN(sext_b, 0b0010011, 0b001, 0b011000000100);
- INSN(sext_h, 0b0010011, 0b001, 0b011000000101);
- INSN(zext_h, 0b0111011, 0b100, 0b000010000000);
+ INSN(_sext_b, 0b0010011, 0b001, 0b011000000100);
+ INSN(_sext_h, 0b0010011, 0b001, 0b011000000101);
+ INSN(_zext_h, 0b0111011, 0b100, 0b000010000000);
INSN(clz, 0b0010011, 0b001, 0b011000000000);
INSN(clzw, 0b0011011, 0b001, 0b011000000000);
INSN(ctz, 0b0010011, 0b001, 0b011000000001);
@@ -2581,6 +2650,15 @@ enum Nf {
return UseRVC && in_compressible_region();
}
+ bool do_compress_zcb(Register reg1 = noreg, Register reg2 = noreg) const {
+ return do_compress() && VM_Version::ext_Zcb.enabled() &&
+ (reg1 == noreg || reg1->is_compressed_valid()) && (reg2 == noreg || reg2->is_compressed_valid());
+ }
+
+ bool do_compress_zcb_zbb(Register reg1 = noreg, Register reg2 = noreg) const {
+ return do_compress_zcb(reg1, reg2) && UseZbb;
+ }
+
// --------------------------
// Load/store register
// --------------------------
@@ -2915,6 +2993,238 @@ enum Nf {
#undef INSN
+// -------------- ZCB Instruction Definitions --------------
+// Zcb additional C instructions
+ private:
+ // Format CLH, c.lh/c.lhu
+ template
+ void c_lh_if(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
+ assert_cond(uimm == 0 || uimm == 2);
+ assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b00);
+ c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
+ c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
+ c_patch((address)&insn, 6, 6, Unsigned ? 0 : 1);
+ c_patch_compressed_reg((address)&insn, 7, Rs1);
+ c_patch((address)&insn, 12, 10, 0b001);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ template
+ void lh_c_mux(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
+ if (do_compress_zcb(Rd_Rs2, Rs1) &&
+ (uimm == 0 || uimm == 2)) {
+ c_lh_if(Rd_Rs2, Rs1, uimm);
+ } else {
+ if (Unsigned) {
+ _lhu(Rd_Rs2, Rs1, uimm);
+ } else {
+ _lh(Rd_Rs2, Rs1, uimm);
+ }
+ }
+ }
+
+ // Format CU, c.[sz]ext.*, c.not
+ template
+ void c_u_if(Register Rs1) {
+ assert_cond(do_compress_zcb(Rs1));
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b01);
+ c_patch((address)&insn, 4, 2, InstructionType);
+ c_patch((address)&insn, 6, 5, 0b11);
+ c_patch_compressed_reg((address)&insn, 7, Rs1);
+ c_patch((address)&insn, 12, 10, 0b111);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ public:
+
+ // Prerequisites: Zcb
+ void c_lh(Register Rd_Rs2, Register Rs1, const int32_t uimm) { c_lh_if(Rd_Rs2, Rs1, uimm); }
+ void lh(Register Rd_Rs2, Register Rs1, const int32_t uimm) { lh_c_mux(Rd_Rs2, Rs1, uimm); }
+
+ // Prerequisites: Zcb
+ void c_lhu(Register Rd_Rs2, Register Rs1, const int32_t uimm) { c_lh_if(Rd_Rs2, Rs1, uimm); }
+ void lhu(Register Rd_Rs2, Register Rs1, const int32_t uimm) { lh_c_mux(Rd_Rs2, Rs1, uimm); }
+
+ // Prerequisites: Zcb
+ // Format CLB, single instruction
+ void c_lbu(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
+ assert_cond(uimm <= 3);
+ assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b00);
+ c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
+ c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
+ c_patch((address)&insn, 6, 6, (uimm & nth_bit(0)) >> 0);
+ c_patch_compressed_reg((address)&insn, 7, Rs1);
+ c_patch((address)&insn, 12, 10, 0b000);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ void lbu(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
+ if (do_compress_zcb(Rd_Rs2, Rs1) &&
+ uimm >= 0 && uimm <= 3) {
+ c_lbu(Rd_Rs2, Rs1, uimm);
+ } else {
+ _lbu(Rd_Rs2, Rs1, uimm);
+ }
+ }
+
+ // Prerequisites: Zcb
+ // Format CSB, single instruction
+ void c_sb(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
+ assert_cond(uimm <= 3);
+ assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b00);
+ c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
+ c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
+ c_patch((address)&insn, 6, 6, (uimm & nth_bit(0)) >> 0);
+ c_patch_compressed_reg((address)&insn, 7, Rs1);
+ c_patch((address)&insn, 12, 10, 0b010);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ void sb(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
+ if (do_compress_zcb(Rd_Rs2, Rs1) &&
+ uimm >= 0 && uimm <= 3) {
+ c_sb(Rd_Rs2, Rs1, uimm);
+ } else {
+ _sb(Rd_Rs2, Rs1, uimm);
+ }
+ }
+
+ // Prerequisites: Zcb
+ // Format CSH, single instruction
+ void c_sh(Register Rd_Rs2, Register Rs1, uint32_t uimm) {
+ assert_cond(uimm == 0 || uimm == 2);
+ assert_cond(do_compress_zcb(Rd_Rs2, Rs1));
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b00);
+ c_patch_compressed_reg((address)&insn, 2, Rd_Rs2);
+ c_patch((address)&insn, 5, 5, (uimm & nth_bit(1)) >> 1);
+ c_patch((address)&insn, 6, 6, 0);
+ c_patch_compressed_reg((address)&insn, 7, Rs1);
+ c_patch((address)&insn, 12, 10, 0b011);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ void sh(Register Rd_Rs2, Register Rs1, const int32_t uimm) {
+ if (do_compress_zcb(Rd_Rs2, Rs1) &&
+ (uimm == 0 || uimm == 2)) {
+ c_sh(Rd_Rs2, Rs1, uimm);
+ } else {
+ _sh(Rd_Rs2, Rs1, uimm);
+ }
+ }
+
+ // Prerequisites: Zcb
+ // Format CS
+ void c_zext_b(Register Rs1) {
+ assert_cond(do_compress_zcb(Rs1));
+ c_u_if<0b000>(Rs1);
+ }
+
+ // Prerequisites: Zbb
+ void sext_b(Register Rd_Rs2, Register Rs1) {
+ assert_cond(UseZbb);
+ if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
+ c_sext_b(Rd_Rs2);
+ } else {
+ _sext_b(Rd_Rs2, Rs1);
+ }
+ }
+
+ // Prerequisites: Zcb, Zbb
+ // Format CS
+ void c_sext_b(Register Rs1) {
+ c_u_if<0b001>(Rs1);
+ }
+
+ // Prerequisites: Zbb
+ void zext_h(Register Rd_Rs2, Register Rs1) {
+ assert_cond(UseZbb);
+ if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
+ c_zext_h(Rd_Rs2);
+ } else {
+ _zext_h(Rd_Rs2, Rs1);
+ }
+ }
+
+ // Prerequisites: Zcb, Zbb
+ // Format CS
+ void c_zext_h(Register Rs1) {
+ c_u_if<0b010>(Rs1);
+ }
+
+ // Prerequisites: Zbb
+ void sext_h(Register Rd_Rs2, Register Rs1) {
+ assert_cond(UseZbb);
+ if (do_compress_zcb_zbb(Rd_Rs2, Rs1) && (Rd_Rs2 == Rs1)) {
+ c_sext_h(Rd_Rs2);
+ } else {
+ _sext_h(Rd_Rs2, Rs1);
+ }
+ }
+
+ // Prerequisites: Zcb, Zbb
+ // Format CS
+ void c_sext_h(Register Rs1) {
+ c_u_if<0b011>(Rs1);
+ }
+
+ // Prerequisites: Zcb, Zba
+ // Format CS
+ void c_zext_w(Register Rs1) {
+ c_u_if<0b100>(Rs1);
+ }
+
+ // Prerequisites: Zcb
+ // Format CS
+ void c_not(Register Rs1) {
+ c_u_if<0b101>(Rs1);
+ }
+
+ // Prerequisites: Zcb (M or Zmmul)
+ // Format CA, c.mul
+ void c_mul(Register Rd_Rs1, Register Rs2) {
+ uint16_t insn = 0;
+ c_patch((address)&insn, 1, 0, 0b01);
+ c_patch_compressed_reg((address)&insn, 2, Rs2);
+ c_patch((address)&insn, 6, 5, 0b10);
+ c_patch_compressed_reg((address)&insn, 7, Rd_Rs1);
+ c_patch((address)&insn, 12, 10, 0b111);
+ c_patch((address)&insn, 15, 13, 0b100);
+ emit_int16(insn);
+ }
+
+ void mul(Register Rd, Register Rs1, Register Rs2) {
+ if (Rd != Rs1 && Rd != Rs2) {
+ // Three registers needed without a mv, emit uncompressed
+ _mul(Rd, Rs1, Rs2);
+ return;
+ }
+
+ // Rd is either Rs1 or Rs2
+ if (!do_compress_zcb(Rs2, Rs1)) {
+ _mul(Rd, Rs1, Rs2);
+ } else {
+ if (Rd == Rs2) {
+ Rs2 = Rs1;
+ } else {
+ assert(Rd == Rs1, "must be");
+ }
+ c_mul(Rd, Rs2);
+ }
+ }
+
// Stack overflow checking
virtual void bang_stack_with_offset(int offset) { Unimplemented(); }
@@ -2940,6 +3250,17 @@ enum Nf {
return uabs(target - branch) < branch_range;
}
+ // Decode the given instruction, checking if it's a 16-bit compressed
+ // instruction and return the address of the next instruction.
+ static address locate_next_instruction(address inst) {
+ // Instruction wider than 16 bits has the two least-significant bits set.
+ if ((0x3 & *inst) == 0x3) {
+ return inst + instruction_size;
+ } else {
+ return inst + compressed_instruction_size;
+ }
+ }
+
Assembler(CodeBuffer* code) : AbstractAssembler(code), _in_compressible_region(true) {}
};
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
index ab84ef7f8a1..711eb210091 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.cpp
@@ -1459,6 +1459,112 @@ void C2_MacroAssembler::string_equals(Register a1, Register a2,
BLOCK_COMMENT("} string_equals");
}
+// jdk.internal.util.ArraysSupport.vectorizedHashCode
+void C2_MacroAssembler::arrays_hashcode(Register ary, Register cnt, Register result,
+ Register tmp1, Register tmp2, Register tmp3,
+ Register tmp4, Register tmp5, Register tmp6,
+ BasicType eltype)
+{
+ assert_different_registers(ary, cnt, result, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, t0, t1);
+
+ const int elsize = arrays_hashcode_elsize(eltype);
+ const int chunks_end_shift = exact_log2(elsize);
+
+ switch (eltype) {
+ case T_BOOLEAN: BLOCK_COMMENT("arrays_hashcode(unsigned byte) {"); break;
+ case T_CHAR: BLOCK_COMMENT("arrays_hashcode(char) {"); break;
+ case T_BYTE: BLOCK_COMMENT("arrays_hashcode(byte) {"); break;
+ case T_SHORT: BLOCK_COMMENT("arrays_hashcode(short) {"); break;
+ case T_INT: BLOCK_COMMENT("arrays_hashcode(int) {"); break;
+ default:
+ ShouldNotReachHere();
+ }
+
+ const int stride = 4;
+ const Register pow31_4 = tmp1;
+ const Register pow31_3 = tmp2;
+ const Register pow31_2 = tmp3;
+ const Register chunks = tmp4;
+ const Register chunks_end = chunks;
+
+ Label DONE, TAIL, TAIL_LOOP, WIDE_LOOP;
+
+ // result has a value initially
+
+ beqz(cnt, DONE);
+
+ andi(chunks, cnt, ~(stride-1));
+ beqz(chunks, TAIL);
+
+ mv(pow31_4, 923521); // [31^^4]
+ mv(pow31_3, 29791); // [31^^3]
+ mv(pow31_2, 961); // [31^^2]
+
+ slli(chunks_end, chunks, chunks_end_shift);
+ add(chunks_end, ary, chunks_end);
+ andi(cnt, cnt, stride-1); // don't forget about tail!
+
+ bind(WIDE_LOOP);
+ mulw(result, result, pow31_4); // 31^^4 * h
+ arrays_hashcode_elload(t0, Address(ary, 0 * elsize), eltype);
+ arrays_hashcode_elload(t1, Address(ary, 1 * elsize), eltype);
+ arrays_hashcode_elload(tmp5, Address(ary, 2 * elsize), eltype);
+ arrays_hashcode_elload(tmp6, Address(ary, 3 * elsize), eltype);
+ mulw(t0, t0, pow31_3); // 31^^3 * ary[i+0]
+ addw(result, result, t0);
+ mulw(t1, t1, pow31_2); // 31^^2 * ary[i+1]
+ addw(result, result, t1);
+ slli(t0, tmp5, 5); // optimize 31^^1 * ary[i+2]
+ subw(tmp5, t0, tmp5); // with ary[i+2]<<5 - ary[i+2]
+ addw(result, result, tmp5);
+ addw(result, result, tmp6); // 31^^4 * h + 31^^3 * ary[i+0] + 31^^2 * ary[i+1]
+ // + 31^^1 * ary[i+2] + 31^^0 * ary[i+3]
+ addi(ary, ary, elsize * stride);
+ bne(ary, chunks_end, WIDE_LOOP);
+ beqz(cnt, DONE);
+
+ bind(TAIL);
+ slli(chunks_end, cnt, chunks_end_shift);
+ add(chunks_end, ary, chunks_end);
+
+ bind(TAIL_LOOP);
+ arrays_hashcode_elload(t0, Address(ary), eltype);
+ slli(t1, result, 5); // optimize 31 * result
+ subw(result, t1, result); // with result<<5 - result
+ addw(result, result, t0);
+ addi(ary, ary, elsize);
+ bne(ary, chunks_end, TAIL_LOOP);
+
+ bind(DONE);
+ BLOCK_COMMENT("} // arrays_hashcode");
+}
+
+int C2_MacroAssembler::arrays_hashcode_elsize(BasicType eltype) {
+ switch (eltype) {
+ case T_BOOLEAN: return sizeof(jboolean);
+ case T_BYTE: return sizeof(jbyte);
+ case T_SHORT: return sizeof(jshort);
+ case T_CHAR: return sizeof(jchar);
+ case T_INT: return sizeof(jint);
+ default:
+ ShouldNotReachHere();
+ return -1;
+ }
+}
+
+void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicType eltype) {
+ switch (eltype) {
+ // T_BOOLEAN used as surrogate for unsigned byte
+ case T_BOOLEAN: lbu(dst, src); break;
+ case T_BYTE: lb(dst, src); break;
+ case T_SHORT: lh(dst, src); break;
+ case T_CHAR: lhu(dst, src); break;
+ case T_INT: lw(dst, src); break;
+ default:
+ ShouldNotReachHere();
+ }
+}
+
typedef void (Assembler::*conditional_branch_insn)(Register op1, Register op2, Label& label, bool is_far);
typedef void (MacroAssembler::*float_conditional_branch_insn)(FloatRegister op1, FloatRegister op2, Label& label,
bool is_far, bool is_unordered);
diff --git a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
index b9a7749631a..4940ce5fe9e 100644
--- a/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c2_MacroAssembler_riscv.hpp
@@ -82,6 +82,15 @@
Register result, Register cnt1,
int elem_size);
+ void arrays_hashcode(Register ary, Register cnt, Register result,
+ Register tmp1, Register tmp2,
+ Register tmp3, Register tmp4,
+ Register tmp5, Register tmp6,
+ BasicType eltype);
+ // helper function for arrays_hashcode
+ int arrays_hashcode_elsize(BasicType eltype);
+ void arrays_hashcode_elload(Register dst, Address src, BasicType eltype);
+
void string_equals(Register r1, Register r2,
Register result, Register cnt1,
int elem_size);
diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
index 4239ca6e383..e29dee56de8 100644
--- a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
+++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
@@ -29,6 +29,7 @@
#include "code/compiledIC.hpp"
#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
@@ -88,9 +89,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/riscv/frame_riscv.hpp b/src/hotspot/cpu/riscv/frame_riscv.hpp
index ae6242a75bd..0c0659d1eee 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.hpp
@@ -189,8 +189,6 @@
static void verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp);
#endif
- const ImmutableOopMap* get_oop_map() const;
-
public:
// Constructors
diff --git a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
index dfec3880192..33727a8c6d0 100644
--- a/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
+++ b/src/hotspot/cpu/riscv/frame_riscv.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -186,7 +186,7 @@ inline bool frame::equal(frame other) const {
unextended_sp() == other.unextended_sp() &&
fp() == other.fp() &&
pc() == other.pc();
- assert(!ret || ret && cb() == other.cb() && _deopt_state == other._deopt_state, "inconsistent construction");
+ assert(!ret || (cb() == other.cb() && _deopt_state == other._deopt_state), "inconsistent construction");
return ret;
}
@@ -345,20 +345,6 @@ inline int frame::sender_sp_ret_address_offset() {
return frame::sender_sp_offset - frame::return_addr_offset;
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
//------------------------------------------------------------------------------
// frame::sender
frame frame::sender(RegisterMap* map) const {
diff --git a/src/hotspot/cpu/riscv/gc/x/x_riscv.ad b/src/hotspot/cpu/riscv/gc/x/x_riscv.ad
index 73d337bc484..3d0273109ac 100644
--- a/src/hotspot/cpu/riscv/gc/x/x_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/x/x_riscv.ad
@@ -52,11 +52,11 @@ static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node
%}
// Load Pointer
-instruct xLoadP(iRegPNoSp dst, memory mem)
+instruct xLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
%{
match(Set dst (LoadP mem));
predicate(UseZGC && !ZGenerational && (n->as_Load()->barrier_data() != 0));
- effect(TEMP dst);
+ effect(TEMP dst, TEMP tmp);
ins_cost(4 * DEFAULT_COST);
@@ -65,17 +65,17 @@ instruct xLoadP(iRegPNoSp dst, memory mem)
ins_encode %{
const Address ref_addr (as_Register($mem$$base), $mem$$disp);
__ ld($dst$$Register, ref_addr);
- x_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data());
+ x_load_barrier(_masm, this, ref_addr, $dst$$Register, $tmp$$Register /* tmp */, barrier_data());
%}
ins_pipe(iload_reg_mem);
%}
-instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(KILL cr, TEMP_DEF res);
+ effect(TEMP_DEF res, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
@@ -86,17 +86,15 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
Label failed;
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
- true /* result_as_bool */);
- __ beqz($res$$Register, failed);
- __ mv(t0, $oldval$$Register);
- __ bind(failed);
+ Assembler::relaxed /* acquire */, Assembler::rl /* release */, $tmp$$Register);
+ __ sub(t0, $tmp$$Register, $oldval$$Register);
+ __ seqz($res$$Register, t0);
if (barrier_data() != XLoadBarrierElided) {
Label good;
- __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
- __ andr(t1, t1, t0);
- __ beqz(t1, good);
- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+ __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+ __ andr(t0, t0, $tmp$$Register);
+ __ beqz(t0, good);
+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
true /* result_as_bool */);
@@ -107,11 +105,11 @@ instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
ins_pipe(pipe_slow);
%}
-instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
- effect(KILL cr, TEMP_DEF res);
+ effect(TEMP_DEF res, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
@@ -122,17 +120,15 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
Label failed;
guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
- Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
- true /* result_as_bool */);
- __ beqz($res$$Register, failed);
- __ mv(t0, $oldval$$Register);
- __ bind(failed);
+ Assembler::aq /* acquire */, Assembler::rl /* release */, $tmp$$Register);
+ __ sub(t0, $tmp$$Register, $oldval$$Register);
+ __ seqz($res$$Register, t0);
if (barrier_data() != XLoadBarrierElided) {
Label good;
- __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
- __ andr(t1, t1, t0);
- __ beqz(t1, good);
- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+ __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+ __ andr(t0, t0, $tmp$$Register);
+ __ beqz(t0, good);
+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $tmp$$Register /* ref */, $res$$Register /* tmp */);
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
true /* result_as_bool */);
@@ -143,10 +139,10 @@ instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_pipe(pipe_slow);
%}
-instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res);
+ effect(TEMP_DEF res, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
@@ -161,7 +157,7 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
__ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
__ andr(t0, t0, $res$$Register);
__ beqz(t0, good);
- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
__ bind(good);
@@ -171,10 +167,10 @@ instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
ins_pipe(pipe_slow);
%}
-instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
- effect(TEMP_DEF res);
+ effect(TEMP_DEF res, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
@@ -189,7 +185,7 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
__ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
__ andr(t0, t0, $res$$Register);
__ beqz(t0, good);
- x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+ x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, $tmp$$Register /* tmp */);
__ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
__ bind(good);
@@ -199,10 +195,10 @@ instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
match(Set prev (GetAndSetP mem newv));
predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, KILL cr);
+ effect(TEMP_DEF prev, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
@@ -210,16 +206,16 @@ instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
ins_encode %{
__ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
- x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+ x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
%}
ins_pipe(pipe_serial);
%}
-instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
match(Set prev (GetAndSetP mem newv));
predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
- effect(TEMP_DEF prev, KILL cr);
+ effect(TEMP_DEF prev, TEMP tmp);
ins_cost(VOLATILE_REF_COST);
@@ -227,7 +223,7 @@ instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr)
ins_encode %{
__ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
- x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+ x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, $tmp$$Register /* tmp */, barrier_data());
%}
ins_pipe(pipe_serial);
%}
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
index 1429f4ad520..29de8234029 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv.ad
@@ -79,7 +79,7 @@ static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address
static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
if (node->barrier_data() == ZBarrierElided) {
- z_color(_masm, node, rnew_zpointer, rnew_zaddress, t0);
+ z_color(_masm, node, rnew_zpointer, rnew_zaddress, tmp);
} else {
bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
@@ -90,11 +90,11 @@ static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address
%}
// Load Pointer
-instruct zLoadP(iRegPNoSp dst, memory mem)
+instruct zLoadP(iRegPNoSp dst, memory mem, iRegPNoSp tmp)
%{
match(Set dst (LoadP mem));
predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
- effect(TEMP dst);
+ effect(TEMP dst, TEMP tmp);
ins_cost(4 * DEFAULT_COST);
@@ -103,34 +103,35 @@ instruct zLoadP(iRegPNoSp dst, memory mem)
ins_encode %{
const Address ref_addr(as_Register($mem$$base), $mem$$disp);
__ ld($dst$$Register, ref_addr);
- z_load_barrier(_masm, this, ref_addr, $dst$$Register, t0);
+ z_load_barrier(_masm, this, ref_addr, $dst$$Register, $tmp$$Register);
%}
ins_pipe(iload_reg_mem);
%}
// Store Pointer
-instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2)
%{
predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
match(Set mem (StoreP mem src));
- effect(TEMP tmp, KILL cr);
+ effect(TEMP tmp1, TEMP tmp2);
ins_cost(125); // XXX
format %{ "sd $mem, $src\t# ptr" %}
ins_encode %{
const Address ref_addr(as_Register($mem$$base), $mem$$disp);
- z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, t1, false /* is_atomic */);
- __ sd($tmp$$Register, ref_addr);
+ z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp1$$Register, $tmp2$$Register, false /* is_atomic */);
+ __ sd($tmp1$$Register, ref_addr);
%}
ins_pipe(pipe_serial);
%}
-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
ins_cost(2 * VOLATILE_REF_COST);
@@ -140,19 +141,20 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
- z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+ z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
%}
ins_pipe(pipe_slow);
%}
-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval,
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
match(Set res (CompareAndSwapP mem (Binary oldval newval)));
match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
ins_cost(2 * VOLATILE_REF_COST);
@@ -162,18 +164,19 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
- z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+ z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
%}
ins_pipe(pipe_slow);
%}
-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
ins_cost(2 * VOLATILE_REF_COST);
@@ -182,8 +185,8 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
- z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+ z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
z_uncolor(_masm, this, $res$$Register);
%}
@@ -191,10 +194,11 @@ instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP n
ins_pipe(pipe_slow);
%}
-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval,
+ iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, iRegPNoSp tmp1) %{
match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
+ effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP tmp1, TEMP_DEF res);
ins_cost(2 * VOLATILE_REF_COST);
@@ -203,8 +207,8 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
ins_encode %{
guarantee($mem$$disp == 0, "impossible encoding");
Address ref_addr($mem$$Register);
- z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
- z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+ z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, $tmp1$$Register);
+ z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, $tmp1$$Register, true /* is_atomic */);
__ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
z_uncolor(_masm, this, $res$$Register);
%}
@@ -212,17 +216,17 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
ins_pipe(pipe_slow);
%}
-instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
match(Set prev (GetAndSetP mem newv));
predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, KILL cr);
+ effect(TEMP_DEF prev, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchg $prev, $newv, [$mem], #@zGetAndSetP" %}
ins_encode %{
- z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+ z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
__ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
z_uncolor(_masm, this, $prev$$Register);
%}
@@ -230,17 +234,17 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
ins_pipe(pipe_serial);
%}
-instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, iRegPNoSp tmp) %{
match(Set prev (GetAndSetP mem newv));
predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
- effect(TEMP_DEF prev, KILL cr);
+ effect(TEMP_DEF prev, TEMP tmp);
ins_cost(2 * VOLATILE_REF_COST);
format %{ "atomic_xchg_acq $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
ins_encode %{
- z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+ z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, $tmp$$Register, true /* is_atomic */);
__ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
z_uncolor(_masm, this, $prev$$Register);
%}
diff --git a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
index f40ffbeefa7..e368bbdc914 100644
--- a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
@@ -50,6 +50,10 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define USE_POINTERS_TO_REGISTER_IMPL_ARRAY
+// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
#endif // CPU_RISCV_GLOBALDEFINITIONS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/globals_riscv.hpp b/src/hotspot/cpu/riscv/globals_riscv.hpp
index 8824b0e1f93..aa95cebec14 100644
--- a/src/hotspot/cpu/riscv/globals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globals_riscv.hpp
@@ -105,6 +105,7 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZba, false, "Use Zba instructions") \
product(bool, UseZbb, false, "Use Zbb instructions") \
product(bool, UseZbs, false, "Use Zbs instructions") \
+ product(bool, UseZacas, false, EXPERIMENTAL, "Use Zacas instructions") \
product(bool, UseZic64b, false, EXPERIMENTAL, "Use Zic64b instructions") \
product(bool, UseZicbom, false, EXPERIMENTAL, "Use Zicbom instructions") \
product(bool, UseZicbop, false, EXPERIMENTAL, "Use Zicbop instructions") \
@@ -112,6 +113,8 @@ define_pd_global(intx, InlineSmallCode, 1000);
product(bool, UseZtso, false, EXPERIMENTAL, "Assume Ztso memory model") \
product(bool, UseZihintpause, false, EXPERIMENTAL, \
"Use Zihintpause instructions") \
+ product(bool, UseZvkn, false, EXPERIMENTAL, \
+ "Use Zvkn group extension, Zvkned, Zvknhb, Zvkb, Zvkt") \
product(bool, UseRVVForBigIntegerShiftIntrinsics, true, \
"Use RVV instructions for left/right shift of BigInteger")
diff --git a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
index 030a6f0fe96..19d665bd421 100644
--- a/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
+++ b/src/hotspot/cpu/riscv/interp_masm_riscv.cpp
@@ -777,7 +777,7 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
assert(lock_offset == 0,
"displached header must be first word in BasicObjectLock");
- cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+ cmpxchg_obj_header(swap_reg, lock_reg, obj_reg, tmp, count, /*fallthrough*/nullptr);
// Test if the oopMark is an obvious stack pointer, i.e.,
// 1) (mark & 7) == 0, and
@@ -891,7 +891,7 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
beqz(header_reg, count);
// Atomic swap back the old header
- cmpxchg_obj_header(swap_reg, header_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+ cmpxchg_obj_header(swap_reg, header_reg, obj_reg, tmp_reg, count, /*fallthrough*/nullptr);
}
// Call the runtime routine for slow case.
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index 3acc010ff85..ce336c16aa7 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -2725,27 +2725,36 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui
void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
Label &succeed, Label *fail) {
- assert_different_registers(addr, tmp);
- assert_different_registers(newv, tmp);
- assert_different_registers(oldv, tmp);
+ assert_different_registers(addr, tmp, t0);
+ assert_different_registers(newv, tmp, t0);
+ assert_different_registers(oldv, tmp, t0);
// oldv holds comparison value
// newv holds value to write in exchange
// addr identifies memory word to compare against/update
- Label retry_load, nope;
- bind(retry_load);
- // Load reserved from the memory location
- load_reserved(tmp, addr, int64, Assembler::aqrl);
- // Fail and exit if it is not what we expect
- bne(tmp, oldv, nope);
- // If the store conditional succeeds, tmp will be zero
- store_conditional(tmp, newv, addr, int64, Assembler::rl);
- beqz(tmp, succeed);
- // Retry only when the store conditional failed
- j(retry_load);
-
- bind(nope);
+ if (UseZacas) {
+ mv(tmp, oldv);
+ atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl);
+ beq(tmp, oldv, succeed);
+ } else {
+ Label retry_load, nope;
+ bind(retry_load);
+ // Load reserved from the memory location
+ load_reserved(tmp, addr, int64, Assembler::aqrl);
+ // Fail and exit if it is not what we expect
+ bne(tmp, oldv, nope);
+ // If the store conditional succeeds, tmp will be zero
+ store_conditional(tmp, newv, addr, int64, Assembler::rl);
+ beqz(tmp, succeed);
+ // Retry only when the store conditional failed
+ j(retry_load);
+
+ bind(nope);
+ }
+
+ // neither amocas nor lr/sc have an implied barrier in the failing case
membar(AnyAny);
+
mv(oldv, tmp);
if (fail != nullptr) {
j(*fail);
@@ -2771,7 +2780,7 @@ void MacroAssembler::load_reserved(Register dst,
break;
case uint32:
lr_w(dst, addr, acquire);
- zero_extend(t0, t0, 32);
+ zero_extend(dst, dst, 32);
break;
default:
ShouldNotReachHere();
@@ -2819,7 +2828,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
}
sll(mask, mask, shift);
- xori(not_mask, mask, -1);
+ notr(not_mask, mask);
sll(expected, expected, shift);
andr(expected, expected, mask);
@@ -2829,7 +2838,7 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
}
// cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps.
-// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w,
+// It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w or amocas.w,
// which are forced to work with 4-byte aligned address.
void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
@@ -2844,14 +2853,29 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
Label retry, fail, done;
bind(retry);
- lr_w(old, aligned_addr, acquire);
- andr(tmp, old, mask);
- bne(tmp, expected, fail);
- andr(tmp, old, not_mask);
- orr(tmp, tmp, new_val);
- sc_w(tmp, tmp, aligned_addr, release);
- bnez(tmp, retry);
+ if (UseZacas) {
+ lw(old, aligned_addr);
+
+ // if old & mask != expected
+ andr(tmp, old, mask);
+ bne(tmp, expected, fail);
+
+ andr(tmp, old, not_mask);
+ orr(tmp, tmp, new_val);
+
+ atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release);
+ bne(tmp, old, retry);
+ } else {
+ lr_w(old, aligned_addr, acquire);
+ andr(tmp, old, mask);
+ bne(tmp, expected, fail);
+
+ andr(tmp, old, not_mask);
+ orr(tmp, tmp, new_val);
+ sc_w(tmp, tmp, aligned_addr, release);
+ bnez(tmp, retry);
+ }
if (result_as_bool) {
mv(result, 1);
@@ -2891,14 +2915,28 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
Label fail, done;
- lr_w(old, aligned_addr, acquire);
- andr(tmp, old, mask);
- bne(tmp, expected, fail);
+ if (UseZacas) {
+ lw(old, aligned_addr);
+
+ // if old & mask != expected
+ andr(tmp, old, mask);
+ bne(tmp, expected, fail);
+
+ andr(tmp, old, not_mask);
+ orr(tmp, tmp, new_val);
+
+ atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release);
+ bne(tmp, old, fail);
+ } else {
+ lr_w(old, aligned_addr, acquire);
+ andr(tmp, old, mask);
+ bne(tmp, expected, fail);
- andr(tmp, old, not_mask);
- orr(tmp, tmp, new_val);
- sc_w(tmp, tmp, aligned_addr, release);
- bnez(tmp, fail);
+ andr(tmp, old, not_mask);
+ orr(tmp, tmp, new_val);
+ sc_w(tmp, tmp, aligned_addr, release);
+ bnez(tmp, fail);
+ }
// Success
mv(result, 1);
@@ -2921,6 +2959,19 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
+ if (UseZacas) {
+ if (result_as_bool) {
+ mv(t0, expected);
+ atomic_cas(t0, new_val, addr, size, acquire, release);
+ xorr(t0, t0, expected);
+ seqz(result, t0);
+ } else {
+ mv(result, expected);
+ atomic_cas(result, new_val, addr, size, acquire, release);
+ }
+ return;
+ }
+
Label retry_load, done, ne_done;
bind(retry_load);
load_reserved(t0, addr, size, acquire);
@@ -2952,6 +3003,11 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result) {
+ if (UseZacas) {
+ cmpxchg(addr, expected, new_val, size, acquire, release, result, true);
+ return;
+ }
+
assert_different_registers(addr, t0);
assert_different_registers(expected, t0);
assert_different_registers(new_val, t0);
@@ -3018,6 +3074,89 @@ ATOMIC_XCHGU(xchgalwu, xchgalw)
#undef ATOMIC_XCHGU
+#define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE) \
+void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \
+ assert(UseZacas, "invariant"); \
+ prev = prev->is_valid() ? prev : zr; \
+ AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \
+ return; \
+}
+
+ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed)
+ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed)
+ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl)
+ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl)
+ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl)
+ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl)
+
+#undef ATOMIC_CAS
+
+#define ATOMIC_CASU(OP1, OP2) \
+void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \
+ atomic_##OP2(prev, newv, addr); \
+ zero_extend(prev, prev, 32); \
+ return; \
+}
+
+ATOMIC_CASU(caswu, casw)
+ATOMIC_CASU(caslwu, caslw)
+ATOMIC_CASU(casalwu, casalw)
+
+#undef ATOMIC_CASU
+
+void MacroAssembler::atomic_cas(
+ Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) {
+ switch (size) {
+ case int64:
+ switch ((Assembler::Aqrl)(acquire | release)) {
+ case Assembler::relaxed:
+ atomic_cas(prev, newv, addr);
+ break;
+ case Assembler::rl:
+ atomic_casl(prev, newv, addr);
+ break;
+ case Assembler::aqrl:
+ atomic_casal(prev, newv, addr);
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+ break;
+ case int32:
+ switch ((Assembler::Aqrl)(acquire | release)) {
+ case Assembler::relaxed:
+ atomic_casw(prev, newv, addr);
+ break;
+ case Assembler::rl:
+ atomic_caslw(prev, newv, addr);
+ break;
+ case Assembler::aqrl:
+ atomic_casalw(prev, newv, addr);
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+ break;
+ case uint32:
+ switch ((Assembler::Aqrl)(acquire | release)) {
+ case Assembler::relaxed:
+ atomic_caswu(prev, newv, addr);
+ break;
+ case Assembler::rl:
+ atomic_caslwu(prev, newv, addr);
+ break;
+ case Assembler::aqrl:
+ atomic_casalwu(prev, newv, addr);
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+}
+
void MacroAssembler::far_jump(const Address &entry, Register tmp) {
assert(ReservedCodeCacheSize < 4*G, "branch out of range");
assert(CodeCache::find_blob(entry.target()) != nullptr,
@@ -4342,6 +4481,57 @@ void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tm
bge(cnt, tmp1, loop);
}
+// java.lang.Math.round(float a)
+// Returns the closest int to the argument, with ties rounding to positive infinity.
+void MacroAssembler::java_round_float(Register dst, FloatRegister src, FloatRegister ftmp) {
+ // this instructions calling sequence provides performance improvement on all tested devices;
+ // don't change it without re-verification
+ Label done;
+ mv(t0, jint_cast(0.5f));
+ fmv_w_x(ftmp, t0);
+
+ // dst = 0 if NaN
+ feq_s(t0, src, src); // replacing fclass with feq as performance optimization
+ mv(dst, zr);
+ beqz(t0, done);
+
+ // dst = (src + 0.5f) rounded down towards negative infinity
+ // Adding 0.5f to some floats exceeds the precision limits for a float and rounding takes place.
+ // RDN is required for fadd_s, RNE gives incorrect results:
+ // --------------------------------------------------------------------
+ // fadd.s rne (src + 0.5f): src = 8388609.000000 ftmp = 8388610.000000
+ // fcvt.w.s rdn: ftmp = 8388610.000000 dst = 8388610
+ // --------------------------------------------------------------------
+ // fadd.s rdn (src + 0.5f): src = 8388609.000000 ftmp = 8388609.000000
+ // fcvt.w.s rdn: ftmp = 8388609.000000 dst = 8388609
+ // --------------------------------------------------------------------
+ fadd_s(ftmp, src, ftmp, RoundingMode::rdn);
+ fcvt_w_s(dst, ftmp, RoundingMode::rdn);
+
+ bind(done);
+}
+
+// java.lang.Math.round(double a)
+// Returns the closest long to the argument, with ties rounding to positive infinity.
+void MacroAssembler::java_round_double(Register dst, FloatRegister src, FloatRegister ftmp) {
+ // this instructions calling sequence provides performance improvement on all tested devices;
+ // don't change it without re-verification
+ Label done;
+ mv(t0, julong_cast(0.5));
+ fmv_d_x(ftmp, t0);
+
+ // dst = 0 if NaN
+ feq_d(t0, src, src); // replacing fclass with feq as performance optimization
+ mv(dst, zr);
+ beqz(t0, done);
+
+ // dst = (src + 0.5) rounded down towards negative infinity
+ fadd_d(ftmp, src, ftmp, RoundingMode::rdn); // RDN is required here otherwise some inputs produce incorrect results
+ fcvt_l_d(dst, ftmp, RoundingMode::rdn);
+
+ bind(done);
+}
+
#define FCVT_SAFE(FLOATCVT, FLOATSIG) \
void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) { \
Label done; \
@@ -4488,41 +4678,54 @@ void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp
}
void MacroAssembler::zero_extend(Register dst, Register src, int bits) {
- if (UseZba && bits == 32) {
- zext_w(dst, src);
- return;
- }
-
- if (UseZbb && bits == 16) {
- zext_h(dst, src);
- return;
- }
-
- if (bits == 8) {
- zext_b(dst, src);
- } else {
- slli(dst, src, XLEN - bits);
- srli(dst, dst, XLEN - bits);
+ switch (bits) {
+ case 32:
+ if (UseZba) {
+ zext_w(dst, src);
+ return;
+ }
+ break;
+ case 16:
+ if (UseZbb) {
+ zext_h(dst, src);
+ return;
+ }
+ break;
+ case 8:
+ if (UseZbb) {
+ zext_b(dst, src);
+ return;
+ }
+ break;
+ default:
+ break;
}
+ slli(dst, src, XLEN - bits);
+ srli(dst, dst, XLEN - bits);
}
void MacroAssembler::sign_extend(Register dst, Register src, int bits) {
- if (UseZbb) {
- if (bits == 8) {
- sext_b(dst, src);
- return;
- } else if (bits == 16) {
- sext_h(dst, src);
+ switch (bits) {
+ case 32:
+ sext_w(dst, src);
return;
- }
- }
-
- if (bits == 32) {
- sext_w(dst, src);
- } else {
- slli(dst, src, XLEN - bits);
- srai(dst, dst, XLEN - bits);
+ case 16:
+ if (UseZbb) {
+ sext_h(dst, src);
+ return;
+ }
+ break;
+ case 8:
+ if (UseZbb) {
+ sext_b(dst, src);
+ return;
+ }
+ break;
+ default:
+ break;
}
+ slli(dst, src, XLEN - bits);
+ srai(dst, dst, XLEN - bits);
}
void MacroAssembler::cmp_x2i(Register dst, Register src1, Register src2,
@@ -4701,9 +4904,9 @@ void MacroAssembler::object_move(OopMap* map,
// A float arg may have to do float reg int reg conversion
void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) {
- assert(src.first()->is_stack() && dst.first()->is_stack() ||
- src.first()->is_reg() && dst.first()->is_reg() ||
- src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error");
+ assert((src.first()->is_stack() && dst.first()->is_stack()) ||
+ (src.first()->is_reg() && dst.first()->is_reg()) ||
+ (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error");
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
lwu(tmp, Address(fp, reg2offset_in(src.first())));
@@ -4745,9 +4948,9 @@ void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) {
// A double move
void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
- assert(src.first()->is_stack() && dst.first()->is_stack() ||
- src.first()->is_reg() && dst.first()->is_reg() ||
- src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error");
+ assert((src.first()->is_stack() && dst.first()->is_stack()) ||
+ (src.first()->is_reg() && dst.first()->is_reg()) ||
+ (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error");
if (src.first()->is_stack()) {
if (dst.first()->is_stack()) {
ld(tmp, Address(fp, reg2offset_in(src.first())));
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index 3b110cd3e28..d283654e6e1 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -473,7 +473,11 @@ class MacroAssembler: public Assembler {
}
inline void notr(Register Rd, Register Rs) {
- xori(Rd, Rs, -1);
+ if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
+ c_not(Rd);
+ } else {
+ xori(Rd, Rs, -1);
+ }
}
inline void neg(Register Rd, Register Rs) {
@@ -489,7 +493,11 @@ class MacroAssembler: public Assembler {
}
inline void zext_b(Register Rd, Register Rs) {
- andi(Rd, Rs, 0xFF);
+ if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
+ c_zext_b(Rd);
+ } else {
+ andi(Rd, Rs, 0xFF);
+ }
}
inline void seqz(Register Rd, Register Rs) {
@@ -511,7 +519,12 @@ class MacroAssembler: public Assembler {
// Bit-manipulation extension pseudo instructions
// zero extend word
inline void zext_w(Register Rd, Register Rs) {
- add_uw(Rd, Rs, zr);
+ assert(UseZba, "must be");
+ if (do_compress_zcb(Rd, Rs) && (Rd == Rs)) {
+ c_zext_w(Rd);
+ } else {
+ add_uw(Rd, Rs, zr);
+ }
}
// Floating-point data-processing pseudo instructions
@@ -1063,6 +1076,19 @@ class MacroAssembler: public Assembler {
void atomic_xchgwu(Register prev, Register newv, Register addr);
void atomic_xchgalwu(Register prev, Register newv, Register addr);
+ void atomic_cas(Register prev, Register newv, Register addr);
+ void atomic_casw(Register prev, Register newv, Register addr);
+ void atomic_casl(Register prev, Register newv, Register addr);
+ void atomic_caslw(Register prev, Register newv, Register addr);
+ void atomic_casal(Register prev, Register newv, Register addr);
+ void atomic_casalw(Register prev, Register newv, Register addr);
+ void atomic_caswu(Register prev, Register newv, Register addr);
+ void atomic_caslwu(Register prev, Register newv, Register addr);
+ void atomic_casalwu(Register prev, Register newv, Register addr);
+
+ void atomic_cas(Register prev, Register newv, Register addr, enum operand_size size,
+ Assembler::Aqrl acquire = Assembler::relaxed, Assembler::Aqrl release = Assembler::relaxed);
+
// Emit a far call/jump. Only invalidates the tmp register which
// is used to keep the entry address for jalr.
// The address must be inside the code cache.
@@ -1252,6 +1278,9 @@ class MacroAssembler: public Assembler {
void fcvt_w_d_safe(Register dst, FloatRegister src, Register tmp = t0);
void fcvt_l_d_safe(Register dst, FloatRegister src, Register tmp = t0);
+ void java_round_float(Register dst, FloatRegister src, FloatRegister ftmp);
+ void java_round_double(Register dst, FloatRegister src, FloatRegister ftmp);
+
// vector load/store unit-stride instructions
void vlex_v(VectorRegister vd, Register base, Assembler::SEW sew, VectorMask vm = unmasked) {
switch (sew) {
@@ -1345,6 +1374,16 @@ class MacroAssembler: public Assembler {
vmfle_vv(vd, vs1, vs2, vm);
}
+ inline void vmsltu_vi(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {
+ guarantee(imm >= 1 && imm <= 16, "imm is invalid");
+ vmsleu_vi(Vd, Vs2, imm-1, vm);
+ }
+
+ inline void vmsgeu_vi(VectorRegister Vd, VectorRegister Vs2, uint32_t imm, VectorMask vm = unmasked) {
+ guarantee(imm >= 1 && imm <= 16, "imm is invalid");
+ vmsgtu_vi(Vd, Vs2, imm-1, vm);
+ }
+
// Copy mask register
inline void vmmv_m(VectorRegister vd, VectorRegister vs) {
vmand_mm(vd, vs, vs);
@@ -1360,6 +1399,10 @@ class MacroAssembler: public Assembler {
vmxnor_mm(vd, vd, vd);
}
+ inline void vnot_v(VectorRegister Vd, VectorRegister Vs, VectorMask vm = unmasked) {
+ vxor_vi(Vd, Vs, -1, vm);
+ }
+
static const int zero_words_block_size;
void cast_primitive_type(BasicType type, Register Rt) {
diff --git a/src/hotspot/cpu/riscv/matcher_riscv.hpp b/src/hotspot/cpu/riscv/matcher_riscv.hpp
index 1da8f003122..08914d4d834 100644
--- a/src/hotspot/cpu/riscv/matcher_riscv.hpp
+++ b/src/hotspot/cpu/riscv/matcher_riscv.hpp
@@ -192,4 +192,9 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ return false;
+ }
+
#endif // CPU_RISCV_MATCHER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index 2b2e5606c81..c29ac1a04fa 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -451,16 +451,27 @@ void NativePostCallNop::make_deopt() {
NativeDeoptInstruction::insert(addr_at(0));
}
-int NativePostCallNop::displacement() const {
+bool NativePostCallNop::decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
// Discard the high 32 bits
- return (int)(intptr_t)MacroAssembler::get_target_of_li32(addr_at(4));
+ int32_t data = (int32_t)(intptr_t)MacroAssembler::get_target_of_li32(addr_at(4));
+ if (data == 0) {
+ return false; // no information encoded
+ }
+ cb_offset = (data & 0xffffff);
+ oopmap_slot = (data >> 24) & 0xff;
+ return true; // decoding succeeded
}
-void NativePostCallNop::patch(jint diff) {
- assert(diff != 0, "must be");
+bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+ if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
+ return false; // cannot encode
+ }
+ int32_t data = (oopmap_slot << 24) | cb_offset;
+ assert(data != 0, "must be");
assert(is_lui_to_zr_at(addr_at(4)) && is_addiw_to_zr_at(addr_at(8)), "must be");
- MacroAssembler::patch_imm_in_li32(addr_at(4), diff);
+ MacroAssembler::patch_imm_in_li32(addr_at(4), data);
+ return true; // successfully encoded
}
void NativeDeoptInstruction::verify() {
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
index 3c6ec1cf588..48bbb2b3b18 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
@@ -591,8 +591,8 @@ class NativePostCallNop: public NativeInstruction {
// an addiw as well.
return is_nop() && is_lui_to_zr_at(addr_at(4));
}
- int displacement() const;
- void patch(jint diff);
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const;
+ bool patch(int32_t oopmap_slot, int32_t cb_offset);
void make_deopt();
};
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index 1096ca80043..7e1291f49d7 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1001,6 +1001,7 @@ definitions %{
int_def LOAD_COST ( 300, 3 * DEFAULT_COST); // load, fpload
int_def STORE_COST ( 100, 1 * DEFAULT_COST); // store, fpstore
int_def XFER_COST ( 300, 3 * DEFAULT_COST); // mfc, mtc, fcvt, fmove, fcmp
+ int_def FMVX_COST ( 100, 1 * DEFAULT_COST); // shuffles with no conversion
int_def BRANCH_COST ( 200, 2 * DEFAULT_COST); // branch, jmp, call
int_def IMUL_COST ( 1000, 10 * DEFAULT_COST); // imul
int_def IDIVSI_COST ( 3400, 34 * DEFAULT_COST); // idivsi
@@ -8417,6 +8418,34 @@ instruct convN2I(iRegINoSp dst, iRegN src)
ins_pipe(ialu_reg);
%}
+instruct round_double_reg(iRegLNoSp dst, fRegD src, fRegD ftmp) %{
+ match(Set dst (RoundD src));
+
+ ins_cost(XFER_COST + BRANCH_COST);
+ effect(TEMP ftmp);
+ format %{ "java_round_double $dst, $src\t#@round_double_reg" %}
+
+ ins_encode %{
+ __ java_round_double($dst$$Register, as_FloatRegister($src$$reg), as_FloatRegister($ftmp$$reg));
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
+instruct round_float_reg(iRegINoSp dst, fRegF src, fRegF ftmp) %{
+ match(Set dst (RoundF src));
+
+ ins_cost(XFER_COST + BRANCH_COST);
+ effect(TEMP ftmp);
+ format %{ "java_round_float $dst, $src\t#@round_float_reg" %}
+
+ ins_encode %{
+ __ java_round_float($dst$$Register, as_FloatRegister($src$$reg), as_FloatRegister($ftmp$$reg));
+ %}
+
+ ins_pipe(pipe_slow);
+%}
+
// Convert oop pointer into compressed form
instruct encodeHeapOop(iRegNNoSp dst, iRegP src) %{
match(Set dst (EncodeP src));
@@ -8646,7 +8675,7 @@ instruct MoveF2I_reg_reg(iRegINoSp dst, fRegF src) %{
effect(DEF dst, USE src);
- ins_cost(XFER_COST);
+ ins_cost(FMVX_COST);
format %{ "fmv.x.w $dst, $src\t#@MoveF2I_reg_reg" %}
@@ -8664,7 +8693,7 @@ instruct MoveI2F_reg_reg(fRegF dst, iRegI src) %{
effect(DEF dst, USE src);
- ins_cost(XFER_COST);
+ ins_cost(FMVX_COST);
format %{ "fmv.w.x $dst, $src\t#@MoveI2F_reg_reg" %}
@@ -8682,7 +8711,7 @@ instruct MoveD2L_reg_reg(iRegLNoSp dst, fRegD src) %{
effect(DEF dst, USE src);
- ins_cost(XFER_COST);
+ ins_cost(FMVX_COST);
format %{ "fmv.x.d $dst, $src\t#@MoveD2L_reg_reg" %}
@@ -8700,7 +8729,7 @@ instruct MoveL2D_reg_reg(fRegD dst, iRegL src) %{
effect(DEF dst, USE src);
- ins_cost(XFER_COST);
+ ins_cost(FMVX_COST);
format %{ "fmv.d.x $dst, $src\t#@MoveL2D_reg_reg" %}
@@ -10371,6 +10400,26 @@ instruct array_equalsC(iRegP_R11 ary1, iRegP_R12 ary2, iRegI_R10 result,
ins_pipe(pipe_class_memory);
%}
+// fast ArraysSupport.vectorizedHashCode
+instruct arrays_hashcode(iRegP_R11 ary, iRegI_R12 cnt, iRegI_R10 result, immI basic_type,
+ iRegLNoSp tmp1, iRegLNoSp tmp2,
+ iRegLNoSp tmp3, iRegLNoSp tmp4,
+ iRegLNoSp tmp5, iRegLNoSp tmp6, rFlagsReg cr)
+%{
+ match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
+ effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
+ USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
+
+ format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
+ ins_encode %{
+ __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
+ $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
+ $tmp4$$Register, $tmp5$$Register, $tmp6$$Register,
+ (BasicType)$basic_type$$constant);
+ %}
+ ins_pipe(pipe_class_memory);
+%}
+
// ============================================================================
// Safepoint Instructions
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index 36430e126f9..01fe307bc27 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -266,7 +266,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
uint int_args = 0;
uint fp_args = 0;
- uint stk_args = 0; // inc by 2 each time
+ uint stk_args = 0;
for (int i = 0; i < total_args_passed; i++) {
switch (sig_bt[i]) {
@@ -278,8 +278,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_VOID:
@@ -295,6 +296,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -303,8 +305,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_DOUBLE:
@@ -312,6 +315,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -321,7 +325,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
}
}
- return align_up(stk_args, 2);
+ return stk_args;
}
// Patch the callers callsite with entry to compiled code if it exists.
@@ -1675,7 +1679,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ sd(swap_reg, Address(lock_reg, mark_word_offset));
// src -> dest if dest == x10 else x10 <- dest
- __ cmpxchg_obj_header(x10, lock_reg, obj_reg, t0, count, /*fallthrough*/nullptr);
+ __ cmpxchg_obj_header(x10, lock_reg, obj_reg, lock_tmp, count, /*fallthrough*/nullptr);
// Test if the oopMark is an obvious stack pointer, i.e.,
// 1) (mark & 3) == 0, and
@@ -1815,7 +1819,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Atomic swap old header if oop still contains the stack lock
Label count;
- __ cmpxchg_obj_header(x10, old_hdr, obj_reg, t0, count, &slow_path_unlock);
+ __ cmpxchg_obj_header(x10, old_hdr, obj_reg, lock_tmp, count, &slow_path_unlock);
__ bind(count);
__ decrement(Address(xthread, JavaThread::held_monitor_count_offset()));
} else {
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index e10357c4a5a..4bd33d08f89 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -3659,8 +3659,394 @@ class StubGenerator: public StubCodeGenerator {
return entry;
}
};
+
#endif // COMPILER2
+#undef __
+#define __ this->
+ class Sha2Generator : public MacroAssembler {
+ StubCodeGenerator* _cgen;
+ public:
+ Sha2Generator(MacroAssembler* masm, StubCodeGenerator* cgen) : MacroAssembler(masm->code()), _cgen(cgen) {}
+ address generate_sha256_implCompress(bool multi_block) {
+ return generate_sha2_implCompress(Assembler::e32, multi_block);
+ }
+ address generate_sha512_implCompress(bool multi_block) {
+ return generate_sha2_implCompress(Assembler::e64, multi_block);
+ }
+ private:
+
+ void vleXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
+ if (vset_sew == Assembler::e32) __ vle32_v(vr, sr);
+ else __ vle64_v(vr, sr);
+ }
+
+ void vseXX_v(Assembler::SEW vset_sew, VectorRegister vr, Register sr) {
+ if (vset_sew == Assembler::e32) __ vse32_v(vr, sr);
+ else __ vse64_v(vr, sr);
+ }
+
+ // Overview of the logic in each "quad round".
+ //
+ // The code below repeats 16/20 times the logic implementing four rounds
+ // of the SHA-256/512 core loop as documented by NIST. 16/20 "quad rounds"
+ // to implementing the 64/80 single rounds.
+ //
+ // // Load four word (u32/64) constants (K[t+3], K[t+2], K[t+1], K[t+0])
+ // // Output:
+ // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
+ // vl1reXX.v vTmp1, ofs
+ //
+ // // Increment word constant address by stride (16/32 bytes, 4*4B/8B, 128b/256b)
+ // addi ofs, ofs, 16/32
+ //
+ // // Add constants to message schedule words:
+ // // Input
+ // // vTmp1 = {K[t+3], K[t+2], K[t+1], K[t+0]}
+ // // vW0 = {W[t+3], W[t+2], W[t+1], W[t+0]}; // Vt0 = W[3:0];
+ // // Output
+ // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
+ // vadd.vv vTmp0, vTmp1, vW0
+ //
+ // // 2 rounds of working variables updates.
+ // // vState1[t+4] <- vState1[t], vState0[t], vTmp0[t]
+ // // Input:
+ // // vState1 = {c[t],d[t],g[t],h[t]} " = vState1[t] "
+ // // vState0 = {a[t],b[t],e[t],f[t]}
+ // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
+ // // Output:
+ // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] "
+ // // = {h[t+4],g[t+4],d[t+4],c[t+4]} " = vState1[t+4] "
+ // vsha2cl.vv vState1, vState0, vTmp0
+ //
+ // // 2 rounds of working variables updates.
+ // // vState0[t+4] <- vState0[t], vState0[t+2], vTmp0[t]
+ // // Input
+ // // vState0 = {a[t],b[t],e[t],f[t]} " = vState0[t] "
+ // // = {h[t+2],g[t+2],d[t+2],c[t+2]} " = vState1[t+2] "
+ // // vState1 = {f[t+2],e[t+2],b[t+2],a[t+2]} " = vState0[t+2] "
+ // // vTmp0 = {W[t+3]+K[t+3], W[t+2]+K[t+2], W[t+1]+K[t+1], W[t+0]+K[t+0]}
+ // // Output:
+ // // vState0 = {f[t+4],e[t+4],b[t+4],a[t+4]} " = vState0[t+4] "
+ // vsha2ch.vv vState0, vState1, vTmp0
+ //
+ // // Combine 2QW into 1QW
+ // //
+ // // To generate the next 4 words, "new_vW0"/"vTmp0" from vW0-vW3, vsha2ms needs
+ // // vW0[0..3], vW1[0], vW2[1..3], vW3[0, 2..3]
+ // // and it can only take 3 vectors as inputs. Hence we need to combine
+ // // vW1[0] and vW2[1..3] in a single vector.
+ // //
+ // // vmerge Vt4, Vt1, Vt2, V0
+ // // Input
+ // // V0 = mask // first word from vW2, 1..3 words from vW1
+ // // vW2 = {Wt-8, Wt-7, Wt-6, Wt-5}
+ // // vW1 = {Wt-12, Wt-11, Wt-10, Wt-9}
+ // // Output
+ // // Vt4 = {Wt-12, Wt-7, Wt-6, Wt-5}
+ // vmerge.vvm vTmp0, vW2, vW1, v0
+ //
+ // // Generate next Four Message Schedule Words (hence allowing for 4 more rounds)
+ // // Input
+ // // vW0 = {W[t+ 3], W[t+ 2], W[t+ 1], W[t+ 0]} W[ 3: 0]
+ // // vW3 = {W[t+15], W[t+14], W[t+13], W[t+12]} W[15:12]
+ // // vTmp0 = {W[t+11], W[t+10], W[t+ 9], W[t+ 4]} W[11: 9,4]
+ // // Output (next four message schedule words)
+ // // vW0 = {W[t+19], W[t+18], W[t+17], W[t+16]} W[19:16]
+ // vsha2ms.vv vW0, vTmp0, vW3
+ //
+ // BEFORE
+ // vW0 - vW3 hold the message schedule words (initially the block words)
+ // vW0 = W[ 3: 0] "oldest"
+ // vW1 = W[ 7: 4]
+ // vW2 = W[11: 8]
+ // vW3 = W[15:12] "newest"
+ //
+ // vt6 - vt7 hold the working state variables
+ // vState0 = {a[t],b[t],e[t],f[t]} // initially {H5,H4,H1,H0}
+ // vState1 = {c[t],d[t],g[t],h[t]} // initially {H7,H6,H3,H2}
+ //
+ // AFTER
+ // vW0 - vW3 hold the message schedule words (initially the block words)
+ // vW1 = W[ 7: 4] "oldest"
+ // vW2 = W[11: 8]
+ // vW3 = W[15:12]
+ // vW0 = W[19:16] "newest"
+ //
+ // vState0 and vState1 hold the working state variables
+ // vState0 = {a[t+4],b[t+4],e[t+4],f[t+4]}
+ // vState1 = {c[t+4],d[t+4],g[t+4],h[t+4]}
+ //
+ // The group of vectors vW0,vW1,vW2,vW3 is "rotated" by one in each quad-round,
+ // hence the uses of those vectors rotate in each round, and we get back to the
+ // initial configuration every 4 quad-rounds. We could avoid those changes at
+ // the cost of moving those vectors at the end of each quad-rounds.
+ void sha2_quad_round(Assembler::SEW vset_sew, VectorRegister rot1, VectorRegister rot2, VectorRegister rot3, VectorRegister rot4,
+ Register scalarconst, VectorRegister vtemp, VectorRegister vtemp2, VectorRegister v_abef, VectorRegister v_cdgh,
+ bool gen_words = true, bool step_const = true) {
+ __ vleXX_v(vset_sew, vtemp, scalarconst);
+ if (step_const) {
+ __ addi(scalarconst, scalarconst, vset_sew == Assembler::e32 ? 16 : 32);
+ }
+ __ vadd_vv(vtemp2, vtemp, rot1);
+ __ vsha2cl_vv(v_cdgh, v_abef, vtemp2);
+ __ vsha2ch_vv(v_abef, v_cdgh, vtemp2);
+ if (gen_words) {
+ __ vmerge_vvm(vtemp2, rot3, rot2);
+ __ vsha2ms_vv(rot1, vtemp2, rot4);
+ }
+ }
+
+ const char* stub_name(Assembler::SEW vset_sew, bool multi_block) {
+ if (vset_sew == Assembler::e32 && !multi_block) return "sha256_implCompress";
+ if (vset_sew == Assembler::e32 && multi_block) return "sha256_implCompressMB";
+ if (vset_sew == Assembler::e64 && !multi_block) return "sha512_implCompress";
+ if (vset_sew == Assembler::e64 && multi_block) return "sha512_implCompressMB";
+ ShouldNotReachHere();
+ return "bad name lookup";
+ }
+
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0 - byte[] source+offset
+ // c_rarg1 - int[] SHA.state
+ // c_rarg2 - int offset
+ // c_rarg3 - int limit
+ //
+ address generate_sha2_implCompress(Assembler::SEW vset_sew, bool multi_block) {
+ alignas(64) static const uint32_t round_consts_256[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+ };
+ alignas(64) static const uint64_t round_consts_512[80] = {
+ 0x428a2f98d728ae22l, 0x7137449123ef65cdl, 0xb5c0fbcfec4d3b2fl,
+ 0xe9b5dba58189dbbcl, 0x3956c25bf348b538l, 0x59f111f1b605d019l,
+ 0x923f82a4af194f9bl, 0xab1c5ed5da6d8118l, 0xd807aa98a3030242l,
+ 0x12835b0145706fbel, 0x243185be4ee4b28cl, 0x550c7dc3d5ffb4e2l,
+ 0x72be5d74f27b896fl, 0x80deb1fe3b1696b1l, 0x9bdc06a725c71235l,
+ 0xc19bf174cf692694l, 0xe49b69c19ef14ad2l, 0xefbe4786384f25e3l,
+ 0x0fc19dc68b8cd5b5l, 0x240ca1cc77ac9c65l, 0x2de92c6f592b0275l,
+ 0x4a7484aa6ea6e483l, 0x5cb0a9dcbd41fbd4l, 0x76f988da831153b5l,
+ 0x983e5152ee66dfabl, 0xa831c66d2db43210l, 0xb00327c898fb213fl,
+ 0xbf597fc7beef0ee4l, 0xc6e00bf33da88fc2l, 0xd5a79147930aa725l,
+ 0x06ca6351e003826fl, 0x142929670a0e6e70l, 0x27b70a8546d22ffcl,
+ 0x2e1b21385c26c926l, 0x4d2c6dfc5ac42aedl, 0x53380d139d95b3dfl,
+ 0x650a73548baf63del, 0x766a0abb3c77b2a8l, 0x81c2c92e47edaee6l,
+ 0x92722c851482353bl, 0xa2bfe8a14cf10364l, 0xa81a664bbc423001l,
+ 0xc24b8b70d0f89791l, 0xc76c51a30654be30l, 0xd192e819d6ef5218l,
+ 0xd69906245565a910l, 0xf40e35855771202al, 0x106aa07032bbd1b8l,
+ 0x19a4c116b8d2d0c8l, 0x1e376c085141ab53l, 0x2748774cdf8eeb99l,
+ 0x34b0bcb5e19b48a8l, 0x391c0cb3c5c95a63l, 0x4ed8aa4ae3418acbl,
+ 0x5b9cca4f7763e373l, 0x682e6ff3d6b2b8a3l, 0x748f82ee5defb2fcl,
+ 0x78a5636f43172f60l, 0x84c87814a1f0ab72l, 0x8cc702081a6439ecl,
+ 0x90befffa23631e28l, 0xa4506cebde82bde9l, 0xbef9a3f7b2c67915l,
+ 0xc67178f2e372532bl, 0xca273eceea26619cl, 0xd186b8c721c0c207l,
+ 0xeada7dd6cde0eb1el, 0xf57d4f7fee6ed178l, 0x06f067aa72176fbal,
+ 0x0a637dc5a2c898a6l, 0x113f9804bef90dael, 0x1b710b35131c471bl,
+ 0x28db77f523047d84l, 0x32caab7b40c72493l, 0x3c9ebe0a15c9bebcl,
+ 0x431d67c49c100d4cl, 0x4cc5d4becb3e42b6l, 0x597f299cfc657e2al,
+ 0x5fcb6fab3ad6faecl, 0x6c44198c4a475817l
+ };
+ const int const_add = vset_sew == Assembler::e32 ? 16 : 32;
+
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(_cgen, "StubRoutines", stub_name(vset_sew, multi_block));
+ address start = __ pc();
+
+ Register buf = c_rarg0;
+ Register state = c_rarg1;
+ Register ofs = c_rarg2;
+ Register limit = c_rarg3;
+ Register consts = t2; // caller saved
+ Register state_c = x28; // caller saved
+ VectorRegister vindex = v2;
+ VectorRegister vW0 = v4;
+ VectorRegister vW1 = v6;
+ VectorRegister vW2 = v8;
+ VectorRegister vW3 = v10;
+ VectorRegister vState0 = v12;
+ VectorRegister vState1 = v14;
+ VectorRegister vHash0 = v16;
+ VectorRegister vHash1 = v18;
+ VectorRegister vTmp0 = v20;
+ VectorRegister vTmp1 = v22;
+
+ Label multi_block_loop;
+
+ __ enter();
+
+ address constant_table = vset_sew == Assembler::e32 ? (address)round_consts_256 : (address)round_consts_512;
+ la(consts, ExternalAddress(constant_table));
+
+ // Register use in this function:
+ //
+ // VECTORS
+ // vW0 - vW3 (512/1024-bits / 4*128/256 bits / 4*4*32/65 bits), hold the message
+ // schedule words (Wt). They start with the message block
+ // content (W0 to W15), then further words in the message
+ // schedule generated via vsha2ms from previous Wt.
+ // Initially:
+ // vW0 = W[ 3:0] = { W3, W2, W1, W0}
+ // vW1 = W[ 7:4] = { W7, W6, W5, W4}
+ // vW2 = W[ 11:8] = {W11, W10, W9, W8}
+ // vW3 = W[15:12] = {W15, W14, W13, W12}
+ //
+ // vState0 - vState1 hold the working state variables (a, b, ..., h)
+ // vState0 = {f[t],e[t],b[t],a[t]}
+ // vState1 = {h[t],g[t],d[t],c[t]}
+ // Initially:
+ // vState0 = {H5i-1, H4i-1, H1i-1 , H0i-1}
+ // vState1 = {H7i-i, H6i-1, H3i-1 , H2i-1}
+ //
+ // v0 = masks for vrgather/vmerge. Single value during the 16 rounds.
+ //
+ // vTmp0 = temporary, Wt+Kt
+ // vTmp1 = temporary, Kt
+ //
+ // vHash0/vHash1 = hold the initial values of the hash, byte-swapped.
+ //
+ // During most of the function the vector state is configured so that each
+ // vector is interpreted as containing four 32/64 bits (e32/e64) elements (128/256 bits).
+
+ // vsha2ch/vsha2cl uses EGW of 4*SEW.
+ // SHA256 SEW = e32, EGW = 128-bits
+ // SHA512 SEW = e64, EGW = 256-bits
+ //
+ // VLEN is required to be at least 128.
+ // For the case of VLEN=128 and SHA512 we need LMUL=2 to work with 4*e64 (EGW = 256)
+ //
+ // m1: LMUL=1/2
+ // ta: tail agnostic (don't care about those lanes)
+ // ma: mask agnostic (don't care about those lanes)
+ // x0 is not written, we known the number of vector elements.
+
+ if (vset_sew == Assembler::e64 && MaxVectorSize == 16) { // SHA512 and VLEN = 128
+ __ vsetivli(x0, 4, vset_sew, Assembler::m2, Assembler::ma, Assembler::ta);
+ } else {
+ __ vsetivli(x0, 4, vset_sew, Assembler::m1, Assembler::ma, Assembler::ta);
+ }
+
+ int64_t indexes = vset_sew == Assembler::e32 ? 0x00041014ul : 0x00082028ul;
+ __ li(t0, indexes);
+ __ vmv_v_x(vindex, t0);
+
+ // Step-over a,b, so we are pointing to c.
+ // const_add is equal to 4x state variable, div by 2 is thus 2, a,b
+ __ addi(state_c, state, const_add/2);
+
+ // Use index-load to get {f,e,b,a},{h,g,d,c}
+ __ vluxei8_v(vState0, state, vindex);
+ __ vluxei8_v(vState1, state_c, vindex);
+
+ __ bind(multi_block_loop);
+
+ // Capture the initial H values in vHash0 and vHash1 to allow for computing
+ // the resulting H', since H' = H+{a',b',c',...,h'}.
+ __ vmv_v_v(vHash0, vState0);
+ __ vmv_v_v(vHash1, vState1);
+
+ // Load the 512/1024-bits of the message block in vW0-vW3 and perform
+ // an endian swap on each 4/8 bytes element.
+ //
+ // If Zvkb is not implemented one can use vrgather
+ // with an index sequence to byte-swap.
+ // sequence = [3 2 1 0 7 6 5 4 11 10 9 8 15 14 13 12]
+ // gives us "N ^ 3" as a nice formula to generate
+ // this sequence. 'vid' gives us the N.
+ __ vleXX_v(vset_sew, vW0, buf);
+ __ vrev8_v(vW0, vW0);
+ __ addi(buf, buf, const_add);
+ __ vleXX_v(vset_sew, vW1, buf);
+ __ vrev8_v(vW1, vW1);
+ __ addi(buf, buf, const_add);
+ __ vleXX_v(vset_sew, vW2, buf);
+ __ vrev8_v(vW2, vW2);
+ __ addi(buf, buf, const_add);
+ __ vleXX_v(vset_sew, vW3, buf);
+ __ vrev8_v(vW3, vW3);
+ __ addi(buf, buf, const_add);
+
+ // Set v0 up for the vmerge that replaces the first word (idx==0)
+ __ vid_v(v0);
+ __ vmseq_vi(v0, v0, 0x0); // v0.mask[i] = (i == 0 ? 1 : 0)
+
+ VectorRegister rotation_regs[] = {vW0, vW1, vW2, vW3};
+ int rot_pos = 0;
+ // Quad-round #0 (+0, vW0->vW1->vW2->vW3) ... #11 (+3, vW3->vW0->vW1->vW2)
+ const int qr_end = vset_sew == Assembler::e32 ? 12 : 16;
+ for (int i = 0; i < qr_end; i++) {
+ sha2_quad_round(vset_sew,
+ rotation_regs[(rot_pos + 0) & 0x3],
+ rotation_regs[(rot_pos + 1) & 0x3],
+ rotation_regs[(rot_pos + 2) & 0x3],
+ rotation_regs[(rot_pos + 3) & 0x3],
+ consts,
+ vTmp1, vTmp0, vState0, vState1);
+ ++rot_pos;
+ }
+ // Quad-round #12 (+0, vW0->vW1->vW2->vW3) ... #15 (+3, vW3->vW0->vW1->vW2)
+ // Note that we stop generating new message schedule words (Wt, vW0-13)
+ // as we already generated all the words we end up consuming (i.e., W[63:60]).
+ const int qr_c_end = qr_end + 4;
+ for (int i = qr_end; i < qr_c_end; i++) {
+ sha2_quad_round(vset_sew,
+ rotation_regs[(rot_pos + 0) & 0x3],
+ rotation_regs[(rot_pos + 1) & 0x3],
+ rotation_regs[(rot_pos + 2) & 0x3],
+ rotation_regs[(rot_pos + 3) & 0x3],
+ consts,
+ vTmp1, vTmp0, vState0, vState1, false, i < (qr_c_end-1));
+ ++rot_pos;
+ }
+
+ //--------------------------------------------------------------------------------
+ // Compute the updated hash value H'
+ // H' = H + {h',g',...,b',a'}
+ // = {h,g,...,b,a} + {h',g',...,b',a'}
+ // = {h+h',g+g',...,b+b',a+a'}
+
+ // H' = H+{a',b',c',...,h'}
+ __ vadd_vv(vState0, vHash0, vState0);
+ __ vadd_vv(vState1, vHash1, vState1);
+
+ if (multi_block) {
+ int total_adds = vset_sew == Assembler::e32 ? 240 : 608;
+ __ addi(consts, consts, -total_adds);
+ __ add(ofs, ofs, vset_sew == Assembler::e32 ? 64 : 128);
+ __ ble(ofs, limit, multi_block_loop);
+ __ mv(c_rarg0, ofs); // return ofs
+ }
+
+ // Store H[0..8] = {a,b,c,d,e,f,g,h} from
+ // vState0 = {f,e,b,a}
+ // vState1 = {h,g,d,c}
+ __ vsuxei8_v(vState0, state, vindex);
+ __ vsuxei8_v(vState1, state_c, vindex);
+
+ __ leave();
+ __ ret();
+
+ return start;
+ }
+ };
+#undef __
+#define __ masm->
+
// Continuation point for throwing of implicit exceptions that are
// not handled in the current activation. Fabricates an exception
// oop and initiates normal exception dispatching in this
@@ -4155,14 +4541,18 @@ class StubGenerator: public StubCodeGenerator {
// to minimize the number of memory operations:
// read the 4 state 4-byte values in pairs, with a single ld,
- // and split them into 2 registers
- __ mv(t0, mask32);
+ // and split them into 2 registers.
+ //
+ // And, as the core algorithm of md5 works on 32-bits words, so
+ // in the following code, it does not care about the content of
+ // higher 32-bits in state[x]. Based on this observation,
+ // we can apply further optimization, which is to just ignore the
+ // higher 32-bits in state0/state2, rather than set the higher
+ // 32-bits of state0/state2 to zero explicitly with extra instructions.
__ ld(state0, Address(state));
__ srli(state1, state0, 32);
- __ andr(state0, state0, t0);
__ ld(state2, Address(state, 8));
__ srli(state3, state2, 32);
- __ andr(state2, state2, t0);
Label md5_loop;
__ BIND(md5_loop);
@@ -4858,6 +5248,18 @@ static const int64_t right_3_bits = right_n_bits(3);
}
#endif // COMPILER2
+ if (UseSHA256Intrinsics) {
+ Sha2Generator sha2(_masm, this);
+ StubRoutines::_sha256_implCompress = sha2.generate_sha256_implCompress(false);
+ StubRoutines::_sha256_implCompressMB = sha2.generate_sha256_implCompress(true);
+ }
+
+ if (UseSHA512Intrinsics) {
+ Sha2Generator sha2(_masm, this);
+ StubRoutines::_sha512_implCompress = sha2.generate_sha512_implCompress(false);
+ StubRoutines::_sha512_implCompressMB = sha2.generate_sha512_implCompress(true);
+ }
+
generate_compare_long_strings();
generate_string_indexof_stubs();
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index fc83e629858..9a72b8d75a1 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -146,26 +146,11 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
- if (UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- }
-
if (UseSHA1Intrinsics) {
warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
}
- if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
- }
-
- if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
- }
-
if (UseSHA3Intrinsics) {
warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
@@ -272,6 +257,10 @@ void VM_Version::initialize() {
// NOTE: Make sure codes dependent on UseRVV are put after c2_initialize(),
// as there are extra checks inside it which could disable UseRVV
// in some situations.
+ if (UseZvkn && !UseRVV) {
+ FLAG_SET_DEFAULT(UseZvkn, false);
+ warning("Cannot enable Zvkn on cpu without RVV support.");
+ }
if (UseRVV) {
if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
@@ -283,6 +272,31 @@ void VM_Version::initialize() {
}
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
+
+ if (!UseZvkn && UseSHA) {
+ warning("SHA instructions are not available on this CPU");
+ FLAG_SET_DEFAULT(UseSHA, false);
+ } else if (UseZvkn && FLAG_IS_DEFAULT(UseSHA)) {
+ FLAG_SET_DEFAULT(UseSHA, true);
+ }
+
+ if (!UseSHA) {
+ if (UseSHA256Intrinsics) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
+ }
+ if (UseSHA512Intrinsics) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
+ }
+ } else {
+ if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
+ }
+ if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
+ }
+ }
}
#ifdef COMPILER2
@@ -315,6 +329,10 @@ void VM_Version::c2_initialize() {
}
}
+ if (FLAG_IS_DEFAULT(UseVectorizedHashCodeIntrinsic)) {
+ FLAG_SET_DEFAULT(UseVectorizedHashCodeIntrinsic, true);
+ }
+
if (!UseZicbop) {
if (!FLAG_IS_DEFAULT(AllocatePrefetchStyle)) {
warning("Zicbop is not available on this CPU");
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.hpp b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
index 3c769ebfe2a..1ea853284ff 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.hpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.hpp
@@ -110,6 +110,9 @@ class VM_Version : public Abstract_VM_Version {
// Zic64b Cache blocks must be 64 bytes in size, naturally aligned in the address space.
// Zihintpause Pause instruction HINT
//
+ // Zc Code Size Reduction - Additional compressed instructions.
+ // Zcb Simple code-size saving instructions
+ //
// Other features and settings
// mvendorid Manufactory JEDEC id encoded, ISA vol 2 3.1.2..
// marchid Id for microarch. Mvendorid plus marchid uniquely identify the microarch.
@@ -117,6 +120,8 @@ class VM_Version : public Abstract_VM_Version {
// unaligned_access Unaligned memory accesses (unknown, unspported, emulated, slow, firmware, fast)
// satp mode SATP bits (number of virtual addr bits) mbare, sv39, sv48, sv57, sv64
+ public:
+
#define RV_NO_FLAG_BIT (BitsPerWord+1) // nth_bit will return 0 on values larger than BitsPerWord
// declaration name , extension name, bit pos ,in str, mapped flag)
@@ -137,11 +142,13 @@ class VM_Version : public Abstract_VM_Version {
decl(ext_Zbb , "Zbb" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbb)) \
decl(ext_Zbc , "Zbc" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zbs , "Zbs" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZbs)) \
+ decl(ext_Zcb , "Zcb" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zicsr , "Zicsr" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zifencei , "Zifencei" , RV_NO_FLAG_BIT, true , NO_UPDATE_DEFAULT) \
decl(ext_Zic64b , "Zic64b" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZic64b)) \
decl(ext_Ztso , "Ztso" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZtso)) \
decl(ext_Zihintpause , "Zihintpause" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZihintpause)) \
+ decl(ext_Zacas , "Zacas" , RV_NO_FLAG_BIT, true , UPDATE_DEFAULT(UseZacas)) \
decl(mvendorid , "VendorId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(marchid , "ArchId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
decl(mimpid , "ImpId" , RV_NO_FLAG_BIT, false, NO_UPDATE_DEFAULT) \
@@ -208,6 +215,9 @@ class VM_Version : public Abstract_VM_Version {
constexpr static bool supports_stack_watermark_barrier() { return true; }
static bool supports_on_spin_wait() { return UseZihintpause; }
+
+ // RISCV64 supports fast class initialization checks
+ static bool supports_fast_class_init_checks() { return true; }
};
#endif // CPU_RISCV_VM_VERSION_RISCV_HPP
diff --git a/src/hotspot/cpu/s390/compiledIC_s390.cpp b/src/hotspot/cpu/s390/compiledIC_s390.cpp
index 09822425e12..7ea90c1de7c 100644
--- a/src/hotspot/cpu/s390/compiledIC_s390.cpp
+++ b/src/hotspot/cpu/s390/compiledIC_s390.cpp
@@ -95,9 +95,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/s390/frame_s390.hpp b/src/hotspot/cpu/s390/frame_s390.hpp
index 3f81cd254d0..06433d9136f 100644
--- a/src/hotspot/cpu/s390/frame_s390.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.hpp
@@ -465,7 +465,6 @@
// Initialize frame members (_pc and _sp must be given)
inline void setup();
- const ImmutableOopMap* get_oop_map() const;
// Constructors
diff --git a/src/hotspot/cpu/s390/frame_s390.inline.hpp b/src/hotspot/cpu/s390/frame_s390.inline.hpp
index 91a6917c319..178f7f90849 100644
--- a/src/hotspot/cpu/s390/frame_s390.inline.hpp
+++ b/src/hotspot/cpu/s390/frame_s390.inline.hpp
@@ -292,20 +292,6 @@ inline intptr_t* frame::real_fp() const {
return fp();
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
inline int frame::compiled_frame_stack_argsize() const {
Unimplemented();
return 0;
diff --git a/src/hotspot/cpu/s390/globalDefinitions_s390.hpp b/src/hotspot/cpu/s390/globalDefinitions_s390.hpp
index 39baf5bf047..236d4c5bf69 100644
--- a/src/hotspot/cpu/s390/globalDefinitions_s390.hpp
+++ b/src/hotspot/cpu/s390/globalDefinitions_s390.hpp
@@ -48,6 +48,9 @@ const bool CCallingConventionRequiresIntsAsLongs = true;
// The expected size in bytes of a cache line, used to pad data structures.
#define DEFAULT_CACHE_LINE_SIZE 256
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
#define SUPPORT_RESERVED_STACK_AREA
#endif // CPU_S390_GLOBALDEFINITIONS_S390_HPP
diff --git a/src/hotspot/cpu/s390/matcher_s390.hpp b/src/hotspot/cpu/s390/matcher_s390.hpp
index e68dca34c3b..450ea35a6cb 100644
--- a/src/hotspot/cpu/s390/matcher_s390.hpp
+++ b/src/hotspot/cpu/s390/matcher_s390.hpp
@@ -184,4 +184,9 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ return false;
+ }
+
#endif // CPU_S390_MATCHER_S390_HPP
diff --git a/src/hotspot/cpu/s390/nativeInst_s390.hpp b/src/hotspot/cpu/s390/nativeInst_s390.hpp
index 65bfe499370..abad50da8b4 100644
--- a/src/hotspot/cpu/s390/nativeInst_s390.hpp
+++ b/src/hotspot/cpu/s390/nativeInst_s390.hpp
@@ -657,8 +657,8 @@ class NativeGeneralJump: public NativeInstruction {
class NativePostCallNop: public NativeInstruction {
public:
bool check() const { Unimplemented(); return false; }
- int displacement() const { return 0; }
- void patch(jint diff) { Unimplemented(); }
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { return false; }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset) { Unimplemented(); return false; }
void make_deopt() { Unimplemented(); }
};
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index 4798f35f19d..ed1795cfa33 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -755,7 +755,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
ShouldNotReachHere();
}
}
- return align_up(stk, 2);
+ return stk;
}
int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp
index d72e2712a40..93d5b11c473 100644
--- a/src/hotspot/cpu/s390/vm_version_s390.hpp
+++ b/src/hotspot/cpu/s390/vm_version_s390.hpp
@@ -410,7 +410,7 @@ class VM_Version: public Abstract_VM_Version {
// Override Abstract_VM_Version implementation
static void print_platform_virtualization_info(outputStream*);
- // s390 supports fast class initialization checks for static methods.
+ // s390 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
// CPU feature query functions
diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp
index cedddaed975..63d6ec9b00f 100644
--- a/src/hotspot/cpu/x86/assembler_x86.cpp
+++ b/src/hotspot/cpu/x86/assembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -889,8 +889,8 @@ address Assembler::locate_operand(address inst, WhichOperand which) {
assert(which == imm_operand || which == disp32_operand,
"which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
#else
- assert((which == call32_operand || which == imm_operand) && is_64bit ||
- which == narrow_oop_operand && !is_64bit,
+ assert(((which == call32_operand || which == imm_operand) && is_64bit) ||
+ (which == narrow_oop_operand && !is_64bit),
"which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, p2i(ip));
#endif // _LP64
return ip;
@@ -920,6 +920,7 @@ address Assembler::locate_operand(address inst, WhichOperand which) {
case 0x11: // movups
case 0x12: // movlps
case 0x28: // movaps
+ case 0x29: // movaps
case 0x2E: // ucomiss
case 0x2F: // comiss
case 0x54: // andps
@@ -969,7 +970,7 @@ address Assembler::locate_operand(address inst, WhichOperand which) {
assert(which == call32_operand, "jcc has no disp32 or imm");
return ip;
default:
- ShouldNotReachHere();
+ fatal("not handled: 0x0F%2X", 0xFF & *(ip-1));
}
break;
@@ -2845,6 +2846,13 @@ void Assembler::kxorbl(KRegister dst, KRegister src1, KRegister src2) {
emit_int16(0x47, (0xC0 | encode));
}
+void Assembler::kxnorwl(KRegister dst, KRegister src1, KRegister src2) {
+ assert(VM_Version::supports_evex(), "");
+ InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
+ int encode = vex_prefix_and_encode(dst->encoding(), src1->encoding(), src2->encoding(), VEX_SIMD_NONE, VEX_OPCODE_0F, &attributes);
+ emit_int16(0x46, (0xC0 | encode));
+}
+
void Assembler::kxorwl(KRegister dst, KRegister src1, KRegister src2) {
assert(VM_Version::supports_evex(), "");
InstructionAttr attributes(AVX_256bit, /* rex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
@@ -7227,7 +7235,7 @@ void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, int vector
// Integer vector arithmetic
void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(VM_Version::supports_avx() && (vector_len == 0) ||
+ assert((VM_Version::supports_avx() && (vector_len == 0)) ||
VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
@@ -7235,7 +7243,7 @@ void Assembler::vphaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int v
}
void Assembler::vphaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
- assert(VM_Version::supports_avx() && (vector_len == 0) ||
+ assert((VM_Version::supports_avx() && (vector_len == 0)) ||
VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
int encode = vex_prefix_and_encode(dst->encoding(), nds->encoding(), src->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
@@ -10770,7 +10778,7 @@ void Assembler::vpgatherdd(XMMRegister dst, Address src, XMMRegister mask, int v
assert(src.isxmmindex(),"expected to be xmm index");
assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
InstructionMark im(this);
- InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0x90);
emit_operand(dst, src, 0);
@@ -10783,7 +10791,7 @@ void Assembler::vpgatherdq(XMMRegister dst, Address src, XMMRegister mask, int v
assert(src.isxmmindex(),"expected to be xmm index");
assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
InstructionMark im(this);
- InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0x90);
emit_operand(dst, src, 0);
@@ -10796,7 +10804,7 @@ void Assembler::vgatherdpd(XMMRegister dst, Address src, XMMRegister mask, int v
assert(src.isxmmindex(),"expected to be xmm index");
assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
InstructionMark im(this);
- InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ true);
+ InstructionAttr attributes(vector_len, /* vex_w */ true, /* legacy_mode */ true, /* no_mask_reg */ true, /* uses_vl */ false);
vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0x92);
emit_operand(dst, src, 0);
@@ -10809,7 +10817,7 @@ void Assembler::vgatherdps(XMMRegister dst, Address src, XMMRegister mask, int v
assert(src.isxmmindex(),"expected to be xmm index");
assert(dst != src.xmmindex(), "instruction will #UD if dst and index are the same");
InstructionMark im(this);
- InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ true);
+ InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ true, /* no_mask_reg */ false, /* uses_vl */ false);
vex_prefix(src, mask->encoding(), dst->encoding(), VEX_SIMD_66, VEX_OPCODE_0F_38, &attributes);
emit_int8((unsigned char)0x92);
emit_operand(dst, src, 0);
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index 1d3e43141f2..df148cb0c7e 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -1524,6 +1524,8 @@ class Assembler : public AbstractAssembler {
void kordl(KRegister dst, KRegister src1, KRegister src2);
void korql(KRegister dst, KRegister src1, KRegister src2);
+ void kxnorwl(KRegister dst, KRegister src1, KRegister src2);
+
void kxorbl(KRegister dst, KRegister src1, KRegister src2);
void kxorwl(KRegister dst, KRegister src1, KRegister src2);
void kxordl(KRegister dst, KRegister src1, KRegister src2);
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 88cee688137..ac07ea759fb 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -5252,8 +5252,8 @@ void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, KRegister m
void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp,
Register tmp, int masklen, BasicType bt, int vec_enc) {
- assert(vec_enc == AVX_128bit && VM_Version::supports_avx() ||
- vec_enc == AVX_256bit && (VM_Version::supports_avx2() || type2aelembytes(bt) >= 4), "");
+ assert((vec_enc == AVX_128bit && VM_Version::supports_avx()) ||
+ (vec_enc == AVX_256bit && (VM_Version::supports_avx2() || type2aelembytes(bt) >= 4)), "");
assert(VM_Version::supports_popcnt(), "");
bool need_clip = false;
diff --git a/src/hotspot/cpu/x86/compiledIC_x86.cpp b/src/hotspot/cpu/x86/compiledIC_x86.cpp
index 39e8aafcd93..8fc001039fb 100644
--- a/src/hotspot/cpu/x86/compiledIC_x86.cpp
+++ b/src/hotspot/cpu/x86/compiledIC_x86.cpp
@@ -28,6 +28,7 @@
#include "code/compiledIC.hpp"
#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
+#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/safepoint.hpp"
@@ -84,9 +85,9 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
+ log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
p2i(instruction_address()),
callee->name_and_sig_as_C_string());
}
diff --git a/src/hotspot/cpu/x86/frame_x86.hpp b/src/hotspot/cpu/x86/frame_x86.hpp
index 8c073116280..27beeeb7f44 100644
--- a/src/hotspot/cpu/x86/frame_x86.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.hpp
@@ -152,8 +152,6 @@
static void verify_deopt_original_pc(CompiledMethod* nm, intptr_t* unextended_sp);
#endif
- const ImmutableOopMap* get_oop_map() const;
-
public:
// Constructors
diff --git a/src/hotspot/cpu/x86/frame_x86.inline.hpp b/src/hotspot/cpu/x86/frame_x86.inline.hpp
index 893538bb7e9..d88c8eab683 100644
--- a/src/hotspot/cpu/x86/frame_x86.inline.hpp
+++ b/src/hotspot/cpu/x86/frame_x86.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -184,7 +184,7 @@ inline bool frame::equal(frame other) const {
&& unextended_sp() == other.unextended_sp()
&& fp() == other.fp()
&& pc() == other.pc();
- assert(!ret || ret && cb() == other.cb() && _deopt_state == other._deopt_state, "inconsistent construction");
+ assert(!ret || (ret && cb() == other.cb() && _deopt_state == other._deopt_state), "inconsistent construction");
return ret;
}
@@ -346,20 +346,6 @@ inline int frame::sender_sp_ret_address_offset() {
return frame::sender_sp_offset - frame::return_addr_offset;
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- if (_cb == nullptr) return nullptr;
- if (_cb->oop_maps() != nullptr) {
- NativePostCallNop* nop = nativePostCallNop_at(_pc);
- if (nop != nullptr && nop->displacement() != 0) {
- int slot = ((nop->displacement() >> 24) & 0xff);
- return _cb->oop_map_for_slot(slot, _pc);
- }
- const ImmutableOopMap* oop_map = OopMapSet::find_map(this);
- return oop_map;
- }
- return nullptr;
-}
-
//------------------------------------------------------------------------------
// frame::sender
diff --git a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
index 7eff43471b5..2e82453b380 100644
--- a/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
+++ b/src/hotspot/cpu/x86/globalDefinitions_x86.hpp
@@ -38,29 +38,18 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define CPU_MULTI_COPY_ATOMIC
-// The expected size in bytes of a cache line, used to pad data structures.
-#if COMPILER1_AND_COMPILER2
- #ifdef _LP64
- // tiered, 64-bit, large machine
- #define DEFAULT_CACHE_LINE_SIZE 128
- #define OM_CACHE_LINE_SIZE 64
- #else
- // tiered, 32-bit, medium machine
- #define DEFAULT_CACHE_LINE_SIZE 64
- #endif
-#elif defined(COMPILER1)
- // pure C1, 32-bit, small machine
- // i486 was the last Intel chip with 16-byte cache line size
- #define DEFAULT_CACHE_LINE_SIZE 32
-#elif defined(COMPILER2)
- #ifdef _LP64
- // pure C2, 64-bit, large machine
- #define DEFAULT_CACHE_LINE_SIZE 128
- #define OM_CACHE_LINE_SIZE 64
- #else
- // pure C2, 32-bit, medium machine
- #define DEFAULT_CACHE_LINE_SIZE 64
- #endif
+// The expected size in bytes of a cache line.
+#define DEFAULT_CACHE_LINE_SIZE 64
+
+// The default padding size for data structures to avoid false sharing.
+#ifdef _LP64
+// The common wisdom is that adjacent cache line prefetchers on some hardware
+// may pull two cache lines on access, so we have to pessimistically assume twice
+// the cache line size for padding. TODO: Check if this is still true for modern
+// hardware. If not, DEFAULT_CACHE_LINE_SIZE might as well suffice.
+#define DEFAULT_PADDING_SIZE (DEFAULT_CACHE_LINE_SIZE*2)
+#else
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
#endif
#if defined(COMPILER2)
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index 96d840adf6d..38fa0eb0671 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -1882,92 +1882,6 @@ void MacroAssembler::cmpoop(Register src1, jobject src2, Register rscratch) {
}
#endif
-void MacroAssembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
- if ((UseAVX > 0) && (dst != src)) {
- xorpd(dst, dst);
- }
- Assembler::cvtss2sd(dst, src);
-}
-
-void MacroAssembler::cvtss2sd(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorpd(dst, dst);
- }
- Assembler::cvtss2sd(dst, src);
-}
-
-void MacroAssembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
- if ((UseAVX > 0) && (dst != src)) {
- xorps(dst, dst);
- }
- Assembler::cvtsd2ss(dst, src);
-}
-
-void MacroAssembler::cvtsd2ss(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorps(dst, dst);
- }
- Assembler::cvtsd2ss(dst, src);
-}
-
-void MacroAssembler::cvtsi2sdl(XMMRegister dst, Register src) {
- if (UseAVX > 0) {
- xorpd(dst, dst);
- }
- Assembler::cvtsi2sdl(dst, src);
-}
-
-void MacroAssembler::cvtsi2sdl(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorpd(dst, dst);
- }
- Assembler::cvtsi2sdl(dst, src);
-}
-
-void MacroAssembler::cvtsi2ssl(XMMRegister dst, Register src) {
- if (UseAVX > 0) {
- xorps(dst, dst);
- }
- Assembler::cvtsi2ssl(dst, src);
-}
-
-void MacroAssembler::cvtsi2ssl(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorps(dst, dst);
- }
- Assembler::cvtsi2ssl(dst, src);
-}
-
-#ifdef _LP64
-void MacroAssembler::cvtsi2sdq(XMMRegister dst, Register src) {
- if (UseAVX > 0) {
- xorpd(dst, dst);
- }
- Assembler::cvtsi2sdq(dst, src);
-}
-
-void MacroAssembler::cvtsi2sdq(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorpd(dst, dst);
- }
- Assembler::cvtsi2sdq(dst, src);
-}
-
-void MacroAssembler::cvtsi2ssq(XMMRegister dst, Register src) {
- if (UseAVX > 0) {
- xorps(dst, dst);
- }
- Assembler::cvtsi2ssq(dst, src);
-}
-
-void MacroAssembler::cvtsi2ssq(XMMRegister dst, Address src) {
- if (UseAVX > 0) {
- xorps(dst, dst);
- }
- Assembler::cvtsi2ssq(dst, src);
-}
-#endif // _LP64
-
void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch) {
assert(rscratch != noreg || always_reachable(adr), "missing");
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index a6f907ee7ee..be1078e09ee 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -145,8 +145,8 @@ class MacroAssembler: public Assembler {
op == 0xE9 /* jmp */ ||
op == 0xEB /* short jmp */ ||
(op & 0xF0) == 0x70 /* short jcc */ ||
- op == 0x0F && (branch[1] & 0xF0) == 0x80 /* jcc */ ||
- op == 0xC7 && branch[1] == 0xF8 /* xbegin */,
+ (op == 0x0F && (branch[1] & 0xF0) == 0x80) /* jcc */ ||
+ (op == 0xC7 && branch[1] == 0xF8) /* xbegin */,
"Invalid opcode at patch point");
if (op == 0xEB || (op & 0xF0) == 0x70) {
@@ -860,23 +860,6 @@ class MacroAssembler: public Assembler {
void cmpxchgptr(Register reg, Address adr);
-
- // cvt instructions
- void cvtss2sd(XMMRegister dst, XMMRegister src);
- void cvtss2sd(XMMRegister dst, Address src);
- void cvtsd2ss(XMMRegister dst, XMMRegister src);
- void cvtsd2ss(XMMRegister dst, Address src);
- void cvtsi2sdl(XMMRegister dst, Register src);
- void cvtsi2sdl(XMMRegister dst, Address src);
- void cvtsi2ssl(XMMRegister dst, Register src);
- void cvtsi2ssl(XMMRegister dst, Address src);
-#ifdef _LP64
- void cvtsi2sdq(XMMRegister dst, Register src);
- void cvtsi2sdq(XMMRegister dst, Address src);
- void cvtsi2ssq(XMMRegister dst, Register src);
- void cvtsi2ssq(XMMRegister dst, Address src);
-#endif
-
void locked_cmpxchgptr(Register reg, AddressLiteral adr, Register rscratch = noreg);
void imulptr(Register dst, Register src) { LP64_ONLY(imulq(dst, src)) NOT_LP64(imull(dst, src)); }
diff --git a/src/hotspot/cpu/x86/matcher_x86.hpp b/src/hotspot/cpu/x86/matcher_x86.hpp
index bc249c0f33a..de844c4be9f 100644
--- a/src/hotspot/cpu/x86/matcher_x86.hpp
+++ b/src/hotspot/cpu/x86/matcher_x86.hpp
@@ -248,4 +248,17 @@
}
}
+ // Is SIMD sort supported for this CPU?
+ static bool supports_simd_sort(BasicType bt) {
+ if (VM_Version::supports_avx512dq()) {
+ return true;
+ }
+ else if (VM_Version::supports_avx2() && !is_double_word_type(bt)) {
+ return true;
+ }
+ else {
+ return false;
+ }
+ }
+
#endif // CPU_X86_MATCHER_X86_HPP
diff --git a/src/hotspot/cpu/x86/nativeInst_x86.cpp b/src/hotspot/cpu/x86/nativeInst_x86.cpp
index 1df8c78c99d..b59f4246256 100644
--- a/src/hotspot/cpu/x86/nativeInst_x86.cpp
+++ b/src/hotspot/cpu/x86/nativeInst_x86.cpp
@@ -684,10 +684,15 @@ void NativePostCallNop::make_deopt() {
ICache::invalidate_range(instr_addr, instruction_size);
}
-void NativePostCallNop::patch(jint diff) {
- assert(diff != 0, "must be");
+bool NativePostCallNop::patch(int32_t oopmap_slot, int32_t cb_offset) {
+ if (((oopmap_slot & 0xff) != oopmap_slot) || ((cb_offset & 0xffffff) != cb_offset)) {
+ return false; // cannot encode
+ }
+ int32_t data = (oopmap_slot << 24) | cb_offset;
+ assert(data != 0, "must be");
int32_t *code_pos = (int32_t *) addr_at(displacement_offset);
- *((int32_t *)(code_pos)) = (int32_t) diff;
+ *((int32_t *)(code_pos)) = (int32_t) data;
+ return true; // successfully encoded
}
void NativeDeoptInstruction::verify() {
diff --git a/src/hotspot/cpu/x86/nativeInst_x86.hpp b/src/hotspot/cpu/x86/nativeInst_x86.hpp
index 938a10cb63d..f8cbf70f189 100644
--- a/src/hotspot/cpu/x86/nativeInst_x86.hpp
+++ b/src/hotspot/cpu/x86/nativeInst_x86.hpp
@@ -735,8 +735,16 @@ class NativePostCallNop: public NativeInstruction {
};
bool check() const { return int_at(0) == 0x841f0f; }
- int displacement() const { return (jint) int_at(displacement_offset); }
- void patch(jint diff);
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const {
+ int32_t data = int_at(displacement_offset);
+ if (data == 0) {
+ return false; // no information encoded
+ }
+ cb_offset = (data & 0xffffff);
+ oopmap_slot = (data >> 24) & 0xff;
+ return true; // decoding succeeded
+ }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset);
void make_deopt();
};
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
index 1d410a2f42b..298f652e971 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
@@ -528,8 +528,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
}
}
- // return value can be odd number of VMRegImpl stack slots make multiple of 2
- return align_up(stack, 2);
+ return stack;
}
const uint SharedRuntime::java_return_convention_max_int = 1;
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index f6da06f9415..3ff7f7cb676 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -499,7 +499,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
uint int_args = 0;
uint fp_args = 0;
- uint stk_args = 0; // inc by 2 each time
+ uint stk_args = 0;
for (int i = 0; i < total_args_passed; i++) {
switch (sig_bt[i]) {
@@ -511,8 +511,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_VOID:
@@ -529,6 +530,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (int_args < Argument::n_int_register_parameters_j) {
regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -537,8 +539,9 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set1(VMRegImpl::stack2reg(stk_args));
- stk_args += 2;
+ stk_args += 1;
}
break;
case T_DOUBLE:
@@ -546,6 +549,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
if (fp_args < Argument::n_float_register_parameters_j) {
regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
} else {
+ stk_args = align_up(stk_args, 2);
regs[i].set2(VMRegImpl::stack2reg(stk_args));
stk_args += 2;
}
@@ -556,7 +560,7 @@ int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
}
}
- return align_up(stk_args, 2);
+ return stk_args;
}
// Same as java_calling_convention() but for multiple return
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index 9363f369037..4adccff65cb 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2334,7 +2334,7 @@ address StubGenerator::generate_base64_decodeBlock() {
const Register isURL = c_rarg5;// Base64 or URL character set
__ movl(isMIME, Address(rbp, 2 * wordSize));
#else
- const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64
+ const Address dp_mem(rbp, 6 * wordSize); // length is on stack on Win64
const Address isURL_mem(rbp, 7 * wordSize);
const Register isURL = r10; // pick the volatile windows register
const Register dp = r12;
@@ -2556,10 +2556,12 @@ address StubGenerator::generate_base64_decodeBlock() {
// output_size in r13
// Strip pad characters, if any, and adjust length and mask
+ __ addq(length, start_offset);
__ cmpb(Address(source, length, Address::times_1, -1), '=');
__ jcc(Assembler::equal, L_padding);
__ BIND(L_donePadding);
+ __ subq(length, start_offset);
// Output size is (64 - output_size), output mask is (all 1s >> output_size).
__ kmovql(input_mask, rax);
@@ -4363,22 +4365,23 @@ void StubGenerator::generate_compiler_stubs() {
= CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
}
- // Load x86_64_sort library on supported hardware to enable avx512 sort and partition intrinsics
- if (VM_Version::is_intel() && VM_Version::supports_avx512dq()) {
+ // Load x86_64_sort library on supported hardware to enable SIMD sort and partition intrinsics
+
+ if (VM_Version::is_intel() && (VM_Version::supports_avx512dq() || VM_Version::supports_avx2())) {
void *libsimdsort = nullptr;
char ebuf_[1024];
char dll_name_simd_sort[JVM_MAXPATHLEN];
if (os::dll_locate_lib(dll_name_simd_sort, sizeof(dll_name_simd_sort), Arguments::get_dll_dir(), "simdsort")) {
libsimdsort = os::dll_load(dll_name_simd_sort, ebuf_, sizeof ebuf_);
}
- // Get addresses for avx512 sort and partition routines
+ // Get addresses for SIMD sort and partition routines
if (libsimdsort != nullptr) {
log_info(library)("Loaded library %s, handle " INTPTR_FORMAT, JNI_LIB_PREFIX "simdsort" JNI_LIB_SUFFIX, p2i(libsimdsort));
- snprintf(ebuf_, sizeof(ebuf_), "avx512_sort");
+ snprintf(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512dq() ? "avx512_sort" : "avx2_sort");
StubRoutines::_array_sort = (address)os::dll_lookup(libsimdsort, ebuf_);
- snprintf(ebuf_, sizeof(ebuf_), "avx512_partition");
+ snprintf(ebuf_, sizeof(ebuf_), VM_Version::supports_avx512dq() ? "avx512_partition" : "avx2_partition");
StubRoutines::_array_partition = (address)os::dll_lookup(libsimdsort, ebuf_);
}
}
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 3aba20498d9..f4d22aaf9ce 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -4339,9 +4339,14 @@ void TemplateTable::_new() {
// get InstanceKlass
__ load_resolved_klass_at_index(rcx, rcx, rdx);
- // make sure klass is initialized & doesn't have finalizer
+ // make sure klass is initialized
+#ifdef _LP64
+ assert(VM_Version::supports_fast_class_init_checks(), "must support fast class initialization checks");
+ __ clinit_barrier(rcx, r15_thread, nullptr /*L_fast_path*/, &slow_case);
+#else
__ cmpb(Address(rcx, InstanceKlass::init_state_offset()), InstanceKlass::fully_initialized);
__ jcc(Assembler::notEqual, slow_case);
+#endif
__ allocate_instance(rcx, rax, rdx, rbx, true, slow_case);
__ jmp(done);
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index c8e633eff25..a5e0ab72102 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -858,7 +858,7 @@ void VM_Version::get_processor_features() {
// Check if processor has Intel Ecore
if (FLAG_IS_DEFAULT(EnableX86ECoreOpts) && is_intel() && cpu_family() == 6 &&
- (_model == 0x97 || _model == 0xAC || _model == 0xAF)) {
+ (_model == 0x97 || _model == 0xAA || _model == 0xAC || _model == 0xAF)) {
FLAG_SET_DEFAULT(EnableX86ECoreOpts, true);
}
@@ -1130,6 +1130,7 @@ void VM_Version::get_processor_features() {
FLAG_SET_DEFAULT(UseGHASHIntrinsics, false);
}
+#ifdef _LP64
// ChaCha20 Intrinsics
// As long as the system supports AVX as a baseline we can do a
// SIMD-enabled block function. StubGenerator makes the determination
@@ -1145,6 +1146,13 @@ void VM_Version::get_processor_features() {
}
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
+#else
+ // No support currently for ChaCha20 intrinsics on 32-bit platforms
+ if (UseChaCha20Intrinsics) {
+ warning("ChaCha20 intrinsics are not available on this CPU.");
+ FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
+ }
+#endif // _LP64
// Base64 Intrinsics (Check the condition for which the intrinsic will be active)
if (UseAVX >= 2) {
@@ -1170,7 +1178,7 @@ void VM_Version::get_processor_features() {
UseMD5Intrinsics = true;
}
- if (supports_sha() LP64_ONLY(|| supports_avx2() && supports_bmi2())) {
+ if (supports_sha() LP64_ONLY(|| (supports_avx2() && supports_bmi2()))) {
if (FLAG_IS_DEFAULT(UseSHA)) {
UseSHA = true;
}
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index 454a8f31255..e521a6ee3bc 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -756,7 +756,7 @@ class VM_Version : public Abstract_VM_Version {
// the intrinsic for java.lang.Thread.onSpinWait()
static bool supports_on_spin_wait() { return supports_sse2(); }
- // x86_64 supports fast class initialization checks for static methods.
+ // x86_64 supports fast class initialization checks
static bool supports_fast_class_init_checks() {
return LP64_ONLY(true) NOT_LP64(false); // not implemented on x86_32
}
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index 72070625638..d5b03610282 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -2532,8 +2532,8 @@ static inline Assembler::ComparisonPredicateFP booltest_pred_to_comparison_pred_
static void vec_mov_helper(CodeBuffer *cbuf, int src_lo, int dst_lo,
int src_hi, int dst_hi, uint ireg, outputStream* st) {
assert(ireg == Op_VecS || // 32bit vector
- (src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
- (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi,
+ ((src_lo & 1) == 0 && (src_lo + 1) == src_hi &&
+ (dst_lo & 1) == 0 && (dst_lo + 1) == dst_hi),
"no non-adjacent vector moves" );
if (cbuf) {
C2_MacroAssembler _masm(cbuf);
@@ -4087,39 +4087,26 @@ instruct gather(legVec dst, memory mem, legVec idx, rRegP tmp, legVec mask) %{
effect(TEMP dst, TEMP tmp, TEMP mask);
format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and $mask as TEMP" %}
ins_encode %{
- assert(UseAVX >= 2, "sanity");
-
int vlen_enc = vector_length_encoding(this);
BasicType elem_bt = Matcher::vector_element_basic_type(this);
-
- assert(Matcher::vector_length_in_bytes(this) >= 16, "sanity");
assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
-
- if (vlen_enc == Assembler::AVX_128bit) {
- __ movdqu($mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), noreg);
- } else {
- __ vmovdqu($mask$$XMMRegister, ExternalAddress(vector_all_bits_set()), noreg);
- }
+ __ vpcmpeqd($mask$$XMMRegister, $mask$$XMMRegister, $mask$$XMMRegister, vlen_enc);
__ lea($tmp$$Register, $mem$$Address);
__ vgather(elem_bt, $dst$$XMMRegister, $tmp$$Register, $idx$$XMMRegister, $mask$$XMMRegister, vlen_enc);
%}
ins_pipe( pipe_slow );
%}
+
instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
match(Set dst (LoadVectorGather mem idx));
effect(TEMP dst, TEMP tmp, TEMP ktmp);
format %{ "load_vector_gather $dst, $mem, $idx\t! using $tmp and ktmp as TEMP" %}
ins_encode %{
- assert(UseAVX > 2, "sanity");
-
int vlen_enc = vector_length_encoding(this);
BasicType elem_bt = Matcher::vector_element_basic_type(this);
-
- assert(!is_subword_type(elem_bt), "sanity"); // T_INT, T_LONG, T_FLOAT, T_DOUBLE
-
- __ kmovwl($ktmp$$KRegister, ExternalAddress(vector_all_bits_set()), noreg);
+ __ kxnorwl($ktmp$$KRegister, $ktmp$$KRegister, $ktmp$$KRegister);
__ lea($tmp$$Register, $mem$$Address);
__ evgather(elem_bt, $dst$$XMMRegister, $ktmp$$KRegister, $tmp$$Register, $idx$$XMMRegister, vlen_enc);
%}
@@ -4127,6 +4114,7 @@ instruct evgather(vec dst, memory mem, vec idx, rRegP tmp, kReg ktmp) %{
%}
instruct evgather_masked(vec dst, memory mem, vec idx, kReg mask, kReg ktmp, rRegP tmp) %{
+ predicate(VM_Version::supports_avx512vl() || Matcher::vector_length_in_bytes(n) == 64);
match(Set dst (LoadVectorGatherMasked mem (Binary idx mask)));
effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp);
format %{ "load_vector_gather_masked $dst, $mem, $idx, $mask\t! using $tmp and ktmp as TEMP" %}
@@ -9011,6 +8999,21 @@ instruct vmasked_store_evex(memory mem, vec src, kReg mask) %{
%}
#ifdef _LP64
+instruct verify_vector_alignment(rRegP addr, immL32 mask, rFlagsReg cr) %{
+ match(Set addr (VerifyVectorAlignment addr mask));
+ effect(KILL cr);
+ format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
+ ins_encode %{
+ Label Lskip;
+ // check if masked bits of addr are zero
+ __ testq($addr$$Register, $mask$$constant);
+ __ jccb(Assembler::equal, Lskip);
+ __ stop("verify_vector_alignment found a misaligned vector memory access");
+ __ bind(Lskip);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct vmask_cmp_node(rRegI dst, vec src1, vec src2, kReg mask, kReg ktmp1, kReg ktmp2, rFlagsReg cr) %{
match(Set dst (VectorCmpMasked src1 (Binary src2 mask)));
effect(TEMP_DEF dst, TEMP ktmp1, TEMP ktmp2, KILL cr);
diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad
index 17b1aea0e51..5a3012f3d81 100644
--- a/src/hotspot/cpu/x86/x86_32.ad
+++ b/src/hotspot/cpu/x86/x86_32.ad
@@ -1,5 +1,5 @@
//
-// Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -11139,7 +11139,7 @@ instruct convI2FPR_mem(regFPR dst, memory mem) %{
// Convert an int to a float in xmm; no rounding step needed.
instruct convI2F_reg(regF dst, rRegI src) %{
- predicate( UseSSE==1 || UseSSE>=2 && !UseXmmI2F );
+ predicate( UseSSE==1 || ( UseSSE>=2 && !UseXmmI2F ));
match(Set dst (ConvI2F src));
format %{ "CVTSI2SS $dst, $src" %}
ins_encode %{
@@ -13158,7 +13158,7 @@ instruct cmovPP_reg_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, eRegP dst, eRe
// Compare 2 longs and CMOVE doubles
instruct cmovDDPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13168,7 +13168,7 @@ instruct cmovDDPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regDPR dst, regD
// Compare 2 longs and CMOVE doubles
instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13177,7 +13177,7 @@ instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src
%}
instruct cmovFFPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13186,7 +13186,7 @@ instruct cmovFFPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regFPR dst, regF
%}
instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
+ predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13349,7 +13349,7 @@ instruct cmovPP_reg_EQNE_U(cmpOpU cmp, flagsReg_ulong_EQNE flags, eRegP dst, eRe
// Compare 2 longs and CMOVE doubles
instruct cmovDDPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13359,7 +13359,7 @@ instruct cmovDDPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regDPR dst, regD
// Compare 2 longs and CMOVE doubles
instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13368,7 +13368,7 @@ instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src
%}
instruct cmovFFPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13377,7 +13377,7 @@ instruct cmovFFPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regFPR dst, regF
%}
instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
+ predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13568,7 +13568,7 @@ instruct cmovPP_reg_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, eRegP
// Compare 2 longs and CMOVE doubles
instruct cmovDDPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13578,7 +13578,7 @@ instruct cmovDDPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regDPR d
// Compare 2 longs and CMOVE doubles
instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13587,7 +13587,7 @@ instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst,
%}
instruct cmovFFPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
@@ -13597,7 +13597,7 @@ instruct cmovFFPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regFPR d
instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt );
+ predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
ins_cost(200);
expand %{
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index d1965f88d2d..19997636412 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -10139,7 +10139,7 @@ instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
instruct convF2D_reg_reg(regD dst, regF src)
%{
match(Set dst (ConvF2D src));
- effect(TEMP dst);
+
format %{ "cvtss2sd $dst, $src" %}
ins_encode %{
__ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
@@ -10161,7 +10161,7 @@ instruct convF2D_reg_mem(regD dst, memory src)
instruct convD2F_reg_reg(regF dst, regD src)
%{
match(Set dst (ConvD2F src));
- effect(TEMP dst);
+
format %{ "cvtsd2ss $dst, $src" %}
ins_encode %{
__ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
diff --git a/src/hotspot/cpu/zero/frame_zero.hpp b/src/hotspot/cpu/zero/frame_zero.hpp
index 87511ab212e..a4815d2c10b 100644
--- a/src/hotspot/cpu/zero/frame_zero.hpp
+++ b/src/hotspot/cpu/zero/frame_zero.hpp
@@ -44,8 +44,6 @@
align_wiggle = 1
};
- const ImmutableOopMap* get_oop_map() const;
-
// Constructor
public:
frame(ZeroFrame* zeroframe, intptr_t* sp);
diff --git a/src/hotspot/cpu/zero/frame_zero.inline.hpp b/src/hotspot/cpu/zero/frame_zero.inline.hpp
index 1b504cfa666..944084f70af 100644
--- a/src/hotspot/cpu/zero/frame_zero.inline.hpp
+++ b/src/hotspot/cpu/zero/frame_zero.inline.hpp
@@ -176,11 +176,6 @@ inline intptr_t* frame::unextended_sp() const {
return (intptr_t *) -1;
}
-inline const ImmutableOopMap* frame::get_oop_map() const {
- Unimplemented();
- return nullptr;
-}
-
inline int frame::compiled_frame_stack_argsize() const {
Unimplemented();
return 0;
diff --git a/src/hotspot/cpu/zero/globalDefinitions_zero.hpp b/src/hotspot/cpu/zero/globalDefinitions_zero.hpp
index 810f7de3cb3..0f78b3eeefb 100644
--- a/src/hotspot/cpu/zero/globalDefinitions_zero.hpp
+++ b/src/hotspot/cpu/zero/globalDefinitions_zero.hpp
@@ -30,8 +30,12 @@
#define SUPPORTS_NATIVE_CX8
#endif
+// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
+// The default padding size for data structures to avoid false sharing.
+#define DEFAULT_PADDING_SIZE DEFAULT_CACHE_LINE_SIZE
+
#define SUPPORT_MONITOR_COUNT
#include
diff --git a/src/hotspot/cpu/zero/nativeInst_zero.hpp b/src/hotspot/cpu/zero/nativeInst_zero.hpp
index 5b2c9743cf9..77a7d511ac5 100644
--- a/src/hotspot/cpu/zero/nativeInst_zero.hpp
+++ b/src/hotspot/cpu/zero/nativeInst_zero.hpp
@@ -214,8 +214,8 @@ inline NativeGeneralJump* nativeGeneralJump_at(address address) {
class NativePostCallNop: public NativeInstruction {
public:
bool check() const { Unimplemented(); return false; }
- int displacement() const { Unimplemented(); return 0; }
- void patch(jint diff) { Unimplemented(); }
+ bool decode(int32_t& oopmap_slot, int32_t& cb_offset) const { Unimplemented(); return false; }
+ bool patch(int32_t oopmap_slot, int32_t cb_offset) { Unimplemented(); return false; }
void make_deopt() { Unimplemented(); }
};
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index 54c538cf367..c75828c0c22 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -2112,11 +2112,6 @@ bool os::can_commit_large_page_memory() {
return false;
}
-bool os::can_execute_large_page_memory() {
- // Does not matter, we do not support huge pages.
- return false;
-}
-
char* os::pd_attempt_map_memory_to_file_at(char* requested_addr, size_t bytes, int file_desc) {
assert(file_desc >= 0, "file_desc is not valid");
char* result = nullptr;
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 18f3cd2b38d..2f7f31157e8 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -979,7 +979,7 @@ bool os::dll_address_to_library_name(address addr, char* buf,
// in case of error it checks if .dll/.so was built for the
// same architecture as Hotspot is running on
-void *os::Bsd::dlopen_helper(const char *filename, int mode) {
+void *os::Bsd::dlopen_helper(const char *filename, int mode, char *ebuf, int ebuflen) {
#ifndef IA32
bool ieee_handling = IEEE_subnormal_handling_OK();
if (!ieee_handling) {
@@ -1005,27 +1005,44 @@ void *os::Bsd::dlopen_helper(const char *filename, int mode) {
assert(rtn == 0, "fegetenv must succeed");
#endif // IA32
- void * result= ::dlopen(filename, RTLD_LAZY);
-
+ void* result;
+ JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
+ result = ::dlopen(filename, RTLD_LAZY);
+ if (result == nullptr) {
+ const char* error_report = ::dlerror();
+ if (error_report == nullptr) {
+ error_report = "dlerror returned no error description";
+ }
+ if (ebuf != nullptr && ebuflen > 0) {
+ ::strncpy(ebuf, error_report, ebuflen-1);
+ ebuf[ebuflen-1]='\0';
+ }
+ Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
+ log_info(os)("shared library load of %s failed, %s", filename, error_report);
+ JFR_ONLY(load_event.set_error_msg(error_report);)
+ } else {
+ Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
+ log_info(os)("shared library load of %s was successful", filename);
#ifndef IA32
- if (result != nullptr && ! IEEE_subnormal_handling_OK()) {
- // We just dlopen()ed a library that mangled the floating-point
- // flags. Silently fix things now.
- int rtn = fesetenv(&default_fenv);
- assert(rtn == 0, "fesetenv must succeed");
- bool ieee_handling_after_issue = IEEE_subnormal_handling_OK();
-
- if (ieee_handling_after_issue) {
- Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
- log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
- } else {
- Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
- log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+ if (! IEEE_subnormal_handling_OK()) {
+ // We just dlopen()ed a library that mangled the floating-point
+ // flags. Silently fix things now.
+ JFR_ONLY(load_event.set_fp_env_correction_attempt(true);)
+ int rtn = fesetenv(&default_fenv);
+ assert(rtn == 0, "fesetenv must succeed");
+
+ if (IEEE_subnormal_handling_OK()) {
+ Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
+ log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
+ JFR_ONLY(load_event.set_fp_env_correction_success(true);)
+ } else {
+ Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
+ log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+ assert(false, "fesetenv didn't work");
+ }
}
-
- assert(ieee_handling_after_issue, "fesetenv didn't work");
- }
#endif // IA32
+ }
return result;
}
@@ -1037,30 +1054,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
#else
log_info(os)("attempting shared library load of %s", filename);
- void* result;
- JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
- result = os::Bsd::dlopen_helper(filename, RTLD_LAZY);
- if (result != nullptr) {
- Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
- // Successful loading
- log_info(os)("shared library load of %s was successful", filename);
- return result;
- }
-
- const char* error_report = ::dlerror();
- if (error_report == nullptr) {
- error_report = "dlerror returned no error description";
- }
- if (ebuf != nullptr && ebuflen > 0) {
- // Read system error message into ebuf
- ::strncpy(ebuf, error_report, ebuflen-1);
- ebuf[ebuflen-1]='\0';
- }
- Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
- log_info(os)("shared library load of %s failed, %s", filename, error_report);
- JFR_ONLY(load_event.set_error_msg(error_report);)
-
- return nullptr;
+ return os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
#endif // STATIC_BUILD
}
#else
@@ -1071,29 +1065,13 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
log_info(os)("attempting shared library load of %s", filename);
void* result;
- JFR_ONLY(NativeLibraryLoadEvent load_event(filename, &result);)
- result = os::Bsd::dlopen_helper(filename, RTLD_LAZY);
+ result = os::Bsd::dlopen_helper(filename, RTLD_LAZY, ebuf, ebuflen);
if (result != nullptr) {
- Events::log_dll_message(nullptr, "Loaded shared library %s", filename);
- // Successful loading
- log_info(os)("shared library load of %s was successful", filename);
return result;
}
- Elf32_Ehdr elf_head;
-
- const char* const error_report = ::dlerror();
- if (error_report == nullptr) {
- error_report = "dlerror returned no error description";
- }
- if (ebuf != nullptr && ebuflen > 0) {
- // Read system error message into ebuf
- ::strncpy(ebuf, error_report, ebuflen-1);
- ebuf[ebuflen-1]='\0';
- }
Events::log_dll_message(nullptr, "Loading shared library %s failed, %s", filename, error_report);
log_info(os)("shared library load of %s failed, %s", filename, error_report);
- JFR_ONLY(load_event.set_error_msg(error_report);)
int diag_msg_max_length=ebuflen-strlen(ebuf);
char* diag_msg_buf=ebuf+strlen(ebuf);
@@ -1102,7 +1080,6 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
return nullptr;
}
-
int file_descriptor= ::open(filename, O_RDONLY | O_NONBLOCK);
if (file_descriptor < 0) {
@@ -1110,6 +1087,7 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
return nullptr;
}
+ Elf32_Ehdr elf_head;
bool failed_to_read_elf_head=
(sizeof(elf_head)!=
(::read(file_descriptor, &elf_head,sizeof(elf_head))));
@@ -1809,11 +1787,6 @@ bool os::can_commit_large_page_memory() {
return false;
}
-bool os::can_execute_large_page_memory() {
- // Does not matter, we do not support huge pages.
- return false;
-}
-
char* os::pd_attempt_map_memory_to_file_at(char* requested_addr, size_t bytes, int file_desc) {
assert(file_desc >= 0, "file_desc is not valid");
char* result = pd_attempt_reserve_memory_at(requested_addr, bytes, !ExecMem);
diff --git a/src/hotspot/os/bsd/os_bsd.hpp b/src/hotspot/os/bsd/os_bsd.hpp
index f79212bc43c..72de9ca5971 100644
--- a/src/hotspot/os/bsd/os_bsd.hpp
+++ b/src/hotspot/os/bsd/os_bsd.hpp
@@ -70,7 +70,7 @@ class os::Bsd {
// Real-time clock functions
static void clock_init(void);
- static void *dlopen_helper(const char *path, int mode);
+ static void *dlopen_helper(const char *path, int mode, char *ebuf, int ebuflen);
// Stack repair handling
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
index 6c5470445f1..3cc2141f176 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
@@ -266,6 +266,8 @@ class CgroupSubsystem: public CHeapObj {
virtual jlong memory_and_swap_limit_in_bytes() = 0;
virtual jlong memory_soft_limit_in_bytes() = 0;
virtual jlong memory_max_usage_in_bytes() = 0;
+ virtual jlong rss_usage_in_bytes() = 0;
+ virtual jlong cache_usage_in_bytes() = 0;
virtual char * cpu_cpuset_cpus() = 0;
virtual char * cpu_cpuset_memory_nodes() = 0;
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
index 146d7237a90..37a1f476bde 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
@@ -214,6 +214,17 @@ jlong CgroupV1Subsystem::memory_max_usage_in_bytes() {
return memmaxusage;
}
+jlong CgroupV1Subsystem::rss_usage_in_bytes() {
+ GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat",
+ "rss", JULONG_FORMAT, JULONG_FORMAT, rss);
+ return rss;
+}
+
+jlong CgroupV1Subsystem::cache_usage_in_bytes() {
+ GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat",
+ "cache", JULONG_FORMAT, JULONG_FORMAT, cache);
+ return cache;
+}
jlong CgroupV1Subsystem::kernel_memory_usage_in_bytes() {
GET_CONTAINER_INFO(jlong, _memory->controller(), "/memory.kmem.usage_in_bytes",
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
index fae65da2a58..c7550136f48 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.hpp
@@ -79,6 +79,8 @@ class CgroupV1Subsystem: public CgroupSubsystem {
jlong memory_soft_limit_in_bytes();
jlong memory_usage_in_bytes();
jlong memory_max_usage_in_bytes();
+ jlong rss_usage_in_bytes();
+ jlong cache_usage_in_bytes();
jlong kernel_memory_usage_in_bytes();
jlong kernel_memory_limit_in_bytes();
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
index 1a02bbe95d2..1c433be1c23 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
@@ -139,6 +139,18 @@ jlong CgroupV2Subsystem::memory_max_usage_in_bytes() {
return OSCONTAINER_ERROR; // not supported
}
+jlong CgroupV2Subsystem::rss_usage_in_bytes() {
+ GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat",
+ "anon", JULONG_FORMAT, JULONG_FORMAT, rss);
+ return rss;
+}
+
+jlong CgroupV2Subsystem::cache_usage_in_bytes() {
+ GET_CONTAINER_INFO_LINE(julong, _memory->controller(), "/memory.stat",
+ "file", JULONG_FORMAT, JULONG_FORMAT, cache);
+ return cache;
+}
+
char* CgroupV2Subsystem::mem_soft_limit_val() {
GET_CONTAINER_INFO_CPTR(cptr, _unified, "/memory.low",
"Memory Soft Limit is: %s", "%1023s", mem_soft_limit_str, 1024);
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
index bb6b538c216..b12b4ce6512 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.hpp
@@ -78,6 +78,8 @@ class CgroupV2Subsystem: public CgroupSubsystem {
jlong memory_soft_limit_in_bytes();
jlong memory_usage_in_bytes();
jlong memory_max_usage_in_bytes();
+ jlong rss_usage_in_bytes();
+ jlong cache_usage_in_bytes();
char * cpu_cpuset_cpus();
char * cpu_cpuset_memory_nodes();
diff --git a/src/hotspot/os/linux/osContainer_linux.cpp b/src/hotspot/os/linux/osContainer_linux.cpp
index c70c96c678a..14bfa5a7678 100644
--- a/src/hotspot/os/linux/osContainer_linux.cpp
+++ b/src/hotspot/os/linux/osContainer_linux.cpp
@@ -92,6 +92,16 @@ jlong OSContainer::memory_max_usage_in_bytes() {
return cgroup_subsystem->memory_max_usage_in_bytes();
}
+jlong OSContainer::rss_usage_in_bytes() {
+ assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
+ return cgroup_subsystem->rss_usage_in_bytes();
+}
+
+jlong OSContainer::cache_usage_in_bytes() {
+ assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
+ return cgroup_subsystem->cache_usage_in_bytes();
+}
+
void OSContainer::print_version_specific_info(outputStream* st) {
assert(cgroup_subsystem != nullptr, "cgroup subsystem not available");
cgroup_subsystem->print_version_specific_info(st);
diff --git a/src/hotspot/os/linux/osContainer_linux.hpp b/src/hotspot/os/linux/osContainer_linux.hpp
index 776167993cb..bb03ba3b005 100644
--- a/src/hotspot/os/linux/osContainer_linux.hpp
+++ b/src/hotspot/os/linux/osContainer_linux.hpp
@@ -55,6 +55,8 @@ class OSContainer: AllStatic {
static jlong memory_soft_limit_in_bytes();
static jlong memory_usage_in_bytes();
static jlong memory_max_usage_in_bytes();
+ static jlong rss_usage_in_bytes();
+ static jlong cache_usage_in_bytes();
static int active_processor_count();
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 3fc025e5100..3a170edd697 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2015, 2022 SAP SE. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1856,18 +1856,19 @@ void * os::Linux::dlopen_helper(const char *filename, char *ebuf, int ebuflen) {
if (! IEEE_subnormal_handling_OK()) {
// We just dlopen()ed a library that mangled the floating-point flags.
// Attempt to fix things now.
+ JFR_ONLY(load_event.set_fp_env_correction_attempt(true);)
int rtn = fesetenv(&default_fenv);
assert(rtn == 0, "fesetenv must succeed");
- bool ieee_handling_after_issue = IEEE_subnormal_handling_OK();
- if (ieee_handling_after_issue) {
+ if (IEEE_subnormal_handling_OK()) {
Events::log_dll_message(nullptr, "IEEE subnormal handling had to be corrected after loading %s", filename);
log_info(os)("IEEE subnormal handling had to be corrected after loading %s", filename);
+ JFR_ONLY(load_event.set_fp_env_correction_success(true);)
} else {
Events::log_dll_message(nullptr, "IEEE subnormal handling could not be corrected after loading %s", filename);
log_info(os)("IEEE subnormal handling could not be corrected after loading %s", filename);
+ assert(false, "fesetenv didn't work");
}
- assert(ieee_handling_after_issue, "fesetenv didn't work");
}
#endif // IA32
}
@@ -2096,7 +2097,6 @@ const char* distro_files[] = {
"/etc/mandrake-release",
"/etc/sun-release",
"/etc/redhat-release",
- "/etc/SuSE-release",
"/etc/lsb-release",
"/etc/turbolinux-release",
"/etc/gentoo-release",
@@ -2104,6 +2104,7 @@ const char* distro_files[] = {
"/etc/angstrom-version",
"/etc/system-release",
"/etc/os-release",
+ "/etc/SuSE-release", // Deprecated in favor of os-release since SuSE 12
nullptr };
void os::Linux::print_distro_info(outputStream* st) {
@@ -2215,6 +2216,8 @@ void os::Linux::print_system_memory_info(outputStream* st) {
// https://www.kernel.org/doc/Documentation/vm/transhuge.txt
_print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/enabled",
"/sys/kernel/mm/transparent_hugepage/enabled", st);
+ _print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/hpage_pmd_size",
+ "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", st);
_print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/shmem_enabled",
"/sys/kernel/mm/transparent_hugepage/shmem_enabled", st);
_print_ascii_file_h("/sys/kernel/mm/transparent_hugepage/defrag (defrag/compaction efforts parameter)",
@@ -2395,6 +2398,8 @@ bool os::Linux::print_container_info(outputStream* st) {
OSContainer::print_container_helper(st, OSContainer::memory_soft_limit_in_bytes(), "memory_soft_limit_in_bytes");
OSContainer::print_container_helper(st, OSContainer::memory_usage_in_bytes(), "memory_usage_in_bytes");
OSContainer::print_container_helper(st, OSContainer::memory_max_usage_in_bytes(), "memory_max_usage_in_bytes");
+ OSContainer::print_container_helper(st, OSContainer::rss_usage_in_bytes(), "rss_usage_in_bytes");
+ OSContainer::print_container_helper(st, OSContainer::cache_usage_in_bytes(), "cache_usage_in_bytes");
OSContainer::print_version_specific_info(st);
@@ -4075,10 +4080,6 @@ bool os::can_commit_large_page_memory() {
return UseTransparentHugePages;
}
-bool os::can_execute_large_page_memory() {
- return UseTransparentHugePages;
-}
-
char* os::pd_attempt_map_memory_to_file_at(char* requested_addr, size_t bytes, int file_desc) {
assert(file_desc >= 0, "file_desc is not valid");
char* result = pd_attempt_reserve_memory_at(requested_addr, bytes, !ExecMem);
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 223bc0bd135..1b5c7850bf2 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -3436,10 +3436,6 @@ bool os::can_commit_large_page_memory() {
return false;
}
-bool os::can_execute_large_page_memory() {
- return true;
-}
-
static char* reserve_large_pages_individually(size_t size, char* req_addr, bool exec) {
log_debug(pagesize)("Reserving large pages individually.");
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index df5a42fd118..fbd7c4eccd4 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -514,7 +514,39 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
extern "C" {
int SpinPause() {
- return 0;
+ // We don't use StubRoutines::aarch64::spin_wait stub in order to
+ // avoid a costly call to os::current_thread_enable_wx() on MacOS.
+ // We should return 1 if SpinPause is implemented, and since there
+ // will be a sequence of 11 instructions for NONE and YIELD and 12
+ // instructions for NOP and ISB, SpinPause will always return 1.
+ uint64_t br_dst;
+ const int instructions_per_case = 2;
+ int64_t off = VM_Version::spin_wait_desc().inst() * instructions_per_case * Assembler::instruction_size;
+
+ assert(VM_Version::spin_wait_desc().inst() >= SpinWait::NONE &&
+ VM_Version::spin_wait_desc().inst() <= SpinWait::YIELD, "must be");
+ assert(-1 == SpinWait::NONE, "must be");
+ assert( 0 == SpinWait::NOP, "must be");
+ assert( 1 == SpinWait::ISB, "must be");
+ assert( 2 == SpinWait::YIELD, "must be");
+
+ asm volatile(
+ " adr %[d], 20 \n" // 20 == PC here + 5 instructions => address
+ // to entry for case SpinWait::NOP
+ " add %[d], %[d], %[o] \n"
+ " br %[d] \n"
+ " b SpinPause_return \n" // case SpinWait::NONE (-1)
+ " nop \n" // padding
+ " nop \n" // case SpinWait::NOP ( 0)
+ " b SpinPause_return \n"
+ " isb \n" // case SpinWait::ISB ( 1)
+ " b SpinPause_return \n"
+ " yield \n" // case SpinWait::YIELD ( 2)
+ "SpinPause_return: \n"
+ : [d]"=&r"(br_dst)
+ : [o]"r"(off)
+ : "memory");
+ return 1;
}
void _Copy_conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
index 8b9d1178ca5..282467bc9e0 100644
--- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
@@ -232,7 +232,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
CompiledMethod* nm = (cb != nullptr) ? cb->as_compiled_method_or_null() : nullptr;
bool is_unsafe_arraycopy = (thread->doing_unsafe_access() && UnsafeCopyMemory::contains_pc(pc));
if ((nm != nullptr && nm->has_unsafe_access()) || is_unsafe_arraycopy) {
- address next_pc = pc + NativeCall::instruction_size;
+ address next_pc = Assembler::locate_next_instruction(pc);
if (is_unsafe_arraycopy) {
next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
}
@@ -271,7 +271,7 @@ bool PosixSignals::pd_hotspot_signal_handler(int sig, siginfo_t* info,
thread->thread_state() == _thread_in_native) &&
sig == SIGBUS && /* info->si_code == BUS_OBJERR && */
thread->doing_unsafe_access()) {
- address next_pc = pc + NativeCall::instruction_size;
+ address next_pc = Assembler::locate_next_instruction(pc);
if (UnsafeCopyMemory::contains_pc(pc)) {
next_pc = UnsafeCopyMemory::page_error_continue_pc(pc);
}
diff --git a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
index 6e93406b1a3..354dbd70bb4 100644
--- a/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/vm_version_linux_riscv.cpp
@@ -240,6 +240,8 @@ void VM_Version::rivos_features() {
ext_Zbb.enable_feature();
ext_Zbs.enable_feature();
+ ext_Zcb.enable_feature();
+
ext_Zicsr.enable_feature();
ext_Zifencei.enable_feature();
ext_Zic64b.enable_feature();
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index 16e3287aa0a..d2aa573cecd 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -794,6 +794,7 @@ bool InstructForm::captures_bottom_type(FormDict &globals) const {
!strcmp(_matrule->_rChild->_opType,"StrInflatedCopy") ||
!strcmp(_matrule->_rChild->_opType,"VectorCmpMasked")||
!strcmp(_matrule->_rChild->_opType,"VectorMaskGen")||
+ !strcmp(_matrule->_rChild->_opType,"VerifyVectorAlignment")||
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeP") ||
!strcmp(_matrule->_rChild->_opType,"CompareAndExchangeN"))) return true;
else if ( is_ideal_load() == Form::idealP ) return true;
diff --git a/src/hotspot/share/c1/c1_Compilation.cpp b/src/hotspot/share/c1/c1_Compilation.cpp
index 1fac664ef1d..d94de804305 100644
--- a/src/hotspot/share/c1/c1_Compilation.cpp
+++ b/src/hotspot/share/c1/c1_Compilation.cpp
@@ -33,10 +33,12 @@
#include "c1/c1_ValueMap.hpp"
#include "c1/c1_ValueStack.hpp"
#include "code/debugInfoRec.hpp"
+#include "compiler/compilationFailureInfo.hpp"
#include "compiler/compilationMemoryStatistic.hpp"
#include "compiler/compilerDirectives.hpp"
#include "compiler/compileLog.hpp"
#include "compiler/compileTask.hpp"
+#include "compiler/compiler_globals.hpp"
#include "compiler/compilerDirectives.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -76,7 +78,6 @@ static const char * timer_name[] = {
};
static elapsedTimer timers[max_phase_timers];
-static uint totalInstructionNodes = 0;
class PhaseTraceTime: public TraceTime {
private:
@@ -389,10 +390,6 @@ int Compilation::compile_java_method() {
BAILOUT_("mdo allocation failed", no_frame_size);
}
- if (method()->is_synchronized()) {
- set_has_monitors(true);
- }
-
{
PhaseTraceTime timeit(_t_buildIR);
build_hir();
@@ -494,7 +491,6 @@ void Compilation::compile_method() {
if (log() != nullptr) // Print code cache state into compiler log
log()->code_cache_state();
- totalInstructionNodes += Instruction::number_of_instructions();
}
@@ -581,9 +577,10 @@ Compilation::Compilation(AbstractCompiler* compiler, ciEnv* env, ciMethod* metho
, _would_profile(false)
, _has_method_handle_invokes(false)
, _has_reserved_stack_access(method->has_reserved_stack_access())
-, _has_monitors(false)
+, _has_monitors(method->is_synchronized() || method->has_monitor_bytecodes())
, _install_code(install_code)
, _bailout_msg(nullptr)
+, _first_failure_details(nullptr)
, _exception_info_list(nullptr)
, _allocator(nullptr)
, _code(buffer_blob)
@@ -634,7 +631,7 @@ Compilation::Compilation(AbstractCompiler* compiler, ciEnv* env, ciMethod* metho
Compilation::~Compilation() {
// simulate crash during compilation
assert(CICrashAt < 0 || (uintx)_env->compile_id() != (uintx)CICrashAt, "just as planned");
-
+ delete _first_failure_details;
_env->set_compiler_data(nullptr);
}
@@ -660,6 +657,9 @@ void Compilation::bailout(const char* msg) {
// keep first bailout message
if (PrintCompilation || PrintBailouts) tty->print_cr("compilation bailout: %s", msg);
_bailout_msg = msg;
+ if (CaptureBailoutInformation) {
+ _first_failure_details = new CompilationFailureInfo(msg);
+ }
}
}
diff --git a/src/hotspot/share/c1/c1_Compilation.hpp b/src/hotspot/share/c1/c1_Compilation.hpp
index e3fe224270d..82b28bd69ae 100644
--- a/src/hotspot/share/c1/c1_Compilation.hpp
+++ b/src/hotspot/share/c1/c1_Compilation.hpp
@@ -35,6 +35,7 @@
#include "runtime/deoptimization.hpp"
#include "runtime/sharedRuntime.hpp"
+class CompilationFailureInfo;
class CompilationResourceObj;
class XHandlers;
class ExceptionInfo;
@@ -86,6 +87,7 @@ class Compilation: public StackObj {
bool _has_monitors; // Fastpath monitors detection for Continuations
bool _install_code;
const char* _bailout_msg;
+ CompilationFailureInfo* _first_failure_details; // Details for the first failure happening during compilation
bool _oom;
ExceptionInfoList* _exception_info_list;
ExceptionHandlerTable _exception_handler_table;
@@ -214,6 +216,7 @@ class Compilation: public StackObj {
void bailout(const char* msg);
bool bailed_out() const { return _bailout_msg != nullptr; }
const char* bailout_msg() const { return _bailout_msg; }
+ const CompilationFailureInfo* first_failure_details() const { return _first_failure_details; }
static uint desired_max_code_buffer_size() {
return (uint)NMethodSizeLimit; // default 64K
diff --git a/src/hotspot/share/c1/c1_FrameMap.cpp b/src/hotspot/share/c1/c1_FrameMap.cpp
index d889fa2b769..fe70c0b31f6 100644
--- a/src/hotspot/share/c1/c1_FrameMap.cpp
+++ b/src/hotspot/share/c1/c1_FrameMap.cpp
@@ -72,7 +72,7 @@ CallingConvention* FrameMap::java_calling_convention(const BasicTypeArray* signa
}
}
- intptr_t out_preserve = SharedRuntime::java_calling_convention(sig_bt, regs, sizeargs);
+ intptr_t out_preserve = align_up(SharedRuntime::java_calling_convention(sig_bt, regs, sizeargs), 2);
LIR_OprList* args = new LIR_OprList(signature->length());
for (i = 0; i < sizeargs;) {
BasicType t = sig_bt[i];
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index acaf9cb60b2..00d03e8ef7e 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1416,8 +1416,8 @@ void GraphBuilder::if_node(Value x, If::Condition cond, Value y, ValueStack* sta
Instruction *i = append(new If(x, cond, false, y, tsux, fsux, (is_bb || compilation()->is_optimistic() || subst_check) ? state_before : nullptr, is_bb, subst_check));
assert(i->as_Goto() == nullptr ||
- (i->as_Goto()->sux_at(0) == tsux && i->as_Goto()->is_safepoint() == tsux->bci() < stream()->cur_bci()) ||
- (i->as_Goto()->sux_at(0) == fsux && i->as_Goto()->is_safepoint() == fsux->bci() < stream()->cur_bci()),
+ (i->as_Goto()->sux_at(0) == tsux && i->as_Goto()->is_safepoint() == (tsux->bci() < stream()->cur_bci())) ||
+ (i->as_Goto()->sux_at(0) == fsux && i->as_Goto()->is_safepoint() == (fsux->bci() < stream()->cur_bci())),
"safepoint state of Goto returned by canonicalizer incorrect");
if (is_profiling()) {
@@ -1552,7 +1552,7 @@ void GraphBuilder::table_switch() {
if (res->as_Goto()) {
for (i = 0; i < l; i++) {
if (sux->at(i) == res->as_Goto()->sux_at(0)) {
- assert(res->as_Goto()->is_safepoint() == sw.dest_offset_at(i) < 0, "safepoint state of Goto returned by canonicalizer incorrect");
+ assert(res->as_Goto()->is_safepoint() == (sw.dest_offset_at(i) < 0), "safepoint state of Goto returned by canonicalizer incorrect");
}
}
}
@@ -1601,7 +1601,7 @@ void GraphBuilder::lookup_switch() {
if (res->as_Goto()) {
for (i = 0; i < l; i++) {
if (sux->at(i) == res->as_Goto()->sux_at(0)) {
- assert(res->as_Goto()->is_safepoint() == sw.pair_at(i).offset() < 0, "safepoint state of Goto returned by canonicalizer incorrect");
+ assert(res->as_Goto()->is_safepoint() == (sw.pair_at(i).offset() < 0), "safepoint state of Goto returned by canonicalizer incorrect");
}
}
}
@@ -2580,7 +2580,6 @@ void GraphBuilder::monitorenter(Value x, int bci) {
// save state before locking in case of deoptimization after a NullPointerException
ValueStack* state_before = copy_state_for_exception_with_bci(bci);
- compilation()->set_has_monitors(true);
append_with_bci(new MonitorEnter(x, state()->lock(x), state_before, maybe_inlinetype), bci);
kill_all();
}
@@ -3780,6 +3779,15 @@ int GraphBuilder::recursive_inline_level(ciMethod* cur_callee) const {
return recur_level;
}
+static void set_flags_for_inlined_callee(Compilation* compilation, ciMethod* callee) {
+ if (callee->has_reserved_stack_access()) {
+ compilation->set_has_reserved_stack_access(true);
+ }
+ if (callee->is_synchronized() || callee->has_monitor_bytecodes()) {
+ compilation->set_has_monitors(true);
+ }
+}
+
bool GraphBuilder::try_inline(ciMethod* callee, bool holder_known, bool ignore_return, Bytecodes::Code bc, Value receiver) {
const char* msg = nullptr;
@@ -3796,9 +3804,7 @@ bool GraphBuilder::try_inline(ciMethod* callee, bool holder_known, bool ignore_r
// method handle invokes
if (callee->is_method_handle_intrinsic()) {
if (try_method_handle_inline(callee, ignore_return)) {
- if (callee->has_reserved_stack_access()) {
- compilation()->set_has_reserved_stack_access(true);
- }
+ set_flags_for_inlined_callee(compilation(), callee);
return true;
}
return false;
@@ -3809,9 +3815,7 @@ bool GraphBuilder::try_inline(ciMethod* callee, bool holder_known, bool ignore_r
callee->check_intrinsic_candidate()) {
if (try_inline_intrinsics(callee, ignore_return)) {
print_inlining(callee, "intrinsic");
- if (callee->has_reserved_stack_access()) {
- compilation()->set_has_reserved_stack_access(true);
- }
+ set_flags_for_inlined_callee(compilation(), callee);
return true;
}
// try normal inlining
@@ -3829,9 +3833,7 @@ bool GraphBuilder::try_inline(ciMethod* callee, bool holder_known, bool ignore_r
bc = code();
}
if (try_inline_full(callee, holder_known, ignore_return, bc, receiver)) {
- if (callee->has_reserved_stack_access()) {
- compilation()->set_has_reserved_stack_access(true);
- }
+ set_flags_for_inlined_callee(compilation(), callee);
return true;
}
@@ -4738,7 +4740,9 @@ void GraphBuilder::append_unsafe_get_and_set(ciMethod* callee, bool is_add) {
#ifndef PRODUCT
void GraphBuilder::print_stats() {
- vmap()->print();
+ if (UseLocalValueNumbering) {
+ vmap()->print();
+ }
}
#endif // PRODUCT
diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp
index 4430720a2f5..30e25f36479 100644
--- a/src/hotspot/share/c1/c1_LinearScan.cpp
+++ b/src/hotspot/share/c1/c1_LinearScan.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -4857,8 +4857,8 @@ void IntervalWalker::next_interval() {
// intervals may start at same position -> prefer fixed interval
kind = fixed != Interval::end() && fixed->from() <= any->from() ? fixedKind : anyKind;
- assert (kind == fixedKind && fixed->from() <= any->from() ||
- kind == anyKind && any->from() <= fixed->from(), "wrong interval!!!");
+ assert((kind == fixedKind && fixed->from() <= any->from()) ||
+ (kind == anyKind && any->from() <= fixed->from()), "wrong interval!!!");
assert(any == Interval::end() || fixed == Interval::end() || any->from() != fixed->from() || kind == fixedKind, "if fixed and any-Interval start at same position, fixed must be processed first");
} else if (fixed != Interval::end()) {
diff --git a/src/hotspot/share/c1/c1_ValueStack.hpp b/src/hotspot/share/c1/c1_ValueStack.hpp
index c120ca6b6e8..c89ddfda7ab 100644
--- a/src/hotspot/share/c1/c1_ValueStack.hpp
+++ b/src/hotspot/share/c1/c1_ValueStack.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -61,7 +61,7 @@ class ValueStack: public CompilationResourceObj {
Values* _locks; // the monitor stack (holding the locked values)
Value check(ValueTag tag, Value t) {
- assert(tag == t->type()->tag() || tag == objectTag && t->type()->tag() == addressTag, "types must correspond");
+ assert(tag == t->type()->tag() || (tag == objectTag && t->type()->tag() == addressTag), "types must correspond");
return t;
}
diff --git a/src/hotspot/share/c1/c1_globals.hpp b/src/hotspot/share/c1/c1_globals.hpp
index 08e9610b2a6..058091b3c43 100644
--- a/src/hotspot/share/c1/c1_globals.hpp
+++ b/src/hotspot/share/c1/c1_globals.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -275,9 +275,11 @@
develop(bool, InstallMethods, true, \
"Install methods at the end of successful compilations") \
\
+ /* The compiler assumes, in many places, that methods are at most 1MB. */ \
+ /* Therefore, we restrict this flag to at most 1MB. */ \
develop(intx, NMethodSizeLimit, (64*K)*wordSize, \
"Maximum size of a compiled method.") \
- range(0, max_jint) \
+ range(0, 1*M) \
\
develop(bool, TraceFPUStack, false, \
"Trace emulation of the FPU stack (intel only)") \
diff --git a/src/hotspot/share/cds/cdsConfig.hpp b/src/hotspot/share/cds/cdsConfig.hpp
index 9193c9f9d15..ba3883f3db5 100644
--- a/src/hotspot/share/cds/cdsConfig.hpp
+++ b/src/hotspot/share/cds/cdsConfig.hpp
@@ -55,7 +55,7 @@ class CDSConfig : public AllStatic {
static void initialize() NOT_CDS_RETURN;
static void check_system_property(const char* key, const char* value) NOT_CDS_RETURN;
static void check_unsupported_dumping_properties() NOT_CDS_RETURN;
- static bool check_vm_args_consistency(bool mode_flag_cmd_line) NOT_CDS_RETURN_(false);
+ static bool check_vm_args_consistency(bool mode_flag_cmd_line) NOT_CDS_RETURN_(true);
static bool module_patching_disables_cds() { return _module_patching_disables_cds; }
static void set_module_patching_disables_cds() { _module_patching_disables_cds = true; }
diff --git a/src/hotspot/share/cds/cdsHeapVerifier.cpp b/src/hotspot/share/cds/cdsHeapVerifier.cpp
index b687ad566b3..29e5af97c6a 100644
--- a/src/hotspot/share/cds/cdsHeapVerifier.cpp
+++ b/src/hotspot/share/cds/cdsHeapVerifier.cpp
@@ -92,11 +92,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
//
// class field type
ADD_EXCL("java/lang/ClassLoader", "scl"); // A
- ADD_EXCL("java/lang/invoke/InvokerBytecodeGenerator", "DONTINLINE_SIG", // B
- "FORCEINLINE_SIG", // B
- "HIDDEN_SIG", // B
- "INJECTEDPROFILE_SIG", // B
- "LF_COMPILED_SIG"); // B
ADD_EXCL("java/lang/Module", "ALL_UNNAMED_MODULE", // A
"ALL_UNNAMED_MODULE_SET", // A
"EVERYONE_MODULE", // A
@@ -106,10 +101,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
ADD_EXCL("java/lang/reflect/AccessFlag$Location", "EMPTY_SET"); // E
ADD_EXCL("java/lang/System", "bootLayer"); // A
- ADD_EXCL("java/lang/VersionProps", "VENDOR_URL_BUG", // C
- "VENDOR_URL_VM_BUG", // C
- "VENDOR_VERSION"); // C
- ADD_EXCL("java/net/URL$DefaultFactory", "PREFIX"); // B FIXME: JDK-8276561
// A dummy object used by HashSet. The value doesn't matter and it's never
// tested for equality.
@@ -118,7 +109,6 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
ADD_EXCL("jdk/internal/loader/ClassLoaders", "BOOT_LOADER", // A
"APP_LOADER", // A
"PLATFORM_LOADER"); // A
- ADD_EXCL("jdk/internal/loader/URLClassPath", "JAVA_VERSION"); // B
ADD_EXCL("jdk/internal/module/Builder", "cachedVersion"); // D
ADD_EXCL("jdk/internal/module/ModuleLoaderMap$Mapper", "APP_CLASSLOADER", // A
"APP_LOADER_INDEX", // A
@@ -128,14 +118,10 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
// This just points to an empty Map
ADD_EXCL("jdk/internal/reflect/Reflection", "methodFilterMap"); // E
- ADD_EXCL("jdk/internal/util/StaticProperty", "FILE_ENCODING", // C
- "JAVA_LOCALE_USE_OLD_ISO_CODES"); // C
// Integer for 0 and 1 are in java/lang/Integer$IntegerCache and are archived
ADD_EXCL("sun/invoke/util/ValueConversions", "ONE_INT", // E
"ZERO_INT"); // E
- ADD_EXCL("sun/security/util/SecurityConstants", "PROVIDER_VER"); // C
-
# undef ADD_EXCL
@@ -229,6 +215,12 @@ inline bool CDSHeapVerifier::do_entry(oop& orig_obj, HeapShared::CachedOopInfo&
StaticFieldInfo* info = _table.get(orig_obj);
if (info != nullptr) {
+ if (value.orig_referrer() == nullptr && java_lang_String::is_instance(orig_obj)) {
+ // This string object is not referenced by any of the archived object graphs. It's archived
+ // only because it's in the interned string table. So we are not in a condition that
+ // should be flagged by CDSHeapVerifier.
+ return true; /* keep on iterating */
+ }
ResourceMark rm;
LogStream ls(Log(cds, heap)::warning());
ls.print_cr("Archive heap points to a static field that may be reinitialized at runtime:");
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index fd073d49425..464b75ce1ba 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -644,12 +644,14 @@ int FileMapInfo::get_module_shared_path_index(Symbol* location) {
const char* file = ClassLoader::skip_uri_protocol(location->as_C_string());
for (int i = ClassLoaderExt::app_module_paths_start_index(); i < get_number_of_shared_paths(); i++) {
SharedClassPathEntry* ent = shared_path(i);
- assert(ent->in_named_module(), "must be");
- bool cond = strcmp(file, ent->name()) == 0;
- log_debug(class, path)("get_module_shared_path_index (%d) %s : %s = %s", i,
- location->as_C_string(), ent->name(), cond ? "same" : "different");
- if (cond) {
- return i;
+ if (!ent->is_non_existent()) {
+ assert(ent->in_named_module(), "must be");
+ bool cond = strcmp(file, ent->name()) == 0;
+ log_debug(class, path)("get_module_shared_path_index (%d) %s : %s = %s", i,
+ location->as_C_string(), ent->name(), cond ? "same" : "different");
+ if (cond) {
+ return i;
+ }
}
}
@@ -1537,7 +1539,7 @@ BitMapView FileMapRegion::ptrmap_view() {
return bitmap_view(false);
}
-bool FileMapRegion::check_region_crc() const {
+bool FileMapRegion::check_region_crc(char* base) const {
// This function should be called after the region has been properly
// loaded into memory via FileMapInfo::map_region() or FileMapInfo::read_region().
// I.e., this->mapped_base() must be valid.
@@ -1546,8 +1548,8 @@ bool FileMapRegion::check_region_crc() const {
return true;
}
- assert(mapped_base() != nullptr, "must be initialized");
- int crc = ClassLoader::crc32(0, mapped_base(), (jint)sz);
+ assert(base != nullptr, "must be initialized");
+ int crc = ClassLoader::crc32(0, base, (jint)sz);
if (crc != this->crc()) {
log_warning(cds)("Checksum verification failed.");
return false;
@@ -1832,13 +1834,13 @@ bool FileMapInfo::read_region(int i, char* base, size_t size, bool do_commit) {
return false;
}
- r->set_mapped_from_file(false);
- r->set_mapped_base(base);
-
- if (VerifySharedSpaces && !r->check_region_crc()) {
+ if (VerifySharedSpaces && !r->check_region_crc(base)) {
return false;
}
+ r->set_mapped_from_file(false);
+ r->set_mapped_base(base);
+
return true;
}
@@ -1875,6 +1877,7 @@ MapArchiveResult FileMapInfo::map_region(int i, intx addr_delta, char* mapped_ba
return MAP_ARCHIVE_OTHER_FAILURE; // oom or I/O error.
} else {
assert(r->mapped_base() != nullptr, "must be initialized");
+ return MAP_ARCHIVE_SUCCESS;
}
} else {
// Note that this may either be a "fresh" mapping into unreserved address
@@ -1889,15 +1892,16 @@ MapArchiveResult FileMapInfo::map_region(int i, intx addr_delta, char* mapped_ba
_memory_mapping_failed = true;
return MAP_ARCHIVE_MMAP_FAILURE;
}
+
+ if (VerifySharedSpaces && !r->check_region_crc(requested_addr)) {
+ return MAP_ARCHIVE_OTHER_FAILURE;
+ }
+
r->set_mapped_from_file(true);
r->set_mapped_base(requested_addr);
- }
- if (VerifySharedSpaces && !r->check_region_crc()) {
- return MAP_ARCHIVE_OTHER_FAILURE;
+ return MAP_ARCHIVE_SUCCESS;
}
-
- return MAP_ARCHIVE_SUCCESS;
}
// The return value is the location of the archive relocation bitmap.
@@ -1915,8 +1919,7 @@ char* FileMapInfo::map_bitmap_region() {
return nullptr;
}
- r->set_mapped_base(bitmap_base);
- if (VerifySharedSpaces && !r->check_region_crc()) {
+ if (VerifySharedSpaces && !r->check_region_crc(bitmap_base)) {
log_error(cds)("relocation bitmap CRC error");
if (!os::unmap_memory(bitmap_base, r->used_aligned())) {
fatal("os::unmap_memory of relocation bitmap failed");
@@ -1925,6 +1928,7 @@ char* FileMapInfo::map_bitmap_region() {
}
r->set_mapped_from_file(true);
+ r->set_mapped_base(bitmap_base);
log_info(cds)("Mapped %s region #%d at base " INTPTR_FORMAT " top " INTPTR_FORMAT " (%s)",
is_static() ? "static " : "dynamic",
MetaspaceShared::bm, p2i(r->mapped_base()), p2i(r->mapped_end()),
@@ -2200,13 +2204,14 @@ bool FileMapInfo::map_heap_region_impl() {
return false;
}
- r->set_mapped_base(base);
- if (VerifySharedSpaces && !r->check_region_crc()) {
+ if (VerifySharedSpaces && !r->check_region_crc(base)) {
dealloc_heap_region();
log_info(cds)("UseSharedSpaces: mapped heap region is corrupt");
return false;
}
+ r->set_mapped_base(base);
+
// If the requested range is different from the range allocated by GC, then
// the pointers need to be patched.
address mapped_start = (address) _mapped_heap_memregion.start();
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 7ad1a2f4c34..150c4badc38 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -90,6 +90,7 @@ class SharedClassPathEntry : public MetaspaceObj {
bool is_dir() const { return _type == dir_entry; }
bool is_modules_image() const { return _type == modules_image_entry; }
bool is_jar() const { return _type == jar_entry; }
+ bool is_non_existent() const { return _type == non_existent_entry; }
bool from_class_path_attr() { return _from_class_path_attr; }
time_t timestamp() const { return _timestamp; }
const char* name() const;
@@ -171,7 +172,7 @@ class FileMapRegion: private CDSFileMapRegion {
BitMapView ptrmap_view();
bool has_ptrmap() { return _ptrmap_size_in_bits != 0; }
- bool check_region_crc() const;
+ bool check_region_crc(char* base) const;
void print(outputStream* st, int region_index);
};
diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp
index 8aeecf1f57f..8a78a9dff70 100644
--- a/src/hotspot/share/ci/ciEnv.hpp
+++ b/src/hotspot/share/ci/ciEnv.hpp
@@ -366,7 +366,7 @@ class ciEnv : StackObj {
// The compiler task which has created this env.
// May be useful to find out compile_id, comp_level, etc.
- CompileTask* task() { return _task; }
+ CompileTask* task() const { return _task; }
// Handy forwards to the task:
int comp_level(); // task()->comp_level()
@@ -448,7 +448,7 @@ class ciEnv : StackObj {
static ciEnv* current(CompilerThread *thread) { return thread->env(); }
// Per-compiler data. (Used by C2 to publish the Compile* pointer.)
- void* compiler_data() { return _compiler_data; }
+ void* compiler_data() const { return _compiler_data; }
void set_compiler_data(void* x) { _compiler_data = x; }
// Notice that a method has been inlined in the current compile;
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index 1e62d0aa0a2..7a022cdf181 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1367,7 +1367,7 @@ const char* java_lang_Class::as_external_name(oop java_class) {
Klass* java_lang_Class::array_klass_acquire(oop java_class) {
Klass* k = ((Klass*)java_class->metadata_field_acquire(_array_klass_offset));
- assert(k == nullptr || k->is_klass() && k->is_array_klass(), "should be array klass");
+ assert(k == nullptr || (k->is_klass() && k->is_array_klass()), "should be array klass");
return k;
}
diff --git a/src/hotspot/share/classfile/stringTable.hpp b/src/hotspot/share/classfile/stringTable.hpp
index 8178e3b3bb7..02723a42f42 100644
--- a/src/hotspot/share/classfile/stringTable.hpp
+++ b/src/hotspot/share/classfile/stringTable.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_CLASSFILE_STRINGTABLE_HPP
#define SHARE_CLASSFILE_STRINGTABLE_HPP
-#include "memory/allocation.hpp"
+#include "memory/allStatic.hpp"
#include "memory/padded.hpp"
#include "oops/oop.hpp"
#include "oops/oopHandle.hpp"
@@ -37,15 +37,11 @@ class DumpedInternedStrings;
class JavaThread;
class SerializeClosure;
-class StringTable;
class StringTableConfig;
-class StringTableCreateEntry;
-class StringTable : public CHeapObj{
+class StringTable : AllStatic {
friend class VMStructs;
- friend class Symbol;
friend class StringTableConfig;
- friend class StringTableCreateEntry;
static volatile bool _has_work;
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index b3ae4a37c92..1011e2595b1 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -600,6 +600,7 @@ class methodHandle;
do_intrinsic(_notifyJvmtiVThreadMount, java_lang_VirtualThread, notifyJvmtiMount_name, bool_void_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadUnmount, java_lang_VirtualThread, notifyJvmtiUnmount_name, bool_void_signature, F_RN) \
do_intrinsic(_notifyJvmtiVThreadHideFrames, java_lang_VirtualThread, notifyJvmtiHideFrames_name, bool_void_signature, F_RN) \
+ do_intrinsic(_notifyJvmtiVThreadDisableSuspend, java_lang_VirtualThread, notifyJvmtiDisableSuspend_name, bool_void_signature, F_RN) \
\
/* support for UnsafeConstants */ \
do_class(jdk_internal_misc_UnsafeConstants, "jdk/internal/misc/UnsafeConstants") \
diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp
index 344c6775bc0..e8ee01f66c2 100644
--- a/src/hotspot/share/classfile/vmSymbols.hpp
+++ b/src/hotspot/share/classfile/vmSymbols.hpp
@@ -424,6 +424,7 @@ class SerializeClosure;
template(notifyJvmtiMount_name, "notifyJvmtiMount") \
template(notifyJvmtiUnmount_name, "notifyJvmtiUnmount") \
template(notifyJvmtiHideFrames_name, "notifyJvmtiHideFrames") \
+ template(notifyJvmtiDisableSuspend_name, "notifyJvmtiDisableSuspend") \
template(doYield_name, "doYield") \
template(enter_name, "enter") \
template(enterSpecial_name, "enterSpecial") \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index 87d2344984b..46d77fee6b4 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -164,7 +164,7 @@ RuntimeBlob::RuntimeBlob(
void RuntimeBlob::free(RuntimeBlob* blob) {
assert(blob != nullptr, "caller must check for nullptr");
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
- blob->purge();
+ blob->purge(true /* free_code_cache_data */, true /* unregister_nmethod */);
{
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
CodeCache::free(blob);
@@ -173,7 +173,7 @@ void RuntimeBlob::free(RuntimeBlob* blob) {
MemoryService::track_code_cache_memory_usage();
}
-void CodeBlob::purge(bool free_code_cache_data) {
+void CodeBlob::purge(bool free_code_cache_data, bool unregister_nmethod) {
if (_oop_maps != nullptr) {
delete _oop_maps;
_oop_maps = nullptr;
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index 5945ff998cd..a91ec482f02 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -143,7 +143,7 @@ class CodeBlob {
static unsigned int align_code_offset(int offset);
// Deletion
- virtual void purge(bool free_code_cache_data = true);
+ virtual void purge(bool free_code_cache_data, bool unregister_nmethod);
// Typing
virtual bool is_buffer_blob() const { return false; }
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index 9b10dab2ef7..bf8c1d84e71 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -355,16 +355,8 @@ void CodeCache::initialize_heaps() {
}
size_t CodeCache::page_size(bool aligned, size_t min_pages) {
- if (os::can_execute_large_page_memory()) {
- if (InitialCodeCacheSize < ReservedCodeCacheSize) {
- // Make sure that the page size allows for an incremental commit of the reserved space
- min_pages = MAX2(min_pages, (size_t)8);
- }
- return aligned ? os::page_size_for_region_aligned(ReservedCodeCacheSize, min_pages) :
- os::page_size_for_region_unaligned(ReservedCodeCacheSize, min_pages);
- } else {
- return os::vm_page_size();
- }
+ return aligned ? os::page_size_for_region_aligned(ReservedCodeCacheSize, min_pages) :
+ os::page_size_for_region_unaligned(ReservedCodeCacheSize, min_pages);
}
ReservedCodeSpace CodeCache::reserve_heap_memory(size_t size, size_t rs_ps) {
@@ -1171,7 +1163,11 @@ void CodeCache::initialize() {
FLAG_SET_ERGO(NonNMethodCodeHeapSize, (uintx)os::vm_page_size());
FLAG_SET_ERGO(ProfiledCodeHeapSize, 0);
FLAG_SET_ERGO(NonProfiledCodeHeapSize, 0);
- ReservedCodeSpace rs = reserve_heap_memory(ReservedCodeCacheSize, page_size(false, 8));
+
+ // If InitialCodeCacheSize is equal to ReservedCodeCacheSize, then it's more likely
+ // users want to use the largest available page.
+ const size_t min_pages = (InitialCodeCacheSize == ReservedCodeCacheSize) ? 1 : 8;
+ ReservedCodeSpace rs = reserve_heap_memory(ReservedCodeCacheSize, page_size(false, min_pages));
// Register CodeHeaps with LSan as we sometimes embed pointers to malloc memory.
LSAN_REGISTER_ROOT_REGION(rs.base(), rs.size());
add_heap(rs, "CodeCache", CodeBlobType::All);
@@ -1819,16 +1815,20 @@ void CodeCache::log_state(outputStream* st) {
}
#ifdef LINUX
-void CodeCache::write_perf_map() {
+void CodeCache::write_perf_map(const char* filename) {
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
- // Perf expects to find the map file at /tmp/perf-.map.
+ // Perf expects to find the map file at /tmp/perf-.map
+ // if the file name is not specified.
char fname[32];
- jio_snprintf(fname, sizeof(fname), "/tmp/perf-%d.map", os::current_process_id());
+ if (filename == nullptr) {
+ jio_snprintf(fname, sizeof(fname), "/tmp/perf-%d.map", os::current_process_id());
+ filename = fname;
+ }
- fileStream fs(fname, "w");
+ fileStream fs(filename, "w");
if (!fs.is_open()) {
- log_warning(codecache)("Failed to create %s for perf map", fname);
+ log_warning(codecache)("Failed to create %s for perf map", filename);
return;
}
diff --git a/src/hotspot/share/code/codeCache.hpp b/src/hotspot/share/code/codeCache.hpp
index 5418a29e58f..103268c8ffc 100644
--- a/src/hotspot/share/code/codeCache.hpp
+++ b/src/hotspot/share/code/codeCache.hpp
@@ -227,7 +227,7 @@ class CodeCache : AllStatic {
static void print_trace(const char* event, CodeBlob* cb, uint size = 0) PRODUCT_RETURN;
static void print_summary(outputStream* st, bool detailed = true); // Prints a summary of the code cache usage
static void log_state(outputStream* st);
- LINUX_ONLY(static void write_perf_map();)
+ LINUX_ONLY(static void write_perf_map(const char* filename = nullptr);)
static const char* get_code_heap_name(CodeBlobType code_blob_type) { return (heap_available(code_blob_type) ? get_code_heap(code_blob_type)->name() : "Unused"); }
static void report_codemem_full(CodeBlobType code_blob_type, bool print);
diff --git a/src/hotspot/share/code/codeCache.inline.hpp b/src/hotspot/share/code/codeCache.inline.hpp
index bea66cf0010..5ab7187cd26 100644
--- a/src/hotspot/share/code/codeCache.inline.hpp
+++ b/src/hotspot/share/code/codeCache.inline.hpp
@@ -37,10 +37,9 @@ inline CodeBlob* CodeCache::find_blob_fast(void* pc) {
inline CodeBlob* CodeCache::find_blob_and_oopmap(void* pc, int& slot) {
NativePostCallNop* nop = nativePostCallNop_at((address) pc);
CodeBlob* cb;
- if (nop != nullptr && nop->displacement() != 0) {
- int offset = (nop->displacement() & 0xffffff);
+ int offset;
+ if (nop != nullptr && nop->decode(slot, offset)) {
cb = (CodeBlob*) ((address) pc - offset);
- slot = ((nop->displacement() >> 24) & 0xff);
assert(cb == CodeCache::find_blob(pc), "must be");
} else {
cb = CodeCache::find_blob(pc);
@@ -52,9 +51,12 @@ inline CodeBlob* CodeCache::find_blob_and_oopmap(void* pc, int& slot) {
inline int CodeCache::find_oopmap_slot_fast(void* pc) {
NativePostCallNop* nop = nativePostCallNop_at((address) pc);
- return (nop != nullptr && nop->displacement() != 0)
- ? ((nop->displacement() >> 24) & 0xff)
- : -1;
+ int oopmap_slot;
+ int cb_offset;
+ if (nop != nullptr && nop->decode(oopmap_slot, cb_offset)) {
+ return oopmap_slot;
+ }
+ return -1;
}
#endif // SHARE_VM_COMPILER_CODECACHE_INLINE_HPP
diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp
index 42cfd94f13c..74e70230f3a 100644
--- a/src/hotspot/share/code/compiledIC.cpp
+++ b/src/hotspot/share/code/compiledIC.cpp
@@ -128,11 +128,13 @@ void CompiledIC::internal_set_ic_destination(address entry_point, bool is_icstub
tty->cr();
}
+#ifdef ASSERT
{
CodeBlob* cb = CodeCache::find_blob(_call->instruction_address());
assert(cb != nullptr && cb->is_compiled(), "must be compiled");
- _call->set_destination_mt_safe(entry_point);
}
+#endif
+ _call->set_destination_mt_safe(entry_point);
if (is_optimized() || is_icstub) {
// Optimized call sites don't have a cache value and ICStub call
@@ -291,10 +293,10 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod
}
}
- if (TraceICs) {
+ {
ResourceMark rm;
assert(call_info->selected_method() != nullptr, "Unexpected null selected method");
- tty->print_cr ("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
p2i(instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
}
@@ -364,10 +366,11 @@ bool CompiledIC::is_call_to_interpreted() const {
bool CompiledIC::set_to_clean(bool in_use) {
assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- if (TraceInlineCacheClearing || TraceICs) {
+ if (TraceInlineCacheClearing) {
tty->print_cr("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
print();
}
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
address entry = _call->get_resolve_call_stub(is_optimized());
@@ -433,9 +436,9 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
methodHandle method (thread, (Method*)info.cached_metadata());
_call->set_to_interpreted(method, info);
- if (TraceICs) {
- ResourceMark rm(thread);
- tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
+ {
+ ResourceMark rm(thread);
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
p2i(instruction_address()),
method->print_value_string());
}
@@ -449,9 +452,9 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
// LSan appears unable to follow malloc-based memory consistently when embedded as an
// immediate in generated machine code. So we have to ignore it.
LSAN_IGNORE_OBJECT(holder);
- if (TraceICs) {
+ {
ResourceMark rm(thread);
- tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
}
}
} else {
@@ -479,10 +482,10 @@ bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
}
}
- if (TraceICs) {
+ {
ResourceMark rm(thread);
assert(info.cached_metadata() == nullptr || info.cached_metadata()->is_klass(), "must be");
- tty->print_cr ("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
p2i(instruction_address()),
(info.cached_metadata() != nullptr) ? ((Klass*)info.cached_metadata())->print_value_string() : "nullptr",
(safe) ? "" : " via stub");
@@ -609,9 +612,9 @@ bool CompiledDirectStaticCall::is_call_to_interpreted() const {
}
void CompiledStaticCall::set_to_compiled(address entry) {
- if (TraceICs) {
+ {
ResourceMark rm;
- tty->print_cr("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
+ log_trace(inlinecache)("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
name(),
p2i(instruction_address()),
p2i(entry));
diff --git a/src/hotspot/share/code/compiledMethod.hpp b/src/hotspot/share/code/compiledMethod.hpp
index a7b1b028c34..dda796dfad9 100644
--- a/src/hotspot/share/code/compiledMethod.hpp
+++ b/src/hotspot/share/code/compiledMethod.hpp
@@ -174,7 +174,7 @@ class CompiledMethod : public CodeBlob {
void* _gc_data;
- virtual void purge(bool free_code_cache_data = true) = 0;
+ virtual void purge(bool free_code_cache_data, bool unregister_nmethod) = 0;
private:
DeoptimizationStatus deoptimization_status() const {
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index 23095b507d9..0784699c64e 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -1146,10 +1146,7 @@ static void install_post_call_nop_displacement(nmethod* nm, address pc) {
int oopmap_slot = nm->oop_maps()->find_slot_for_offset(int((intptr_t) pc - (intptr_t) nm->code_begin()));
if (oopmap_slot < 0) { // this can happen at asynchronous (non-safepoint) stackwalks
log_debug(codecache)("failed to find oopmap for cb: " INTPTR_FORMAT " offset: %d", cbaddr, (int) offset);
- } else if (((oopmap_slot & 0xff) == oopmap_slot) && ((offset & 0xffffff) == offset)) {
- jint value = (oopmap_slot << 24) | (jint) offset;
- nop->patch(value);
- } else {
+ } else if (!nop->patch(oopmap_slot, offset)) {
log_debug(codecache)("failed to encode %d %d", oopmap_slot, (int) offset);
}
}
@@ -1453,7 +1450,9 @@ void nmethod::unlink() {
ClassUnloadingContext::context()->register_unlinked_nmethod(this);
}
-void nmethod::purge(bool free_code_cache_data) {
+void nmethod::purge(bool free_code_cache_data, bool unregister_nmethod) {
+ assert(!free_code_cache_data, "must only call not freeing code cache data");
+
MutexLocker ml(CodeCache_lock, Mutex::_no_safepoint_check_flag);
// completely deallocate this method
@@ -1473,13 +1472,13 @@ void nmethod::purge(bool free_code_cache_data) {
ec = next;
}
- Universe::heap()->unregister_nmethod(this);
+ if (unregister_nmethod) {
+ Universe::heap()->unregister_nmethod(this);
+ }
+
CodeCache::unregister_old_nmethod(this);
- CodeBlob::purge();
- if (free_code_cache_data) {
- CodeCache::free(this);
- }
+ CodeBlob::purge(free_code_cache_data, unregister_nmethod);
}
oop nmethod::oop_at(int index) const {
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index a8c92d7e290..5e27ee665fa 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -529,7 +529,7 @@ class nmethod : public CompiledMethod {
void unlink();
// Deallocate this nmethod - called by the GC
- void purge(bool free_code_cache_data = true);
+ void purge(bool free_code_cache_data, bool unregister_nmethod);
// See comment at definition of _last_seen_on_stack
void mark_as_maybe_on_stack();
diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp
index d2e70fbd6db..bfb3db72d72 100644
--- a/src/hotspot/share/code/relocInfo.cpp
+++ b/src/hotspot/share/code/relocInfo.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -150,8 +150,8 @@ void RelocIterator::initialize(CompiledMethod* nm, address begin, address limit)
RelocIterator::RelocIterator(CodeSection* cs, address begin, address limit) {
initialize_misc();
- assert((cs->locs_start() != nullptr) && (cs->locs_end() != nullptr) ||
- (cs->locs_start() == nullptr) && (cs->locs_end() == nullptr), "valid start and end pointer");
+ assert(((cs->locs_start() != nullptr) && (cs->locs_end() != nullptr)) ||
+ ((cs->locs_start() == nullptr) && (cs->locs_end() == nullptr)), "valid start and end pointer");
_current = cs->locs_start()-1;
_end = cs->locs_end();
_addr = cs->start();
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.cpp b/src/hotspot/share/compiler/compilationFailureInfo.cpp
new file mode 100644
index 00000000000..e3f3353589e
--- /dev/null
+++ b/src/hotspot/share/compiler/compilationFailureInfo.cpp
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#if defined(COMPILER1) || defined(COMPILER2)
+
+#ifdef COMPILER1
+#include "c1/c1_Compilation.hpp"
+#endif
+#include "ci/ciEnv.hpp"
+#include "compiler/abstractCompiler.hpp"
+#include "compiler/compilationFailureInfo.hpp"
+#include "compiler/compileTask.hpp"
+#ifdef COMPILER2
+#include "opto/node.hpp"
+#include "opto/compile.hpp"
+#endif
+#include "runtime/os.hpp"
+#include "utilities/ostream.hpp"
+#include "utilities/nativeCallStack.hpp"
+
+CompilationFailureInfo::CompilationFailureInfo(const char* failure_reason) :
+ _stack(2),
+ _failure_reason(os::strdup(failure_reason)),
+ _elapsed_seconds(os::elapsedTime()),
+ _compile_id(ciEnv::current()->task()->compile_id())
+{}
+
+CompilationFailureInfo::~CompilationFailureInfo() {
+ os::free(_failure_reason);
+}
+
+void CompilationFailureInfo::print_on(outputStream* st) const {
+ st->print(" Time: ");
+ os::print_elapsed_time(st, _elapsed_seconds);
+ st->print_cr(" Compile id: %d", _compile_id);
+ st->print_cr(" Reason: '%s'", _failure_reason);
+ st->print_cr(" Callstack: ");
+ _stack.print_on(st);
+ st->cr();
+}
+
+// Convenience function to print current compile failure iff
+// current thread is compiler thread and there is a pending failure.
+// Otherwise prints nothing.
+bool CompilationFailureInfo::print_pending_compilation_failure(outputStream* st) {
+
+ const CompilationFailureInfo* info = nullptr;
+
+ // Carefully tiptoeing because we are called from the error reporter and
+ // nothing is certain.
+
+ const Thread* const t = Thread::current();
+ if (t == nullptr || !t->is_Compiler_thread()) {
+ return false;
+ }
+
+ const ciEnv* const env = ciEnv::current();
+ if (env == nullptr) {
+ return false;
+ }
+
+ const CompileTask* const task = env->task();
+ if (task == nullptr) {
+ return false;
+ }
+
+ const AbstractCompiler* const compiler = task->compiler();
+ if (compiler == nullptr) {
+ return false;
+ }
+
+#ifdef COMPILER1
+ if (compiler->type() == compiler_c1) {
+ const Compilation* const C = (Compilation*)env->compiler_data();
+ if (C != nullptr) {
+ info = C->first_failure_details();
+ }
+ }
+#endif
+#ifdef COMPILER2
+ if (compiler->type() == compiler_c2) {
+ const Compile* const C = (Compile*)env->compiler_data();
+ if (C != nullptr) {
+ info = C->first_failure_details();
+ }
+ }
+#endif
+
+ if (info != nullptr) {
+ st->print_cr("Pending compilation failure details for thread " PTR_FORMAT ":", p2i(t));
+ info->print_on(st);
+ }
+
+ return true;
+}
+
+#endif // defined(COMPILER1) || defined(COMPILER2)
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.hpp b/src/hotspot/share/compiler/compilationFailureInfo.hpp
new file mode 100644
index 00000000000..3de62eb69da
--- /dev/null
+++ b/src/hotspot/share/compiler/compilationFailureInfo.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_COMPILER_COMPILATIONFAILUREINFO_HPP
+#define SHARE_COMPILER_COMPILATIONFAILUREINFO_HPP
+
+#if defined(COMPILER1) || defined(COMPILER2)
+
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/nativeCallStack.hpp"
+
+class outputStream;
+class Symbol;
+
+class CompilationFailureInfo : public CHeapObj {
+ NativeCallStack _stack;
+ char* const _failure_reason;
+ const double _elapsed_seconds;
+ const int _compile_id;
+public:
+ CompilationFailureInfo(const char* failure_reason);
+ ~CompilationFailureInfo();
+ void print_on(outputStream* st) const;
+
+ // Convenience function to print, safely, current compile failure iff
+ // current thread is compiler thread and there is an ongoing compilation
+ // and a pending failure.
+ // Otherwise prints nothing.
+ static bool print_pending_compilation_failure(outputStream* st);
+};
+
+#endif // defined(COMPILER1) || defined(COMPILER2)
+
+#endif // SHARE_COMPILER_COMPILATIONFAILUREINFO_HPP
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index 39e45ec199a..6e992299744 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -1788,7 +1788,7 @@ bool CompileBroker::init_compiler_runtime() {
void CompileBroker::free_buffer_blob_if_allocated(CompilerThread* thread) {
BufferBlob* blob = thread->get_buffer_blob();
if (blob != nullptr) {
- blob->purge();
+ blob->purge(true /* free_code_cache_data */, true /* unregister_nmethod */);
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
CodeCache::free(blob);
}
diff --git a/src/hotspot/share/compiler/compilerDefinitions.inline.hpp b/src/hotspot/share/compiler/compilerDefinitions.inline.hpp
index ed456a42157..d490387672d 100644
--- a/src/hotspot/share/compiler/compilerDefinitions.inline.hpp
+++ b/src/hotspot/share/compiler/compilerDefinitions.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -57,7 +57,7 @@ inline bool CompilerConfig::is_c1_only() {
}
inline bool CompilerConfig::is_c1_or_interpreter_only_no_jvmci() {
- assert(is_jvmci_compiler() && is_jvmci() || !is_jvmci_compiler(), "JVMCI compiler implies enabled JVMCI");
+ assert(!is_jvmci_compiler() || is_jvmci(), "JVMCI compiler implies enabled JVMCI");
return !is_jvmci() && (is_interpreter_only() || is_c1_only());
}
@@ -114,7 +114,7 @@ inline bool CompilerConfig::is_c2_or_jvmci_compiler_only() {
// Tiered is basically C1 & (C2 | JVMCI) minus all the odd cases with restrictions.
inline bool CompilerConfig::is_tiered() {
- assert(is_c1_simple_only() && is_c1_only() || !is_c1_simple_only(), "c1 simple mode must imply c1-only mode");
+ assert(!is_c1_simple_only() || is_c1_only(), "c1 simple mode must imply c1-only mode");
return has_tiered() && !is_interpreter_only() && !is_c1_only() && !is_c2_or_jvmci_compiler_only();
}
diff --git a/src/hotspot/share/compiler/compiler_globals.hpp b/src/hotspot/share/compiler/compiler_globals.hpp
index 7f1f08b68ef..71b475392cf 100644
--- a/src/hotspot/share/compiler/compiler_globals.hpp
+++ b/src/hotspot/share/compiler/compiler_globals.hpp
@@ -379,6 +379,10 @@
"Don't compile methods larger than this if " \
"+DontCompileHugeMethods") \
\
+ product(bool, CaptureBailoutInformation, trueInDebug, DIAGNOSTIC, \
+ "If compilation is stopped with an error, capture diagnostic " \
+ "information at the bailout point") \
+ \
// end of COMPILER_FLAGS
diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp
index b8a7c677f7f..79b50cd0ea9 100644
--- a/src/hotspot/share/gc/g1/g1Arguments.cpp
+++ b/src/hotspot/share/gc/g1/g1Arguments.cpp
@@ -126,7 +126,6 @@ void G1Arguments::initialize_mark_stack_size() {
FLAG_SET_ERGO(MarkStackSize, mark_stack_size);
}
- log_trace(gc)("MarkStackSize: %uk MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
}
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
index 59d877554ea..3b933dfa7f9 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
@@ -187,6 +187,15 @@ class G1CodeRootSetHashTable : public CHeapObj {
}
}
+ // Removes dead/unlinked entries.
+ void bulk_remove() {
+ auto delete_check = [&] (nmethod** value) {
+ return (*value)->is_unlinked();
+ };
+
+ clean(delete_check);
+ }
+
// Calculate the log2 of the table size we want to shrink to.
size_t log2_target_shrink_size(size_t current_size) const {
// A table with the new size should be at most filled by this factor. Otherwise
@@ -255,6 +264,11 @@ bool G1CodeRootSet::remove(nmethod* method) {
return _table->remove(method);
}
+void G1CodeRootSet::bulk_remove() {
+ assert(!_is_iterating, "should not mutate while iterating the table");
+ _table->bulk_remove();
+}
+
bool G1CodeRootSet::contains(nmethod* method) {
return _table->contains(method);
}
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.hpp b/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
index d1051e6e42e..9c5ccdd1202 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.hpp
@@ -44,6 +44,7 @@ class G1CodeRootSet {
void add(nmethod* method);
bool remove(nmethod* method);
+ void bulk_remove();
bool contains(nmethod* method);
void clear();
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 25d7ed13ab0..7a630c27f79 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -2517,6 +2517,7 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
GCTraceTime(Debug, gc, phases) debug(description, timer);
ClassUnloadingContext ctx(workers()->active_workers(),
+ false /* unregister_nmethods_during_purge */,
false /* lock_codeblob_free_separately */);
{
CodeCache::UnlinkingScope scope(is_alive);
@@ -2528,6 +2529,10 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
GCTraceTime(Debug, gc, phases) t("Purge Unlinked NMethods", timer);
ctx.purge_nmethods();
}
+ {
+ GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", timer);
+ G1CollectedHeap::heap()->bulk_unregister_nmethods();
+ }
{
GCTraceTime(Debug, gc, phases) t("Free Code Blobs", timer);
ctx.free_code_blobs();
@@ -2539,6 +2544,33 @@ void G1CollectedHeap::unload_classes_and_code(const char* description, BoolObjec
}
}
+class G1BulkUnregisterNMethodTask : public WorkerTask {
+ HeapRegionClaimer _hrclaimer;
+
+ class UnregisterNMethodsHeapRegionClosure : public HeapRegionClosure {
+ public:
+
+ bool do_heap_region(HeapRegion* hr) {
+ hr->rem_set()->bulk_remove_code_roots();
+ return false;
+ }
+ } _cl;
+
+public:
+ G1BulkUnregisterNMethodTask(uint num_workers)
+ : WorkerTask("G1 Remove Unlinked NMethods From Code Root Set Task"),
+ _hrclaimer(num_workers) { }
+
+ void work(uint worker_id) {
+ G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&_cl, &_hrclaimer, worker_id);
+ }
+};
+
+void G1CollectedHeap::bulk_unregister_nmethods() {
+ uint num_workers = workers()->active_workers();
+ G1BulkUnregisterNMethodTask t(num_workers);
+ workers()->run_task(&t);
+}
bool G1STWSubjectToDiscoveryClosure::do_object_b(oop obj) {
assert(obj != nullptr, "must not be null");
@@ -2963,31 +2995,6 @@ class RegisterNMethodOopClosure: public OopClosure {
void do_oop(narrowOop* p) { ShouldNotReachHere(); }
};
-class UnregisterNMethodOopClosure: public OopClosure {
- G1CollectedHeap* _g1h;
- nmethod* _nm;
-
-public:
- UnregisterNMethodOopClosure(G1CollectedHeap* g1h, nmethod* nm) :
- _g1h(g1h), _nm(nm) {}
-
- void do_oop(oop* p) {
- oop heap_oop = RawAccess<>::oop_load(p);
- if (!CompressedOops::is_null(heap_oop)) {
- oop obj = CompressedOops::decode_not_null(heap_oop);
- HeapRegion* hr = _g1h->heap_region_containing(obj);
- assert(!hr->is_continues_humongous(),
- "trying to remove code root " PTR_FORMAT " in continuation of humongous region " HR_FORMAT
- " starting at " HR_FORMAT,
- p2i(_nm), HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()));
-
- hr->remove_code_root(_nm);
- }
- }
-
- void do_oop(narrowOop* p) { ShouldNotReachHere(); }
-};
-
void G1CollectedHeap::register_nmethod(nmethod* nm) {
guarantee(nm != nullptr, "sanity");
RegisterNMethodOopClosure reg_cl(this, nm);
@@ -2995,9 +3002,8 @@ void G1CollectedHeap::register_nmethod(nmethod* nm) {
}
void G1CollectedHeap::unregister_nmethod(nmethod* nm) {
- guarantee(nm != nullptr, "sanity");
- UnregisterNMethodOopClosure reg_cl(this, nm);
- nm->oops_do(®_cl, true);
+ // We always unregister nmethods in bulk during code unloading only.
+ ShouldNotReachHere();
}
void G1CollectedHeap::update_used_after_gc(bool evacuation_failed) {
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 1d25ea278de..58ac36a734f 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -1270,6 +1270,8 @@ class G1CollectedHeap : public CollectedHeap {
void unload_classes_and_code(const char* description, BoolObjectClosure* cl, GCTimer* timer);
+ void bulk_unregister_nmethods();
+
// Verification
// Perform any cleanup actions necessary before allowing a verification.
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index 78e1bec3438..533817f9724 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -75,6 +75,7 @@
#include "utilities/align.hpp"
#include "utilities/formatBuffer.hpp"
#include "utilities/growableArray.hpp"
+#include "utilities/powerOfTwo.hpp"
bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
assert(addr < _cm->finger(), "invariant");
@@ -94,80 +95,173 @@ bool G1CMBitMapClosure::do_addr(HeapWord* const addr) {
}
G1CMMarkStack::G1CMMarkStack() :
- _max_chunk_capacity(0),
- _base(nullptr),
- _chunk_capacity(0) {
+ _chunk_allocator() {
set_empty();
}
-bool G1CMMarkStack::resize(size_t new_capacity) {
- assert(is_empty(), "Only resize when stack is empty.");
- assert(new_capacity <= _max_chunk_capacity,
- "Trying to resize stack to " SIZE_FORMAT " chunks when the maximum is " SIZE_FORMAT, new_capacity, _max_chunk_capacity);
+size_t G1CMMarkStack::capacity_alignment() {
+ return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
+}
- TaskQueueEntryChunk* new_base = MmapArrayAllocator::allocate_or_null(new_capacity, mtGC);
+bool G1CMMarkStack::initialize() {
+ guarantee(_chunk_allocator.capacity() == 0, "G1CMMarkStack already initialized.");
- if (new_base == nullptr) {
- log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
- return false;
+ size_t initial_capacity = MarkStackSize;
+ size_t max_capacity = MarkStackSizeMax;
+
+ size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
+
+ size_t max_num_chunks = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
+ size_t initial_num_chunks = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
+
+ initial_num_chunks = round_up_power_of_2(initial_num_chunks);
+ max_num_chunks = MAX2(initial_num_chunks, max_num_chunks);
+
+ size_t limit = (INT_MAX - 1);
+ max_capacity = MIN2((max_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
+ initial_capacity = MIN2((initial_num_chunks * TaskEntryChunkSizeInVoidStar), limit);
+
+ FLAG_SET_ERGO(MarkStackSizeMax, max_capacity);
+ FLAG_SET_ERGO(MarkStackSize, initial_capacity);
+
+ log_trace(gc)("MarkStackSize: %uk MarkStackSizeMax: %uk", (uint)(MarkStackSize / K), (uint)(MarkStackSizeMax / K));
+
+ log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
+ initial_num_chunks, max_capacity);
+
+ return _chunk_allocator.initialize(initial_num_chunks, max_num_chunks);
+}
+
+G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_chunk() {
+ if (_size >= _max_capacity) {
+ return nullptr;
}
- // Release old mapping.
- if (_base != nullptr) {
- MmapArrayAllocator::free(_base, _chunk_capacity);
+
+ size_t cur_idx = Atomic::fetch_then_add(&_size, 1u);
+
+ if (cur_idx >= _max_capacity) {
+ return nullptr;
}
- _base = new_base;
- _chunk_capacity = new_capacity;
- set_empty();
+ size_t bucket = get_bucket(cur_idx);
+ if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
+ if (!_should_grow) {
+ // Prefer to restart the CM.
+ return nullptr;
+ }
- return true;
-}
+ MutexLocker x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
+ if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
+ if (!expand()) {
+ return nullptr;
+ }
+ }
+ }
-size_t G1CMMarkStack::capacity_alignment() {
- return (size_t)lcm(os::vm_allocation_granularity(), sizeof(TaskQueueEntryChunk)) / sizeof(G1TaskQueueEntry);
+ size_t bucket_idx = get_bucket_index(cur_idx);
+ TaskQueueEntryChunk* result = ::new (&_buckets[bucket][bucket_idx]) TaskQueueEntryChunk;
+ result->next = nullptr;
+ return result;
}
-bool G1CMMarkStack::initialize(size_t initial_capacity, size_t max_capacity) {
- guarantee(_max_chunk_capacity == 0, "G1CMMarkStack already initialized.");
+G1CMMarkStack::ChunkAllocator::ChunkAllocator() :
+ _min_capacity(0),
+ _max_capacity(0),
+ _capacity(0),
+ _num_buckets(0),
+ _should_grow(false),
+ _buckets(nullptr),
+ _size(0)
+{ }
- size_t const TaskEntryChunkSizeInVoidStar = sizeof(TaskQueueEntryChunk) / sizeof(G1TaskQueueEntry);
+bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t max_capacity) {
+ guarantee(is_power_of_2(initial_capacity), "Invalid initial_capacity");
- _max_chunk_capacity = align_up(max_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
- size_t initial_chunk_capacity = align_up(initial_capacity, capacity_alignment()) / TaskEntryChunkSizeInVoidStar;
+ _min_capacity = initial_capacity;
+ _max_capacity = max_capacity;
+ _num_buckets = get_bucket(_max_capacity) + 1;
- guarantee(initial_chunk_capacity <= _max_chunk_capacity,
- "Maximum chunk capacity " SIZE_FORMAT " smaller than initial capacity " SIZE_FORMAT,
- _max_chunk_capacity,
- initial_chunk_capacity);
+ _buckets = NEW_C_HEAP_ARRAY(TaskQueueEntryChunk*, _num_buckets, mtGC);
- log_debug(gc)("Initialize mark stack with " SIZE_FORMAT " chunks, maximum " SIZE_FORMAT,
- initial_chunk_capacity, _max_chunk_capacity);
+ for (size_t i = 0; i < _num_buckets; i++) {
+ _buckets[i] = nullptr;
+ }
+
+ size_t new_capacity = bucket_size(0);
- return resize(initial_chunk_capacity);
+ if (!reserve(new_capacity)) {
+ log_warning(gc)("Failed to reserve memory for new overflow mark stack with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.", new_capacity, new_capacity * sizeof(TaskQueueEntryChunk));
+ return false;
+ }
+ return true;
}
-void G1CMMarkStack::expand() {
- if (_chunk_capacity == _max_chunk_capacity) {
- log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _chunk_capacity);
- return;
+bool G1CMMarkStack::ChunkAllocator::expand() {
+ if (_capacity == _max_capacity) {
+ log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _capacity);
+ return false;
}
- size_t old_capacity = _chunk_capacity;
- // Double capacity if possible
- size_t new_capacity = MIN2(old_capacity * 2, _max_chunk_capacity);
+ size_t old_capacity = _capacity;
+ // Double capacity if possible.
+ size_t new_capacity = MIN2(old_capacity * 2, _max_capacity);
- if (resize(new_capacity)) {
- log_debug(gc)("Expanded mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
+ if (reserve(new_capacity)) {
+ log_debug(gc)("Expanded the mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
old_capacity, new_capacity);
- } else {
- log_warning(gc)("Failed to expand mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
- old_capacity, new_capacity);
+ return true;
+ }
+ return false;
+}
+
+G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
+ if (_buckets == nullptr) {
+ return;
}
+
+ for (size_t i = 0; i < _num_buckets; i++) {
+ if (_buckets[i] != nullptr) {
+ MmapArrayAllocator::free(_buckets[i], bucket_size(i));
+ _buckets[i] = nullptr;
+ }
+ }
+
+ FREE_C_HEAP_ARRAY(TaskQueueEntryChunk*, _buckets);
}
-G1CMMarkStack::~G1CMMarkStack() {
- if (_base != nullptr) {
- MmapArrayAllocator::free(_base, _chunk_capacity);
+bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
+ assert(new_capacity <= _max_capacity, "Cannot expand overflow mark stack beyond the max_capacity" SIZE_FORMAT " chunks.", _max_capacity);
+
+ size_t highest_bucket = get_bucket(new_capacity - 1);
+ size_t i = get_bucket(_capacity);
+
+ for (; i <= highest_bucket; i++) {
+ if (Atomic::load_acquire(&_buckets[i]) != nullptr) {
+ continue; // Skip over already allocated buckets.
+ }
+
+ size_t bucket_capacity = bucket_size(i);
+
+ // Trim bucket size so that we do not exceed the _max_capacity.
+ bucket_capacity = (_capacity + bucket_capacity) <= _max_capacity ?
+ bucket_capacity :
+ _max_capacity - _capacity;
+
+
+ TaskQueueEntryChunk* bucket_base = MmapArrayAllocator::allocate_or_null(bucket_capacity, mtGC);
+
+ if (bucket_base == nullptr) {
+ log_warning(gc)("Failed to reserve memory for increasing the overflow mark stack capacity with " SIZE_FORMAT " chunks and size " SIZE_FORMAT "B.",
+ bucket_capacity, bucket_capacity * sizeof(TaskQueueEntryChunk));
+ return false;
+ }
+ _capacity += bucket_capacity;
+ Atomic::release_store(&_buckets[i], bucket_base);
}
+ return true;
+}
+
+void G1CMMarkStack::expand() {
+ _chunk_allocator.expand();
}
void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
@@ -208,31 +302,13 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::remove_chunk_from_free_list()
return remove_chunk_from_list(&_free_list);
}
-G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::allocate_new_chunk() {
- // This dirty read of _hwm is okay because we only ever increase the _hwm in parallel code.
- // Further this limits _hwm to a value of _chunk_capacity + #threads, avoiding
- // wraparound of _hwm.
- if (_hwm >= _chunk_capacity) {
- return nullptr;
- }
-
- size_t cur_idx = Atomic::fetch_then_add(&_hwm, 1u);
- if (cur_idx >= _chunk_capacity) {
- return nullptr;
- }
-
- TaskQueueEntryChunk* result = ::new (&_base[cur_idx]) TaskQueueEntryChunk;
- result->next = nullptr;
- return result;
-}
-
bool G1CMMarkStack::par_push_chunk(G1TaskQueueEntry* ptr_arr) {
// Get a new chunk.
TaskQueueEntryChunk* new_chunk = remove_chunk_from_free_list();
if (new_chunk == nullptr) {
// Did not get a chunk from the free list. Allocate from backing memory.
- new_chunk = allocate_new_chunk();
+ new_chunk = _chunk_allocator.allocate_new_chunk();
if (new_chunk == nullptr) {
return false;
@@ -261,9 +337,9 @@ bool G1CMMarkStack::par_pop_chunk(G1TaskQueueEntry* ptr_arr) {
void G1CMMarkStack::set_empty() {
_chunks_in_chunk_list = 0;
- _hwm = 0;
_chunk_list = nullptr;
_free_list = nullptr;
+ _chunk_allocator.reset();
}
G1CMRootMemRegions::G1CMRootMemRegions(uint const max_regions) :
@@ -440,7 +516,7 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
_concurrent_workers->initialize_workers();
_num_concurrent_workers = _concurrent_workers->active_workers();
- if (!_global_mark_stack.initialize(MarkStackSize, MarkStackSizeMax)) {
+ if (!_global_mark_stack.initialize()) {
vm_exit_during_initialization("Failed to allocate initial concurrent mark overflow mark stack.");
}
@@ -1635,6 +1711,9 @@ void G1ConcurrentMark::weak_refs_work() {
assert(_global_mark_stack.is_empty(), "mark stack should be empty");
+ // Prefer to grow the stack until the max capacity.
+ _global_mark_stack.set_should_grow();
+
// We need at least one active thread. If reference processing
// is not multi-threaded we use the current (VMThread) thread,
// otherwise we use the workers from the G1CollectedHeap and
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index 4799f32e0c8..a4315ec5f6c 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -136,24 +136,108 @@ class G1CMMarkStack {
G1TaskQueueEntry data[EntriesPerChunk];
};
- size_t _max_chunk_capacity; // Maximum number of TaskQueueEntryChunk elements on the stack.
+ class ChunkAllocator {
+ // The chunk allocator relies on a growable array data structure that allows resizing without the
+ // need to copy existing items. The basic approach involves organizing the array into chunks,
+ // essentially creating an "array of arrays"; referred to as buckets in this implementation. To
+ // facilitate efficient indexing, the size of the first bucket is set to a power of 2. This choice
+ // allows for quick conversion of an array index into a bucket index and the corresponding offset
+ // within the bucket. Additionally, each new bucket added to the growable array doubles the capacity of
+ // the growable array.
+ //
+ // Illustration of the Growable Array data structure.
+ //
+ // +----+ +----+----+
+ // | |------->| | |
+ // | | +----+----+
+ // +----+ +----+----+
+ // | |------->| | |
+ // | | +----+----+
+ // +----+ +-----+-----+-----+-----+
+ // | |------->| | | | |
+ // | | +-----+-----+-----+-----+
+ // +----+ +-----+-----+-----+-----+-----+-----+-----+----+
+ // | |------->| | | | | | | | |
+ // | | +-----+-----+-----+-----+-----+-----+-----+----+
+ // +----+
+ //
+ size_t _min_capacity;
+ size_t _max_capacity;
+ size_t _capacity;
+ size_t _num_buckets;
+ bool _should_grow;
+ TaskQueueEntryChunk* volatile* _buckets;
+ char _pad0[DEFAULT_PADDING_SIZE];
+ volatile size_t _size;
+ char _pad4[DEFAULT_PADDING_SIZE - sizeof(size_t)];
+
+ size_t bucket_size(size_t bucket) {
+ return (bucket == 0) ?
+ _min_capacity :
+ _min_capacity * ( 1ULL << (bucket -1));
+ }
+
+ static unsigned int find_highest_bit(uintptr_t mask) {
+ return count_leading_zeros(mask) ^ (BitsPerWord - 1U);
+ }
+
+ size_t get_bucket(size_t array_idx) {
+ if (array_idx < _min_capacity) {
+ return 0;
+ }
- TaskQueueEntryChunk* _base; // Bottom address of allocated memory area.
- size_t _chunk_capacity; // Current maximum number of TaskQueueEntryChunk elements.
+ return find_highest_bit(array_idx) - find_highest_bit(_min_capacity) + 1;
+ }
+
+ size_t get_bucket_index(size_t array_idx) {
+ if (array_idx < _min_capacity) {
+ return array_idx;
+ }
+ return array_idx - (1ULL << find_highest_bit(array_idx));
+ }
+
+ bool reserve(size_t new_capacity);
+
+ public:
+ ChunkAllocator();
+
+ ~ChunkAllocator();
+
+ bool initialize(size_t initial_capacity, size_t max_capacity);
+
+ void reset() {
+ _size = 0;
+ _should_grow = false;
+ }
- char _pad0[DEFAULT_CACHE_LINE_SIZE];
+ // During G1CMConcurrentMarkingTask or finalize_marking phases, we prefer to restart the marking when
+ // the G1CMMarkStack overflows. Attempts to expand the G1CMMarkStack should be followed with a restart
+ // of the marking. On failure to allocate a new chuck, the caller just returns and forces a restart.
+ // This approach offers better memory utilization for the G1CMMarkStack, as each iteration of the
+ // marking potentially involves traversing fewer unmarked nodes in the graph.
+
+ // However, during the reference processing phase, instead of restarting the marking process, the
+ // G1CMMarkStack is expanded upon failure to allocate a new chunk. The decision between these two
+ // modes of expansion is determined by the _should_grow parameter.
+ void set_should_grow() {
+ _should_grow = true;
+ }
+
+ size_t capacity() const { return _capacity; }
+
+ bool expand();
+
+ TaskQueueEntryChunk* allocate_new_chunk();
+ };
+
+ ChunkAllocator _chunk_allocator;
+
+ char _pad0[DEFAULT_PADDING_SIZE];
TaskQueueEntryChunk* volatile _free_list; // Linked list of free chunks that can be allocated by users.
- char _pad1[DEFAULT_CACHE_LINE_SIZE - sizeof(TaskQueueEntryChunk*)];
+ char _pad1[DEFAULT_PADDING_SIZE - sizeof(TaskQueueEntryChunk*)];
TaskQueueEntryChunk* volatile _chunk_list; // List of chunks currently containing data.
volatile size_t _chunks_in_chunk_list;
- char _pad2[DEFAULT_CACHE_LINE_SIZE - sizeof(TaskQueueEntryChunk*) - sizeof(size_t)];
-
- volatile size_t _hwm; // High water mark within the reserved space.
- char _pad4[DEFAULT_CACHE_LINE_SIZE - sizeof(size_t)];
-
- // Allocate a new chunk from the reserved memory, using the high water mark. Returns
- // null if out of memory.
- TaskQueueEntryChunk* allocate_new_chunk();
+ char _pad2[DEFAULT_PADDING_SIZE - sizeof(TaskQueueEntryChunk*) - sizeof(size_t)];
// Atomically add the given chunk to the list.
void add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem);
@@ -167,19 +251,15 @@ class G1CMMarkStack {
TaskQueueEntryChunk* remove_chunk_from_chunk_list();
TaskQueueEntryChunk* remove_chunk_from_free_list();
- // Resizes the mark stack to the given new capacity. Releases any previous
- // memory if successful.
- bool resize(size_t new_capacity);
-
public:
G1CMMarkStack();
- ~G1CMMarkStack();
+ ~G1CMMarkStack() = default;
// Alignment and minimum capacity of this mark stack in number of oops.
static size_t capacity_alignment();
- // Allocate and initialize the mark stack with the given number of oops.
- bool initialize(size_t initial_capacity, size_t max_capacity);
+ // Allocate and initialize the mark stack.
+ bool initialize();
// Pushes the given buffer containing at most EntriesPerChunk elements on the mark
// stack. If less than EntriesPerChunk elements are to be pushed, the array must
@@ -197,7 +277,11 @@ class G1CMMarkStack {
// _chunk_list.
bool is_empty() const { return _chunk_list == nullptr; }
- size_t capacity() const { return _chunk_capacity; }
+ size_t capacity() const { return _chunk_allocator.capacity(); }
+
+ void set_should_grow() {
+ _chunk_allocator.set_should_grow();
+ }
// Expand the stack, typically in response to an overflow condition
void expand();
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
index aebdd2c43f5..b2ac4b25e90 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
@@ -58,56 +58,46 @@ G1ConcurrentRefineThread* G1ConcurrentRefineThreadControl::create_refinement_thr
return result;
}
-G1ConcurrentRefineThreadControl::G1ConcurrentRefineThreadControl() :
+G1ConcurrentRefineThreadControl::G1ConcurrentRefineThreadControl(uint max_num_threads) :
_cr(nullptr),
- _threads(nullptr),
- _max_num_threads(0)
+ _threads(max_num_threads)
{}
G1ConcurrentRefineThreadControl::~G1ConcurrentRefineThreadControl() {
- if (_threads != nullptr) {
- for (uint i = 0; i < _max_num_threads; i++) {
- G1ConcurrentRefineThread* t = _threads[i];
- if (t == nullptr) {
-#ifdef ASSERT
- for (uint j = i + 1; j < _max_num_threads; ++j) {
- assert(_threads[j] == nullptr, "invariant");
- }
-#endif // ASSERT
- break;
- } else {
- delete t;
- }
+ while (_threads.is_nonempty()) {
+ delete _threads.pop();
+ }
+}
+
+bool G1ConcurrentRefineThreadControl::ensure_threads_created(uint worker_id, bool initializing) {
+ assert(worker_id < max_num_threads(), "precondition");
+
+ while ((uint)_threads.length() <= worker_id) {
+ G1ConcurrentRefineThread* rt = create_refinement_thread(_threads.length(), initializing);
+ if (rt == nullptr) {
+ return false;
}
- FREE_C_HEAP_ARRAY(G1ConcurrentRefineThread*, _threads);
+ _threads.push(rt);
}
+
+ return true;
}
-jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr, uint max_num_threads) {
+jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr) {
assert(cr != nullptr, "G1ConcurrentRefine must not be null");
_cr = cr;
- _max_num_threads = max_num_threads;
-
- if (max_num_threads > 0) {
- _threads = NEW_C_HEAP_ARRAY(G1ConcurrentRefineThread*, max_num_threads, mtGC);
- _threads[0] = create_refinement_thread(0, true);
- if (_threads[0] == nullptr) {
+ if (max_num_threads() > 0) {
+ _threads.push(create_refinement_thread(0, true));
+ if (_threads.at(0) == nullptr) {
vm_shutdown_during_initialization("Could not allocate primary refinement thread");
return JNI_ENOMEM;
}
- if (UseDynamicNumberOfGCThreads) {
- for (uint i = 1; i < max_num_threads; ++i) {
- _threads[i] = nullptr;
- }
- } else {
- for (uint i = 1; i < max_num_threads; ++i) {
- _threads[i] = create_refinement_thread(i, true);
- if (_threads[i] == nullptr) {
- vm_shutdown_during_initialization("Could not allocate refinement threads.");
- return JNI_ENOMEM;
- }
+ if (!UseDynamicNumberOfGCThreads) {
+ if (!ensure_threads_created(max_num_threads() - 1, true)) {
+ vm_shutdown_during_initialization("Could not allocate refinement threads");
+ return JNI_ENOMEM;
}
}
}
@@ -117,38 +107,28 @@ jint G1ConcurrentRefineThreadControl::initialize(G1ConcurrentRefine* cr, uint ma
#ifdef ASSERT
void G1ConcurrentRefineThreadControl::assert_current_thread_is_primary_refinement_thread() const {
- assert(_threads != nullptr, "No threads");
- assert(Thread::current() == _threads[0], "Not primary thread");
+ assert(Thread::current() == _threads.at(0), "Not primary thread");
}
#endif // ASSERT
bool G1ConcurrentRefineThreadControl::activate(uint worker_id) {
- assert(worker_id < _max_num_threads, "precondition");
- G1ConcurrentRefineThread* thread_to_activate = _threads[worker_id];
- if (thread_to_activate == nullptr) {
- thread_to_activate = create_refinement_thread(worker_id, false);
- if (thread_to_activate == nullptr) {
- return false;
- }
- _threads[worker_id] = thread_to_activate;
+ if (ensure_threads_created(worker_id, false)) {
+ _threads.at(worker_id)->activate();
+ return true;
}
- thread_to_activate->activate();
- return true;
+
+ return false;
}
void G1ConcurrentRefineThreadControl::worker_threads_do(ThreadClosure* tc) {
- for (uint i = 0; i < _max_num_threads; i++) {
- if (_threads[i] != nullptr) {
- tc->do_thread(_threads[i]);
- }
+ for (G1ConcurrentRefineThread* t : _threads) {
+ tc->do_thread(t);
}
}
void G1ConcurrentRefineThreadControl::stop() {
- for (uint i = 0; i < _max_num_threads; i++) {
- if (_threads[i] != nullptr) {
- _threads[i]->stop();
- }
+ for (G1ConcurrentRefineThread* t : _threads) {
+ t->stop();
}
}
@@ -170,12 +150,12 @@ G1ConcurrentRefine::G1ConcurrentRefine(G1Policy* policy) :
_last_adjust(),
_needs_adjust(false),
_threads_needed(policy, adjust_threads_period_ms()),
- _thread_control(),
+ _thread_control(G1ConcRefinementThreads),
_dcqs(G1BarrierSet::dirty_card_queue_set())
{}
jint G1ConcurrentRefine::initialize() {
- return _thread_control.initialize(this, max_num_threads());
+ return _thread_control.initialize(this);
}
G1ConcurrentRefine* G1ConcurrentRefine::create(G1Policy* policy, jint* ecode) {
@@ -199,10 +179,6 @@ void G1ConcurrentRefine::threads_do(ThreadClosure *tc) {
_thread_control.worker_threads_do(tc);
}
-uint G1ConcurrentRefine::max_num_threads() {
- return G1ConcRefinementThreads;
-}
-
void G1ConcurrentRefine::update_pending_cards_target(double logged_cards_time_ms,
size_t processed_logged_cards,
size_t predicted_thread_buffer_cards,
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
index 30a8fe7f757..dd0b62a22ea 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.hpp
@@ -30,6 +30,7 @@
#include "memory/allocation.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
+#include "utilities/growableArray.hpp"
#include "utilities/macros.hpp"
// Forward decl
@@ -43,24 +44,25 @@ class ThreadClosure;
// iterate over them.
class G1ConcurrentRefineThreadControl {
G1ConcurrentRefine* _cr;
- G1ConcurrentRefineThread** _threads;
- uint _max_num_threads;
+ GrowableArrayCHeap _threads;
// Create the refinement thread for the given worker id.
// If initializing is true, ignore InjectGCWorkerCreationFailure.
G1ConcurrentRefineThread* create_refinement_thread(uint worker_id, bool initializing);
+ bool ensure_threads_created(uint worker_id, bool initializing);
+
NONCOPYABLE(G1ConcurrentRefineThreadControl);
public:
- G1ConcurrentRefineThreadControl();
+ G1ConcurrentRefineThreadControl(uint max_num_threads);
~G1ConcurrentRefineThreadControl();
- jint initialize(G1ConcurrentRefine* cr, uint max_num_threads);
+ jint initialize(G1ConcurrentRefine* cr);
void assert_current_thread_is_primary_refinement_thread() const NOT_DEBUG_RETURN;
- uint max_num_threads() const { return _max_num_threads; }
+ uint max_num_threads() const { return _threads.capacity(); }
// Activate the indicated thread. If the thread has not yet been allocated,
// allocate and then activate. If allocation is needed and fails, return
@@ -215,9 +217,6 @@ class G1ConcurrentRefine : public CHeapObj {
// Iterate over all concurrent refinement threads applying the given closure.
void threads_do(ThreadClosure *tc);
-
- // Maximum number of refinement threads.
- static uint max_num_threads();
};
#endif // SHARE_GC_G1_G1CONCURRENTREFINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp b/src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp
index bf25e3828d1..1dd3a9aface 100644
--- a/src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp
+++ b/src/hotspot/share/gc/g1/g1DirtyCardQueue.hpp
@@ -135,7 +135,7 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
// and install the next list, and meanwhile there can be a thread dealing
// with the previous list.
PausedList* volatile _plist;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(PausedList*));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(PausedList*));
NONCOPYABLE(PausedBuffers);
@@ -157,19 +157,19 @@ class G1DirtyCardQueueSet: public PtrQueueSet {
HeadTail take_all();
};
- DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0);
// Upper bound on the number of cards in the completed and paused buffers.
volatile size_t _num_cards;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(size_t));
// If the queue contains more cards than configured here, the
// mutator must start doing some of the concurrent refinement work.
volatile size_t _mutator_refinement_threshold;
- DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
+ DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(size_t));
// Buffers ready for refinement.
// NonblockingQueue has inner padding of one cache line.
NonblockingQueue _completed;
// Add a trailer padding after NonblockingQueue.
- DEFINE_PAD_MINUS_SIZE(3, DEFAULT_CACHE_LINE_SIZE, sizeof(BufferNode*));
+ DEFINE_PAD_MINUS_SIZE(3, DEFAULT_PADDING_SIZE, sizeof(BufferNode*));
// Buffers for which refinement is temporarily paused.
// PausedBuffers has inner padding, including trailer.
PausedBuffers _paused;
diff --git a/src/hotspot/share/gc/g1/g1FromCardCache.cpp b/src/hotspot/share/gc/g1/g1FromCardCache.cpp
index d3e7af768b5..062130f40ab 100644
--- a/src/hotspot/share/gc/g1/g1FromCardCache.cpp
+++ b/src/hotspot/share/gc/g1/g1FromCardCache.cpp
@@ -81,7 +81,7 @@ void G1FromCardCache::print(outputStream* out) {
#endif
uint G1FromCardCache::num_par_rem_sets() {
- return G1DirtyCardQueueSet::num_par_ids() + G1ConcurrentRefine::max_num_threads() + MAX2(ConcGCThreads, ParallelGCThreads);
+ return G1DirtyCardQueueSet::num_par_ids() + G1ConcRefinementThreads + MAX2(ConcGCThreads, ParallelGCThreads);
}
void G1FromCardCache::clear(uint region_idx) {
diff --git a/src/hotspot/share/gc/g1/g1HeapTransition.cpp b/src/hotspot/share/gc/g1/g1HeapTransition.cpp
index a2097041b20..4f759b4a606 100644
--- a/src/hotspot/share/gc/g1/g1HeapTransition.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapTransition.cpp
@@ -62,7 +62,7 @@ G1HeapTransition::Data::~Data() {
G1HeapTransition::G1HeapTransition(G1CollectedHeap* g1_heap) : _g1_heap(g1_heap), _before(g1_heap) { }
-struct DetailedUsage : public StackObj {
+struct G1HeapTransition::DetailedUsage : public StackObj {
size_t _eden_used;
size_t _survivor_used;
size_t _old_used;
@@ -79,7 +79,7 @@ struct DetailedUsage : public StackObj {
_humongous_region_count(0) {}
};
-class DetailedUsageClosure: public HeapRegionClosure {
+class G1HeapTransition::DetailedUsageClosure: public HeapRegionClosure {
public:
DetailedUsage _usage;
bool do_heap_region(HeapRegion* r) {
diff --git a/src/hotspot/share/gc/g1/g1HeapTransition.hpp b/src/hotspot/share/gc/g1/g1HeapTransition.hpp
index 09d901f283c..54f87fd3b71 100644
--- a/src/hotspot/share/gc/g1/g1HeapTransition.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapTransition.hpp
@@ -31,6 +31,9 @@
class G1CollectedHeap;
class G1HeapTransition {
+ struct DetailedUsage;
+ class DetailedUsageClosure;
+
struct Data {
size_t _eden_length;
size_t _survivor_length;
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArena.hpp b/src/hotspot/share/gc/g1/g1MonotonicArena.hpp
index 8a6bb88e008..586766a8c8f 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArena.hpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArena.hpp
@@ -125,7 +125,7 @@ class G1MonotonicArena::Segment {
char* _bottom; // Actual data.
// Do not add class member variables beyond this point
- static size_t header_size() { return align_up(sizeof(Segment), DEFAULT_CACHE_LINE_SIZE); }
+ static size_t header_size() { return align_up(sizeof(Segment), DEFAULT_PADDING_SIZE); }
static size_t payload_size(uint slot_size, uint num_slots) {
// The cast (size_t) is required to guard against overflow wrap around.
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index 1f3348614b1..8063d9652aa 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -95,7 +95,7 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
// entries, since entry 0 keeps track of surviving bytes for non-young regions.
// We also add a few elements at the beginning and at the end in
// an attempt to eliminate cache contention
- const size_t padding_elem_num = (DEFAULT_CACHE_LINE_SIZE / sizeof(size_t));
+ const size_t padding_elem_num = (DEFAULT_PADDING_SIZE / sizeof(size_t));
size_t array_length = padding_elem_num + _surviving_words_length + padding_elem_num;
_surviving_young_words_base = NEW_C_HEAP_ARRAY(size_t, array_length, mtGC);
diff --git a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
index f270e31f772..970cce1623e 100644
--- a/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
+++ b/src/hotspot/share/gc/g1/g1RedirtyCardsQueue.hpp
@@ -66,11 +66,11 @@ class G1RedirtyCardsLocalQueueSet : private PtrQueueSet {
// collected (and processed) buffers reverts back to collecting, allowing
// the set to be reused for another round of redirtying.
class G1RedirtyCardsQueueSet : public PtrQueueSet {
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, 0);
BufferNode::Stack _list;
- DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(size_t));
+ DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(size_t));
volatile size_t _entry_count;
- DEFINE_PAD_MINUS_SIZE(3, DEFAULT_CACHE_LINE_SIZE, sizeof(BufferNode*));
+ DEFINE_PAD_MINUS_SIZE(3, DEFAULT_PADDING_SIZE, sizeof(BufferNode*));
BufferNode* _tail;
DEBUG_ONLY(mutable bool _collecting;)
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index 2ff7e835d33..ac445850085 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -91,11 +91,6 @@ class G1RemSetScanState : public CHeapObj {
size_t _max_reserved_regions;
- // Has this region that is part of the regions in the collection set been processed yet.
- typedef bool G1RemsetIterState;
-
- G1RemsetIterState volatile* _collection_set_iter_state;
-
// Card table iteration claim for each heap region, from 0 (completely unscanned)
// to (>=) HeapRegion::CardsPerRegion (completely scanned).
uint volatile* _card_table_scan_state;
diff --git a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
index cedbefe1fe8..4156e85fbfa 100644
--- a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
@@ -67,7 +67,7 @@ double G1RemSetSummary::rs_thread_vtime(uint thread) const {
}
G1RemSetSummary::G1RemSetSummary(bool should_update) :
- _num_vtimes(G1ConcurrentRefine::max_num_threads()),
+ _num_vtimes(G1ConcRefinementThreads),
_rs_threads_vtimes(NEW_C_HEAP_ARRAY(double, _num_vtimes, mtGC)) {
memset(_rs_threads_vtimes, 0, sizeof(double) * _num_vtimes);
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
index 6c084a72d94..c227b625b8d 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
@@ -38,18 +38,16 @@ bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {
}
void G1RemSetTrackingPolicy::update_at_allocate(HeapRegion* r) {
- if (r->is_young()) {
- // Always collect remembered set for young regions.
- r->rem_set()->set_state_complete();
- } else if (r->is_humongous()) {
- // Collect remembered sets for humongous regions by default to allow eager reclaim.
- r->rem_set()->set_state_complete();
- } else if (r->is_old()) {
+ assert(r->is_young() || r->is_humongous() || r->is_old(),
+ "Region %u with unexpected heap region type %s", r->hrm_index(), r->get_type_str());
+ if (r->is_old()) {
// By default, do not create remembered set for new old regions.
r->rem_set()->set_state_untracked();
- } else {
- guarantee(false, "Unhandled region %u with heap region type %s", r->hrm_index(), r->get_type_str());
+ return;
}
+ // Always collect remembered set for young regions and for humongous regions.
+ // Humongous regions need that for eager reclaim.
+ r->rem_set()->set_state_complete();
}
void G1RemSetTrackingPolicy::update_at_free(HeapRegion* r) {
diff --git a/src/hotspot/share/gc/g1/g1_globals.hpp b/src/hotspot/share/gc/g1/g1_globals.hpp
index c46c1b47332..5f7f3a27d5b 100644
--- a/src/hotspot/share/gc/g1/g1_globals.hpp
+++ b/src/hotspot/share/gc/g1/g1_globals.hpp
@@ -113,11 +113,11 @@
"of the optimal occupancy to start marking.") \
range(1, max_intx) \
\
- product(uint, G1ConfidencePercent, 50, \
+ product(uint, G1ConfidencePercent, 50, \
"Confidence level for MMU/pause predictions") \
range(0, 100) \
\
- product(intx, G1SummarizeRSetStatsPeriod, 0, DIAGNOSTIC, \
+ product(uintx, G1SummarizeRSetStatsPeriod, 0, DIAGNOSTIC, \
"The period (in number of GCs) at which we will generate " \
"update buffer processing info " \
"(0 means do not periodically generate this info); " \
@@ -148,7 +148,7 @@
"Number of entries in an SATB log buffer.") \
constraint(G1SATBBufferSizeConstraintFunc, AtParse) \
\
- develop(intx, G1SATBProcessCompletedThreshold, 20, \
+ develop(uintx, G1SATBProcessCompletedThreshold, 20, \
"Number of completed buffers that triggers log processing.") \
range(0, max_jint) \
\
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp b/src/hotspot/share/gc/g1/heapRegionRemSet.cpp
index 41cce2fa661..19b383a08a3 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp
+++ b/src/hotspot/share/gc/g1/heapRegionRemSet.cpp
@@ -115,6 +115,10 @@ void HeapRegionRemSet::remove_code_root(nmethod* nm) {
guarantee(!_code_roots.contains(nm), "duplicate entry found");
}
+void HeapRegionRemSet::bulk_remove_code_roots() {
+ _code_roots.bulk_remove();
+}
+
void HeapRegionRemSet::code_roots_do(CodeBlobClosure* blk) const {
_code_roots.nmethods_do(blk);
}
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp b/src/hotspot/share/gc/g1/heapRegionRemSet.hpp
index d75f52baa95..2e5ae7b1c30 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp
+++ b/src/hotspot/share/gc/g1/heapRegionRemSet.hpp
@@ -150,6 +150,7 @@ class HeapRegionRemSet : public CHeapObj {
// the heap region that owns this RSet.
void add_code_root(nmethod* nm);
void remove_code_root(nmethod* nm);
+ void bulk_remove_code_roots();
// Applies blk->do_code_blob() to each of the entries in _code_roots
void code_roots_do(CodeBlobClosure* blk) const;
diff --git a/src/hotspot/share/gc/parallel/gcAdaptivePolicyCounters.hpp b/src/hotspot/share/gc/parallel/gcAdaptivePolicyCounters.hpp
index 57b48d49f95..e83813c78f4 100644
--- a/src/hotspot/share/gc/parallel/gcAdaptivePolicyCounters.hpp
+++ b/src/hotspot/share/gc/parallel/gcAdaptivePolicyCounters.hpp
@@ -60,7 +60,6 @@ class GCAdaptivePolicyCounters : public GCPolicyCounters {
PerfVariable* _decrease_for_footprint_counter;
PerfVariable* _minor_pause_young_slope_counter;
- PerfVariable* _major_pause_old_slope_counter;
PerfVariable* _decide_at_full_gc_counter;
diff --git a/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp b/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
index 77ecb4da466..18271fabd05 100644
--- a/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
+++ b/src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
@@ -184,8 +184,6 @@ class MutableNUMASpace : public MutableSpace {
virtual size_t used_in_words() const;
virtual size_t free_in_words() const;
- using MutableSpace::capacity_in_words;
-
virtual size_t tlab_capacity(Thread* thr) const;
virtual size_t tlab_used(Thread* thr) const;
virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
index 748a5658843..2531ecb2d5c 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
@@ -532,6 +532,14 @@ void ParallelScavengeHeap::resize_all_tlabs() {
CollectedHeap::resize_all_tlabs();
}
+void ParallelScavengeHeap::prune_scavengable_nmethods() {
+ ScavengableNMethods::prune_nmethods_not_into_young();
+}
+
+void ParallelScavengeHeap::prune_unlinked_nmethods() {
+ ScavengableNMethods::prune_unlinked_nmethods();
+}
+
// This method is used by System.gc() and JVMTI.
void ParallelScavengeHeap::collect(GCCause::Cause cause) {
assert(!Heap_lock->owned_by_self(),
@@ -863,10 +871,6 @@ void ParallelScavengeHeap::verify_nmethod(nmethod* nm) {
ScavengableNMethods::verify_nmethod(nm);
}
-void ParallelScavengeHeap::prune_scavengable_nmethods() {
- ScavengableNMethods::prune_nmethods();
-}
-
GrowableArray ParallelScavengeHeap::memory_managers() {
GrowableArray memory_managers(2);
memory_managers.append(_young_manager);
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
index d30effd5e5b..eec32a2c1c2 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
@@ -176,6 +176,7 @@ class ParallelScavengeHeap : public CollectedHeap {
void verify_nmethod(nmethod* nm) override;
void prune_scavengable_nmethods();
+ void prune_unlinked_nmethods();
size_t max_capacity() const override;
diff --git a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
index 3e255c15a4e..be6d9a501b5 100644
--- a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
+++ b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.cpp
@@ -51,7 +51,6 @@ PSAdaptiveSizePolicy::PSAdaptiveSizePolicy(size_t init_eden_size,
_avg_major_pause(new AdaptivePaddedAverage(AdaptiveTimeWeight, PausePadding)),
_avg_base_footprint(new AdaptiveWeightedAverage(AdaptiveSizePolicyWeight)),
_gc_stats(),
- _collection_cost_margin_fraction(AdaptiveSizePolicyCollectionCostMargin / 100.0),
_major_pause_old_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
_major_pause_young_estimator(new LinearLeastSquareFit(AdaptiveSizePolicyWeight)),
_latest_major_mutator_interval_seconds(0),
@@ -281,11 +280,11 @@ void PSAdaptiveSizePolicy::compute_eden_space_size(
//
// Make changes only to affect one of the pauses (the larger)
// at a time.
- adjust_eden_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size);
+ adjust_eden_for_pause_time(&desired_eden_size);
} else if (_avg_minor_pause->padded_average() > gc_minor_pause_goal_sec()) {
// Adjust only for the minor pause time goal
- adjust_eden_for_minor_pause_time(is_full_gc, &desired_eden_size);
+ adjust_eden_for_minor_pause_time(&desired_eden_size);
} else if(adjusted_mutator_cost() < _throughput_goal) {
// This branch used to require that (mutator_cost() > 0.0 in 1.4.2.
@@ -456,7 +455,7 @@ void PSAdaptiveSizePolicy::compute_old_gen_free_space(
// at a time.
if (is_full_gc) {
set_decide_at_full_gc(decide_at_full_gc_true);
- adjust_promo_for_pause_time(is_full_gc, &desired_promo_size, &desired_eden_size);
+ adjust_promo_for_pause_time(&desired_promo_size);
}
} else if (adjusted_mutator_cost() < _throughput_goal) {
// This branch used to require that (mutator_cost() > 0.0 in 1.4.2.
@@ -571,9 +570,7 @@ void PSAdaptiveSizePolicy::decay_supplemental_growth(bool is_full_gc) {
}
}
-void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc,
- size_t* desired_eden_size_ptr) {
-
+void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(size_t* desired_eden_size_ptr) {
// Adjust the young generation size to reduce pause time of
// of collections.
//
@@ -586,25 +583,23 @@ void PSAdaptiveSizePolicy::adjust_eden_for_minor_pause_time(bool is_full_gc,
decrease_young_gen_for_min_pauses_true);
*desired_eden_size_ptr = *desired_eden_size_ptr -
eden_decrement_aligned_down(*desired_eden_size_ptr);
- } else {
- // EXPERIMENTAL ADJUSTMENT
- // Only record that the estimator indicated such an action.
- // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta;
- set_change_young_gen_for_min_pauses(
- increase_young_gen_for_min_pauses_true);
+ } else {
+ // EXPERIMENTAL ADJUSTMENT
+ // Only record that the estimator indicated such an action.
+ // *desired_eden_size_ptr = *desired_eden_size_ptr + eden_heap_delta;
+ set_change_young_gen_for_min_pauses(
+ increase_young_gen_for_min_pauses_true);
}
}
-void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc,
- size_t* desired_promo_size_ptr,
- size_t* desired_eden_size_ptr) {
+void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(size_t* desired_promo_size_ptr) {
size_t promo_heap_delta = 0;
// Add some checks for a threshold for a change. For example,
// a change less than the required alignment is probably not worth
// attempting.
- if (_avg_minor_pause->padded_average() <= _avg_major_pause->padded_average() && is_full_gc) {
+ if (_avg_minor_pause->padded_average() <= _avg_major_pause->padded_average()) {
// Adjust for the major pause time only at full gc's because the
// affects of a change can only be seen at full gc's.
@@ -631,16 +626,14 @@ void PSAdaptiveSizePolicy::adjust_promo_for_pause_time(bool is_full_gc,
*desired_promo_size_ptr, promo_heap_delta);
}
-void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(bool is_full_gc,
- size_t* desired_promo_size_ptr,
- size_t* desired_eden_size_ptr) {
+void PSAdaptiveSizePolicy::adjust_eden_for_pause_time(size_t* desired_eden_size_ptr) {
size_t eden_heap_delta = 0;
// Add some checks for a threshold for a change. For example,
// a change less than the required alignment is probably not worth
// attempting.
if (_avg_minor_pause->padded_average() > _avg_major_pause->padded_average()) {
- adjust_eden_for_minor_pause_time(is_full_gc, desired_eden_size_ptr);
+ adjust_eden_for_minor_pause_time(desired_eden_size_ptr);
}
log_trace(gc, ergo)(
"PSAdaptiveSizePolicy::adjust_eden_for_pause_time "
@@ -880,17 +873,6 @@ size_t PSAdaptiveSizePolicy::scale_down(size_t change,
return reduced_change;
}
-size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden,
- uint percent_change) {
- size_t eden_heap_delta;
- eden_heap_delta = cur_eden / 100 * percent_change;
- return eden_heap_delta;
-}
-
-size_t PSAdaptiveSizePolicy::eden_increment(size_t cur_eden) {
- return eden_increment(cur_eden, YoungGenerationSizeIncrement);
-}
-
size_t PSAdaptiveSizePolicy::eden_increment_with_supplement_aligned_up(
size_t cur_eden) {
size_t result = eden_increment(cur_eden,
@@ -903,23 +885,6 @@ size_t PSAdaptiveSizePolicy::eden_decrement_aligned_down(size_t cur_eden) {
return align_down(eden_heap_delta, _space_alignment);
}
-size_t PSAdaptiveSizePolicy::eden_decrement(size_t cur_eden) {
- size_t eden_heap_delta = eden_increment(cur_eden) /
- AdaptiveSizeDecrementScaleFactor;
- return eden_heap_delta;
-}
-
-size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo,
- uint percent_change) {
- size_t promo_heap_delta;
- promo_heap_delta = cur_promo / 100 * percent_change;
- return promo_heap_delta;
-}
-
-size_t PSAdaptiveSizePolicy::promo_increment(size_t cur_promo) {
- return promo_increment(cur_promo, TenuredGenerationSizeIncrement);
-}
-
size_t PSAdaptiveSizePolicy::promo_increment_with_supplement_aligned_up(
size_t cur_promo) {
size_t result = promo_increment(cur_promo,
@@ -932,12 +897,6 @@ size_t PSAdaptiveSizePolicy::promo_decrement_aligned_down(size_t cur_promo) {
return align_down(promo_heap_delta, _space_alignment);
}
-size_t PSAdaptiveSizePolicy::promo_decrement(size_t cur_promo) {
- size_t promo_heap_delta = promo_increment(cur_promo);
- promo_heap_delta = promo_heap_delta / AdaptiveSizeDecrementScaleFactor;
- return promo_heap_delta;
-}
-
uint PSAdaptiveSizePolicy::compute_survivor_space_size_and_threshold(
bool is_survivor_overflow,
uint tenuring_threshold,
diff --git a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
index 580ff9d06c0..f6d4be029fb 100644
--- a/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
+++ b/src/hotspot/share/gc/parallel/psAdaptiveSizePolicy.hpp
@@ -75,8 +75,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
// Statistical data gathered for GC
GCStats _gc_stats;
- const double _collection_cost_margin_fraction;
-
// Variable for estimating the major and minor pause times.
// These variables represent linear least-squares fits of
// the data.
@@ -116,19 +114,13 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
private:
// Accessors
- AdaptivePaddedAverage* avg_major_pause() const { return _avg_major_pause; }
double gc_minor_pause_goal_sec() const { return _gc_minor_pause_goal_sec; }
- void adjust_eden_for_minor_pause_time(bool is_full_gc,
- size_t* desired_eden_size_ptr);
+ void adjust_eden_for_minor_pause_time(size_t* desired_eden_size_ptr);
// Change the generation sizes to achieve a GC pause time goal
// Returned sizes are not necessarily aligned.
- void adjust_promo_for_pause_time(bool is_full_gc,
- size_t* desired_promo_size_ptr,
- size_t* desired_eden_size_ptr);
- void adjust_eden_for_pause_time(bool is_full_gc,
- size_t* desired_promo_size_ptr,
- size_t* desired_eden_size_ptr);
+ void adjust_promo_for_pause_time(size_t* desired_promo_size_ptr);
+ void adjust_eden_for_pause_time(size_t* desired_eden_size_ptr);
// Change the generation sizes to achieve an application throughput goal
// Returned sizes are not necessarily aligned.
void adjust_promo_for_throughput(bool is_full_gc,
@@ -143,14 +135,10 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
size_t desired_total);
// Size in bytes for an increment or decrement of eden.
- virtual size_t eden_increment(size_t cur_eden, uint percent_change);
- virtual size_t eden_decrement(size_t cur_eden);
size_t eden_decrement_aligned_down(size_t cur_eden);
size_t eden_increment_with_supplement_aligned_up(size_t cur_eden);
// Size in bytes for an increment or decrement of the promotion area
- virtual size_t promo_increment(size_t cur_promo, uint percent_change);
- virtual size_t promo_decrement(size_t cur_promo);
size_t promo_decrement_aligned_down(size_t cur_promo);
size_t promo_increment_with_supplement_aligned_up(size_t cur_promo);
@@ -160,7 +148,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
size_t scale_down(size_t change, double part, double total);
protected:
- // Time accessors
// Footprint accessors
size_t live_space() const {
@@ -175,9 +162,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
void set_promo_size(size_t new_size) {
_promo_size = new_size;
}
- void set_survivor_size(size_t new_size) {
- _survivor_size = new_size;
- }
// Update estimators
void update_minor_pause_old_estimator(double minor_pause_in_ms);
@@ -185,9 +169,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
virtual GCPolicyKind kind() const { return _gc_ps_adaptive_size_policy; }
public:
- virtual size_t eden_increment(size_t cur_eden);
- virtual size_t promo_increment(size_t cur_promo);
-
// Accessors for use by performance counters
AdaptivePaddedNoZeroDevAverage* avg_promoted() const {
return _gc_stats.avg_promoted();
@@ -226,10 +207,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
size_t calculated_old_free_size_in_bytes() const;
- size_t average_old_live_in_bytes() const {
- return (size_t) avg_old_live()->average();
- }
-
size_t average_promoted_in_bytes() const {
return (size_t)avg_promoted()->average();
}
@@ -252,40 +229,6 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
_change_old_gen_for_min_pauses = v;
}
- // Return true if the old generation size was changed
- // to try to reach a pause time goal.
- bool old_gen_changed_for_pauses() {
- bool result = _change_old_gen_for_maj_pauses != 0 ||
- _change_old_gen_for_min_pauses != 0;
- return result;
- }
-
- // Return true if the young generation size was changed
- // to try to reach a pause time goal.
- bool young_gen_changed_for_pauses() {
- bool result = _change_young_gen_for_min_pauses != 0 ||
- _change_young_gen_for_maj_pauses != 0;
- return result;
- }
- // end flags for pause goal
-
- // Return true if the old generation size was changed
- // to try to reach a throughput goal.
- bool old_gen_changed_for_throughput() {
- bool result = _change_old_gen_for_throughput != 0;
- return result;
- }
-
- // Return true if the young generation size was changed
- // to try to reach a throughput goal.
- bool young_gen_changed_for_throughput() {
- bool result = _change_young_gen_for_throughput != 0;
- return result;
- }
-
- int decrease_for_footprint() { return _decrease_for_footprint; }
-
-
// Accessors for estimators. The slope of the linear fit is
// currently all that is used for making decisions.
@@ -293,18 +236,12 @@ class PSAdaptiveSizePolicy : public AdaptiveSizePolicy {
return _major_pause_old_estimator;
}
- LinearLeastSquareFit* major_pause_young_estimator() {
- return _major_pause_young_estimator;
- }
-
-
virtual void clear_generation_free_space_flags();
double major_pause_old_slope() { return _major_pause_old_estimator->slope(); }
double major_pause_young_slope() {
return _major_pause_young_estimator->slope();
}
- double major_collection_slope() { return _major_collection_estimator->slope();}
// Calculates optimal (free) space sizes for both the young and old
// generations. Stores results in _eden_size and _promo_size.
diff --git a/src/hotspot/share/gc/parallel/psCardTable.cpp b/src/hotspot/share/gc/parallel/psCardTable.cpp
index ecff82acbce..91fcf026069 100644
--- a/src/hotspot/share/gc/parallel/psCardTable.cpp
+++ b/src/hotspot/share/gc/parallel/psCardTable.cpp
@@ -111,7 +111,7 @@ void PSCardTable::scan_obj_with_limit(PSPromotionManager* pm,
}
}
-void PSCardTable::pre_scavenge(HeapWord* old_gen_bottom, uint active_workers) {
+void PSCardTable::pre_scavenge(uint active_workers) {
_preprocessing_active_workers = active_workers;
}
diff --git a/src/hotspot/share/gc/parallel/psCardTable.hpp b/src/hotspot/share/gc/parallel/psCardTable.hpp
index 3c510267bc3..70c32d23b7f 100644
--- a/src/hotspot/share/gc/parallel/psCardTable.hpp
+++ b/src/hotspot/share/gc/parallel/psCardTable.hpp
@@ -73,7 +73,7 @@ class PSCardTable: public CardTable {
_preprocessing_active_workers(0) {}
// Scavenge support
- void pre_scavenge(HeapWord* old_gen_bottom, uint active_workers);
+ void pre_scavenge(uint active_workers);
// Scavenge contents of stripes with the given index.
void scavenge_contents_parallel(ObjectStartArray* start_array,
HeapWord* old_gen_bottom,
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
index 0774597dbfa..c0eb5460228 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
@@ -47,9 +47,9 @@ class PCMarkAndPushClosure: public OopClosure {
public:
PCMarkAndPushClosure(ParCompactionManager* cm) : _compaction_manager(cm) { }
- template void do_oop_nv(T* p) { _compaction_manager->mark_and_push(p); }
- virtual void do_oop(oop* p) { do_oop_nv(p); }
- virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
+ template void do_oop_work(T* p) { _compaction_manager->mark_and_push(p); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
};
class PCIterateMarkAndPushClosure: public ClaimMetadataVisitingOopIterateClosure {
@@ -60,9 +60,9 @@ class PCIterateMarkAndPushClosure: public ClaimMetadataVisitingOopIterateClosure
ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_stw_fullgc_mark, rp),
_compaction_manager(cm) { }
- template void do_oop_nv(T* p) { _compaction_manager->mark_and_push(p); }
- virtual void do_oop(oop* p) { do_oop_nv(p); }
- virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
+ template void do_oop_work(T* p) { _compaction_manager->mark_and_push(p); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
};
inline bool ParCompactionManager::steal(int queue_num, oop& t) {
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
index edb6723bf0a..0bc10f8652f 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
@@ -416,8 +416,8 @@ print_initial_summary_data(ParallelCompactData& summary_data,
#endif // #ifndef PRODUCT
ParallelCompactData::ParallelCompactData() :
- _region_start(nullptr),
- DEBUG_ONLY(_region_end(nullptr) COMMA)
+ _heap_start(nullptr),
+ DEBUG_ONLY(_heap_end(nullptr) COMMA)
_region_vspace(nullptr),
_reserved_byte_size(0),
_region_data(nullptr),
@@ -426,16 +426,16 @@ ParallelCompactData::ParallelCompactData() :
_block_data(nullptr),
_block_count(0) {}
-bool ParallelCompactData::initialize(MemRegion covered_region)
+bool ParallelCompactData::initialize(MemRegion reserved_heap)
{
- _region_start = covered_region.start();
- const size_t region_size = covered_region.word_size();
- DEBUG_ONLY(_region_end = _region_start + region_size;)
+ _heap_start = reserved_heap.start();
+ const size_t heap_size = reserved_heap.word_size();
+ DEBUG_ONLY(_heap_end = _heap_start + heap_size;)
- assert(region_align_down(_region_start) == _region_start,
+ assert(region_align_down(_heap_start) == _heap_start,
"region start not aligned");
- bool result = initialize_region_data(region_size) && initialize_block_data();
+ bool result = initialize_region_data(heap_size) && initialize_block_data();
return result;
}
@@ -468,12 +468,11 @@ ParallelCompactData::create_vspace(size_t count, size_t element_size)
return 0;
}
-bool ParallelCompactData::initialize_region_data(size_t region_size)
+bool ParallelCompactData::initialize_region_data(size_t heap_size)
{
- assert((region_size & RegionSizeOffsetMask) == 0,
- "region size not a multiple of RegionSize");
+ assert(is_aligned(heap_size, RegionSize), "precondition");
- const size_t count = region_size >> Log2RegionSize;
+ const size_t count = heap_size >> Log2RegionSize;
_region_vspace = create_vspace(count, sizeof(RegionData));
if (_region_vspace != 0) {
_region_data = (RegionData*)_region_vspace->reserved_low_addr();
@@ -531,7 +530,7 @@ HeapWord* ParallelCompactData::partial_obj_end(size_t region_idx) const
void ParallelCompactData::add_obj(HeapWord* addr, size_t len)
{
- const size_t obj_ofs = pointer_delta(addr, _region_start);
+ const size_t obj_ofs = pointer_delta(addr, _heap_start);
const size_t beg_region = obj_ofs >> Log2RegionSize;
// end_region is inclusive
const size_t end_region = (obj_ofs + len - 1) >> Log2RegionSize;
@@ -1770,6 +1769,7 @@ bool PSParallelCompact::invoke_no_policy(bool maximum_heap_compaction) {
ref_processor()->start_discovery(maximum_heap_compaction);
ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
+ false /* unregister_nmethods_during_purge */,
false /* lock_codeblob_free_separately */);
marking_phase(&_gc_tracer);
@@ -2079,6 +2079,10 @@ void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
// Release unloaded nmethod's memory.
ctx->purge_nmethods();
}
+ {
+ GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", &_gc_timer);
+ ParallelScavengeHeap::heap()->prune_unlinked_nmethods();
+ }
{
GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
ctx->free_code_blobs();
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.hpp
index 4cee322a589..94995bd4f19 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.hpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.hpp
@@ -123,7 +123,6 @@ class SplitInfo
// The index of the split region, the size of the partial object on that
// region and the destination of the partial object.
- size_t src_region_idx() const { return _src_region_idx; }
size_t partial_obj_size() const { return _partial_obj_size; }
HeapWord* destination() const { return _destination; }
@@ -283,9 +282,6 @@ class ParallelCompactData
// The location of the java heap data that corresponds to this region.
inline HeapWord* data_location() const;
- // The highest address referenced by objects in this region.
- inline HeapWord* highest_ref() const;
-
// Whether this region is available to be claimed, has been claimed, or has
// been completed.
//
@@ -314,7 +310,6 @@ class ParallelCompactData
// These are atomic.
inline void add_live_obj(size_t words);
- inline void set_highest_ref(HeapWord* addr);
inline void decrement_destination_count();
inline bool claim();
@@ -399,7 +394,7 @@ class ParallelCompactData
public:
ParallelCompactData();
- bool initialize(MemRegion covered_region);
+ bool initialize(MemRegion reserved_heap);
size_t region_count() const { return _region_count; }
size_t reserved_byte_size() const { return _reserved_byte_size; }
@@ -443,26 +438,16 @@ class ParallelCompactData
inline size_t addr_to_region_idx(const HeapWord* addr) const;
inline RegionData* addr_to_region_ptr(const HeapWord* addr) const;
inline HeapWord* region_to_addr(size_t region) const;
- inline HeapWord* region_to_addr(size_t region, size_t offset) const;
inline HeapWord* region_to_addr(const RegionData* region) const;
inline HeapWord* region_align_down(HeapWord* addr) const;
inline HeapWord* region_align_up(HeapWord* addr) const;
inline bool is_region_aligned(HeapWord* addr) const;
- // Analogous to region_offset() for blocks.
- size_t block_offset(const HeapWord* addr) const;
size_t addr_to_block_idx(const HeapWord* addr) const;
- size_t addr_to_block_idx(const oop obj) const {
- return addr_to_block_idx(cast_from_oop(obj));
- }
inline BlockData* addr_to_block_ptr(const HeapWord* addr) const;
- inline HeapWord* block_to_addr(size_t block) const;
- inline size_t region_to_block_idx(size_t region) const;
inline HeapWord* block_align_down(HeapWord* addr) const;
- inline HeapWord* block_align_up(HeapWord* addr) const;
- inline bool is_block_aligned(HeapWord* addr) const;
// Return the address one past the end of the partial object.
HeapWord* partial_obj_end(size_t region_idx) const;
@@ -481,13 +466,12 @@ class ParallelCompactData
private:
bool initialize_block_data();
- bool initialize_region_data(size_t region_size);
+ bool initialize_region_data(size_t heap_size);
PSVirtualSpace* create_vspace(size_t count, size_t element_size);
-private:
- HeapWord* _region_start;
+ HeapWord* _heap_start;
#ifdef ASSERT
- HeapWord* _region_end;
+ HeapWord* _heap_end;
#endif // #ifdef ASSERT
PSVirtualSpace* _region_vspace;
@@ -564,12 +548,6 @@ inline HeapWord* ParallelCompactData::RegionData::data_location() const
NOT_DEBUG(return nullptr;)
}
-inline HeapWord* ParallelCompactData::RegionData::highest_ref() const
-{
- DEBUG_ONLY(return _highest_ref;)
- NOT_DEBUG(return nullptr;)
-}
-
inline void ParallelCompactData::RegionData::set_data_location(HeapWord* addr)
{
DEBUG_ONLY(_data_location = addr;)
@@ -598,16 +576,6 @@ inline void ParallelCompactData::RegionData::add_live_obj(size_t words)
Atomic::add(&_dc_and_los, static_cast(words));
}
-inline void ParallelCompactData::RegionData::set_highest_ref(HeapWord* addr)
-{
-#ifdef ASSERT
- HeapWord* tmp = _highest_ref;
- while (addr > tmp) {
- tmp = Atomic::cmpxchg(&_highest_ref, tmp, addr);
- }
-#endif // #ifdef ASSERT
-}
-
inline bool ParallelCompactData::RegionData::claim()
{
const region_sz_t los = static_cast(live_obj_size());
@@ -662,18 +630,18 @@ ParallelCompactData::block(size_t n) const {
inline size_t
ParallelCompactData::region_offset(const HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr");
+ assert(addr >= _heap_start, "bad addr");
// would mistakenly return 0 for _region_end
- assert(addr < _region_end, "bad addr");
+ assert(addr < _heap_end, "bad addr");
return (size_t(addr) & RegionAddrOffsetMask) >> LogHeapWordSize;
}
inline size_t
ParallelCompactData::addr_to_region_idx(const HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr " PTR_FORMAT " _region_start " PTR_FORMAT, p2i(addr), p2i(_region_start));
- assert(addr <= _region_end, "bad addr " PTR_FORMAT " _region_end " PTR_FORMAT, p2i(addr), p2i(_region_end));
- return pointer_delta(addr, _region_start) >> Log2RegionSize;
+ assert(addr >= _heap_start, "bad addr " PTR_FORMAT " _region_start " PTR_FORMAT, p2i(addr), p2i(_heap_start));
+ assert(addr <= _heap_end, "bad addr " PTR_FORMAT " _region_end " PTR_FORMAT, p2i(addr), p2i(_heap_end));
+ return pointer_delta(addr, _heap_start) >> Log2RegionSize;
}
inline ParallelCompactData::RegionData*
@@ -686,7 +654,7 @@ inline HeapWord*
ParallelCompactData::region_to_addr(size_t region) const
{
assert(region <= _region_count, "region out of range");
- return _region_start + (region << Log2RegionSize);
+ return _heap_start + (region << Log2RegionSize);
}
inline HeapWord*
@@ -696,27 +664,19 @@ ParallelCompactData::region_to_addr(const RegionData* region) const
sizeof(RegionData)));
}
-inline HeapWord*
-ParallelCompactData::region_to_addr(size_t region, size_t offset) const
-{
- assert(region <= _region_count, "region out of range");
- assert(offset < RegionSize, "offset too big"); // This may be too strict.
- return region_to_addr(region) + offset;
-}
-
inline HeapWord*
ParallelCompactData::region_align_down(HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr");
- assert(addr < _region_end + RegionSize, "bad addr");
+ assert(addr >= _heap_start, "bad addr");
+ assert(addr < _heap_end + RegionSize, "bad addr");
return (HeapWord*)(size_t(addr) & RegionAddrMask);
}
inline HeapWord*
ParallelCompactData::region_align_up(HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr");
- assert(addr <= _region_end, "bad addr");
+ assert(addr >= _heap_start, "bad addr");
+ assert(addr <= _heap_end, "bad addr");
return region_align_down(addr + RegionSizeOffsetMask);
}
@@ -726,20 +686,12 @@ ParallelCompactData::is_region_aligned(HeapWord* addr) const
return (size_t(addr) & RegionAddrOffsetMask) == 0;
}
-inline size_t
-ParallelCompactData::block_offset(const HeapWord* addr) const
-{
- assert(addr >= _region_start, "bad addr");
- assert(addr <= _region_end, "bad addr");
- return (size_t(addr) & BlockAddrOffsetMask) >> LogHeapWordSize;
-}
-
inline size_t
ParallelCompactData::addr_to_block_idx(const HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr");
- assert(addr <= _region_end, "bad addr");
- return pointer_delta(addr, _region_start) >> Log2BlockSize;
+ assert(addr >= _heap_start, "bad addr");
+ assert(addr <= _heap_end, "bad addr");
+ return pointer_delta(addr, _heap_start) >> Log2BlockSize;
}
inline ParallelCompactData::BlockData*
@@ -748,41 +700,14 @@ ParallelCompactData::addr_to_block_ptr(const HeapWord* addr) const
return block(addr_to_block_idx(addr));
}
-inline HeapWord*
-ParallelCompactData::block_to_addr(size_t block) const
-{
- assert(block < _block_count, "block out of range");
- return _region_start + (block << Log2BlockSize);
-}
-
-inline size_t
-ParallelCompactData::region_to_block_idx(size_t region) const
-{
- return region << Log2BlocksPerRegion;
-}
-
inline HeapWord*
ParallelCompactData::block_align_down(HeapWord* addr) const
{
- assert(addr >= _region_start, "bad addr");
- assert(addr < _region_end + RegionSize, "bad addr");
+ assert(addr >= _heap_start, "bad addr");
+ assert(addr < _heap_end + RegionSize, "bad addr");
return (HeapWord*)(size_t(addr) & BlockAddrMask);
}
-inline HeapWord*
-ParallelCompactData::block_align_up(HeapWord* addr) const
-{
- assert(addr >= _region_start, "bad addr");
- assert(addr <= _region_end, "bad addr");
- return block_align_down(addr + BlockSizeOffsetMask);
-}
-
-inline bool
-ParallelCompactData::is_block_aligned(HeapWord* addr) const
-{
- return block_offset(addr) == 0;
-}
-
// Abstract closure for use with ParMarkBitMap::iterate(), which will invoke the
// do_addr() method.
//
@@ -1171,15 +1096,6 @@ class PSParallelCompact : AllStatic {
static inline HeapWord* dense_prefix(SpaceId space_id);
static inline ObjectStartArray* start_array(SpaceId space_id);
- // Process the end of the given region range in the dense prefix.
- // This includes saving any object not updated.
- static void dense_prefix_regions_epilogue(ParCompactionManager* cm,
- size_t region_start_index,
- size_t region_end_index,
- idx_t exiting_object_offset,
- idx_t region_offset_start,
- idx_t region_offset_end);
-
// Update a region in the dense prefix. For each live object
// in the region, update it's interior references. For each
// dead object, fill it with deadwood. Dead space at the end
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
index ccf01ea3087..ff6052d5070 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
@@ -130,9 +130,9 @@ class PCAdjustPointerClosure: public BasicOopIterateClosure {
public:
PCAdjustPointerClosure(ParCompactionManager* cm) : _cm(cm) {
}
- template void do_oop_nv(T* p) { PSParallelCompact::adjust_pointer(p, _cm); }
- virtual void do_oop(oop* p) { do_oop_nv(p); }
- virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
+ template void do_oop_work(T* p) { PSParallelCompact::adjust_pointer(p, _cm); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
virtual ReferenceIterationMode reference_iteration_mode() { return DO_FIELDS; }
private:
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index 2c52a123d2c..294f2d099ae 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -174,13 +174,9 @@ void PSPromotionManager::reset_stats() {
#endif // TASKQUEUE_STATS
PSPromotionManager::PSPromotionManager() {
- ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
-
// We set the old lab's start array.
_old_lab.set_start_array(old_gen()->start_array());
- uint queue_size = claimed_stack_depth()->max_elems();
-
if (ParallelGCThreads == 1) {
_target_stack_size = 0;
} else {
@@ -277,9 +273,7 @@ template void PSPromotionManager::process_array_chunk_work(
T* p = base + start;
T* const chunk_end = base + end;
while (p < chunk_end) {
- if (PSScavenge::should_scavenge(p)) {
- claim_or_forward_depth(p);
- }
+ claim_or_forward_depth(p);
++p;
}
}
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index c1cbeb0f597..730d31893d5 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -56,11 +56,14 @@ inline void PSPromotionManager::push_depth(ScannerTask task) {
template
inline void PSPromotionManager::claim_or_forward_depth(T* p) {
- assert(should_scavenge(p, true), "revisiting object?");
assert(ParallelScavengeHeap::heap()->is_in(p), "pointer outside heap");
- oop obj = RawAccess::oop_load(p);
- Prefetch::write(obj->mark_addr(), 0);
- push_depth(ScannerTask(p));
+ T heap_oop = RawAccess<>::oop_load(p);
+ if (PSScavenge::is_obj_in_young(heap_oop)) {
+ oop obj = CompressedOops::decode_not_null(heap_oop);
+ assert(!PSScavenge::is_obj_in_to_space(obj), "revisiting object?");
+ Prefetch::write(obj->mark_addr(), 0);
+ push_depth(ScannerTask(p));
+ }
}
inline void PSPromotionManager::promotion_trace_event(oop new_obj, oop old_obj,
@@ -95,14 +98,12 @@ class PSPushContentsClosure: public BasicOopIterateClosure {
public:
PSPushContentsClosure(PSPromotionManager* pm) : BasicOopIterateClosure(PSScavenge::reference_processor()), _pm(pm) {}
- template void do_oop_nv(T* p) {
- if (PSScavenge::should_scavenge(p)) {
- _pm->claim_or_forward_depth(p);
- }
+ template void do_oop_work(T* p) {
+ _pm->claim_or_forward_depth(p);
}
- virtual void do_oop(oop* p) { do_oop_nv(p); }
- virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
+ virtual void do_oop(oop* p) { do_oop_work(p); }
+ virtual void do_oop(narrowOop* p) { do_oop_work(p); }
};
//
diff --git a/src/hotspot/share/gc/parallel/psScavenge.cpp b/src/hotspot/share/gc/parallel/psScavenge.cpp
index a783dfe7155..c864ded21c5 100644
--- a/src/hotspot/share/gc/parallel/psScavenge.cpp
+++ b/src/hotspot/share/gc/parallel/psScavenge.cpp
@@ -302,7 +302,7 @@ class ScavengeRootsTask : public WorkerTask {
if (!_is_old_gen_empty) {
PSCardTable* card_table = ParallelScavengeHeap::heap()->card_table();
- card_table->pre_scavenge(_old_gen->object_space()->bottom(), active_workers);
+ card_table->pre_scavenge(active_workers);
}
}
diff --git a/src/hotspot/share/gc/parallel/psScavenge.hpp b/src/hotspot/share/gc/parallel/psScavenge.hpp
index 69fa18aa0be..0c61bc9c30e 100644
--- a/src/hotspot/share/gc/parallel/psScavenge.hpp
+++ b/src/hotspot/share/gc/parallel/psScavenge.hpp
@@ -33,12 +33,10 @@
#include "oops/oop.hpp"
#include "utilities/stack.hpp"
-class OopStack;
class ReferenceProcessor;
class ParallelScavengeHeap;
class ParallelScavengeTracer;
class PSIsAliveClosure;
-class PSRefProcTaskExecutor;
class STWGCTimer;
class PSScavenge: AllStatic {
diff --git a/src/hotspot/share/gc/parallel/psVirtualspace.cpp b/src/hotspot/share/gc/parallel/psVirtualspace.cpp
index f0ccd9a751c..a4d686d8670 100644
--- a/src/hotspot/share/gc/parallel/psVirtualspace.cpp
+++ b/src/hotspot/share/gc/parallel/psVirtualspace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -113,7 +113,7 @@ void PSVirtualSpace::verify() const {
// Reserved region must be non-empty or both addrs must be 0.
assert(reserved_low_addr() < reserved_high_addr() ||
- reserved_low_addr() == nullptr && reserved_high_addr() == nullptr,
+ (reserved_low_addr() == nullptr && reserved_high_addr() == nullptr),
"bad reserved addrs");
assert(committed_low_addr() <= committed_high_addr(), "bad committed addrs");
diff --git a/src/hotspot/share/gc/serial/cardTableRS.cpp b/src/hotspot/share/gc/serial/cardTableRS.cpp
index 4df9fbad212..c4af2c8edb1 100644
--- a/src/hotspot/share/gc/serial/cardTableRS.cpp
+++ b/src/hotspot/share/gc/serial/cardTableRS.cpp
@@ -382,20 +382,17 @@ CardTable::CardValue* CardTableRS::find_first_dirty_card(CardValue* const start_
return end_card;
}
-// Because non-objArray objs can be imprecisely-marked (only obj-start card is
-// dirty instead of the part containing old-to-young pointers), if the
-// obj-start of a non-objArray is dirty, all cards that obj completely resides
-// on are considered as dirty, since that obj will be iterated (scanned for
-// old-to-young pointers) as a whole.
+// Because non-objArray objs can be imprecisely marked (only the obj-start card
+// is dirty instead of the part containing old-to-young pointers), if the
+// obj-start of a non-objArray is dirty, all cards that the obj resides on,
+// except the final one, are unconditionally considered as dirty. This is
+// because that obj will be iterated (scanned for old-to-young pointers) as a
+// whole.
template
CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_card,
CardValue* const end_card,
CardTableRS* ct,
Func& object_start) {
-
- // end_card might be just beyond the heap, so need to use the _raw variant.
- HeapWord* end_address = ct->addr_for_raw(end_card);
-
for (CardValue* current_card = start_card; current_card < end_card; /* empty */) {
if (is_dirty(current_card)) {
current_card++;
@@ -418,21 +415,14 @@ CardTable::CardValue* CardTableRS::find_first_clean_card(CardValue* const start_
return current_card;
}
- // This might be the last object in this area, avoid trying to access the
- // card beyond the allowed area.
- HeapWord* next_address = obj_start_addr + obj->size();
- if (next_address >= end_address) {
- break;
- }
-
- // Card occupied by next obj.
- CardValue* next_obj_card = ct->byte_for(next_address);
- if (is_clean(next_obj_card)) {
- return next_obj_card;
+ // Final card occupied by obj.
+ CardValue* obj_final_card = ct->byte_for(obj_start_addr + obj->size() - 1);
+ if (is_clean(obj_final_card)) {
+ return obj_final_card;
}
// Continue the search after this known-dirty card...
- current_card = next_obj_card + 1;
+ current_card = obj_final_card + 1;
}
return end_card;
diff --git a/src/hotspot/share/gc/serial/genMarkSweep.cpp b/src/hotspot/share/gc/serial/genMarkSweep.cpp
index 0defffdb824..28ba4615f73 100644
--- a/src/hotspot/share/gc/serial/genMarkSweep.cpp
+++ b/src/hotspot/share/gc/serial/genMarkSweep.cpp
@@ -36,6 +36,7 @@
#include "gc/serial/defNewGeneration.hpp"
#include "gc/serial/generation.hpp"
#include "gc/serial/genMarkSweep.hpp"
+#include "gc/serial/markSweep.inline.hpp"
#include "gc/serial/serialGcRefProcProxyTask.hpp"
#include "gc/serial/serialHeap.hpp"
#include "gc/shared/classUnloadingContext.hpp"
@@ -48,7 +49,7 @@
#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
-#include "gc/shared/space.hpp"
+#include "gc/shared/space.inline.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/weakProcessor.hpp"
#include "memory/universe.hpp"
@@ -57,6 +58,7 @@
#include "prims/jvmtiExport.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/javaThread.hpp"
+#include "runtime/prefetch.inline.hpp"
#include "runtime/synchronizer.hpp"
#include "runtime/vmThread.hpp"
#include "utilities/copy.hpp"
@@ -66,98 +68,281 @@
#include "jvmci/jvmci.hpp"
#endif
-void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
- assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
+class DeadSpacer : StackObj {
+ size_t _allowed_deadspace_words;
+ bool _active;
+ ContiguousSpace* _space;
- SerialHeap* gch = SerialHeap::heap();
-#ifdef ASSERT
- if (gch->soft_ref_policy()->should_clear_all_soft_refs()) {
- assert(clear_all_softrefs, "Policy should have been checked earlier");
+public:
+ DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
+ size_t ratio = _space->allowed_dead_ratio();
+ _active = ratio > 0;
+
+ if (_active) {
+ // We allow some amount of garbage towards the bottom of the space, so
+ // we don't start compacting before there is a significant gain to be made.
+ // Occasionally, we want to ensure a full compaction, which is determined
+ // by the MarkSweepAlwaysCompactCount parameter.
+ if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) {
+ _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
+ } else {
+ _active = false;
+ }
+ }
}
-#endif
- gch->trace_heap_before_gc(_gc_tracer);
-
- // Increment the invocation count
- _total_invocations++;
+ bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
+ if (!_active) {
+ return false;
+ }
- // Capture used regions for each generation that will be
- // subject to collection, so that card table adjustments can
- // be made intelligently (see clear / invalidate further below).
- gch->save_used_regions();
+ size_t dead_length = pointer_delta(dead_end, dead_start);
+ if (_allowed_deadspace_words >= dead_length) {
+ _allowed_deadspace_words -= dead_length;
+ CollectedHeap::fill_with_object(dead_start, dead_length);
+ oop obj = cast_to_oop(dead_start);
+ // obj->set_mark(obj->mark().set_marked());
+
+ assert(dead_length == obj->size(), "bad filler object size");
+ log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b",
+ p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
+
+ return true;
+ } else {
+ _active = false;
+ return false;
+ }
+ }
+};
- allocate_stacks();
+// Implement the "compaction" part of the mark-compact GC algorithm.
+class Compacter {
+ // There are four spaces in total, but only the first three can be used after
+ // compact. IOW, old and eden/from must be enough for all live objs
+ static constexpr uint max_num_spaces = 4;
+
+ struct CompactionSpace {
+ ContiguousSpace* _space;
+ // Will be the new top after compaction is complete.
+ HeapWord* _compaction_top;
+ // The first dead word in this contiguous space. It's an optimization to
+ // skip large chunk of live objects at the beginning.
+ HeapWord* _first_dead;
+
+ void init(ContiguousSpace* space) {
+ _space = space;
+ _compaction_top = space->bottom();
+ _first_dead = nullptr;
+ }
+ };
- mark_sweep_phase1(clear_all_softrefs);
+ CompactionSpace _spaces[max_num_spaces];
+ // The num of spaces to be compacted, i.e. containing live objs.
+ uint _num_spaces;
- mark_sweep_phase2();
+ uint _index;
- // Don't add any more derived pointers during phase3
-#if COMPILER2_OR_JVMCI
- assert(DerivedPointerTable::is_active(), "Sanity");
- DerivedPointerTable::set_active(false);
-#endif
+ HeapWord* get_compaction_top(uint index) const {
+ return _spaces[index]._compaction_top;
+ }
- mark_sweep_phase3();
+ HeapWord* get_first_dead(uint index) const {
+ return _spaces[index]._first_dead;
+ }
- mark_sweep_phase4();
+ ContiguousSpace* get_space(uint index) const {
+ return _spaces[index]._space;
+ }
- restore_marks();
+ void record_first_dead(uint index, HeapWord* first_dead) {
+ assert(_spaces[index]._first_dead == nullptr, "should write only once");
+ _spaces[index]._first_dead = first_dead;
+ }
- // Set saved marks for allocation profiler (and other things? -- dld)
- // (Should this be in general part?)
- gch->save_marks();
+ HeapWord* alloc(size_t words) {
+ while (true) {
+ if (words <= pointer_delta(_spaces[_index]._space->end(),
+ _spaces[_index]._compaction_top)) {
+ HeapWord* result = _spaces[_index]._compaction_top;
+ _spaces[_index]._compaction_top += words;
+ if (_index == 0) {
+ // old-gen requires BOT update
+ static_cast(_spaces[0]._space)->update_for_block(result, result + words);
+ }
+ return result;
+ }
+
+ // out-of-memory in this space
+ _index++;
+ assert(_index < max_num_spaces - 1, "the last space should not be used");
+ }
+ }
- deallocate_stacks();
+ static void prefetch_read_scan(void* p) {
+ if (PrefetchScanIntervalInBytes >= 0) {
+ Prefetch::read(p, PrefetchScanIntervalInBytes);
+ }
+ }
- MarkSweep::_string_dedup_requests->flush();
+ static void prefetch_write_scan(void* p) {
+ if (PrefetchScanIntervalInBytes >= 0) {
+ Prefetch::write(p, PrefetchScanIntervalInBytes);
+ }
+ }
- bool is_young_gen_empty = (gch->young_gen()->used() == 0);
- gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
+ static void prefetch_write_copy(void* p) {
+ if (PrefetchCopyIntervalInBytes >= 0) {
+ Prefetch::write(p, PrefetchCopyIntervalInBytes);
+ }
+ }
- gch->prune_scavengable_nmethods();
+ static void forward_obj(oop obj, HeapWord* new_addr) {
+ prefetch_write_scan(obj);
+ if (cast_from_oop(obj) != new_addr) {
+ obj->forward_to(cast_to_oop(new_addr));
+ } else {
+ assert(obj->is_gc_marked(), "inv");
+ // This obj will stay in-place. Fix the markword.
+ obj->init_mark();
+ }
+ }
- // Update heap occupancy information which is used as
- // input to soft ref clearing policy at the next gc.
- Universe::heap()->update_capacity_and_used_at_gc();
+ static HeapWord* find_next_live_addr(HeapWord* start, HeapWord* end) {
+ for (HeapWord* i_addr = start; i_addr < end; /* empty */) {
+ prefetch_read_scan(i_addr);
+ oop obj = cast_to_oop(i_addr);
+ if (obj->is_gc_marked()) {
+ return i_addr;
+ }
+ i_addr += obj->size();
+ }
+ return end;
+ };
- // Signal that we have completed a visit to all live objects.
- Universe::heap()->record_whole_heap_examined_timestamp();
+ static size_t relocate(HeapWord* addr) {
+ // Prefetch source and destination
+ prefetch_read_scan(addr);
- gch->trace_heap_after_gc(_gc_tracer);
-}
+ oop obj = cast_to_oop(addr);
+ oop new_obj = obj->forwardee();
+ HeapWord* new_addr = cast_from_oop(new_obj);
+ assert(addr != new_addr, "inv");
+ prefetch_write_copy(new_addr);
-void GenMarkSweep::allocate_stacks() {
- void* scratch = nullptr;
- size_t num_words;
- DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
- young_gen->contribute_scratch(scratch, num_words);
+ size_t obj_size = obj->size();
+ Copy::aligned_conjoint_words(addr, new_addr, obj_size);
+ new_obj->init_mark();
- if (scratch != nullptr) {
- _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
- } else {
- _preserved_count_max = 0;
+ return obj_size;
}
- _preserved_marks = (PreservedMark*)scratch;
- _preserved_count = 0;
-
- _preserved_overflow_stack_set.init(1);
-}
+public:
+ explicit Compacter(SerialHeap* heap) {
+ // In this order so that heap is compacted towards old-gen.
+ _spaces[0].init(heap->old_gen()->space());
+ _spaces[1].init(heap->young_gen()->eden());
+ _spaces[2].init(heap->young_gen()->from());
+
+ bool is_promotion_failed = (heap->young_gen()->from()->next_compaction_space() != nullptr);
+ if (is_promotion_failed) {
+ _spaces[3].init(heap->young_gen()->to());
+ _num_spaces = 4;
+ } else {
+ _num_spaces = 3;
+ }
+ _index = 0;
+ }
+ void phase2_calculate_new_addr() {
+ for (uint i = 0; i < _num_spaces; ++i) {
+ ContiguousSpace* space = get_space(i);
+ HeapWord* cur_addr = space->bottom();
+ HeapWord* top = space->top();
+
+ bool record_first_dead_done = false;
+
+ DeadSpacer dead_spacer(space);
+
+ while (cur_addr < top) {
+ oop obj = cast_to_oop(cur_addr);
+ size_t obj_size = obj->size();
+ if (obj->is_gc_marked()) {
+ HeapWord* new_addr = alloc(obj_size);
+ forward_obj(obj, new_addr);
+ cur_addr += obj_size;
+ } else {
+ // Skipping the current known-unmarked obj
+ HeapWord* next_live_addr = find_next_live_addr(cur_addr + obj_size, top);
+ if (dead_spacer.insert_deadspace(cur_addr, next_live_addr)) {
+ // Register space for the filler obj
+ alloc(pointer_delta(next_live_addr, cur_addr));
+ } else {
+ if (!record_first_dead_done) {
+ record_first_dead(i, cur_addr);
+ record_first_dead_done = true;
+ }
+ *(HeapWord**)cur_addr = next_live_addr;
+ }
+ cur_addr = next_live_addr;
+ }
+ }
+
+ if (!record_first_dead_done) {
+ record_first_dead(i, top);
+ }
+ }
+ }
-void GenMarkSweep::deallocate_stacks() {
- if (_preserved_count_max != 0) {
- DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
- young_gen->reset_scratch();
+ void phase3_adjust_pointers() {
+ for (uint i = 0; i < _num_spaces; ++i) {
+ ContiguousSpace* space = get_space(i);
+ HeapWord* cur_addr = space->bottom();
+ HeapWord* const top = space->top();
+ HeapWord* const first_dead = get_first_dead(i);
+
+ while (cur_addr < top) {
+ prefetch_write_scan(cur_addr);
+ if (cur_addr < first_dead || cast_to_oop(cur_addr)->is_gc_marked()) {
+ size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_addr));
+ cur_addr += size;
+ } else {
+ assert(*(HeapWord**)cur_addr > cur_addr, "forward progress");
+ cur_addr = *(HeapWord**)cur_addr;
+ }
+ }
+ }
}
- _preserved_overflow_stack_set.reclaim();
- _marking_stack.clear();
- _objarray_stack.clear(true);
-}
+ void phase4_compact() {
+ for (uint i = 0; i < _num_spaces; ++i) {
+ ContiguousSpace* space = get_space(i);
+ HeapWord* cur_addr = space->bottom();
+ HeapWord* top = space->top();
+
+ // Check if the first obj inside this space is forwarded.
+ if (!cast_to_oop(cur_addr)->is_forwarded()) {
+ // Jump over consecutive (in-place) live-objs-chunk
+ cur_addr = get_first_dead(i);
+ }
+
+ while (cur_addr < top) {
+ if (!cast_to_oop(cur_addr)->is_forwarded()) {
+ cur_addr = *(HeapWord**) cur_addr;
+ continue;
+ }
+ cur_addr += relocate(cur_addr);
+ }
+
+ // Reset top and unused memory
+ space->set_top(get_compaction_top(i));
+ if (ZapUnusedHeapArea) {
+ space->mangle_unused_area();
+ }
+ }
+ }
+};
-void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
+void GenMarkSweep::phase1_mark(bool clear_all_softrefs) {
// Recursively traverse all live objects and mark them
GCTraceTime(Info, gc, phases) tm("Phase 1: Mark live objects", _gc_timer);
@@ -219,6 +404,10 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
// Release unloaded nmethod's memory.
ctx->purge_nmethods();
}
+ {
+ GCTraceTime(Debug, gc, phases) ur("Unregister NMethods", gc_timer());
+ gch->prune_unlinked_nmethods();
+ }
{
GCTraceTime(Debug, gc, phases) t("Free Code Blobs", gc_timer());
ctx->free_code_blobs();
@@ -237,54 +426,121 @@ void GenMarkSweep::mark_sweep_phase1(bool clear_all_softrefs) {
}
}
+void GenMarkSweep::invoke_at_safepoint(bool clear_all_softrefs) {
+ assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint");
-void GenMarkSweep::mark_sweep_phase2() {
- // Now all live objects are marked, compute the new object addresses.
- GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
+ SerialHeap* gch = SerialHeap::heap();
+#ifdef ASSERT
+ if (gch->soft_ref_policy()->should_clear_all_soft_refs()) {
+ assert(clear_all_softrefs, "Policy should have been checked earlier");
+ }
+#endif
- SerialHeap::heap()->prepare_for_compaction();
-}
+ gch->trace_heap_before_gc(_gc_tracer);
-class GenAdjustPointersClosure: public SerialHeap::GenClosure {
-public:
- void do_generation(Generation* gen) {
- gen->adjust_pointers();
+ // Increment the invocation count
+ _total_invocations++;
+
+ // Capture used regions for each generation that will be
+ // subject to collection, so that card table adjustments can
+ // be made intelligently (see clear / invalidate further below).
+ gch->save_used_regions();
+
+ allocate_stacks();
+
+ phase1_mark(clear_all_softrefs);
+
+ Compacter compacter{gch};
+
+ {
+ // Now all live objects are marked, compute the new object addresses.
+ GCTraceTime(Info, gc, phases) tm("Phase 2: Compute new object addresses", _gc_timer);
+
+ compacter.phase2_calculate_new_addr();
}
-};
-void GenMarkSweep::mark_sweep_phase3() {
- SerialHeap* gch = SerialHeap::heap();
+ // Don't add any more derived pointers during phase3
+#if COMPILER2_OR_JVMCI
+ assert(DerivedPointerTable::is_active(), "Sanity");
+ DerivedPointerTable::set_active(false);
+#endif
+
+ {
+ // Adjust the pointers to reflect the new locations
+ GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
+
+ ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
+
+ CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
+ gch->process_roots(SerialHeap::SO_AllCodeCache,
+ &adjust_pointer_closure,
+ &adjust_cld_closure,
+ &adjust_cld_closure,
+ &code_closure);
+
+ WeakProcessor::oops_do(&adjust_pointer_closure);
+
+ adjust_marks();
+ compacter.phase3_adjust_pointers();
+ }
+
+ {
+ // All pointers are now adjusted, move objects accordingly
+ GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
+
+ compacter.phase4_compact();
+ }
- // Adjust the pointers to reflect the new locations
- GCTraceTime(Info, gc, phases) tm("Phase 3: Adjust pointers", gc_timer());
+ restore_marks();
+
+ // Set saved marks for allocation profiler (and other things? -- dld)
+ // (Should this be in general part?)
+ gch->save_marks();
+
+ deallocate_stacks();
- ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_stw_fullgc_adjust);
+ MarkSweep::_string_dedup_requests->flush();
- CodeBlobToOopClosure code_closure(&adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
- gch->process_roots(SerialHeap::SO_AllCodeCache,
- &adjust_pointer_closure,
- &adjust_cld_closure,
- &adjust_cld_closure,
- &code_closure);
+ bool is_young_gen_empty = (gch->young_gen()->used() == 0);
+ gch->rem_set()->maintain_old_to_young_invariant(gch->old_gen(), is_young_gen_empty);
- gch->gen_process_weak_roots(&adjust_pointer_closure);
+ gch->prune_scavengable_nmethods();
- adjust_marks();
- GenAdjustPointersClosure blk;
- gch->generation_iterate(&blk, true);
+ // Update heap occupancy information which is used as
+ // input to soft ref clearing policy at the next gc.
+ Universe::heap()->update_capacity_and_used_at_gc();
+
+ // Signal that we have completed a visit to all live objects.
+ Universe::heap()->record_whole_heap_examined_timestamp();
+
+ gch->trace_heap_after_gc(_gc_tracer);
}
-class GenCompactClosure: public SerialHeap::GenClosure {
-public:
- void do_generation(Generation* gen) {
- gen->compact();
+void GenMarkSweep::allocate_stacks() {
+ void* scratch = nullptr;
+ size_t num_words;
+ DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
+ young_gen->contribute_scratch(scratch, num_words);
+
+ if (scratch != nullptr) {
+ _preserved_count_max = num_words * HeapWordSize / sizeof(PreservedMark);
+ } else {
+ _preserved_count_max = 0;
}
-};
-void GenMarkSweep::mark_sweep_phase4() {
- // All pointers are now adjusted, move objects accordingly
- GCTraceTime(Info, gc, phases) tm("Phase 4: Move objects", _gc_timer);
+ _preserved_marks = (PreservedMark*)scratch;
+ _preserved_count = 0;
+
+ _preserved_overflow_stack_set.init(1);
+}
- GenCompactClosure blk;
- SerialHeap::heap()->generation_iterate(&blk, true);
+void GenMarkSweep::deallocate_stacks() {
+ if (_preserved_count_max != 0) {
+ DefNewGeneration* young_gen = (DefNewGeneration*)SerialHeap::heap()->young_gen();
+ young_gen->reset_scratch();
+ }
+
+ _preserved_overflow_stack_set.reclaim();
+ _marking_stack.clear();
+ _objarray_stack.clear(true);
}
diff --git a/src/hotspot/share/gc/serial/genMarkSweep.hpp b/src/hotspot/share/gc/serial/genMarkSweep.hpp
index be830318bd0..520e8010573 100644
--- a/src/hotspot/share/gc/serial/genMarkSweep.hpp
+++ b/src/hotspot/share/gc/serial/genMarkSweep.hpp
@@ -32,15 +32,8 @@ class GenMarkSweep : public MarkSweep {
static void invoke_at_safepoint(bool clear_all_softrefs);
private:
-
// Mark live objects
- static void mark_sweep_phase1(bool clear_all_softrefs);
- // Calculate new addresses
- static void mark_sweep_phase2();
- // Update pointers
- static void mark_sweep_phase3();
- // Move objects to new positions
- static void mark_sweep_phase4();
+ static void phase1_mark(bool clear_all_softrefs);
// Temporary data structures for traversal and storing/restoring marks
static void allocate_stacks();
diff --git a/src/hotspot/share/gc/serial/generation.cpp b/src/hotspot/share/gc/serial/generation.cpp
index aa3c804ba3c..302f8f231e7 100644
--- a/src/hotspot/share/gc/serial/generation.cpp
+++ b/src/hotspot/share/gc/serial/generation.cpp
@@ -173,18 +173,6 @@ HeapWord* Generation::block_start(const void* p) const {
return blk._start;
}
-class GenerationBlockSizeClosure : public SpaceClosure {
- public:
- const HeapWord* _p;
- size_t size;
- virtual void do_space(Space* s) {
- if (size == 0 && s->is_in_reserved(_p)) {
- size = s->block_size(_p);
- }
- }
- GenerationBlockSizeClosure(const HeapWord* p) { _p = p; size = 0; }
-};
-
class GenerationBlockIsObjClosure : public SpaceClosure {
public:
const HeapWord* _p;
@@ -218,34 +206,3 @@ void Generation::object_iterate(ObjectClosure* cl) {
GenerationObjIterateClosure blk(cl);
space_iterate(&blk);
}
-
-void Generation::prepare_for_compaction(CompactPoint* cp) {
- // Generic implementation, can be specialized
- ContiguousSpace* space = first_compaction_space();
- while (space != nullptr) {
- space->prepare_for_compaction(cp);
- space = space->next_compaction_space();
- }
-}
-
-class AdjustPointersClosure: public SpaceClosure {
- public:
- void do_space(Space* sp) {
- sp->adjust_pointers();
- }
-};
-
-void Generation::adjust_pointers() {
- // Note that this is done over all spaces, not just the compactible
- // ones.
- AdjustPointersClosure blk;
- space_iterate(&blk, true);
-}
-
-void Generation::compact() {
- ContiguousSpace* sp = first_compaction_space();
- while (sp != nullptr) {
- sp->compact();
- sp = sp->next_compaction_space();
- }
-}
diff --git a/src/hotspot/share/gc/serial/generation.hpp b/src/hotspot/share/gc/serial/generation.hpp
index 63eefae720b..d93c2b66df7 100644
--- a/src/hotspot/share/gc/serial/generation.hpp
+++ b/src/hotspot/share/gc/serial/generation.hpp
@@ -51,7 +51,7 @@
class DefNewGeneration;
class GCMemoryManager;
class ContiguousSpace;
-class CompactPoint;
+
class OopClosure;
class GCStats;
@@ -286,13 +286,6 @@ class Generation: public CHeapObj {
GCStats* gc_stats() const { return _gc_stats; }
virtual void update_gc_stats(Generation* current_generation, bool full) {}
- // Mark sweep support phase2
- virtual void prepare_for_compaction(CompactPoint* cp);
- // Mark sweep support phase3
- virtual void adjust_pointers();
- // Mark sweep support phase4
- virtual void compact();
-
// Accessing "marks".
// This function gives a generation a chance to note a point between
diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp
index 8f126c3129e..9361a4e5eb9 100644
--- a/src/hotspot/share/gc/serial/serialHeap.cpp
+++ b/src/hotspot/share/gc/serial/serialHeap.cpp
@@ -28,6 +28,7 @@
#include "gc/serial/tenuredGeneration.inline.hpp"
#include "gc/shared/gcLocker.inline.hpp"
#include "gc/shared/genMemoryPools.hpp"
+#include "gc/shared/scavengableNMethods.hpp"
#include "gc/shared/strongRootsScope.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "memory/universe.hpp"
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.hpp b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
index 75e8bf84860..6b2a5891c64 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.hpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
@@ -70,8 +70,6 @@ class TenuredGeneration: public Generation {
GenerationCounters* _gen_counters;
CSpaceCounters* _space_counters;
- // Accessing spaces
- TenuredSpace* space() const { return _the_space; }
// Attempt to expand the generation by "bytes". Expand by at a
// minimum "expand_bytes". Return true if some amount (not
@@ -85,6 +83,8 @@ class TenuredGeneration: public Generation {
public:
virtual void compute_new_size();
+ TenuredSpace* space() const { return _the_space; }
+
// Grow generation with specified size (returns false if unable to grow)
bool grow_by(size_t bytes);
// Grow generation to reserved size.
diff --git a/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp b/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
index ca5458cd49c..959fb310258 100644
--- a/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
+++ b/src/hotspot/share/gc/shared/adaptiveSizePolicy.hpp
@@ -51,15 +51,11 @@ class AdaptiveSizePolicy : public CHeapObj {
decrease_old_gen_for_throughput_true = -7,
decrease_young_gen_for_througput_true = -6,
- increase_old_gen_for_min_pauses_true = -5,
- decrease_old_gen_for_min_pauses_true = -4,
- decrease_young_gen_for_maj_pauses_true = -3,
increase_young_gen_for_min_pauses_true = -2,
increase_old_gen_for_maj_pauses_true = -1,
decrease_young_gen_for_min_pauses_true = 1,
decrease_old_gen_for_maj_pauses_true = 2,
- increase_young_gen_for_maj_pauses_true = 3,
increase_old_gen_for_throughput_true = 4,
increase_young_gen_for_througput_true = 5,
@@ -267,12 +263,12 @@ class AdaptiveSizePolicy : public CHeapObj {
// to use minor_collection_end() in its current form.
}
- virtual size_t eden_increment(size_t cur_eden);
- virtual size_t eden_increment(size_t cur_eden, uint percent_change);
- virtual size_t eden_decrement(size_t cur_eden);
- virtual size_t promo_increment(size_t cur_eden);
- virtual size_t promo_increment(size_t cur_eden, uint percent_change);
- virtual size_t promo_decrement(size_t cur_eden);
+ size_t eden_increment(size_t cur_eden);
+ size_t eden_increment(size_t cur_eden, uint percent_change);
+ size_t eden_decrement(size_t cur_eden);
+ size_t promo_increment(size_t cur_eden);
+ size_t promo_increment(size_t cur_eden, uint percent_change);
+ size_t promo_decrement(size_t cur_eden);
virtual void clear_generation_free_space_flags();
@@ -344,17 +340,11 @@ class AdaptiveSizePolicy : public CHeapObj {
AdaptiveWeightedAverage* avg_eden_live() const { return _avg_eden_live; }
AdaptiveWeightedAverage* avg_old_live() const { return _avg_old_live; }
- AdaptivePaddedAverage* avg_survived() const { return _avg_survived; }
- AdaptivePaddedNoZeroDevAverage* avg_pretenured() { return _avg_pretenured; }
-
// Methods indicating events of interest to the adaptive size policy,
// called by GC algorithms. It is the responsibility of users of this
// policy to call these methods at the correct times!
virtual void minor_collection_begin();
virtual void minor_collection_end(GCCause::Cause gc_cause);
- virtual LinearLeastSquareFit* minor_pause_old_estimator() const {
- return _minor_pause_old_estimator;
- }
LinearLeastSquareFit* minor_pause_young_estimator() {
return _minor_pause_young_estimator;
@@ -404,10 +394,6 @@ class AdaptiveSizePolicy : public CHeapObj {
_overhead_checker.set_gc_overhead_limit_exceeded(v);
}
- bool gc_overhead_limit_near() {
- return _overhead_checker.gc_overhead_limit_near();
- }
-
void reset_gc_overhead_limit_count() {
_overhead_checker.reset_gc_overhead_limit_count();
}
diff --git a/src/hotspot/share/gc/shared/cardTable.hpp b/src/hotspot/share/gc/shared/cardTable.hpp
index 61858075ca5..db6f9796ac1 100644
--- a/src/hotspot/share/gc/shared/cardTable.hpp
+++ b/src/hotspot/share/gc/shared/cardTable.hpp
@@ -83,14 +83,6 @@ class CardTable: public CHeapObj {
return cards_required(_whole_heap.word_size()) - 1;
}
- // Mapping from card marking array entry to address of first word without checks.
- HeapWord* addr_for_raw(const CardValue* p) const {
- // As _byte_map_base may be "negative" (the card table has been allocated before
- // the heap in memory), do not use pointer_delta() to avoid the assertion failure.
- size_t delta = p - _byte_map_base;
- return (HeapWord*) (delta << _card_shift);
- }
-
private:
void initialize_covered_region(void* region0_start, void* region1_start);
@@ -152,13 +144,16 @@ class CardTable: public CHeapObj {
return byte_after(p);
}
- // Mapping from card marking array entry to address of first word.
+ // Mapping from card marking array entry to address of first word
HeapWord* addr_for(const CardValue* p) const {
assert(p >= _byte_map && p < _byte_map + _byte_map_size,
"out of bounds access to card marking array. p: " PTR_FORMAT
" _byte_map: " PTR_FORMAT " _byte_map + _byte_map_size: " PTR_FORMAT,
p2i(p), p2i(_byte_map), p2i(_byte_map + _byte_map_size));
- HeapWord* result = addr_for_raw(p);
+ // As _byte_map_base may be "negative" (the card table has been allocated before
+ // the heap in memory), do not use pointer_delta() to avoid the assertion failure.
+ size_t delta = p - _byte_map_base;
+ HeapWord* result = (HeapWord*) (delta << _card_shift);
assert(_whole_heap.contains(result),
"Returning result = " PTR_FORMAT " out of bounds of "
" card marking array's _whole_heap = [" PTR_FORMAT "," PTR_FORMAT ")",
diff --git a/src/hotspot/share/gc/shared/classUnloadingContext.cpp b/src/hotspot/share/gc/shared/classUnloadingContext.cpp
index f624eaa6e44..6d9674ef801 100644
--- a/src/hotspot/share/gc/shared/classUnloadingContext.cpp
+++ b/src/hotspot/share/gc/shared/classUnloadingContext.cpp
@@ -32,10 +32,13 @@
ClassUnloadingContext* ClassUnloadingContext::_context = nullptr;
-ClassUnloadingContext::ClassUnloadingContext(uint num_workers, bool lock_codeblob_free_separately) :
+ClassUnloadingContext::ClassUnloadingContext(uint num_workers,
+ bool unregister_nmethods_during_purge,
+ bool lock_codeblob_free_separately) :
_cld_head(nullptr),
_num_nmethod_unlink_workers(num_workers),
_unlinked_nmethods(nullptr),
+ _unregister_nmethods_during_purge(unregister_nmethods_during_purge),
_lock_codeblob_free_separately(lock_codeblob_free_separately) {
assert(_context == nullptr, "context already set");
@@ -113,7 +116,7 @@ void ClassUnloadingContext::purge_nmethods() {
NMethodSet* set = _unlinked_nmethods[i];
for (nmethod* nm : *set) {
freed_memory += nm->size();
- nm->purge(false /* free_code_cache_data */);
+ nm->purge(false /* free_code_cache_data */, _unregister_nmethods_during_purge);
}
}
diff --git a/src/hotspot/share/gc/shared/classUnloadingContext.hpp b/src/hotspot/share/gc/shared/classUnloadingContext.hpp
index 42a8f764fa4..30930967d38 100644
--- a/src/hotspot/share/gc/shared/classUnloadingContext.hpp
+++ b/src/hotspot/share/gc/shared/classUnloadingContext.hpp
@@ -42,6 +42,7 @@ class ClassUnloadingContext : public CHeapObj {
using NMethodSet = GrowableArrayCHeap;
NMethodSet** _unlinked_nmethods;
+ bool _unregister_nmethods_during_purge;
bool _lock_codeblob_free_separately;
public:
@@ -49,10 +50,14 @@ class ClassUnloadingContext : public CHeapObj {
// Num_nmethod_unlink_workers configures the maximum numbers of threads unlinking
// nmethods.
+ // unregister_nmethods_during_purge determines whether unloaded nmethods should
+ // be unregistered from the garbage collector during purge. If not, ,the caller
+ // is responsible to do that later.
// lock_codeblob_free_separately determines whether freeing the code blobs takes
// the CodeCache_lock during the whole operation (=false) or per code blob
// free operation (=true).
ClassUnloadingContext(uint num_nmethod_unlink_workers,
+ bool unregister_nmethods_during_purge,
bool lock_codeblob_free_separately);
~ClassUnloadingContext();
diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp
index 422fef24318..1b65ee90b5d 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.cpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.cpp
@@ -558,9 +558,13 @@ void CollectedHeap::record_whole_heap_examined_timestamp() {
void CollectedHeap::full_gc_dump(GCTimer* timer, bool before) {
assert(timer != nullptr, "timer is null");
+ static uint count = 0;
if ((HeapDumpBeforeFullGC && before) || (HeapDumpAfterFullGC && !before)) {
- GCTraceTime(Info, gc) tm(before ? "Heap Dump (before full gc)" : "Heap Dump (after full gc)", timer);
- HeapDumper::dump_heap();
+ if (FullGCHeapDumpLimit == 0 || count < FullGCHeapDumpLimit) {
+ GCTraceTime(Info, gc) tm(before ? "Heap Dump (before full gc)" : "Heap Dump (after full gc)", timer);
+ HeapDumper::dump_heap();
+ count++;
+ }
}
LogTarget(Trace, gc, classhisto) lt;
diff --git a/src/hotspot/share/gc/shared/freeListAllocator.hpp b/src/hotspot/share/gc/shared/freeListAllocator.hpp
index d8ab4c4e8bc..74b896963a7 100644
--- a/src/hotspot/share/gc/shared/freeListAllocator.hpp
+++ b/src/hotspot/share/gc/shared/freeListAllocator.hpp
@@ -109,10 +109,10 @@ class FreeListAllocator {
typedef LockFreeStack Stack;
FreeListConfig* _config;
- char _name[DEFAULT_CACHE_LINE_SIZE - sizeof(FreeListConfig*)]; // Use name as padding.
+ char _name[DEFAULT_PADDING_SIZE - sizeof(FreeListConfig*)]; // Use name as padding.
#define DECLARE_PADDED_MEMBER(Id, Type, Name) \
- Type Name; DEFINE_PAD_MINUS_SIZE(Id, DEFAULT_CACHE_LINE_SIZE, sizeof(Type))
+ Type Name; DEFINE_PAD_MINUS_SIZE(Id, DEFAULT_PADDING_SIZE, sizeof(Type))
DECLARE_PADDED_MEMBER(1, volatile size_t, _free_count);
DECLARE_PADDED_MEMBER(2, Stack, _free_list);
DECLARE_PADDED_MEMBER(3, volatile bool, _transfer_lock);
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index e2b30bc615e..ae8674c84a9 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -408,11 +408,6 @@
"Allowed collection cost difference between generations") \
range(0, 100) \
\
- product(uint, AdaptiveSizePolicyCollectionCostMargin, 50, \
- "If collection costs are within margin, reduce both by full " \
- "delta") \
- range(0, 100) \
- \
product(uint, YoungGenerationSizeIncrement, 20, \
"Adaptive size percentage change in young generation") \
range(0, 100) \
diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.cpp b/src/hotspot/share/gc/shared/genCollectedHeap.cpp
index b81ce4b7782..cf101caae38 100644
--- a/src/hotspot/share/gc/shared/genCollectedHeap.cpp
+++ b/src/hotspot/share/gc/shared/genCollectedHeap.cpp
@@ -524,6 +524,7 @@ void GenCollectedHeap::do_collection(bool full,
CodeCache::on_gc_marking_cycle_start();
ClassUnloadingContext ctx(1 /* num_nmethod_unlink_workers */,
+ false /* unregister_nmethods_during_purge */,
false /* lock_codeblob_free_separately */);
collect_generation(_old_gen,
@@ -582,7 +583,11 @@ void GenCollectedHeap::verify_nmethod(nmethod* nm) {
}
void GenCollectedHeap::prune_scavengable_nmethods() {
- ScavengableNMethods::prune_nmethods();
+ ScavengableNMethods::prune_nmethods_not_into_young();
+}
+
+void GenCollectedHeap::prune_unlinked_nmethods() {
+ ScavengableNMethods::prune_unlinked_nmethods();
}
HeapWord* GenCollectedHeap::satisfy_failed_allocation(size_t size, bool is_tlab) {
@@ -708,10 +713,6 @@ void GenCollectedHeap::process_roots(ScanningOption so,
DEBUG_ONLY(ScavengableNMethods::asserted_non_scavengable_nmethods_do(&assert_code_is_non_scavengable));
}
-void GenCollectedHeap::gen_process_weak_roots(OopClosure* root_closure) {
- WeakProcessor::oops_do(root_closure);
-}
-
bool GenCollectedHeap::no_allocs_since_save_marks() {
return _young_gen->no_allocs_since_save_marks() &&
_old_gen->no_allocs_since_save_marks();
@@ -906,15 +907,6 @@ GenCollectedHeap* GenCollectedHeap::heap() {
return named_heap(CollectedHeap::Serial);
}
-#if INCLUDE_SERIALGC
-void GenCollectedHeap::prepare_for_compaction() {
- // Start by compacting into same gen.
- CompactPoint cp(_old_gen);
- _old_gen->prepare_for_compaction(&cp);
- _young_gen->prepare_for_compaction(&cp);
-}
-#endif // INCLUDE_SERIALGC
-
void GenCollectedHeap::verify(VerifyOption option /* ignored */) {
log_debug(gc, verify)("%s", _old_gen->name());
_old_gen->verify();
diff --git a/src/hotspot/share/gc/shared/genCollectedHeap.hpp b/src/hotspot/share/gc/shared/genCollectedHeap.hpp
index 0f4f95f90f7..1c1759f1422 100644
--- a/src/hotspot/share/gc/shared/genCollectedHeap.hpp
+++ b/src/hotspot/share/gc/shared/genCollectedHeap.hpp
@@ -182,6 +182,7 @@ class GenCollectedHeap : public CollectedHeap {
void verify_nmethod(nmethod* nm) override;
void prune_scavengable_nmethods();
+ void prune_unlinked_nmethods();
// Iteration functions.
void object_iterate(ObjectClosure* cl) override;
@@ -291,11 +292,6 @@ class GenCollectedHeap : public CollectedHeap {
CLDClosure* weak_cld_closure,
CodeBlobToOopClosure* code_roots);
- // Apply "root_closure" to all the weak roots of the system.
- // These include JNI weak roots, string table,
- // and referents of reachable weak refs.
- void gen_process_weak_roots(OopClosure* root_closure);
-
// Set the saved marks of generations, if that makes sense.
// In particular, if any generation might iterate over the oops
// in other generations, it should call this method.
@@ -339,13 +335,6 @@ class GenCollectedHeap : public CollectedHeap {
HeapWord* mem_allocate_work(size_t size,
bool is_tlab);
-#if INCLUDE_SERIALGC
- // For use by mark-sweep. As implemented, mark-sweep-compact is global
- // in an essential way: compaction is performed across generations, by
- // iterating over spaces.
- void prepare_for_compaction();
-#endif
-
// Save the tops of the spaces in all generations
void record_gen_tops_before_GC() PRODUCT_RETURN;
diff --git a/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp b/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
index 8ac9a3fc783..6c6f83177d2 100644
--- a/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
+++ b/src/hotspot/share/gc/shared/jvmFlagConstraintsGC.cpp
@@ -93,13 +93,7 @@ JVMFlag::Error YoungPLABSizeConstraintFunc(size_t value, bool verbose) {
}
JVMFlag::Error OldPLABSizeConstraintFunc(size_t value, bool verbose) {
- JVMFlag::Error status = JVMFlag::SUCCESS;
-
- {
- status = MinMaxPLABSizeBounds("OldPLABSize", value, verbose);
- }
-
- return status;
+ return MinMaxPLABSizeBounds("OldPLABSize", value, verbose);
}
JVMFlag::Error MinHeapFreeRatioConstraintFunc(uintx value, bool verbose) {
diff --git a/src/hotspot/share/gc/shared/satbMarkQueue.hpp b/src/hotspot/share/gc/shared/satbMarkQueue.hpp
index d92523d7e4f..78fe2033296 100644
--- a/src/hotspot/share/gc/shared/satbMarkQueue.hpp
+++ b/src/hotspot/share/gc/shared/satbMarkQueue.hpp
@@ -85,7 +85,7 @@ class SATBMarkQueue: public PtrQueue {
class SATBMarkQueueSet: public PtrQueueSet {
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, 0);
PaddedEnd _list;
volatile size_t _count_and_process_flag;
// These are rarely (if ever) changed, so same cache line as count.
@@ -93,7 +93,7 @@ class SATBMarkQueueSet: public PtrQueueSet {
size_t _buffer_enqueue_threshold;
// SATB is only active during marking. Enqueuing is only done when active.
bool _all_active;
- DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, 4 * sizeof(size_t));
+ DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, 4 * sizeof(size_t));
BufferNode* get_completed_buffer();
void abandon_completed_buffers();
diff --git a/src/hotspot/share/gc/shared/scavengableNMethods.cpp b/src/hotspot/share/gc/shared/scavengableNMethods.cpp
index ec9983da4a9..9f961ff4bf8 100644
--- a/src/hotspot/share/gc/shared/scavengableNMethods.cpp
+++ b/src/hotspot/share/gc/shared/scavengableNMethods.cpp
@@ -59,18 +59,8 @@ void ScavengableNMethods::register_nmethod(nmethod* nm) {
}
void ScavengableNMethods::unregister_nmethod(nmethod* nm) {
- assert_locked_or_safepoint(CodeCache_lock);
-
- if (gc_data(nm).on_list()) {
- nmethod* prev = nullptr;
- for (nmethod* cur = _head; cur != nullptr; cur = gc_data(cur).next()) {
- if (cur == nm) {
- unlist_nmethod(cur, prev);
- return;
- }
- prev = cur;
- }
- }
+ // All users of this method only unregister in bulk during code unloading.
+ ShouldNotReachHere();
}
#ifndef PRODUCT
@@ -172,10 +162,37 @@ void ScavengableNMethods::nmethods_do_and_prune(CodeBlobToOopClosure* cl) {
debug_only(verify_unlisted_nmethods(nullptr));
}
-void ScavengableNMethods::prune_nmethods() {
+void ScavengableNMethods::prune_nmethods_not_into_young() {
nmethods_do_and_prune(nullptr /* No closure */);
}
+void ScavengableNMethods::prune_unlinked_nmethods() {
+ assert_locked_or_safepoint(CodeCache_lock);
+
+ debug_only(mark_on_list_nmethods());
+
+ nmethod* prev = nullptr;
+ nmethod* cur = _head;
+ while (cur != nullptr) {
+ ScavengableNMethodsData data = gc_data(cur);
+ debug_only(data.clear_marked());
+ assert(data.on_list(), "else shouldn't be on this list");
+
+ nmethod* const next = data.next();
+
+ if (cur->is_unlinked()) {
+ unlist_nmethod(cur, prev);
+ } else {
+ prev = cur;
+ }
+
+ cur = next;
+ }
+
+ // Check for stray marks.
+ debug_only(verify_unlisted_nmethods(nullptr));
+}
+
// Walk the list of methods which might contain oops to the java heap.
void ScavengableNMethods::nmethods_do(CodeBlobToOopClosure* cl) {
nmethods_do_and_prune(cl);
@@ -218,8 +235,9 @@ void ScavengableNMethods::mark_on_list_nmethods() {
nmethod* nm = iter.method();
ScavengableNMethodsData data = gc_data(nm);
assert(data.not_marked(), "clean state");
- if (data.on_list())
+ if (data.on_list()) {
data.set_marked();
+ }
}
}
@@ -230,7 +248,10 @@ void ScavengableNMethods::verify_unlisted_nmethods(CodeBlobClosure* cl) {
while(iter.next()) {
nmethod* nm = iter.method();
- verify_nmethod(nm);
+ // Can not verify already unlinked nmethods as they are partially invalid already.
+ if (!nm->is_unlinked()) {
+ verify_nmethod(nm);
+ }
if (cl != nullptr && !gc_data(nm).on_list()) {
cl->do_code_blob(nm);
diff --git a/src/hotspot/share/gc/shared/scavengableNMethods.hpp b/src/hotspot/share/gc/shared/scavengableNMethods.hpp
index 4852e6d32fb..94d594cd529 100644
--- a/src/hotspot/share/gc/shared/scavengableNMethods.hpp
+++ b/src/hotspot/share/gc/shared/scavengableNMethods.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -46,8 +46,10 @@ class ScavengableNMethods : public AllStatic {
static void unregister_nmethod(nmethod* nm);
static void verify_nmethod(nmethod* nm);
- // Remove nmethods that no longer have scavengable oops.
- static void prune_nmethods();
+ // Remove nmethods that no longer have oops into young gen.
+ static void prune_nmethods_not_into_young();
+ // Remvoe unlinked (dead) nmethods.
+ static void prune_unlinked_nmethods();
// Apply closure to every scavengable nmethod.
// Remove nmethods that no longer have scavengable oops.
diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp
index 1c2afaa8b70..67d3ad53383 100644
--- a/src/hotspot/share/gc/shared/space.cpp
+++ b/src/hotspot/share/gc/shared/space.cpp
@@ -35,19 +35,13 @@
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
#include "runtime/java.hpp"
-#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
-#if INCLUDE_SERIALGC
-#include "gc/serial/serialBlockOffsetTable.inline.hpp"
-#include "gc/serial/defNewGeneration.hpp"
-#endif
ContiguousSpace::ContiguousSpace(): Space(),
- _compaction_top(nullptr),
_next_compaction_space(nullptr),
_top(nullptr) {
_mangler = new GenSpaceMangler(this);
@@ -59,8 +53,7 @@ ContiguousSpace::~ContiguousSpace() {
void ContiguousSpace::initialize(MemRegion mr,
bool clear_space,
- bool mangle_space)
-{
+ bool mangle_space) {
HeapWord* bottom = mr.start();
HeapWord* end = mr.end();
assert(Universe::on_page_boundary(bottom) && Universe::on_page_boundary(end),
@@ -70,7 +63,6 @@ void ContiguousSpace::initialize(MemRegion mr,
if (clear_space) {
clear(mangle_space);
}
- set_compaction_top(bottom);
_next_compaction_space = nullptr;
}
@@ -80,11 +72,6 @@ void ContiguousSpace::clear(bool mangle_space) {
if (ZapUnusedHeapArea && mangle_space) {
mangle_unused_area();
}
- _compaction_top = bottom();
-}
-
-bool ContiguousSpace::is_free_block(const HeapWord* p) const {
- return p >= _top;
}
#ifndef PRODUCT
@@ -115,230 +102,6 @@ void ContiguousSpace::mangle_unused_area_complete() {
#endif // NOT_PRODUCT
-HeapWord* ContiguousSpace::forward(oop q, size_t size,
- CompactPoint* cp, HeapWord* compact_top) {
- // q is alive
- // First check if we should switch compaction space
- assert(this == cp->space, "'this' should be current compaction space.");
- size_t compaction_max_size = pointer_delta(end(), compact_top);
- while (size > compaction_max_size) {
- // switch to next compaction space
- cp->space->set_compaction_top(compact_top);
- cp->space = cp->space->next_compaction_space();
- if (cp->space == nullptr) {
- cp->gen = GenCollectedHeap::heap()->young_gen();
- assert(cp->gen != nullptr, "compaction must succeed");
- cp->space = cp->gen->first_compaction_space();
- assert(cp->space != nullptr, "generation must have a first compaction space");
- }
- compact_top = cp->space->bottom();
- cp->space->set_compaction_top(compact_top);
- compaction_max_size = pointer_delta(cp->space->end(), compact_top);
- }
-
- // store the forwarding pointer into the mark word
- if (cast_from_oop(q) != compact_top) {
- q->forward_to(cast_to_oop(compact_top));
- assert(q->is_gc_marked(), "encoding the pointer should preserve the mark");
- } else {
- // if the object isn't moving we can just set the mark to the default
- // mark and handle it specially later on.
- q->init_mark();
- assert(!q->is_forwarded(), "should not be forwarded");
- }
-
- compact_top += size;
-
- // We need to update the offset table so that the beginnings of objects can be
- // found during scavenge. Note that we are updating the offset table based on
- // where the object will be once the compaction phase finishes.
- cp->space->update_for_block(compact_top - size, compact_top);
- return compact_top;
-}
-
-#if INCLUDE_SERIALGC
-
-void ContiguousSpace::prepare_for_compaction(CompactPoint* cp) {
- // Compute the new addresses for the live objects and store it in the mark
- // Used by universe::mark_sweep_phase2()
-
- // We're sure to be here before any objects are compacted into this
- // space, so this is a good time to initialize this:
- set_compaction_top(bottom());
-
- if (cp->space == nullptr) {
- assert(cp->gen != nullptr, "need a generation");
- assert(cp->gen->first_compaction_space() == this, "just checking");
- cp->space = cp->gen->first_compaction_space();
- cp->space->set_compaction_top(cp->space->bottom());
- }
-
- HeapWord* compact_top = cp->space->compaction_top(); // This is where we are currently compacting to.
-
- DeadSpacer dead_spacer(this);
-
- HeapWord* end_of_live = bottom(); // One byte beyond the last byte of the last live object.
- HeapWord* first_dead = nullptr; // The first dead object.
-
- const intx interval = PrefetchScanIntervalInBytes;
-
- HeapWord* cur_obj = bottom();
- HeapWord* scan_limit = top();
-
- while (cur_obj < scan_limit) {
- if (cast_to_oop(cur_obj)->is_gc_marked()) {
- // prefetch beyond cur_obj
- Prefetch::write(cur_obj, interval);
- size_t size = cast_to_oop(cur_obj)->size();
- compact_top = cp->space->forward(cast_to_oop(cur_obj), size, cp, compact_top);
- cur_obj += size;
- end_of_live = cur_obj;
- } else {
- // run over all the contiguous dead objects
- HeapWord* end = cur_obj;
- do {
- // prefetch beyond end
- Prefetch::write(end, interval);
- end += cast_to_oop(end)->size();
- } while (end < scan_limit && !cast_to_oop(end)->is_gc_marked());
-
- // see if we might want to pretend this object is alive so that
- // we don't have to compact quite as often.
- if (cur_obj == compact_top && dead_spacer.insert_deadspace(cur_obj, end)) {
- oop obj = cast_to_oop(cur_obj);
- compact_top = cp->space->forward(obj, obj->size(), cp, compact_top);
- end_of_live = end;
- } else {
- // otherwise, it really is a free region.
-
- // cur_obj is a pointer to a dead object. Use this dead memory to store a pointer to the next live object.
- *(HeapWord**)cur_obj = end;
-
- // see if this is the first dead region.
- if (first_dead == nullptr) {
- first_dead = cur_obj;
- }
- }
-
- // move on to the next object
- cur_obj = end;
- }
- }
-
- assert(cur_obj == scan_limit, "just checking");
- _end_of_live = end_of_live;
- if (first_dead != nullptr) {
- _first_dead = first_dead;
- } else {
- _first_dead = end_of_live;
- }
-
- // save the compaction_top of the compaction space.
- cp->space->set_compaction_top(compact_top);
-}
-
-void ContiguousSpace::adjust_pointers() {
- // Check first is there is any work to do.
- if (used() == 0) {
- return; // Nothing to do.
- }
-
- // adjust all the interior pointers to point at the new locations of objects
- // Used by MarkSweep::mark_sweep_phase3()
-
- HeapWord* cur_obj = bottom();
- HeapWord* const end_of_live = _end_of_live; // Established by prepare_for_compaction().
- HeapWord* const first_dead = _first_dead; // Established by prepare_for_compaction().
-
- assert(first_dead <= end_of_live, "Stands to reason, no?");
-
- const intx interval = PrefetchScanIntervalInBytes;
-
- debug_only(HeapWord* prev_obj = nullptr);
- while (cur_obj < end_of_live) {
- Prefetch::write(cur_obj, interval);
- if (cur_obj < first_dead || cast_to_oop(cur_obj)->is_gc_marked()) {
- // cur_obj is alive
- // point all the oops to the new location
- size_t size = MarkSweep::adjust_pointers(cast_to_oop(cur_obj));
- debug_only(prev_obj = cur_obj);
- cur_obj += size;
- } else {
- debug_only(prev_obj = cur_obj);
- // cur_obj is not a live object, instead it points at the next live object
- cur_obj = *(HeapWord**)cur_obj;
- assert(cur_obj > prev_obj, "we should be moving forward through memory, cur_obj: " PTR_FORMAT ", prev_obj: " PTR_FORMAT, p2i(cur_obj), p2i(prev_obj));
- }
- }
-
- assert(cur_obj == end_of_live, "just checking");
-}
-
-void ContiguousSpace::compact() {
- // Copy all live objects to their new location
- // Used by MarkSweep::mark_sweep_phase4()
-
- verify_up_to_first_dead(this);
-
- HeapWord* const start = bottom();
- HeapWord* const end_of_live = _end_of_live;
-
- assert(_first_dead <= end_of_live, "Invariant. _first_dead: " PTR_FORMAT " <= end_of_live: " PTR_FORMAT, p2i(_first_dead), p2i(end_of_live));
- if (_first_dead == end_of_live && (start == end_of_live || !cast_to_oop(start)->is_gc_marked())) {
- // Nothing to compact. The space is either empty or all live object should be left in place.
- clear_empty_region(this);
- return;
- }
-
- const intx scan_interval = PrefetchScanIntervalInBytes;
- const intx copy_interval = PrefetchCopyIntervalInBytes;
-
- assert(start < end_of_live, "bottom: " PTR_FORMAT " should be < end_of_live: " PTR_FORMAT, p2i(start), p2i(end_of_live));
- HeapWord* cur_obj = start;
- if (_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) {
- // All object before _first_dead can be skipped. They should not be moved.
- // A pointer to the first live object is stored at the memory location for _first_dead.
- cur_obj = *(HeapWord**)(_first_dead);
- }
-
- debug_only(HeapWord* prev_obj = nullptr);
- while (cur_obj < end_of_live) {
- if (!cast_to_oop(cur_obj)->is_forwarded()) {
- debug_only(prev_obj = cur_obj);
- // The first word of the dead object contains a pointer to the next live object or end of space.
- cur_obj = *(HeapWord**)cur_obj;
- assert(cur_obj > prev_obj, "we should be moving forward through memory");
- } else {
- // prefetch beyond q
- Prefetch::read(cur_obj, scan_interval);
-
- // size and destination
- size_t size = cast_to_oop(cur_obj)->size();
- HeapWord* compaction_top = cast_from_oop(cast_to_oop(cur_obj)->forwardee());
-
- // prefetch beyond compaction_top
- Prefetch::write(compaction_top, copy_interval);
-
- // copy object and reinit its mark
- assert(cur_obj != compaction_top, "everything in this pass should be moving");
- Copy::aligned_conjoint_words(cur_obj, compaction_top, size);
- oop new_obj = cast_to_oop(compaction_top);
-
- ContinuationGCSupport::transform_stack_chunk(new_obj);
-
- new_obj->init_mark();
- assert(new_obj->klass() != nullptr, "should have a class");
-
- debug_only(prev_obj = cur_obj);
- cur_obj += size;
- }
- }
-
- clear_empty_region(this);
-}
-
-#endif // INCLUDE_SERIALGC
-
void Space::print_short() const { print_short_on(tty); }
void Space::print_short_on(outputStream* st) const {
@@ -481,10 +244,6 @@ HeapWord* ContiguousSpace::par_allocate(size_t size) {
}
#if INCLUDE_SERIALGC
-void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) {
- _offsets.update_for_block(start, end);
-}
-
HeapWord* TenuredSpace::block_start_const(const void* addr) const {
HeapWord* cur_block = _offsets.block_start_reaching_into_card(addr);
diff --git a/src/hotspot/share/gc/shared/space.hpp b/src/hotspot/share/gc/shared/space.hpp
index 5d559f74f5e..85f870ba1e2 100644
--- a/src/hotspot/share/gc/shared/space.hpp
+++ b/src/hotspot/share/gc/shared/space.hpp
@@ -51,7 +51,6 @@ class Generation;
class ContiguousSpace;
class CardTableRS;
class DirtyCardToOopClosure;
-class FilteringClosure;
// A Space describes a heap area. Class Space is an abstract
// base class.
@@ -85,12 +84,6 @@ class Space: public CHeapObj {
void set_saved_mark_word(HeapWord* p) { _saved_mark_word = p; }
- // Returns true if this object has been allocated since a
- // generation's "save_marks" call.
- bool obj_allocated_since_save_marks(const oop obj) const {
- return cast_from_oop(obj) >= saved_mark_word();
- }
-
// Returns a subregion of the space containing only the allocated objects in
// the space.
virtual MemRegion used_region() const = 0;
@@ -134,9 +127,6 @@ class Space: public CHeapObj {
// given address.
bool is_in_reserved(const void* p) const { return _bottom <= p && p < _end; }
- // Returns true iff the given block is not allocated.
- virtual bool is_free_block(const HeapWord* p) const = 0;
-
// Test whether p is double-aligned
static bool is_aligned(void* p) {
return ::is_aligned(p, sizeof(double));
@@ -184,29 +174,12 @@ class Space: public CHeapObj {
// Allocation (return null if full). Enforces mutual exclusion internally.
virtual HeapWord* par_allocate(size_t word_size) = 0;
-#if INCLUDE_SERIALGC
- // Mark-sweep-compact support: all spaces can update pointers to objects
- // moving as a part of compaction.
- virtual void adjust_pointers() = 0;
-#endif
-
void print() const;
virtual void print_on(outputStream* st) const;
void print_short() const;
void print_short_on(outputStream* st) const;
};
-// A structure to represent a point at which objects are being copied
-// during compaction.
-class CompactPoint : public StackObj {
-public:
- Generation* gen;
- ContiguousSpace* space;
-
- CompactPoint(Generation* g = nullptr) :
- gen(g), space(nullptr) {}
-};
-
class GenSpaceMangler;
// A space in which the free area is contiguous. It therefore supports
@@ -215,26 +188,13 @@ class ContiguousSpace: public Space {
friend class VMStructs;
private:
- HeapWord* _compaction_top;
ContiguousSpace* _next_compaction_space;
- static inline void verify_up_to_first_dead(ContiguousSpace* space) NOT_DEBUG_RETURN;
-
- static inline void clear_empty_region(ContiguousSpace* space);
-
- protected:
+protected:
HeapWord* _top;
// A helper for mangling the unused area of the space in debug builds.
GenSpaceMangler* _mangler;
- // Used during compaction.
- HeapWord* _first_dead;
- HeapWord* _end_of_live;
-
- // This the function to invoke when an allocation of an object covering
- // "start" to "end" occurs to update other internal data structures.
- virtual void update_for_block(HeapWord* start, HeapWord* the_end) { }
-
GenSpaceMangler* mangler() { return _mangler; }
// Allocation helpers (return null if full).
@@ -254,23 +214,13 @@ class ContiguousSpace: public Space {
// The "clear" method must be called on a region that may have
// had allocation performed in it, but is now to be considered empty.
- virtual void clear(bool mangle_space);
-
- // Used temporarily during a compaction phase to hold the value
- // top should have when compaction is complete.
- HeapWord* compaction_top() const { return _compaction_top; }
-
- void set_compaction_top(HeapWord* value) {
- assert(value == nullptr || (value >= bottom() && value <= end()),
- "should point inside space");
- _compaction_top = value;
- }
+ void clear(bool mangle_space);
// Returns the next space (in the current generation) to be compacted in
// the global compaction order. Also is used to select the next
// space into which to compact.
- virtual ContiguousSpace* next_compaction_space() const {
+ ContiguousSpace* next_compaction_space() const {
return _next_compaction_space;
}
@@ -278,42 +228,10 @@ class ContiguousSpace: public Space {
_next_compaction_space = csp;
}
-#if INCLUDE_SERIALGC
- // MarkSweep support phase2
-
- // Start the process of compaction of the current space: compute
- // post-compaction addresses, and insert forwarding pointers. The fields
- // "cp->gen" and "cp->compaction_space" are the generation and space into
- // which we are currently compacting. This call updates "cp" as necessary,
- // and leaves the "compaction_top" of the final value of
- // "cp->compaction_space" up-to-date. Offset tables may be updated in
- // this phase as if the final copy had occurred; if so, "cp->threshold"
- // indicates when the next such action should be taken.
- void prepare_for_compaction(CompactPoint* cp);
- // MarkSweep support phase3
- void adjust_pointers() override;
- // MarkSweep support phase4
- virtual void compact();
-#endif // INCLUDE_SERIALGC
-
// The maximum percentage of objects that can be dead in the compacted
// live part of a compacted space ("deadwood" support.)
virtual size_t allowed_dead_ratio() const { return 0; };
- // "q" is an object of the given "size" that should be forwarded;
- // "cp" names the generation ("gen") and containing "this" (which must
- // also equal "cp->space"). "compact_top" is where in "this" the
- // next object should be forwarded to. If there is room in "this" for
- // the object, insert an appropriate forwarding pointer in "q".
- // If not, go to the next compaction space (there must
- // be one, since compaction must succeed -- we go to the first space of
- // the previous generation if necessary, updating "cp"), reset compact_top
- // and then forward. In either case, returns the new value of "compact_top".
- // Invokes the "update_for_block" function of the then-current compaction
- // space.
- virtual HeapWord* forward(oop q, size_t size, CompactPoint* cp,
- HeapWord* compact_top);
-
// Accessors
HeapWord* top() const { return _top; }
void set_top(HeapWord* value) { _top = value; }
@@ -346,8 +264,6 @@ class ContiguousSpace: public Space {
size_t used() const override { return byte_size(bottom(), top()); }
size_t free() const override { return byte_size(top(), end()); }
- bool is_free_block(const HeapWord* p) const override;
-
// In a contiguous space we have a more obvious bound on what parts
// contain objects.
MemRegion used_region() const override { return MemRegion(bottom(), top()); }
@@ -359,12 +275,6 @@ class ContiguousSpace: public Space {
// Iteration
void object_iterate(ObjectClosure* blk) override;
- // Compaction support
- void reset_after_compaction() {
- assert(compaction_top() >= bottom() && compaction_top() <= end(), "should point inside space");
- set_top(compaction_top());
- }
-
// Apply "blk->do_oop" to the addresses of all reference fields in objects
// starting with the _saved_mark_word, which was noted during a generation's
// save_marks and is required to denote the head of an object.
@@ -388,7 +298,6 @@ class ContiguousSpace: public Space {
// Addresses for inlined allocation
HeapWord** top_addr() { return &_top; }
- HeapWord** end_addr() { return &_end; }
void print_on(outputStream* st) const override;
@@ -419,8 +328,7 @@ class TenuredSpace: public ContiguousSpace {
inline HeapWord* allocate(size_t word_size) override;
inline HeapWord* par_allocate(size_t word_size) override;
- // MarkSweep support phase3
- void update_for_block(HeapWord* start, HeapWord* end) override;
+ inline void update_for_block(HeapWord* start, HeapWord* end);
void print_on(outputStream* st) const override;
};
diff --git a/src/hotspot/share/gc/shared/space.inline.hpp b/src/hotspot/share/gc/shared/space.inline.hpp
index 9c4c2864ced..bfaf84e8fac 100644
--- a/src/hotspot/share/gc/shared/space.inline.hpp
+++ b/src/hotspot/share/gc/shared/space.inline.hpp
@@ -27,17 +27,12 @@
#include "gc/shared/space.hpp"
-#include "gc/serial/generation.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/spaceDecorator.hpp"
#include "oops/oop.inline.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/prefetch.inline.hpp"
#include "runtime/safepoint.hpp"
-#if INCLUDE_SERIALGC
-#include "gc/serial/serialBlockOffsetTable.inline.hpp"
-#include "gc/serial/markSweep.inline.hpp"
-#endif
inline HeapWord* Space::block_start(const void* p) {
return block_start_const(p);
@@ -60,90 +55,8 @@ inline HeapWord* TenuredSpace::par_allocate(size_t size) {
return res;
}
-class DeadSpacer : StackObj {
- size_t _allowed_deadspace_words;
- bool _active;
- ContiguousSpace* _space;
-
-public:
- DeadSpacer(ContiguousSpace* space) : _allowed_deadspace_words(0), _space(space) {
- size_t ratio = _space->allowed_dead_ratio();
- _active = ratio > 0;
-
- if (_active) {
- assert(!UseG1GC, "G1 should not be using dead space");
-
- // We allow some amount of garbage towards the bottom of the space, so
- // we don't start compacting before there is a significant gain to be made.
- // Occasionally, we want to ensure a full compaction, which is determined
- // by the MarkSweepAlwaysCompactCount parameter.
- if ((MarkSweep::total_invocations() % MarkSweepAlwaysCompactCount) != 0) {
- _allowed_deadspace_words = (space->capacity() * ratio / 100) / HeapWordSize;
- } else {
- _active = false;
- }
- }
- }
-
- bool insert_deadspace(HeapWord* dead_start, HeapWord* dead_end) {
- if (!_active) {
- return false;
- }
-
- size_t dead_length = pointer_delta(dead_end, dead_start);
- if (_allowed_deadspace_words >= dead_length) {
- _allowed_deadspace_words -= dead_length;
- CollectedHeap::fill_with_object(dead_start, dead_length);
- oop obj = cast_to_oop(dead_start);
- obj->set_mark(obj->mark().set_marked());
-
- assert(dead_length == obj->size(), "bad filler object size");
- log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b",
- p2i(dead_start), p2i(dead_end), dead_length * HeapWordSize);
-
- return true;
- } else {
- _active = false;
- return false;
- }
- }
-};
-
-#ifdef ASSERT
-inline void ContiguousSpace::verify_up_to_first_dead(ContiguousSpace* space) {
- HeapWord* cur_obj = space->bottom();
-
- if (cur_obj < space->_end_of_live && space->_first_dead > cur_obj && !cast_to_oop(cur_obj)->is_gc_marked()) {
- // we have a chunk of the space which hasn't moved and we've reinitialized
- // the mark word during the previous pass, so we can't use is_gc_marked for
- // the traversal.
- HeapWord* prev_obj = nullptr;
-
- while (cur_obj < space->_first_dead) {
- size_t size = cast_to_oop(cur_obj)->size();
- assert(!cast_to_oop(cur_obj)->is_gc_marked(), "should be unmarked (special dense prefix handling)");
- prev_obj = cur_obj;
- cur_obj += size;
- }
- }
-}
-#endif
-
-inline void ContiguousSpace::clear_empty_region(ContiguousSpace* space) {
- // Let's remember if we were empty before we did the compaction.
- bool was_empty = space->used_region().is_empty();
- // Reset space after compaction is complete
- space->reset_after_compaction();
- // We do this clear, below, since it has overloaded meanings for some
- // space subtypes. For example, TenuredSpace's that were
- // compacted into will have had their offset table thresholds updated
- // continuously, but those that weren't need to have their thresholds
- // re-initialized. Also mangles unused area for debugging.
- if (space->used_region().is_empty()) {
- if (!was_empty) space->clear(SpaceDecorator::Mangle);
- } else {
- if (ZapUnusedHeapArea) space->mangle_unused_area();
- }
+inline void TenuredSpace::update_for_block(HeapWord* start, HeapWord* end) {
+ _offsets.update_for_block(start, end);
}
#endif // INCLUDE_SERIALGC
diff --git a/src/hotspot/share/gc/shared/taskTerminator.hpp b/src/hotspot/share/gc/shared/taskTerminator.hpp
index aac4e065a5c..540d91c88bb 100644
--- a/src/hotspot/share/gc/shared/taskTerminator.hpp
+++ b/src/hotspot/share/gc/shared/taskTerminator.hpp
@@ -72,9 +72,9 @@ class TaskTerminator : public CHeapObj {
uint _n_threads;
TaskQueueSetSuper* _queue_set;
- DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0);
volatile uint _offered_termination;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile uint));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(volatile uint));
Monitor _blocker;
Thread* _spin_master;
diff --git a/src/hotspot/share/gc/shared/taskqueue.hpp b/src/hotspot/share/gc/shared/taskqueue.hpp
index a5e3dfdf807..2e21ba33b0b 100644
--- a/src/hotspot/share/gc/shared/taskqueue.hpp
+++ b/src/hotspot/share/gc/shared/taskqueue.hpp
@@ -233,11 +233,11 @@ class TaskQueueSuper: public CHeapObj {
}
private:
- DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0);
// Index of the first free element after the last one pushed (mod N).
volatile uint _bottom;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(uint));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(uint));
// top() is the index of the oldest pushed element (mod N), and tag()
// is the associated epoch, to distinguish different modifications of
@@ -245,7 +245,7 @@ class TaskQueueSuper: public CHeapObj {
// (_bottom - top()) mod N == N-1; the latter indicates underflow
// during concurrent pop_local/pop_global.
volatile Age _age;
- DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(Age));
+ DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(Age));
NONCOPYABLE(TaskQueueSuper);
@@ -396,7 +396,7 @@ class GenericTaskQueue: public TaskQueueSuper {
// Element array.
E* _elems;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(E*));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(E*));
// Queue owner local variables. Not to be accessed by other threads.
static const uint InvalidQueueId = uint(-1);
@@ -404,7 +404,7 @@ class GenericTaskQueue: public TaskQueueSuper {
int _seed; // Current random seed used for selecting a random queue during stealing.
- DEFINE_PAD_MINUS_SIZE(2, DEFAULT_CACHE_LINE_SIZE, sizeof(uint) + sizeof(int));
+ DEFINE_PAD_MINUS_SIZE(2, DEFAULT_PADDING_SIZE, sizeof(uint) + sizeof(int));
public:
int next_random_queue_id();
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index 696cdc00dc5..a5d671a3b92 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -99,10 +99,6 @@
nonstatic_field(CollectedHeap, _is_gc_active, bool) \
nonstatic_field(CollectedHeap, _total_collections, unsigned int) \
\
- nonstatic_field(ContiguousSpace, _compaction_top, HeapWord*) \
- nonstatic_field(ContiguousSpace, _first_dead, HeapWord*) \
- nonstatic_field(ContiguousSpace, _end_of_live, HeapWord*) \
- \
nonstatic_field(ContiguousSpace, _top, HeapWord*) \
nonstatic_field(ContiguousSpace, _saved_mark_word, HeapWord*) \
\
diff --git a/src/hotspot/share/gc/shared/workerPolicy.cpp b/src/hotspot/share/gc/shared/workerPolicy.cpp
index 6db711a959e..4ab178dd6a7 100644
--- a/src/hotspot/share/gc/shared/workerPolicy.cpp
+++ b/src/hotspot/share/gc/shared/workerPolicy.cpp
@@ -73,11 +73,7 @@ uint WorkerPolicy::calc_parallel_worker_threads() {
uint WorkerPolicy::parallel_worker_threads() {
if (!_parallel_worker_threads_initialized) {
- if (FLAG_IS_DEFAULT(ParallelGCThreads)) {
- _parallel_worker_threads = WorkerPolicy::calc_parallel_worker_threads();
- } else {
- _parallel_worker_threads = ParallelGCThreads;
- }
+ _parallel_worker_threads = WorkerPolicy::calc_parallel_worker_threads();
_parallel_worker_threads_initialized = true;
}
return _parallel_worker_threads;
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
index f9ca3278a42..238f57b1538 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -2155,7 +2155,7 @@ void MemoryGraphFixer::collect_memory_nodes() {
assert(m != nullptr || (c->is_Loop() && j == LoopNode::LoopBackControl && iteration == 1) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "expect memory state");
if (m != nullptr) {
if (m == prev_region && ((c->is_Loop() && j == LoopNode::LoopBackControl) || (prev_region->is_Phi() && prev_region->in(0) == c))) {
- assert(c->is_Loop() && j == LoopNode::LoopBackControl || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
+ assert((c->is_Loop() && j == LoopNode::LoopBackControl) || _phase->C->has_irreducible_loop() || has_never_branch(_phase->C->root()), "");
// continue
} else if (unique == nullptr) {
unique = m;
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
index b2e812bb666..c36ff59d07e 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp
@@ -357,7 +357,7 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) {
if (EVAC && obj == fwd) {
fwd = _heap->evacuate_object(obj, thread);
}
- assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
+ shenandoah_assert_forwarded_except(elem_ptr, obj, _heap->cancelled_gc());
ShenandoahHeap::atomic_update_oop(fwd, elem_ptr, o);
obj = fwd;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
index b548073be33..13371f5e194 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp
@@ -53,7 +53,7 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
if (EVAC && obj == fwd) {
fwd = _heap->evacuate_object(obj, _thread);
}
- assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
+ shenandoah_assert_forwarded_except(p, obj, _heap->cancelled_gc());
ShenandoahHeap::atomic_update_oop(fwd, p, o);
obj = fwd;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
index 7564af5f6b7..c7284fbcead 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp
@@ -705,10 +705,8 @@ void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
if (resolved == obj) {
resolved = _heap->evacuate_object(obj, _thread);
}
+ shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
ShenandoahHeap::atomic_update_oop(resolved, p, obj);
- assert(_heap->cancelled_gc() ||
- _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
- "Sanity");
}
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
index d8398bb1ed8..e7cf402a527 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp
@@ -131,6 +131,27 @@ void ShenandoahDegenGC::op_degenerated() {
// and we can do evacuation. Otherwise, it would be the shortcut cycle.
if (heap->is_evacuation_in_progress()) {
+ if (_degen_point == _degenerated_evac) {
+ // Degeneration under oom-evac protocol allows the mutator LRB to expose
+ // references to from-space objects. This is okay, in theory, because we
+ // will come to the safepoint here to complete the evacuations and update
+ // the references. However, if the from-space reference is written to a
+ // region that was EC during final mark or was recycled after final mark
+ // it will not have TAMS or UWM updated. Such a region is effectively
+ // skipped during update references which can lead to crashes and corruption
+ // if the from-space reference is accessed.
+ if (UseTLAB) {
+ heap->labs_make_parsable();
+ }
+
+ for (size_t i = 0; i < heap->num_regions(); i++) {
+ ShenandoahHeapRegion* r = heap->get_region(i);
+ if (r->is_active() && r->top() > r->get_update_watermark()) {
+ r->set_update_watermark_at_safepoint(r->top());
+ }
+ }
+ }
+
// Degeneration under oom-evac protocol might have left some objects in
// collection set un-evacuated. Restart evacuation from the beginning to
// capture all objects. For all the objects that are already evacuated,
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index 18fd09ead0a..4cef5378d30 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -910,6 +910,9 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
// Make empty regions that have been allocated into regular
if (r->is_empty() && live > 0) {
r->make_regular_bypass();
+ if (ZapUnusedHeapArea) {
+ SpaceMangler::mangle_region(MemRegion(r->top(), r->end()));
+ }
}
// Reclaim regular regions that became empty
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index 67d7f0eb4fa..12eb2aad5e8 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -504,6 +504,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
_num_regions(0),
_regions(nullptr),
_update_refs_iterator(this),
+ _gc_state_changed(false),
_gc_no_progress_count(0),
_control_thread(nullptr),
_shenandoah_policy(policy),
@@ -1741,27 +1742,32 @@ void ShenandoahHeap::prepare_update_heap_references(bool concurrent) {
_update_refs_iterator.reset();
}
-void ShenandoahHeap::set_gc_state_all_threads(char state) {
- for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
- ShenandoahThreadLocalData::set_gc_state(t, state);
+void ShenandoahHeap::propagate_gc_state_to_java_threads() {
+ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
+ if (_gc_state_changed) {
+ _gc_state_changed = false;
+ char state = gc_state();
+ for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
+ ShenandoahThreadLocalData::set_gc_state(t, state);
+ }
}
}
-void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) {
- assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should really be Shenandoah safepoint");
+void ShenandoahHeap::set_gc_state(uint mask, bool value) {
+ assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
_gc_state.set_cond(mask, value);
- set_gc_state_all_threads(_gc_state.raw_value());
+ _gc_state_changed = true;
}
void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) {
assert(!has_forwarded_objects(), "Not expected before/after mark phase");
- set_gc_state_mask(MARKING, in_progress);
+ set_gc_state(MARKING, in_progress);
ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress);
}
void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
- set_gc_state_mask(EVACUATION, in_progress);
+ set_gc_state(EVACUATION, in_progress);
}
void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
@@ -1773,7 +1779,7 @@ void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
}
void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
- set_gc_state_mask(WEAK_ROOTS, cond);
+ set_gc_state(WEAK_ROOTS, cond);
}
GCTracer* ShenandoahHeap::tracer() {
@@ -1822,6 +1828,7 @@ void ShenandoahHeap::stop() {
void ShenandoahHeap::stw_unload_classes(bool full_gc) {
if (!unload_classes()) return;
ClassUnloadingContext ctx(_workers->active_workers(),
+ true /* unregister_nmethods_during_purge */,
false /* lock_codeblob_free_separately */);
// Unload classes and purge SystemDictionary.
@@ -1899,7 +1906,7 @@ void ShenandoahHeap::parallel_cleaning(bool full_gc) {
}
void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
- set_gc_state_mask(HAS_FORWARDED, cond);
+ set_gc_state(HAS_FORWARDED, cond);
}
void ShenandoahHeap::set_unload_classes(bool uc) {
@@ -1938,7 +1945,7 @@ void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
}
void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
- set_gc_state_mask(UPDATEREFS, in_progress);
+ set_gc_state(UPDATEREFS, in_progress);
}
void ShenandoahHeap::register_nmethod(nmethod* nm) {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
index f5030efa748..deb0a972167 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp
@@ -124,6 +124,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
friend class ShenandoahGCStateResetter;
friend class ShenandoahParallelObjectIterator;
friend class ShenandoahSafepoint;
+
// Supported GC
friend class ShenandoahConcurrentGC;
friend class ShenandoahDegenGC;
@@ -283,6 +284,7 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
};
private:
+ bool _gc_state_changed;
ShenandoahSharedBitmap _gc_state;
ShenandoahSharedFlag _degenerated_gc_in_progress;
ShenandoahSharedFlag _full_gc_in_progress;
@@ -291,12 +293,20 @@ class ShenandoahHeap : public CollectedHeap, public ShenandoahSpaceInfo {
size_t _gc_no_progress_count;
- void set_gc_state_all_threads(char state);
- void set_gc_state_mask(uint mask, bool value);
+ // This updates the singlular, global gc state. This must happen on a safepoint.
+ void set_gc_state(uint mask, bool value);
public:
char gc_state() const;
+ // This copies the global gc state into a thread local variable for java threads.
+ // It is primarily intended to support quick access at barriers.
+ void propagate_gc_state_to_java_threads();
+
+ // This is public to support assertions that the state hasn't been changed off of
+ // a safepoint and that any changes were propagated to java threads after the safepoint.
+ bool has_gc_state_changed() const { return _gc_state_changed; }
+
void set_concurrent_mark_in_progress(bool in_progress);
void set_evacuation_in_progress(bool in_progress);
void set_update_refs_in_progress(bool in_progress);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp
index f99de804bd5..f395119d46a 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp
@@ -362,7 +362,7 @@ bool ShenandoahReferenceProcessor::discover_reference(oop reference, ReferenceTy
log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
uint worker_id = WorkerThread::worker_id();
- _ref_proc_thread_locals->inc_encountered(type);
+ _ref_proc_thread_locals[worker_id].inc_encountered(type);
if (UseCompressedOops) {
return discover(reference, type, worker_id);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
index 422595e9313..9eec573cc56 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp
@@ -89,6 +89,7 @@ class ShenandoahThreadLocalData {
}
static char gc_state(Thread* thread) {
+ assert(thread->is_Java_thread(), "GC state is only synchronized to java threads");
return data(thread)->_gc_state;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
index 81eee97f25f..bb13e9b8e22 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
@@ -140,6 +140,7 @@ void ShenandoahUnload::unload() {
assert(heap->is_concurrent_weak_root_in_progress(), "Filtered by caller");
ClassUnloadingContext ctx(heap->workers()->active_workers(),
+ true /* unregister_nmethods_during_purge */,
true /* lock_codeblob_free_separately */);
// Unlink stale metadata and nmethods
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp
index 4a97e599f3e..eeeb1dcad19 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp
@@ -35,12 +35,23 @@
#include "interpreter/oopMapCache.hpp"
#include "memory/universe.hpp"
+bool VM_ShenandoahOperation::doit_prologue() {
+ assert(!ShenandoahHeap::heap()->has_gc_state_changed(), "GC State can only be changed on a safepoint.");
+ return true;
+}
+
+void VM_ShenandoahOperation::doit_epilogue() {
+ assert(!ShenandoahHeap::heap()->has_gc_state_changed(), "GC State was not synchronized to java threads.");
+}
+
bool VM_ShenandoahReferenceOperation::doit_prologue() {
+ VM_ShenandoahOperation::doit_prologue();
Heap_lock->lock();
return true;
}
void VM_ShenandoahReferenceOperation::doit_epilogue() {
+ VM_ShenandoahOperation::doit_epilogue();
OopMapCache::cleanup_old_entries();
if (Universe::has_reference_pending_list()) {
Heap_lock->notify_all();
@@ -51,34 +62,41 @@ void VM_ShenandoahReferenceOperation::doit_epilogue() {
void VM_ShenandoahInitMark::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Init Mark", SvcGCMarker::CONCURRENT);
_gc->entry_init_mark();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahFinalMarkStartEvac::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Final Mark", SvcGCMarker::CONCURRENT);
_gc->entry_final_mark();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahFullGC::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Full GC", SvcGCMarker::FULL);
_full_gc->entry_full(_gc_cause);
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahDegeneratedGC::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Degenerated GC", SvcGCMarker::CONCURRENT);
_gc->entry_degenerated();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahInitUpdateRefs::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Init Update Refs", SvcGCMarker::CONCURRENT);
_gc->entry_init_updaterefs();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahFinalUpdateRefs::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Final Update Refs", SvcGCMarker::CONCURRENT);
_gc->entry_final_updaterefs();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
void VM_ShenandoahFinalRoots::doit() {
ShenandoahGCPauseMark mark(_gc_id, "Final Roots", SvcGCMarker::CONCURRENT);
_gc->entry_final_roots();
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp
index 65ddd8b1f11..1b78766935f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp
@@ -47,14 +47,16 @@ class VM_ShenandoahOperation : public VM_Operation {
uint _gc_id;
public:
VM_ShenandoahOperation() : _gc_id(GCId::current()) {};
- virtual bool skip_thread_oop_barriers() const { return true; }
+ bool skip_thread_oop_barriers() const override { return true; }
+ bool doit_prologue() override;
+ void doit_epilogue() override;
};
class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation {
public:
VM_ShenandoahReferenceOperation() : VM_ShenandoahOperation() {};
- bool doit_prologue();
- void doit_epilogue();
+ bool doit_prologue() override;
+ void doit_epilogue() override;
};
class VM_ShenandoahInitMark: public VM_ShenandoahOperation {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
index 1d5d962a4ec..f67cafdb8fe 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp
@@ -620,6 +620,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label,
guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens");
guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize");
+ ShenandoahHeap::heap()->propagate_gc_state_to_java_threads();
+
// Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check
ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/);
diff --git a/src/hotspot/share/gc/x/xHeap.cpp b/src/hotspot/share/gc/x/xHeap.cpp
index 65da4e1c983..75d9bc08460 100644
--- a/src/hotspot/share/gc/x/xHeap.cpp
+++ b/src/hotspot/share/gc/x/xHeap.cpp
@@ -249,7 +249,7 @@ void XHeap::mark_start() {
// Enter mark phase
XGlobalPhase = XPhaseMark;
- // Reset marking information and mark roots
+ // Reset marking information
_mark.start();
// Update statistics
@@ -322,6 +322,7 @@ void XHeap::process_non_strong_references() {
_weak_roots_processor.process_weak_roots();
ClassUnloadingContext ctx(_workers.active_workers(),
+ true /* unregister_nmethods_during_purge */,
true /* lock_codeblob_free_separately */);
// Unlink stale metadata and nmethods
diff --git a/src/hotspot/share/gc/z/zArguments.cpp b/src/hotspot/share/gc/z/zArguments.cpp
index 5ebfeb75f89..192cad86e67 100644
--- a/src/hotspot/share/gc/z/zArguments.cpp
+++ b/src/hotspot/share/gc/z/zArguments.cpp
@@ -143,6 +143,9 @@ void ZArguments::initialize() {
FLAG_SET_DEFAULT(ZFragmentationLimit, 5.0);
}
+ // Set medium page size here because MaxTenuringThreshold may use it.
+ ZHeuristics::set_medium_page_size();
+
if (!FLAG_IS_DEFAULT(ZTenuringThreshold) && ZTenuringThreshold != -1) {
FLAG_SET_ERGO_IF_DEFAULT(MaxTenuringThreshold, ZTenuringThreshold);
if (MaxTenuringThreshold == 0) {
diff --git a/src/hotspot/share/gc/z/zBarrierSet.cpp b/src/hotspot/share/gc/z/zBarrierSet.cpp
index 160673e8059..a82cc67754b 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.cpp
@@ -152,6 +152,19 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
deoptimize_allocation(thread);
}
+void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj, size_t size) {
+ volatile zpointer* src = (volatile zpointer*)src_obj->base();
+ volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
+
+ for (const zpointer* const end = cast_from_oop(src_obj) + size; src < end; src++, dst++) {
+ zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
+ // We avoid healing here because the store below colors the pointer store good,
+ // hence avoiding the cost of a CAS.
+ ZBarrier::store_barrier_on_heap_oop_field(dst, false /* heal */);
+ Atomic::store(dst, ZAddress::store_good(elem));
+ }
+}
+
void ZBarrierSet::print_on(outputStream* st) const {
st->print_cr("ZBarrierSet");
}
diff --git a/src/hotspot/share/gc/z/zBarrierSet.hpp b/src/hotspot/share/gc/z/zBarrierSet.hpp
index 3bb067daebb..9ac8535d9d7 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.hpp
@@ -39,6 +39,8 @@ class ZBarrierSet : public BarrierSet {
static ZBarrierSetAssembler* assembler();
static bool barrier_needed(DecoratorSet decorators, BasicType type);
+ static void clone_obj_array(objArrayOop src, objArrayOop dst, size_t size);
+
virtual void on_thread_create(Thread* thread);
virtual void on_thread_destroy(Thread* thread);
virtual void on_thread_attach(Thread* thread);
diff --git a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
index 02328a2bd02..5e8cc3a8032 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
@@ -425,14 +425,13 @@ inline void ZBarrierSet::AccessBarrier::oop_arraycopy_i
}
}
-class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
+class ZColorStoreGoodOopClosure : public BasicOopIterateClosure {
public:
virtual void do_oop(oop* p_) {
volatile zpointer* const p = (volatile zpointer*)p_;
const zpointer ptr = ZBarrier::load_atomic(p);
const zaddress addr = ZPointer::uncolor(ptr);
- ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
- *p = ZAddress::store_good(addr);
+ Atomic::store(p, ZAddress::store_good(addr));
}
virtual void do_oop(narrowOop* p) {
@@ -455,6 +454,17 @@ template
inline void ZBarrierSet::AccessBarrier::clone_in_heap(oop src, oop dst, size_t size) {
assert_is_valid(to_zaddress(src));
+ if (dst->is_objArray()) {
+ // Cloning an object array is similar to performing array copy.
+ // If an array is large enough to have its allocation segmented,
+ // this operation might require GC barriers. However, the intrinsics
+ // for cloning arrays transform the clone to an optimized allocation
+ // and arraycopy sequence, so the performance of this runtime call
+ // does not matter for object arrays.
+ clone_obj_array(objArrayOop(src), objArrayOop(dst), size);
+ return;
+ }
+
// Fix the oops
ZLoadBarrierOopClosure cl;
ZIterator::oop_iterate(src, &cl);
@@ -462,10 +472,10 @@ inline void ZBarrierSet::AccessBarrier::clone_in_heap(o
// Clone the object
Raw::clone_in_heap(src, dst, size);
- assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
+ assert(dst->is_typeArray() || ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
// Color store good before handing out
- ZStoreBarrierOopClosure cl_sg;
+ ZColorStoreGoodOopClosure cl_sg;
ZIterator::oop_iterate(dst, &cl_sg);
}
diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp
index 4621101eb88..5e42fbd7ef8 100644
--- a/src/hotspot/share/gc/z/zGeneration.cpp
+++ b/src/hotspot/share/gc/z/zGeneration.cpp
@@ -861,7 +861,7 @@ void ZGenerationYoung::mark_start() {
// Enter mark phase
set_phase(Phase::Mark);
- // Reset marking information and mark roots
+ // Reset marking information
_mark.start();
// Flip remembered set bits
@@ -1213,7 +1213,7 @@ void ZGenerationOld::mark_start() {
// Enter mark phase
set_phase(Phase::Mark);
- // Reset marking information and mark roots
+ // Reset marking information
_mark.start();
// Update statistics
@@ -1318,6 +1318,7 @@ void ZGenerationOld::process_non_strong_references() {
_weak_roots_processor.process_weak_roots();
ClassUnloadingContext ctx(_workers.active_workers(),
+ true /* unregister_nmethods_during_purge */,
true /* lock_codeblob_free_separately */);
// Unlink stale metadata and nmethods
diff --git a/src/hotspot/share/gc/z/zHeuristics.cpp b/src/hotspot/share/gc/z/zHeuristics.cpp
index bcd9dd84405..df2f5f68489 100644
--- a/src/hotspot/share/gc/z/zHeuristics.cpp
+++ b/src/hotspot/share/gc/z/zHeuristics.cpp
@@ -60,8 +60,8 @@ size_t ZHeuristics::relocation_headroom() {
}
bool ZHeuristics::use_per_cpu_shared_small_pages() {
- // Use per-CPU shared small pages only if these pages occupy at most 3.125%
- // of the max heap size. Otherwise fall back to using a single shared small
+ // Use per-CPU shared small pages only if these pages don't have a significant
+ // heap overhead. Otherwise fall back to using a single shared small
// page. This is useful when using small heaps on large machines.
const size_t per_cpu_share = significant_heap_overhead() / ZCPU::count();
return per_cpu_share >= ZPageSizeSmall;
diff --git a/src/hotspot/share/gc/z/zInitialize.cpp b/src/hotspot/share/gc/z/zInitialize.cpp
index 0c0dc6e87a6..bf8cb96a4cb 100644
--- a/src/hotspot/share/gc/z/zInitialize.cpp
+++ b/src/hotspot/share/gc/z/zInitialize.cpp
@@ -28,7 +28,6 @@
#include "gc/z/zDriver.hpp"
#include "gc/z/zGCIdPrinter.hpp"
#include "gc/z/zGlobals.hpp"
-#include "gc/z/zHeuristics.hpp"
#include "gc/z/zInitialize.hpp"
#include "gc/z/zJNICritical.hpp"
#include "gc/z/zLargePages.hpp"
@@ -54,7 +53,6 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
ZThreadLocalAllocBuffer::initialize();
ZTracer::initialize();
ZLargePages::initialize();
- ZHeuristics::set_medium_page_size();
ZBarrierSet::set_barrier_set(barrier_set);
ZJNICritical::initialize();
ZDriver::initialize();
diff --git a/src/hotspot/share/gc/z/zReferenceProcessor.cpp b/src/hotspot/share/gc/z/zReferenceProcessor.cpp
index 1037093523d..df8cb2b0e95 100644
--- a/src/hotspot/share/gc/z/zReferenceProcessor.cpp
+++ b/src/hotspot/share/gc/z/zReferenceProcessor.cpp
@@ -113,6 +113,7 @@ static void list_append(zaddress& head, zaddress& tail, zaddress reference) {
ZReferenceProcessor::ZReferenceProcessor(ZWorkers* workers)
: _workers(workers),
_soft_reference_policy(nullptr),
+ _clear_all_soft_refs(false),
_encountered_count(),
_discovered_count(),
_enqueued_count(),
@@ -124,8 +125,9 @@ void ZReferenceProcessor::set_soft_reference_policy(bool clear) {
static AlwaysClearPolicy always_clear_policy;
static LRUMaxHeapPolicy lru_max_heap_policy;
+ _clear_all_soft_refs = clear;
+
if (clear) {
- log_info(gc, ref)("Clearing All SoftReferences");
_soft_reference_policy = &always_clear_policy;
} else {
_soft_reference_policy = &lru_max_heap_policy;
@@ -438,6 +440,10 @@ class ZReferenceProcessorTask : public ZTask {
void ZReferenceProcessor::process_references() {
ZStatTimerOld timer(ZSubPhaseConcurrentReferencesProcess);
+ if (_clear_all_soft_refs) {
+ log_info(gc, ref)("Clearing All SoftReferences");
+ }
+
// Process discovered lists
ZReferenceProcessorTask task(this);
_workers->run(&task);
diff --git a/src/hotspot/share/gc/z/zReferenceProcessor.hpp b/src/hotspot/share/gc/z/zReferenceProcessor.hpp
index d39cc8634cd..7a8900827da 100644
--- a/src/hotspot/share/gc/z/zReferenceProcessor.hpp
+++ b/src/hotspot/share/gc/z/zReferenceProcessor.hpp
@@ -41,6 +41,7 @@ class ZReferenceProcessor : public ReferenceDiscoverer {
ZWorkers* const _workers;
ReferencePolicy* _soft_reference_policy;
+ bool _clear_all_soft_refs;
ZPerWorker _encountered_count;
ZPerWorker _discovered_count;
ZPerWorker _enqueued_count;
diff --git a/src/hotspot/share/include/jvm.h b/src/hotspot/share/include/jvm.h
index e59db550fe9..5a58e6d57d8 100644
--- a/src/hotspot/share/include/jvm.h
+++ b/src/hotspot/share/include/jvm.h
@@ -1169,6 +1169,9 @@ JVM_VirtualThreadUnmount(JNIEnv* env, jobject vthread, jboolean hide);
JNIEXPORT void JNICALL
JVM_VirtualThreadHideFrames(JNIEnv* env, jobject vthread, jboolean hide);
+JNIEXPORT void JNICALL
+JVM_VirtualThreadDisableSuspend(JNIEnv* env, jobject vthread, jboolean enter);
+
/*
* Core reflection support.
*/
diff --git a/src/hotspot/share/interpreter/bytecodes.cpp b/src/hotspot/share/interpreter/bytecodes.cpp
index 24c9f918a5c..0d1be1dca7b 100644
--- a/src/hotspot/share/interpreter/bytecodes.cpp
+++ b/src/hotspot/share/interpreter/bytecodes.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -538,7 +538,7 @@ jchar Bytecodes::compute_flags(const char* format, jchar more_flags) {
}
guarantee(has_size == 0 || // no field yet
this_size == has_size || // same size
- this_size < has_size && *fp == '\0', // last field can be short
+ (this_size < has_size && *fp == '\0'), // last field can be short
"mixed field sizes in format");
has_size = this_size;
}
diff --git a/src/hotspot/share/jfr/jfr.cpp b/src/hotspot/share/jfr/jfr.cpp
index 45848fb0243..03b47bcd21d 100644
--- a/src/hotspot/share/jfr/jfr.cpp
+++ b/src/hotspot/share/jfr/jfr.cpp
@@ -67,7 +67,9 @@ void Jfr::on_create_vm_3() {
}
void Jfr::on_unloading_classes() {
- JfrCheckpointManager::on_unloading_classes();
+ if (JfrRecorder::is_created() || JfrRecorder::is_started_on_commandline()) {
+ JfrCheckpointManager::on_unloading_classes();
+ }
}
bool Jfr::is_excluded(Thread* t) {
diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
index 866f8c1df13..0b3097ca1fa 100644
--- a/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.cpp
@@ -525,6 +525,12 @@ const char* JfrJavaSupport::c_str(jstring string, Thread* thread, bool c_heap /*
return string != nullptr ? c_str(resolve_non_null(string), thread, c_heap) : nullptr;
}
+void JfrJavaSupport::free_c_str(const char* str, bool c_heap) {
+ if (c_heap) {
+ FREE_C_HEAP_ARRAY(char, str);
+ }
+}
+
static Symbol** allocate_symbol_array(bool c_heap, int length, Thread* thread) {
return c_heap ?
NEW_C_HEAP_ARRAY(Symbol*, length, mtTracing) :
@@ -546,6 +552,7 @@ Symbol** JfrJavaSupport::symbol_array(jobjectArray string_array, JavaThread* thr
if (object != nullptr) {
const char* text = c_str(arrayOop->obj_at(i), thread, c_heap);
symbol = SymbolTable::new_symbol(text);
+ free_c_str(text, c_heap);
}
result_array[i] = symbol;
}
diff --git a/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp b/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp
index 8b4ecf18dc1..19ff332c118 100644
--- a/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp
+++ b/src/hotspot/share/jfr/jni/jfrJavaSupport.hpp
@@ -86,6 +86,7 @@ class JfrJavaSupport : public AllStatic {
static Klass* klass(const jobject handle);
static const char* c_str(jstring string, Thread* thread, bool c_heap = false);
static const char* c_str(oop string, Thread* thread, bool c_heap = false);
+ static void free_c_str(const char* str, bool c_heap);
static Symbol** symbol_array(jobjectArray string_array, JavaThread* thread, intptr_t* result_size, bool c_heap = false);
// exceptions
diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml
index c1f626a8192..53fa17555c6 100644
--- a/src/hotspot/share/jfr/metadata/metadata.xml
+++ b/src/hotspot/share/jfr/metadata/metadata.xml
@@ -957,6 +957,8 @@
initialize(thread_local_buffer_size,
- thread_local_buffer_prealloc_count,
- thread_local_buffer_prealloc_count)) {
+ thread_local_buffer_prealloc_count,
+ thread_local_buffer_prealloc_count)) {
return false;
}
assert(_virtual_thread_local_mspace == nullptr, "invariant");
_virtual_thread_local_mspace = new JfrThreadLocalCheckpointMspace();
if (_virtual_thread_local_mspace == nullptr || !_virtual_thread_local_mspace->initialize(virtual_thread_local_buffer_size,
- JFR_MSPACE_UNLIMITED_CACHE_SIZE,
- virtual_thread_local_buffer_prealloc_count)) {
+ JFR_MSPACE_UNLIMITED_CACHE_SIZE,
+ virtual_thread_local_buffer_prealloc_count)) {
return false;
}
return true;
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.hpp b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.hpp
index f99159b3a51..a8ec2fd70f5 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointWriter.hpp
@@ -55,6 +55,7 @@ struct JfrCheckpointContext {
class JfrCheckpointWriter : public JfrCheckpointWriterBase {
friend class JfrCheckpointManager;
+ friend class JfrDeprecationManager;
friend class JfrSerializerRegistration;
friend class JfrTypeManager;
private:
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
index 94ac0772969..8df2f6593ef 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
@@ -66,20 +66,38 @@ static bool _class_unload = false;
static bool _flushpoint = false;
static bool _initial_type_set = true;
-static bool current_epoch() {
- return _class_unload || _flushpoint;
+static inline bool flushpoint() {
+ return _flushpoint;
}
-static bool previous_epoch() {
+static inline bool unloading() {
+ return _class_unload;
+}
+
+static inline bool current_epoch() {
+ return flushpoint() || unloading();
+}
+
+static inline bool previous_epoch() {
return !current_epoch();
}
-static bool is_initial_typeset_for_chunk() {
- return _initial_type_set && !_class_unload;
+template
+static inline bool used(const T* ptr) {
+ assert(ptr != nullptr, "invariant");
+ return current_epoch() ? USED_THIS_EPOCH(ptr) : USED_PREVIOUS_EPOCH(ptr);
+}
+
+template
+static inline bool not_used(const T* ptr) {
+ return !used(ptr);
}
-static bool is_complete() {
- return !_artifacts->has_klass_entries() && current_epoch();
+template
+static void do_artifact(const T* ptr) {
+ if (used(ptr)) {
+ _subsystem_callback->do_artifact(ptr);
+ }
}
static traceid mark_symbol(KlassPtr klass, bool leakp) {
@@ -94,66 +112,107 @@ static traceid get_bootstrap_name(bool leakp) {
return _artifacts->bootstrap_name(leakp);
}
-static const char* primitive_name(KlassPtr type_array_klass) {
- switch (type_array_klass->name()->base()[1]) {
- case JVM_SIGNATURE_BOOLEAN: return "boolean";
- case JVM_SIGNATURE_BYTE: return "byte";
- case JVM_SIGNATURE_CHAR: return "char";
- case JVM_SIGNATURE_SHORT: return "short";
- case JVM_SIGNATURE_INT: return "int";
- case JVM_SIGNATURE_LONG: return "long";
- case JVM_SIGNATURE_FLOAT: return "float";
- case JVM_SIGNATURE_DOUBLE: return "double";
+template
+static traceid artifact_id(const T* ptr) {
+ assert(ptr != nullptr, "invariant");
+ return JfrTraceId::load_raw(ptr);
+}
+
+template
+static traceid artifact_tag(const T* ptr, bool leakp) {
+ assert(ptr != nullptr, "invariant");
+ if (leakp) {
+ if (IS_NOT_LEAKP(ptr)) {
+ SET_LEAKP(ptr);
+ }
+ assert(IS_LEAKP(ptr), "invariant");
+ return artifact_id(ptr);
}
- assert(false, "invalid type array klass");
- return nullptr;
+ if (not_used(ptr)) {
+ SET_TRANSIENT(ptr);
+ }
+ assert(used(ptr), "invariant");
+ return artifact_id(ptr);
}
-static Symbol* primitive_symbol(KlassPtr type_array_klass) {
- if (type_array_klass == nullptr) {
- // void.class
- static Symbol* const void_class_name = SymbolTable::probe("void", 4);
- assert(void_class_name != nullptr, "invariant");
- return void_class_name;
+static inline bool should_do_cld_klass(const Klass* klass, bool leakp) {
+ return klass != nullptr && _artifacts->should_do_cld_klass(klass, leakp);
+}
+
+static inline KlassPtr get_cld_klass(CldPtr cld, bool leakp) {
+ if (cld == nullptr) {
+ return nullptr;
}
- const char* const primitive_type_str = primitive_name(type_array_klass);
- assert(primitive_type_str != nullptr, "invariant");
- Symbol* const primitive_type_sym = SymbolTable::probe(primitive_type_str, (int)strlen(primitive_type_str));
- assert(primitive_type_sym != nullptr, "invariant");
- return primitive_type_sym;
+ assert(leakp ? IS_LEAKP(cld) : used(cld), "invariant");
+ KlassPtr cld_klass = cld->class_loader_klass();
+ if (cld_klass == nullptr) {
+ return nullptr;
+ }
+ if (should_do_cld_klass(cld_klass, leakp)) {
+ if (current_epoch()) {
+ // This will enqueue the klass, which is important for
+ // reachability when doing clear and reset at rotation.
+ JfrTraceId::load(cld_klass);
+ } else {
+ artifact_tag(cld_klass, leakp);
+ }
+ return cld_klass;
+ }
+ return nullptr;
}
-template
-static traceid artifact_id(const T* ptr) {
- assert(ptr != nullptr, "invariant");
- return JfrTraceId::load_raw(ptr);
+static inline CldPtr get_cld(ModPtr mod) {
+ return mod != nullptr ? mod->loader_data() : nullptr;
}
-static traceid package_id(KlassPtr klass, bool leakp) {
+static ClassLoaderData* get_cld(const Klass* klass) {
assert(klass != nullptr, "invariant");
- PkgPtr pkg_entry = klass->package();
- if (pkg_entry == nullptr) {
- return 0;
- }
- if (leakp) {
- SET_LEAKP(pkg_entry);
+ if (klass->is_objArray_klass()) {
+ klass = ObjArrayKlass::cast(klass)->bottom_klass();
}
- // package implicitly tagged already
- return artifact_id(pkg_entry);
+ return klass->is_non_strong_hidden() ? nullptr : klass->class_loader_data();
+}
+
+static inline ModPtr get_module(PkgPtr pkg) {
+ return pkg != nullptr ? pkg->module() : nullptr;
+}
+
+static inline PkgPtr get_package(KlassPtr klass) {
+ return klass != nullptr ? klass->package() : nullptr;
+}
+
+static inline KlassPtr get_module_cld_klass(KlassPtr klass, bool leakp) {
+ assert(klass != nullptr, "invariant");
+ return get_cld_klass(get_cld(get_module(get_package(klass))), leakp);
+}
+
+static traceid cld_id(CldPtr cld, bool leakp) {
+ assert(cld != nullptr, "invariant");
+ return artifact_tag(cld, leakp);
}
static traceid module_id(PkgPtr pkg, bool leakp) {
assert(pkg != nullptr, "invariant");
- ModPtr module_entry = pkg->module();
- if (module_entry == nullptr) {
+ ModPtr mod = get_module(pkg);
+ if (mod == nullptr) {
return 0;
}
- if (leakp) {
- SET_LEAKP(module_entry);
- } else {
- SET_TRANSIENT(module_entry);
+ CldPtr cld = get_cld(mod);
+ if (cld != nullptr) {
+ cld_id(cld, leakp);
}
- return artifact_id(module_entry);
+ return artifact_tag(mod, leakp);
+}
+
+static traceid package_id(KlassPtr klass, bool leakp) {
+ assert(klass != nullptr, "invariant");
+ PkgPtr pkg = get_package(klass);
+ if (pkg == nullptr) {
+ return 0;
+ }
+ // Ensure module and its CLD gets tagged.
+ module_id(pkg, leakp);
+ return artifact_tag(pkg, leakp);
}
static traceid method_id(KlassPtr klass, MethodPtr method) {
@@ -162,16 +221,6 @@ static traceid method_id(KlassPtr klass, MethodPtr method) {
return METHOD_ID(klass, method);
}
-static traceid cld_id(CldPtr cld, bool leakp) {
- assert(cld != nullptr, "invariant");
- if (leakp) {
- SET_LEAKP(cld);
- } else {
- SET_TRANSIENT(cld);
- }
- return artifact_id(cld);
-}
-
template
static s4 get_flags(const T* ptr) {
assert(ptr != nullptr, "invariant");
@@ -183,73 +232,203 @@ static u4 get_primitive_flags() {
return JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC;
}
-static ClassLoaderData* get_cld(const Klass* klass) {
- assert(klass != nullptr, "invariant");
- if (klass->is_objArray_klass()) {
- klass = ObjArrayKlass::cast(klass)->bottom_klass();
+class PackageFieldSelector {
+ public:
+ typedef PkgPtr TypePtr;
+ static TypePtr select(KlassPtr klass) {
+ assert(klass != nullptr, "invariant");
+ return klass->package();
}
- if (klass->is_non_strong_hidden()) return nullptr;
- return klass->class_loader_data();
-}
+};
+
+class ModuleFieldSelector {
+ public:
+ typedef ModPtr TypePtr;
+ static TypePtr select(KlassPtr klass) {
+ assert(klass != nullptr, "invariant");
+ PkgPtr pkg = klass->package();
+ return pkg != nullptr ? pkg->module() : nullptr;
+ }
+};
+
+class KlassCldFieldSelector {
+ public:
+ typedef CldPtr TypePtr;
+ static TypePtr select(KlassPtr klass) {
+ assert(klass != nullptr, "invariant");
+ return get_cld(klass);
+ }
+};
+
+class ModuleCldFieldSelector {
+ public:
+ typedef CldPtr TypePtr;
+ static TypePtr select(KlassPtr klass) {
+ assert(klass != nullptr, "invariant");
+ ModPtr mod = ModuleFieldSelector::select(klass);
+ return mod != nullptr ? mod->loader_data() : nullptr;
+ }
+};
+
+template
+class SerializePredicate {
+ bool _class_unload;
+ public:
+ SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
+ bool operator()(T const& value) {
+ assert(value != nullptr, "invariant");
+ return _class_unload ? _artifacts->should_do_unloading_artifact(value) : IS_NOT_SERIALIZED(value);
+ }
+};
+
+template <>
+class SerializePredicate {
+ bool _class_unload;
+public:
+ SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
+ bool operator()(const Klass* klass) {
+ assert(klass != nullptr, "invariant");
+ return _class_unload ? true : IS_NOT_SERIALIZED(klass);
+ }
+};
+
+template <>
+class SerializePredicate {
+ bool _class_unload;
+public:
+ SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
+ bool operator()(const Method* method) {
+ assert(method != nullptr, "invariant");
+ return _class_unload ? true : METHOD_IS_NOT_SERIALIZED(method);
+ }
+};
template
static void set_serialized(const T* ptr) {
assert(ptr != nullptr, "invariant");
- SET_SERIALIZED(ptr);
- assert(IS_SERIALIZED(ptr), "invariant");
if (current_epoch()) {
CLEAR_THIS_EPOCH_CLEARED_BIT(ptr);
}
+ SET_SERIALIZED(ptr);
+ assert(IS_SERIALIZED(ptr), "invariant");
}
/*
- * In C++03, functions used as template parameters must have external linkage;
- * this restriction was removed in C++11. Change back to "static" and
- * rename functions when C++11 becomes available.
+ *********************** Klasses *************************
+ *
+ * When we process a Klass, we need to process its transitive closure.
+ *
+ * This includes two branches:
+ *
+ * [1] Klass -> CLD -> class_loader_Klass
+ * [2] Klass -> PackageEntry -> ModuleEntry -> CLD -> class_loader_Klass
+ *
+ * A Klass viewed as this closure becomes a node in a binary tree:
+ *
+ * Klass
+ * O
+ * / \
+ * / \
+ * [1] O O [2]
+ *
+ * We write the Klass and tag the artifacts in its closure (subtree)
+ * using preorder traversal by recursing the class_loader_Klass(es).
*
- * The weird naming is an effort to decrease the risk of name clashes.
*/
-static int write_klass(JfrCheckpointWriter* writer, KlassPtr klass, bool leakp) {
+static void do_write_klass(JfrCheckpointWriter* writer, CldPtr cld, KlassPtr klass, bool leakp) {
assert(writer != nullptr, "invariant");
assert(_artifacts != nullptr, "invariant");
assert(klass != nullptr, "invariant");
writer->write(artifact_id(klass));
- ClassLoaderData* cld = get_cld(klass);
writer->write(cld != nullptr ? cld_id(cld, leakp) : 0);
writer->write(mark_symbol(klass, leakp));
writer->write(package_id(klass, leakp));
writer->write(klass->modifier_flags());
writer->write(klass->is_hidden());
- return 1;
+ if (!leakp) {
+ set_serialized(klass);
+ }
+}
+
+static inline bool should_write_cld_klass(KlassPtr klass, bool leakp) {
+ return klass != nullptr && (leakp || IS_NOT_SERIALIZED(klass));
+}
+
+static void write_klass(JfrCheckpointWriter* writer, KlassPtr klass, bool leakp, int& elements) {
+ assert(elements >= 0, "invariant");
+ ClassLoaderData* cld = get_cld(klass);
+ do_write_klass(writer, cld, klass, leakp);
+ ++elements;
+ if (cld != nullptr) {
+ // Write the klass for the direct cld.
+ KlassPtr cld_klass = get_cld_klass(cld, leakp);
+ if (should_write_cld_klass(cld_klass, leakp)) {
+ write_klass(writer, cld_klass, leakp, elements);
+ }
+ }
+ KlassPtr mod_klass = get_module_cld_klass(klass, leakp);
+ if (should_write_cld_klass(mod_klass, leakp)) {
+ // Write the klass for the module cld.
+ write_klass(writer, mod_klass, leakp, elements);
+ }
}
+/*
+ * In C++03, functions used as template parameters must have external linkage;
+ * this restriction was removed in C++11. Change back to "static" and
+ * rename functions when C++11 becomes available.
+ *
+ * The weird naming is an effort to decrease the risk of name clashes.
+ */
int write__klass(JfrCheckpointWriter* writer, const void* k) {
assert(k != nullptr, "invariant");
- KlassPtr klass = (KlassPtr)k;
- set_serialized(klass);
- return write_klass(writer, klass, false);
+ KlassPtr klass = static_cast(k);
+ int elements = 0;
+ write_klass(writer, klass, false, elements);
+ return elements;
}
int write__klass__leakp(JfrCheckpointWriter* writer, const void* k) {
assert(k != nullptr, "invariant");
- KlassPtr klass = (KlassPtr)k;
+ KlassPtr klass = static_cast(k);
CLEAR_LEAKP(klass);
- return write_klass(writer, klass, true);
+ int elements = 0;
+ write_klass(writer, klass, true, elements);
+ return elements;
}
-static bool is_implied(const Klass* klass) {
- assert(klass != nullptr, "invariant");
- return klass->is_subclass_of(vmClasses::ClassLoader_klass()) || klass == vmClasses::Object_klass();
-}
+static int primitives_count = 9;
-static void do_klass(Klass* klass) {
- assert(klass != nullptr, "invariant");
- assert(_flushpoint ? USED_THIS_EPOCH(klass) : USED_PREVIOUS_EPOCH(klass), "invariant");
- assert(_subsystem_callback != nullptr, "invariant");
- _subsystem_callback->do_artifact(klass);
+static const char* primitive_name(KlassPtr type_array_klass) {
+ switch (type_array_klass->name()->base()[1]) {
+ case JVM_SIGNATURE_BOOLEAN: return "boolean";
+ case JVM_SIGNATURE_BYTE: return "byte";
+ case JVM_SIGNATURE_CHAR: return "char";
+ case JVM_SIGNATURE_SHORT: return "short";
+ case JVM_SIGNATURE_INT: return "int";
+ case JVM_SIGNATURE_LONG: return "long";
+ case JVM_SIGNATURE_FLOAT: return "float";
+ case JVM_SIGNATURE_DOUBLE: return "double";
+ }
+ assert(false, "invalid type array klass");
+ return nullptr;
}
+static Symbol* primitive_symbol(KlassPtr type_array_klass) {
+ if (type_array_klass == nullptr) {
+ // void.class
+ static Symbol* const void_class_name = SymbolTable::probe("void", 4);
+ assert(void_class_name != nullptr, "invariant");
+ return void_class_name;
+ }
+ const char* const primitive_type_str = primitive_name(type_array_klass);
+ assert(primitive_type_str != nullptr, "invariant");
+ Symbol* const primitive_type_sym = SymbolTable::probe(primitive_type_str,
+ (int)strlen(primitive_type_str));
+ assert(primitive_type_sym != nullptr, "invariant");
+ return primitive_type_sym;
+}
static traceid primitive_id(KlassPtr array_klass) {
if (array_klass == nullptr) {
@@ -271,148 +450,69 @@ static void write_primitive(JfrCheckpointWriter* writer, KlassPtr type_array_kla
writer->write(false);
}
-static void do_loader_klass(const Klass* klass) {
- if (klass != nullptr && _artifacts->should_do_loader_klass(klass)) {
- if (_leakp_writer != nullptr) {
- SET_LEAKP(klass);
- }
- SET_TRANSIENT(klass);
- _subsystem_callback->do_artifact(klass);
- }
-}
-
-static bool register_klass_unload(Klass* klass) {
- assert(klass != nullptr, "invariant");
- return JfrKlassUnloading::on_unload(klass);
-}
-
-static void do_unloading_klass(Klass* klass) {
- assert(klass != nullptr, "invariant");
- assert(_subsystem_callback != nullptr, "invariant");
- if (register_klass_unload(klass)) {
- _subsystem_callback->do_artifact(klass);
- do_loader_klass(klass->class_loader_data()->class_loader_klass());
- }
-}
-
-/*
- * Abstract klasses are filtered out unconditionally.
- * If a klass is not yet initialized, i.e yet to run its
- * it is also filtered out so we don't accidentally
- * trigger initialization.
- */
-static bool is_classloader_klass_allowed(const Klass* k) {
- assert(k != nullptr, "invariant");
- return !(k->is_abstract() || k->should_be_initialized());
-}
-
-static void do_classloaders() {
- for (ClassHierarchyIterator iter(vmClasses::ClassLoader_klass()); !iter.done(); iter.next()) {
- Klass* subk = iter.klass();
- if (is_classloader_klass_allowed(subk)) {
- do_loader_klass(subk);
- }
- }
+static bool is_initial_typeset_for_chunk() {
+ return _initial_type_set && !unloading();
}
-static int primitives_count = 9;
-
// A mirror representing a primitive class (e.g. int.class) has no reified Klass*,
// instead it has an associated TypeArrayKlass* (e.g. int[].class).
// We can use the TypeArrayKlass* as a proxy for deriving the id of the primitive class.
// The exception is the void.class, which has neither a Klass* nor a TypeArrayKlass*.
// It will use a reserved constant.
static void do_primitives() {
- // Only write the primitive classes once per chunk.
- if (is_initial_typeset_for_chunk()) {
- write_primitive(_writer, Universe::boolArrayKlassObj());
- write_primitive(_writer, Universe::byteArrayKlassObj());
- write_primitive(_writer, Universe::charArrayKlassObj());
- write_primitive(_writer, Universe::shortArrayKlassObj());
- write_primitive(_writer, Universe::intArrayKlassObj());
- write_primitive(_writer, Universe::longArrayKlassObj());
- write_primitive(_writer, Universe::floatArrayKlassObj());
- write_primitive(_writer, Universe::doubleArrayKlassObj());
- write_primitive(_writer, nullptr); // void.class
- }
+ assert(is_initial_typeset_for_chunk(), "invariant");
+ write_primitive(_writer, Universe::boolArrayKlassObj());
+ write_primitive(_writer, Universe::byteArrayKlassObj());
+ write_primitive(_writer, Universe::charArrayKlassObj());
+ write_primitive(_writer, Universe::shortArrayKlassObj());
+ write_primitive(_writer, Universe::intArrayKlassObj());
+ write_primitive(_writer, Universe::longArrayKlassObj());
+ write_primitive(_writer, Universe::floatArrayKlassObj());
+ write_primitive(_writer, Universe::doubleArrayKlassObj());
+ write_primitive(_writer, nullptr); // void.class
}
-static void do_object() {
- SET_TRANSIENT(vmClasses::Object_klass());
- do_klass(vmClasses::Object_klass());
-}
-
-static void do_klasses() {
- if (_class_unload) {
- ClassLoaderDataGraph::classes_unloading_do(&do_unloading_klass);
- return;
- }
- JfrTraceIdLoadBarrier::do_klasses(&do_klass, previous_epoch());
- do_classloaders();
- do_primitives();
- do_object();
-}
-
-template
-static void do_previous_epoch_artifact(JfrArtifactClosure* callback, T* value) {
- assert(callback != nullptr, "invariant");
- assert(value != nullptr, "invariant");
- if (USED_PREVIOUS_EPOCH(value)) {
- callback->do_artifact(value);
- }
- if (IS_SERIALIZED(value)) {
- CLEAR_SERIALIZED(value);
- }
- assert(IS_NOT_SERIALIZED(value), "invariant");
-}
-
-static void do_previous_epoch_klass(JfrArtifactClosure* callback, const Klass* value) {
- assert(callback != nullptr, "invariant");
- assert(value != nullptr, "invariant");
- if (USED_PREVIOUS_EPOCH(value)) {
- callback->do_artifact(value);
+static void do_unloading_klass(Klass* klass) {
+ assert(klass != nullptr, "invariant");
+ assert(_subsystem_callback != nullptr, "invariant");
+ if (JfrKlassUnloading::on_unload(klass)) {
+ _subsystem_callback->do_artifact(klass);
}
}
-static void do_klass_on_clear(Klass* klass) {
+static void do_klass(Klass* klass) {
assert(klass != nullptr, "invariant");
+ assert(used(klass), "invariant");
assert(_subsystem_callback != nullptr, "invariant");
- do_previous_epoch_klass(_subsystem_callback, klass);
+ _subsystem_callback->do_artifact(klass);
}
-static void do_loader_klass_on_clear(const Klass* klass) {
- if (klass != nullptr && _artifacts->should_do_loader_klass(klass)) {
- if (_leakp_writer != nullptr) {
- SET_LEAKP(klass);
- }
- SET_TRANSIENT(klass);
- do_previous_epoch_klass(_subsystem_callback, klass);
+static void do_klasses() {
+ if (unloading()) {
+ ClassLoaderDataGraph::classes_unloading_do(&do_unloading_klass);
+ return;
}
-}
-
-static void do_classloaders_on_clear() {
- for (ClassHierarchyIterator iter(vmClasses::ClassLoader_klass()); !iter.done(); iter.next()) {
- Klass* subk = iter.klass();
- if (is_classloader_klass_allowed(subk)) {
- do_loader_klass_on_clear(subk);
- }
+ if (is_initial_typeset_for_chunk()) {
+ // Only write the primitive classes once per chunk.
+ do_primitives();
}
+ JfrTraceIdLoadBarrier::do_klasses(&do_klass, previous_epoch());
}
-static void do_object_on_clear() {
- SET_TRANSIENT(vmClasses::Object_klass());
- do_klass_on_clear(vmClasses::Object_klass());
+static void do_klass_on_clear(Klass* klass) {
+ do_artifact(klass);
}
static void do_all_klasses() {
ClassLoaderDataGraph::classes_do(&do_klass_on_clear);
- do_classloaders_on_clear();
- do_object_on_clear();
}
+// KlassWriter.
typedef SerializePredicate KlassPredicate;
typedef JfrPredicatedTypeWriterImplHost KlassWriterImpl;
typedef JfrTypeWriterHost KlassWriter;
+
+// Klass registration.
typedef CompositeFunctor KlassWriterRegistration;
typedef JfrArtifactCallbackHost KlassCallback;
@@ -422,29 +522,31 @@ class LeakPredicate {
LeakPredicate(bool class_unload) {}
bool operator()(const Klass* klass) {
assert(klass != nullptr, "invariant");
- return IS_LEAKP(klass) || is_implied(klass);
+ return IS_LEAKP(klass);
}
};
+// KlassWriter for leakp. Only used during start or rotation, i.e. the previous epoch.
typedef LeakPredicate LeakKlassPredicate;
typedef JfrPredicatedTypeWriterImplHost LeakKlassWriterImpl;
typedef JfrTypeWriterHost LeakKlassWriter;
+// Composite KlassWriter with registration.
typedef CompositeFunctor CompositeKlassWriter;
typedef CompositeFunctor CompositeKlassWriterRegistration;
typedef JfrArtifactCallbackHost CompositeKlassCallback;
-static bool write_klasses() {
+static void write_klasses() {
assert(!_artifacts->has_klass_entries(), "invariant");
assert(_writer != nullptr, "invariant");
KlassArtifactRegistrator reg(_artifacts);
- KlassWriter kw(_writer, _class_unload);
+ KlassWriter kw(_writer, unloading());
KlassWriterRegistration kwr(&kw, ®);
if (_leakp_writer == nullptr) {
KlassCallback callback(&_subsystem_callback, &kwr);
do_klasses();
} else {
- LeakKlassWriter lkw(_leakp_writer, _class_unload);
+ LeakKlassWriter lkw(_leakp_writer, unloading());
CompositeKlassWriter ckw(&lkw, &kw);
CompositeKlassWriterRegistration ckwr(&ckw, ®);
CompositeKlassCallback callback(&_subsystem_callback, &ckwr);
@@ -455,32 +557,26 @@ static bool write_klasses() {
// their count is not automatically incremented.
kw.add(primitives_count);
}
- if (is_complete()) {
- return false;
- }
_artifacts->tally(kw);
- return true;
}
-static bool write_klasses_on_clear() {
+static void write_klasses_on_clear() {
assert(!_artifacts->has_klass_entries(), "invariant");
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
KlassArtifactRegistrator reg(_artifacts);
- KlassWriter kw(_writer, _class_unload);
+ KlassWriter kw(_writer, unloading());
KlassWriterRegistration kwr(&kw, ®);
- LeakKlassWriter lkw(_leakp_writer, _class_unload);
+ LeakKlassWriter lkw(_leakp_writer, unloading());
CompositeKlassWriter ckw(&lkw, &kw);
CompositeKlassWriterRegistration ckwr(&ckw, ®);
CompositeKlassCallback callback(&_subsystem_callback, &ckwr);
do_all_klasses();
- if (is_complete()) {
- return false;
- }
_artifacts->tally(kw);
- return true;
}
+/***** Packages *****/
+
static int write_package(JfrCheckpointWriter* writer, PkgPtr pkg, bool leakp) {
assert(writer != nullptr, "invariant");
assert(_artifacts != nullptr, "invariant");
@@ -494,98 +590,101 @@ static int write_package(JfrCheckpointWriter* writer, PkgPtr pkg, bool leakp) {
int write__package(JfrCheckpointWriter* writer, const void* p) {
assert(p != nullptr, "invariant");
- PkgPtr pkg = (PkgPtr)p;
+ PkgPtr pkg = static_cast(p);
set_serialized(pkg);
return write_package(writer, pkg, false);
}
int write__package__leakp(JfrCheckpointWriter* writer, const void* p) {
assert(p != nullptr, "invariant");
- PkgPtr pkg = (PkgPtr)p;
+ PkgPtr pkg = static_cast(p);
CLEAR_LEAKP(pkg);
return write_package(writer, pkg, true);
}
-static void do_package(PackageEntry* entry) {
- do_previous_epoch_artifact(_subsystem_callback, entry);
-}
-
-static void do_packages() {
- ClassLoaderDataGraph::packages_do(&do_package);
-}
-
-class PackageFieldSelector {
- public:
- typedef PkgPtr TypePtr;
- static TypePtr select(KlassPtr klass) {
- assert(klass != nullptr, "invariant");
- return klass->package();
- }
-};
+// PackageWriter.
typedef SerializePredicate PackagePredicate;
typedef JfrPredicatedTypeWriterImplHost PackageWriterImpl;
typedef JfrTypeWriterHost PackageWriter;
-typedef CompositeFunctor > PackageWriterWithClear;
+typedef JfrArtifactCallbackHost PackageCallback;
+
+// PackageWriter used during flush or unloading i.e. the current epoch.
typedef KlassToFieldEnvelope KlassPackageWriter;
-typedef JfrArtifactCallbackHost PackageCallback;
+// PackageWriter with clear. Only used during start or rotation, i.e. the previous epoch.
+typedef CompositeFunctor > PackageWriterWithClear;
+typedef JfrArtifactCallbackHost PackageClearCallback;
+
+// PackageWriter for leakp. Only used during start or rotation, i.e. the previous epoch.
typedef LeakPredicate LeakPackagePredicate;
typedef JfrPredicatedTypeWriterImplHost LeakPackageWriterImpl;
typedef JfrTypeWriterHost LeakPackageWriter;
+// Composite PackageWriter with clear. Only used during start or rotation, i.e. the previous epoch.
typedef CompositeFunctor CompositePackageWriter;
-typedef KlassToFieldEnvelope KlassCompositePackageWriter;
-typedef KlassToFieldEnvelope KlassPackageWriterWithClear;
typedef CompositeFunctor > CompositePackageWriterWithClear;
-typedef JfrArtifactCallbackHost CompositePackageCallback;
+typedef JfrArtifactCallbackHost CompositePackageClearCallback;
+
+static void do_package(PackageEntry* pkg) {
+ do_artifact(pkg);
+}
+
+static void do_all_packages() {
+ ClassLoaderDataGraph::packages_do(&do_package);
+}
+
+static void do_all_packages(PackageWriter& pw) {
+ do_all_packages();
+ _artifacts->tally(pw);
+}
+
+static void do_packages(PackageWriter& pw) {
+ KlassPackageWriter kpw(&pw);
+ _artifacts->iterate_klasses(kpw);
+ _artifacts->tally(pw);
+}
+
+static void write_packages_with_leakp(PackageWriter& pw) {
+ assert(_writer != nullptr, "invariant");
+ assert(_leakp_writer != nullptr, "invariant");
+ assert(previous_epoch(), "invariant");
+ LeakPackageWriter lpw(_leakp_writer, unloading());
+ CompositePackageWriter cpw(&lpw, &pw);
+ ClearArtifact clear;
+ CompositePackageWriterWithClear cpwwc(&cpw, &clear);
+ CompositePackageClearCallback callback(&_subsystem_callback, &cpwwc);
+ do_all_packages(pw);
+}
static void write_packages() {
assert(_writer != nullptr, "invariant");
- PackageWriter pw(_writer, _class_unload);
- KlassPackageWriter kpw(&pw);
+ PackageWriter pw(_writer, unloading());
if (current_epoch()) {
- _artifacts->iterate_klasses(kpw);
- _artifacts->tally(pw);
+ do_packages(pw);
return;
}
assert(previous_epoch(), "invariant");
if (_leakp_writer == nullptr) {
- _artifacts->iterate_klasses(kpw);
ClearArtifact clear;
PackageWriterWithClear pwwc(&pw, &clear);
- PackageCallback callback(&_subsystem_callback, &pwwc);
- do_packages();
- } else {
- LeakPackageWriter lpw(_leakp_writer, _class_unload);
- CompositePackageWriter cpw(&lpw, &pw);
- KlassCompositePackageWriter kcpw(&cpw);
- _artifacts->iterate_klasses(kcpw);
- ClearArtifact clear;
- CompositePackageWriterWithClear cpwwc(&cpw, &clear);
- CompositePackageCallback callback(&_subsystem_callback, &cpwwc);
- do_packages();
+ PackageClearCallback callback(&_subsystem_callback, &pwwc);
+ do_all_packages(pw);
+ return;
}
- _artifacts->tally(pw);
+ write_packages_with_leakp(pw);
}
static void write_packages_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- PackageWriter pw(_writer, _class_unload);
- KlassPackageWriter kpw(&pw);
- LeakPackageWriter lpw(_leakp_writer, _class_unload);
- CompositePackageWriter cpw(&lpw, &pw);
- KlassCompositePackageWriter kcpw(&cpw);
- _artifacts->iterate_klasses(kcpw);
- ClearArtifact clear;
- CompositePackageWriterWithClear cpwwc(&cpw, &clear);
- CompositePackageCallback callback(&_subsystem_callback, &cpwwc);
- do_packages();
- _artifacts->tally(pw);
+ PackageWriter pw(_writer, unloading());
+ write_packages_with_leakp(pw);
}
+/***** Modules *****/
+
static int write_module(JfrCheckpointWriter* writer, ModPtr mod, bool leakp) {
assert(mod != nullptr, "invariant");
assert(_artifacts != nullptr, "invariant");
@@ -599,99 +698,101 @@ static int write_module(JfrCheckpointWriter* writer, ModPtr mod, bool leakp) {
int write__module(JfrCheckpointWriter* writer, const void* m) {
assert(m != nullptr, "invariant");
- ModPtr mod = (ModPtr)m;
+ ModPtr mod = static_cast(m);
set_serialized(mod);
return write_module(writer, mod, false);
}
int write__module__leakp(JfrCheckpointWriter* writer, const void* m) {
assert(m != nullptr, "invariant");
- ModPtr mod = (ModPtr)m;
+ ModPtr mod = static_cast(m);
CLEAR_LEAKP(mod);
return write_module(writer, mod, true);
}
-static void do_module(ModuleEntry* entry) {
- do_previous_epoch_artifact(_subsystem_callback, entry);
-}
-
-static void do_modules() {
- ClassLoaderDataGraph::modules_do(&do_module);
-}
-
-class ModuleFieldSelector {
- public:
- typedef ModPtr TypePtr;
- static TypePtr select(KlassPtr klass) {
- assert(klass != nullptr, "invariant");
- PkgPtr pkg = klass->package();
- return pkg != nullptr ? pkg->module() : nullptr;
- }
-};
-
+// ModuleWriter.
typedef SerializePredicate ModulePredicate;
typedef JfrPredicatedTypeWriterImplHost ModuleWriterImpl;
typedef JfrTypeWriterHost ModuleWriter;
-typedef CompositeFunctor > ModuleWriterWithClear;
-typedef JfrArtifactCallbackHost ModuleCallback;
+typedef JfrArtifactCallbackHost ModuleCallback;
+
+// ModuleWriter used during flush or unloading i.e. the current epoch.
typedef KlassToFieldEnvelope KlassModuleWriter;
+// ModuleWriter with clear. Only used during start or rotation, i.e. the previous epoch.
+typedef CompositeFunctor > ModuleWriterWithClear;
+typedef JfrArtifactCallbackHost ModuleClearCallback;
+
+// ModuleWriter for leakp. Only used during start or rotation, i.e. the previous epoch.
typedef LeakPredicate LeakModulePredicate;
typedef JfrPredicatedTypeWriterImplHost LeakModuleWriterImpl;
typedef JfrTypeWriterHost LeakModuleWriter;
+// Composite ModuleWriter with clear. Only used during start or rotation, i.e. the previous epoch.
typedef CompositeFunctor CompositeModuleWriter;
-typedef KlassToFieldEnvelope KlassCompositeModuleWriter;
typedef CompositeFunctor > CompositeModuleWriterWithClear;
-typedef JfrArtifactCallbackHost CompositeModuleCallback;
+typedef JfrArtifactCallbackHost CompositeModuleClearCallback;
+
+static void do_module(ModuleEntry* mod) {
+ do_artifact(mod);
+}
+
+static void do_all_modules() {
+ ClassLoaderDataGraph::modules_do(&do_module);
+}
+
+static void do_all_modules(ModuleWriter& mw) {
+ do_all_modules();
+ _artifacts->tally(mw);
+}
+
+static void do_modules(ModuleWriter& mw) {
+ KlassModuleWriter kmw(&mw);
+ _artifacts->iterate_klasses(kmw);
+ _artifacts->tally(mw);
+}
+
+static void write_modules_with_leakp(ModuleWriter& mw) {
+ assert(_writer != nullptr, "invariant");
+ assert(_leakp_writer != nullptr, "invariant");
+ assert(previous_epoch(), "invariant");
+ LeakModuleWriter lmw(_leakp_writer, unloading());
+ CompositeModuleWriter cmw(&lmw, &mw);
+ ClearArtifact clear;
+ CompositeModuleWriterWithClear cmwwc(&cmw, &clear);
+ CompositeModuleClearCallback callback(&_subsystem_callback, &cmwwc);
+ do_all_modules(mw);
+}
static void write_modules() {
assert(_writer != nullptr, "invariant");
- ModuleWriter mw(_writer, _class_unload);
- KlassModuleWriter kmw(&mw);
+ ModuleWriter mw(_writer, unloading());
if (current_epoch()) {
- _artifacts->iterate_klasses(kmw);
- _artifacts->tally(mw);
+ do_modules(mw);
return;
}
assert(previous_epoch(), "invariant");
if (_leakp_writer == nullptr) {
- _artifacts->iterate_klasses(kmw);
ClearArtifact clear;
ModuleWriterWithClear mwwc(&mw, &clear);
- ModuleCallback callback(&_subsystem_callback, &mwwc);
- do_modules();
- } else {
- LeakModuleWriter lmw(_leakp_writer, _class_unload);
- CompositeModuleWriter cmw(&lmw, &mw);
- KlassCompositeModuleWriter kcpw(&cmw);
- _artifacts->iterate_klasses(kcpw);
- ClearArtifact clear;
- CompositeModuleWriterWithClear cmwwc(&cmw, &clear);
- CompositeModuleCallback callback(&_subsystem_callback, &cmwwc);
- do_modules();
+ ModuleClearCallback callback(&_subsystem_callback, &mwwc);
+ do_all_modules(mw);
+ return;
}
- _artifacts->tally(mw);
+ write_modules_with_leakp(mw);
}
static void write_modules_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- ModuleWriter mw(_writer, _class_unload);
- KlassModuleWriter kmw(&mw);
- LeakModuleWriter lmw(_leakp_writer, _class_unload);
- CompositeModuleWriter cmw(&lmw, &mw);
- KlassCompositeModuleWriter kcpw(&cmw);
- _artifacts->iterate_klasses(kcpw);
- ClearArtifact clear;
- CompositeModuleWriterWithClear cmwwc(&cmw, &clear);
- CompositeModuleCallback callback(&_subsystem_callback, &cmwwc);
- do_modules();
- _artifacts->tally(mw);
+ ModuleWriter mw(_writer, unloading());
+ write_modules_with_leakp(mw);
}
-static int write_classloader(JfrCheckpointWriter* writer, CldPtr cld, bool leakp) {
+/***** ClassLoaderData - CLD *****/
+
+static int write_cld(JfrCheckpointWriter* writer, CldPtr cld, bool leakp) {
assert(cld != nullptr, "invariant");
// class loader type
const Klass* class_loader_klass = cld->class_loader_klass();
@@ -701,7 +802,7 @@ static int write_classloader(JfrCheckpointWriter* writer, CldPtr cld, bool leakp
writer->write((traceid)0); // class loader type id (absence of)
writer->write(get_bootstrap_name(leakp)); // maps to synthetic name -> "bootstrap"
} else {
- assert(_class_unload ? true : IS_SERIALIZED(class_loader_klass), "invariant");
+ assert(IS_SERIALIZED(class_loader_klass), "invariant");
writer->write(artifact_id(cld)); // class loader instance id
writer->write(artifact_id(class_loader_klass)); // class loader type id
writer->write(mark_symbol(cld->name(), leakp)); // class loader instance name
@@ -709,146 +810,127 @@ static int write_classloader(JfrCheckpointWriter* writer, CldPtr cld, bool leakp
return 1;
}
-int write__classloader(JfrCheckpointWriter* writer, const void* c) {
+int write__cld(JfrCheckpointWriter* writer, const void* c) {
assert(c != nullptr, "invariant");
- CldPtr cld = (CldPtr)c;
+ CldPtr cld = static_cast(c);
set_serialized(cld);
- return write_classloader(writer, cld, false);
+ return write_cld(writer, cld, false);
}
-int write__classloader__leakp(JfrCheckpointWriter* writer, const void* c) {
+int write__cld__leakp(JfrCheckpointWriter* writer, const void* c) {
assert(c != nullptr, "invariant");
- CldPtr cld = (CldPtr)c;
+ CldPtr cld = static_cast(c);
CLEAR_LEAKP(cld);
- return write_classloader(writer, cld, true);
+ return write_cld(writer, cld, true);
}
-static void do_class_loader_data(ClassLoaderData* cld) {
- do_previous_epoch_artifact(_subsystem_callback, cld);
-}
+// CldWriter.
+typedef SerializePredicate CldPredicate;
+typedef JfrPredicatedTypeWriterImplHost CldWriterImpl;
+typedef JfrTypeWriterHost CldWriter;
+typedef JfrArtifactCallbackHost CldCallback;
-class KlassCldFieldSelector {
- public:
- typedef CldPtr TypePtr;
- static TypePtr select(KlassPtr klass) {
- assert(klass != nullptr, "invariant");
- return get_cld(klass);
- }
-};
+// CldWriter used during flush or unloading i.e. the current epoch.
+typedef KlassToFieldEnvelope KlassCldWriter;
+typedef KlassToFieldEnvelope ModuleCldWriter;
+typedef CompositeFunctor KlassAndModuleCldWriter;
-class ModuleCldFieldSelector {
-public:
- typedef CldPtr TypePtr;
- static TypePtr select(KlassPtr klass) {
- assert(klass != nullptr, "invariant");
- ModPtr mod = ModuleFieldSelector::select(klass);
- return mod != nullptr ? mod->loader_data() : nullptr;
- }
-};
+// CldWriter with clear. Only used during start or rotation, i.e. the previous epoch.
+typedef CompositeFunctor > CldWriterWithClear;
+typedef JfrArtifactCallbackHost CldClearCallback;
+
+// CldWriter for leakp. Only used during start or rotation, i.e. the previous epoch.
+typedef LeakPredicate LeakCldPredicate;
+typedef JfrPredicatedTypeWriterImplHost LeakCldWriterImpl;
+typedef JfrTypeWriterHost LeakCldWriter;
+
+// Composite CldWriter with clear. Only used during start or rotation, i.e. the previous epoch.
+typedef CompositeFunctor CompositeCldWriter;
+typedef CompositeFunctor > CompositeCldWriterWithClear;
+typedef JfrArtifactCallbackHost CompositeCldClearCallback;
class CLDCallback : public CLDClosure {
public:
- CLDCallback() {}
void do_cld(ClassLoaderData* cld) {
assert(cld != nullptr, "invariant");
- if (cld->has_class_mirror_holder()) {
- return;
+ if (!cld->has_class_mirror_holder()) {
+ do_artifact(cld);
}
- do_class_loader_data(cld);
}
};
-static void do_class_loaders() {
+static void do_all_clds() {
CLDCallback cld_cb;
ClassLoaderDataGraph::loaded_cld_do(&cld_cb);
}
-typedef SerializePredicate CldPredicate;
-typedef JfrPredicatedTypeWriterImplHost CldWriterImpl;
-typedef JfrTypeWriterHost CldWriter;
-typedef CompositeFunctor > CldWriterWithClear;
-typedef JfrArtifactCallbackHost CldCallback;
-typedef KlassToFieldEnvelope KlassCldWriter;
-typedef KlassToFieldEnvelope ModuleCldWriter;
-typedef CompositeFunctor KlassAndModuleCldWriter;
-
-typedef LeakPredicate LeakCldPredicate;
-typedef JfrPredicatedTypeWriterImplHost LeakCldWriterImpl;
-typedef JfrTypeWriterHost LeakCldWriter;
-
-typedef CompositeFunctor CompositeCldWriter;
-typedef KlassToFieldEnvelope KlassCompositeCldWriter;
-typedef KlassToFieldEnvelope ModuleCompositeCldWriter;
-typedef CompositeFunctor KlassAndModuleCompositeCldWriter;
-typedef CompositeFunctor > CompositeCldWriterWithClear;
-typedef JfrArtifactCallbackHost CompositeCldCallback;
+static void do_all_clds(CldWriter& cldw) {
+ do_all_clds();
+ _artifacts->tally(cldw);
+}
-static void write_classloaders() {
- assert(_writer != nullptr, "invariant");
- CldWriter cldw(_writer, _class_unload);
+static void do_clds(CldWriter& cldw) {
KlassCldWriter kcw(&cldw);
ModuleCldWriter mcw(&cldw);
KlassAndModuleCldWriter kmcw(&kcw, &mcw);
+ _artifacts->iterate_klasses(kmcw);
+ _artifacts->tally(cldw);
+}
+
+static void write_clds_with_leakp(CldWriter& cldw) {
+ assert(_writer != nullptr, "invariant");
+ assert(_leakp_writer != nullptr, "invariant");
+ assert(previous_epoch(), "invariant");
+ LeakCldWriter lcldw(_leakp_writer, unloading());
+ CompositeCldWriter ccldw(&lcldw, &cldw);
+ ClearArtifact clear;
+ CompositeCldWriterWithClear ccldwwc(&ccldw, &clear);
+ CompositeCldClearCallback callback(&_subsystem_callback, &ccldwwc);
+ do_all_clds(cldw);
+}
+
+static void write_clds() {
+ assert(_writer != nullptr, "invariant");
+ CldWriter cldw(_writer, unloading());
if (current_epoch()) {
- _artifacts->iterate_klasses(kmcw);
- _artifacts->tally(cldw);
+ do_clds(cldw);
return;
}
assert(previous_epoch(), "invariant");
if (_leakp_writer == nullptr) {
- _artifacts->iterate_klasses(kmcw);
ClearArtifact clear;
CldWriterWithClear cldwwc(&cldw, &clear);
- CldCallback callback(&_subsystem_callback, &cldwwc);
- do_class_loaders();
- } else {
- LeakCldWriter lcldw(_leakp_writer, _class_unload);
- CompositeCldWriter ccldw(&lcldw, &cldw);
- KlassCompositeCldWriter kccldw(&ccldw);
- ModuleCompositeCldWriter mccldw(&ccldw);
- KlassAndModuleCompositeCldWriter kmccldw(&kccldw, &mccldw);
- _artifacts->iterate_klasses(kmccldw);
- ClearArtifact clear;
- CompositeCldWriterWithClear ccldwwc(&ccldw, &clear);
- CompositeCldCallback callback(&_subsystem_callback, &ccldwwc);
- do_class_loaders();
+ CldClearCallback callback(&_subsystem_callback, &cldwwc);
+ do_all_clds(cldw);
+ return;
}
- _artifacts->tally(cldw);
+ write_clds_with_leakp(cldw);
}
-static void write_classloaders_on_clear() {
+static void write_clds_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
- CldWriter cldw(_writer, _class_unload);
- KlassCldWriter kcw(&cldw);
- ModuleCldWriter mcw(&cldw);
- KlassAndModuleCldWriter kmcw(&kcw, &mcw);
- LeakCldWriter lcldw(_leakp_writer, _class_unload);
- CompositeCldWriter ccldw(&lcldw, &cldw);
- KlassCompositeCldWriter kccldw(&ccldw);
- ModuleCompositeCldWriter mccldw(&ccldw);
- KlassAndModuleCompositeCldWriter kmccldw(&kccldw, &mccldw);
- _artifacts->iterate_klasses(kmccldw);
- ClearArtifact clear;
- CompositeCldWriterWithClear ccldwwc(&ccldw, &clear);
- CompositeCldCallback callback(&_subsystem_callback, &ccldwwc);
- do_class_loaders();
- _artifacts->tally(cldw);
+ assert(previous_epoch(), "invariant");
+ CldWriter cldw(_writer, unloading());
+ write_clds_with_leakp(cldw);
}
-static u1 get_visibility(MethodPtr method) {
- assert(method != nullptr, "invariant");
- return const_cast(method)->is_hidden() ? (u1)1 : (u1)0;
-}
+/***** Methods *****/
template <>
void set_serialized(MethodPtr method) {
assert(method != nullptr, "invariant");
- SET_METHOD_SERIALIZED(method);
- assert(METHOD_IS_SERIALIZED(method), "invariant");
if (current_epoch()) {
CLEAR_THIS_EPOCH_METHOD_CLEARED_BIT(method);
}
+ assert(unloading() ? true : METHOD_IS_NOT_SERIALIZED(method), "invariant");
+ SET_METHOD_SERIALIZED(method);
+ assert(METHOD_IS_SERIALIZED(method), "invariant");
+}
+
+static inline u1 get_visibility(MethodPtr method) {
+ assert(method != nullptr, "invariant");
+ return const_cast(method)->is_hidden() ? (u1)1 : (u1)0;
}
static int write_method(JfrCheckpointWriter* writer, MethodPtr method, bool leakp) {
@@ -857,58 +939,36 @@ static int write_method(JfrCheckpointWriter* writer, MethodPtr method, bool leak
assert(_artifacts != nullptr, "invariant");
KlassPtr klass = method->method_holder();
assert(klass != nullptr, "invariant");
+ assert(used(klass), "invariant");
+ assert(IS_SERIALIZED(klass), "invariant");
writer->write(method_id(klass, method));
writer->write(artifact_id(klass));
writer->write(mark_symbol(method->name(), leakp));
writer->write(mark_symbol(method->signature(), leakp));
- writer->write((u2)get_flags(method));
+ writer->write(static_cast(get_flags(method)));
writer->write(get_visibility(method));
return 1;
}
int write__method(JfrCheckpointWriter* writer, const void* m) {
assert(m != nullptr, "invariant");
- MethodPtr method = (MethodPtr)m;
+ MethodPtr method = static_cast(m);
set_serialized(method);
return write_method(writer, method, false);
}
int write__method__leakp(JfrCheckpointWriter* writer, const void* m) {
assert(m != nullptr, "invariant");
- MethodPtr method = (MethodPtr)m;
+ MethodPtr method = static_cast(m);
CLEAR_LEAKP_METHOD(method);
return write_method(writer, method, true);
}
-
-class BitMapFilter {
- ResourceBitMap _bitmap;
- public:
- explicit BitMapFilter(int length = 0) : _bitmap((size_t)length) {}
- bool operator()(size_t idx) {
- if (_bitmap.size() == 0) {
- return true;
- }
- if (_bitmap.at(idx)) {
- return false;
- }
- _bitmap.set_bit(idx);
- return true;
- }
-};
-
-class AlwaysTrue {
- public:
- explicit AlwaysTrue(int length = 0) {}
- bool operator()(size_t idx) {
- return true;
- }
-};
-
-template
+template
class MethodIteratorHost {
private:
MethodCallback _method_cb;
KlassCallback _klass_cb;
+ KlassUsedPredicate _klass_used_predicate;
MethodUsedPredicate _method_used_predicate;
MethodFlagPredicate _method_flag_predicate;
public:
@@ -918,6 +978,7 @@ class MethodIteratorHost {
bool skip_header = false) :
_method_cb(writer, class_unload, skip_header),
_klass_cb(writer, class_unload, skip_header),
+ _klass_used_predicate(current_epoch),
_method_used_predicate(current_epoch),
_method_flag_predicate(current_epoch) {}
@@ -937,7 +998,7 @@ class MethodIteratorHost {
ik = ik->previous_versions();
}
}
- return _klass_cb(klass);
+ return _klass_used_predicate(klass) ? _klass_cb(klass) : true;
}
int count() const { return _method_cb.count(); }
@@ -964,37 +1025,42 @@ typedef SerializePredicate MethodPredicate;
typedef JfrPredicatedTypeWriterImplHost MethodWriterImplTarget;
typedef Wrapper KlassCallbackStub;
typedef JfrTypeWriterHost MethodWriterImpl;
-typedef MethodIteratorHost MethodWriter;
+typedef MethodIteratorHost MethodWriter;
typedef LeakPredicate LeakMethodPredicate;
typedef JfrPredicatedTypeWriterImplHost LeakMethodWriterImplTarget;
typedef JfrTypeWriterHost LeakMethodWriterImpl;
-typedef MethodIteratorHost LeakMethodWriter;
-typedef MethodIteratorHost LeakMethodWriter;
+typedef MethodIteratorHost LeakMethodWriter;
+typedef MethodIteratorHost LeakMethodWriter;
typedef CompositeFunctor CompositeMethodWriter;
+static void write_methods_with_leakp(MethodWriter& mw) {
+ assert(_writer != nullptr, "invariant");
+ assert(_leakp_writer != nullptr, "invariant");
+ assert(previous_epoch(), "invariant");
+ LeakMethodWriter lpmw(_leakp_writer, current_epoch(), unloading());
+ CompositeMethodWriter cmw(&lpmw, &mw);
+ _artifacts->iterate_klasses(cmw);
+ _artifacts->tally(mw);
+}
+
static void write_methods() {
assert(_writer != nullptr, "invariant");
- MethodWriter mw(_writer, current_epoch(), _class_unload);
+ MethodWriter mw(_writer, current_epoch(), unloading());
if (_leakp_writer == nullptr) {
_artifacts->iterate_klasses(mw);
- } else {
- LeakMethodWriter lpmw(_leakp_writer, current_epoch(), _class_unload);
- CompositeMethodWriter cmw(&lpmw, &mw);
- _artifacts->iterate_klasses(cmw);
+ _artifacts->tally(mw);
+ return;
}
- _artifacts->tally(mw);
+ write_methods_with_leakp(mw);
}
static void write_methods_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- MethodWriter mw(_writer, current_epoch(), _class_unload);
- LeakMethodWriter lpmw(_leakp_writer, current_epoch(), _class_unload);
- CompositeMethodWriter cmw(&lpmw, &mw);
- _artifacts->iterate_klasses(cmw);
- _artifacts->tally(mw);
+ MethodWriter mw(_writer, current_epoch(), unloading());
+ write_methods_with_leakp(mw);
}
template <>
@@ -1022,14 +1088,14 @@ static int write_symbol(JfrCheckpointWriter* writer, SymbolEntryPtr entry, bool
int write__symbol(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
- SymbolEntryPtr entry = (SymbolEntryPtr)e;
+ SymbolEntryPtr entry = static_cast(e);
set_serialized(entry);
return write_symbol(writer, entry, false);
}
int write__symbol__leakp(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
- SymbolEntryPtr entry = (SymbolEntryPtr)e;
+ SymbolEntryPtr entry = static_cast(e);
return write_symbol(writer, entry, true);
}
@@ -1043,14 +1109,14 @@ static int write_string(JfrCheckpointWriter* writer, StringEntryPtr entry, bool
int write__string(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
- StringEntryPtr entry = (StringEntryPtr)e;
+ StringEntryPtr entry = static_cast(e);
set_serialized(entry);
return write_string(writer, entry, false);
}
int write__string__leakp(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
- StringEntryPtr entry = (StringEntryPtr)e;
+ StringEntryPtr entry = static_cast(e);
return write_string(writer, entry, true);
}
@@ -1071,30 +1137,15 @@ typedef JfrTypeWriterHost LeakStringEntr
typedef CompositeFunctor CompositeStringWriter;
static void write_symbols_with_leakp() {
- assert(_leakp_writer != nullptr, "invariant");
- SymbolEntryWriter sw(_writer, _class_unload);
- LeakSymbolEntryWriter lsw(_leakp_writer, _class_unload);
- CompositeSymbolWriter csw(&lsw, &sw);
- _artifacts->iterate_symbols(csw);
- StringEntryWriter sew(_writer, _class_unload, true); // skip header
- LeakStringEntryWriter lsew(_leakp_writer, _class_unload, true); // skip header
- CompositeStringWriter csew(&lsew, &sew);
- _artifacts->iterate_strings(csew);
- sw.add(sew.count());
- lsw.add(lsew.count());
- _artifacts->tally(sw);
-}
-
-static void write_symbols_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- SymbolEntryWriter sw(_writer, _class_unload);
- LeakSymbolEntryWriter lsw(_leakp_writer, _class_unload);
+ SymbolEntryWriter sw(_writer, unloading());
+ LeakSymbolEntryWriter lsw(_leakp_writer, unloading());
CompositeSymbolWriter csw(&lsw, &sw);
_artifacts->iterate_symbols(csw);
- StringEntryWriter sew(_writer, _class_unload, true); // skip header
- LeakStringEntryWriter lsew(_leakp_writer, _class_unload, true); // skip header
+ StringEntryWriter sew(_writer, unloading(), true); // skip header
+ LeakStringEntryWriter lsew(_leakp_writer, unloading(), true); // skip header
CompositeStringWriter csew(&lsew, &sew);
_artifacts->iterate_strings(csew);
sw.add(sew.count());
@@ -1108,17 +1159,24 @@ static void write_symbols() {
write_symbols_with_leakp();
return;
}
- SymbolEntryWriter sw(_writer, _class_unload);
+ SymbolEntryWriter sw(_writer, unloading());
_artifacts->iterate_symbols(sw);
- StringEntryWriter sew(_writer, _class_unload, true); // skip header
+ StringEntryWriter sew(_writer, unloading(), true); // skip header
_artifacts->iterate_strings(sew);
sw.add(sew.count());
_artifacts->tally(sw);
}
+static void write_symbols_on_clear() {
+ assert(_writer != nullptr, "invariant");
+ assert(_leakp_writer != nullptr, "invariant");
+ assert(previous_epoch(), "invariant");
+ write_symbols_with_leakp();
+}
+
typedef Wrapper ClearKlassBits;
typedef Wrapper ClearMethodFlag;
-typedef MethodIteratorHost ClearKlassAndMethods;
+typedef MethodIteratorHost ClearKlassAndMethods;
static void clear_klasses_and_methods() {
ClearKlassAndMethods clear(_writer);
@@ -1165,19 +1223,17 @@ size_t JfrTypeSet::serialize(JfrCheckpointWriter* writer, JfrCheckpointWriter* l
setup(writer, leakp_writer, class_unload, flushpoint);
// write order is important because an individual write step
// might tag an artifact to be written in a subsequent step
- if (!write_klasses()) {
- return 0;
- }
+ write_klasses();
write_packages();
write_modules();
- write_classloaders();
+ write_clds();
write_methods();
write_symbols();
return teardown();
}
/**
- * Clear all tags from the previous epoch.
+ * Clear all tags from the previous epoch. Reset support structures.
*/
void JfrTypeSet::clear(JfrCheckpointWriter* writer, JfrCheckpointWriter* leakp_writer) {
ResourceMark rm;
@@ -1185,7 +1241,7 @@ void JfrTypeSet::clear(JfrCheckpointWriter* writer, JfrCheckpointWriter* leakp_w
write_klasses_on_clear();
write_packages_on_clear();
write_modules_on_clear();
- write_classloaders_on_clear();
+ write_clds_on_clear();
write_methods_on_clear();
write_symbols_on_clear();
teardown();
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
index 0876281d53f..883821f853c 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
@@ -32,7 +32,8 @@
JfrArtifactSet::JfrArtifactSet(bool class_unload) : _symbol_table(nullptr),
_klass_list(nullptr),
- _total_count(0) {
+ _total_count(0),
+ _class_unload(class_unload) {
initialize(class_unload);
assert(_klass_list != nullptr, "invariant");
}
@@ -41,6 +42,7 @@ static const size_t initial_klass_list_size = 256;
const int initial_klass_loader_set_size = 64;
void JfrArtifactSet::initialize(bool class_unload) {
+ _class_unload = class_unload;
if (_symbol_table == nullptr) {
_symbol_table = JfrSymbolTable::create();
assert(_symbol_table != nullptr, "invariant");
@@ -51,6 +53,11 @@ void JfrArtifactSet::initialize(bool class_unload) {
// resource allocation
_klass_list = new GrowableArray(initial_klass_list_size);
_klass_loader_set = new GrowableArray(initial_klass_loader_set_size);
+ _klass_loader_leakp_set = new GrowableArray(initial_klass_loader_set_size);
+
+ if (class_unload) {
+ _unloading_set = new GrowableArray(initial_klass_list_size);
+ }
}
void JfrArtifactSet::clear() {
@@ -97,10 +104,27 @@ int JfrArtifactSet::entries() const {
return _klass_list->length();
}
-bool JfrArtifactSet::should_do_loader_klass(const Klass* k) {
+static inline bool not_in_set(GrowableArray* set, const Klass* k) {
+ assert(set != nullptr, "invariant");
+ assert(k != nullptr, "invariant");
+ return !JfrMutablePredicate::test(set, k);
+}
+
+bool JfrArtifactSet::should_do_cld_klass(const Klass* k, bool leakp) {
assert(k != nullptr, "invariant");
assert(_klass_loader_set != nullptr, "invariant");
- return !JfrMutablePredicate::test(_klass_loader_set, k);
+ assert(_klass_loader_leakp_set != nullptr, "invariant");
+ return not_in_set(leakp ? _klass_loader_leakp_set : _klass_loader_set, k);
+}
+
+bool JfrArtifactSet::should_do_unloading_artifact(const void* ptr) {
+ assert(ptr != nullptr, "invariant");
+ assert(_class_unload, "invariant");
+ assert(_unloading_set != nullptr, "invariant");
+ // The incoming pointers are of all kinds of different types.
+ // However, we are only interested in set membership.
+ // Treat them uniformly as const Klass* for simplicity and code reuse.
+ return not_in_set(_unloading_set, static_cast(ptr));
}
void JfrArtifactSet::register_klass(const Klass* k) {
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
index b0c2d989de5..24424fdef3a 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
@@ -114,28 +114,6 @@ class ClearArtifact {
}
};
-template
-class SerializePredicate {
- bool _class_unload;
- public:
- SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
- bool operator()(T const& value) {
- assert(value != nullptr, "invariant");
- return _class_unload ? true : IS_NOT_SERIALIZED(value);
- }
-};
-
-template <>
-class SerializePredicate {
- bool _class_unload;
- public:
- SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
- bool operator()(const Method* method) {
- assert(method != nullptr, "invariant");
- return _class_unload ? true : METHOD_IS_NOT_SERIALIZED(method);
- }
-};
-
template
class SymbolPredicate {
bool _class_unload;
@@ -150,11 +128,23 @@ class SymbolPredicate {
}
};
+class KlassUsedPredicate {
+ bool _current_epoch;
+public:
+ KlassUsedPredicate(bool current_epoch) : _current_epoch(current_epoch) {}
+ bool operator()(const Klass* klass) {
+ return _current_epoch ? USED_THIS_EPOCH(klass) : USED_PREVIOUS_EPOCH(klass);
+ }
+};
+
class MethodUsedPredicate {
bool _current_epoch;
public:
MethodUsedPredicate(bool current_epoch) : _current_epoch(current_epoch) {}
bool operator()(const Klass* klass) {
+ if (!klass->is_instance_klass()) {
+ return false;
+ }
return _current_epoch ? METHOD_USED_THIS_EPOCH(klass) : METHOD_USED_PREVIOUS_EPOCH(klass);
}
};
@@ -210,7 +200,10 @@ class JfrArtifactSet : public JfrCHeapObj {
JfrSymbolTable* _symbol_table;
GrowableArray* _klass_list;
GrowableArray* _klass_loader_set;
+ GrowableArray* _klass_loader_leakp_set;
+ GrowableArray* _unloading_set;
size_t _total_count;
+ bool _class_unload;
public:
JfrArtifactSet(bool class_unload);
@@ -235,14 +228,20 @@ class JfrArtifactSet : public JfrCHeapObj {
int entries() const;
size_t total_count() const;
void register_klass(const Klass* k);
- bool should_do_loader_klass(const Klass* k);
+ bool should_do_cld_klass(const Klass* k, bool leakp);
+ bool should_do_unloading_artifact(const void* ptr);
void increment_checkpoint_id();
template
void iterate_klasses(Functor& functor) const {
for (int i = 0; i < _klass_list->length(); ++i) {
if (!functor(_klass_list->at(i))) {
- break;
+ return;
+ }
+ }
+ for (int i = 0; i < _klass_loader_set->length(); ++i) {
+ if (!functor(_klass_loader_set->at(i))) {
+ return;
}
}
}
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
index e5b9a33ef56..13853e14a13 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
@@ -33,6 +33,7 @@
#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdBits.inline.hpp"
#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdMacros.hpp"
+#include "jfr/support/jfrKlassExtension.hpp"
#include "oops/klass.hpp"
#include "oops/method.hpp"
#include "runtime/javaThread.hpp"
@@ -66,10 +67,14 @@ inline traceid set_used_and_get(const T* type) {
return TRACE_ID(type);
}
+// We set the 'method_and_class' bits to have a consistent
+// bit pattern set always. This is because the tag is non-atomic,
+// hence, we always need the same bit pattern in an epoch to avoid losing information.
inline void JfrTraceIdLoadBarrier::load_barrier(const Klass* klass) {
- SET_USED_THIS_EPOCH(klass);
- enqueue(klass);
- JfrTraceIdEpoch::set_changed_tag_state();
+ SET_METHOD_AND_CLASS_USED_THIS_EPOCH(klass);
+ assert(METHOD_AND_CLASS_USED_THIS_EPOCH(klass), "invariant");
+ enqueue(klass);
+ JfrTraceIdEpoch::set_changed_tag_state();
}
inline traceid JfrTraceIdLoadBarrier::load(const Klass* klass) {
@@ -113,26 +118,36 @@ inline traceid JfrTraceIdLoadBarrier::load_no_enqueue(const Klass* klass, const
return (METHOD_ID(klass, method));
}
-inline traceid JfrTraceIdLoadBarrier::load(const ModuleEntry* module) {
- return set_used_and_get(module);
-}
-
-inline traceid JfrTraceIdLoadBarrier::load(const PackageEntry* package) {
- return set_used_and_get(package);
-}
-
inline traceid JfrTraceIdLoadBarrier::load(const ClassLoaderData* cld) {
assert(cld != nullptr, "invariant");
if (cld->has_class_mirror_holder()) {
return 0;
}
const Klass* const class_loader_klass = cld->class_loader_klass();
- if (class_loader_klass != nullptr && should_tag(class_loader_klass)) {
- load_barrier(class_loader_klass);
+ if (class_loader_klass != nullptr) {
+ load(class_loader_klass);
}
return set_used_and_get(cld);
}
+inline traceid JfrTraceIdLoadBarrier::load(const ModuleEntry* module) {
+ assert(module != nullptr, "invariant");
+ const ClassLoaderData* cld = module->loader_data();
+ if (cld != nullptr) {
+ load(cld);
+ }
+ return set_used_and_get(module);
+}
+
+inline traceid JfrTraceIdLoadBarrier::load(const PackageEntry* package) {
+ assert(package != nullptr, "invariant");
+ const ModuleEntry* const module_entry = package->module();
+ if (module_entry != nullptr) {
+ load(module_entry);
+ }
+ return set_used_and_get(package);
+}
+
inline traceid JfrTraceIdLoadBarrier::load_leakp(const Klass* klass, const Method* method) {
assert(klass != nullptr, "invariant");
assert(METHOD_AND_CLASS_USED_THIS_EPOCH(klass), "invariant");
diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
index 40c3d7a8c4f..cc460f8c2aa 100644
--- a/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
+++ b/src/hotspot/share/jfr/recorder/jfrRecorder.cpp
@@ -86,9 +86,6 @@ bool JfrRecorder::create_oop_storages() {
return ObjectSampler::create_oop_storage();
}
-// Subsystem
-static JfrCheckpointManager* _checkpoint_manager = nullptr;
-
bool JfrRecorder::on_create_vm_1() {
if (!is_disabled()) {
if (FlightRecorder || is_started_on_commandline()) {
@@ -99,9 +96,10 @@ bool JfrRecorder::on_create_vm_1() {
return false;
}
- _checkpoint_manager = JfrCheckpointManager::create();
- if (_checkpoint_manager == nullptr || !_checkpoint_manager->initialize_early()) {
- return false;
+ if (is_started_on_commandline()) {
+ if (!create_checkpoint_manager()) {
+ return false;
+ }
}
// fast time initialization
@@ -292,7 +290,7 @@ bool JfrRecorder::create_components() {
if (!create_storage()) {
return false;
}
- if (!create_checkpoint_manager()) {
+ if (!initialize_checkpoint_manager()) {
return false;
}
if (!create_stacktrace_repository()) {
@@ -321,6 +319,7 @@ static JfrStackTraceRepository* _stack_trace_repository;
static JfrStringPool* _stringpool = nullptr;
static JfrOSInterface* _os_interface = nullptr;
static JfrThreadSampling* _thread_sampling = nullptr;
+static JfrCheckpointManager* _checkpoint_manager = nullptr;
bool JfrRecorder::create_java_event_writer() {
return JfrJavaEventWriter::initialize();
@@ -357,6 +356,17 @@ bool JfrRecorder::create_storage() {
}
bool JfrRecorder::create_checkpoint_manager() {
+ assert(_checkpoint_manager == nullptr, "invariant");
+ _checkpoint_manager = JfrCheckpointManager::create();
+ return _checkpoint_manager != nullptr && _checkpoint_manager->initialize_early();
+}
+
+bool JfrRecorder::initialize_checkpoint_manager() {
+ if (_checkpoint_manager == nullptr) {
+ if (!create_checkpoint_manager()) {
+ return false;
+ }
+ }
assert(_checkpoint_manager != nullptr, "invariant");
assert(_repository != nullptr, "invariant");
return _checkpoint_manager->initialize(&_repository->chunkwriter());
diff --git a/src/hotspot/share/jfr/recorder/jfrRecorder.hpp b/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
index 3e2541fad98..9f4969b0187 100644
--- a/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
+++ b/src/hotspot/share/jfr/recorder/jfrRecorder.hpp
@@ -42,6 +42,7 @@ class JfrRecorder : public JfrCHeapObj {
static bool on_create_vm_2();
static bool on_create_vm_3();
static bool create_checkpoint_manager();
+ static bool initialize_checkpoint_manager();
static bool create_chunk_repository();
static bool create_java_event_writer();
static bool create_jvmti_agent();
diff --git a/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp b/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp
index 09a452caaa5..49ced300e55 100644
--- a/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp
+++ b/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp
@@ -626,8 +626,8 @@ inline bool ReinitializeAllReleaseRetiredOp::process(typename
const bool retired = node->retired();
node->reinitialize();
assert(node->empty(), "invariant");
- assert(!node->retired(), "invariant");
if (retired) {
+ assert(!node->retired(), "invariant");
_prev = _list.excise(_prev, node);
node->release();
mspace_release(node, _mspace);
diff --git a/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp b/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp
index a36b85dbb7e..5f5c87d239c 100644
--- a/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp
+++ b/src/hotspot/share/jfr/support/jfrDeprecationManager.cpp
@@ -371,8 +371,10 @@ void JfrDeprecationManager::write_edges(JfrChunkWriter& cw, Thread* thread, bool
void JfrDeprecationManager::on_type_set(JfrCheckpointWriter& writer, JfrChunkWriter* cw, Thread* thread) {
assert(_pending_list.is_empty(), "invariant");
- if (writer.has_data() && _pending_head != nullptr) {
+ if (_pending_head != nullptr) {
save_type_set_blob(writer);
+ } else {
+ writer.cancel();
}
if (cw != nullptr) {
write_edges(*cw, thread);
diff --git a/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.cpp b/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.cpp
index c26592763d4..e4c3a43c3b9 100644
--- a/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.cpp
+++ b/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.cpp
@@ -69,7 +69,7 @@ static inline JfrTicksWrapper* allocate_start_time() {
return EventType::is_enabled() ? new JfrTicksWrapper() : nullptr;
}
-NativeLibraryLoadEvent::NativeLibraryLoadEvent(const char* name, void** result) : JfrNativeLibraryEventBase(name), _result(result) {
+NativeLibraryLoadEvent::NativeLibraryLoadEvent(const char* name, void** result) : JfrNativeLibraryEventBase(name), _result(result), _fp_env_correction_attempt(false), _fp_env_correction_success(false) {
assert(_result != nullptr, "invariant");
_start_time = allocate_start_time();
}
@@ -90,8 +90,17 @@ void NativeLibraryUnloadEvent::set_result(bool result) {
_result = result;
}
+static void set_additional_data(EventNativeLibraryLoad& event, const NativeLibraryLoadEvent& helper) {
+ event.set_fpEnvCorrectionAttempt(helper.get_fp_env_correction_attempt());
+ event.set_fpEnvCorrectionSuccess(helper.get_fp_env_correction_success());
+}
+
+static void set_additional_data(EventNativeLibraryUnload& event, const NativeLibraryUnloadEvent& helper) {
+ // no additional entries atm. for the unload event
+}
+
template
-static void commit(HelperType& helper) {
+static void commit(const HelperType& helper) {
if (!helper.has_start_time()) {
return;
}
@@ -101,17 +110,20 @@ static void commit(HelperType& helper) {
event.set_name(helper.name());
event.set_errorMessage(helper.error_msg());
event.set_success(helper.success());
+ set_additional_data(event, helper);
Thread* thread = Thread::current();
assert(thread != nullptr, "invariant");
if (thread->is_Java_thread()) {
JavaThread* jt = JavaThread::cast(thread);
- if (jt->thread_state() != _thread_in_vm) {
- assert(jt->thread_state() == _thread_in_native, "invariant");
+ if (jt->thread_state() == _thread_in_native) {
// For a JavaThread to take a JFR stacktrace, it must be in _thread_in_vm. Can safepoint here.
ThreadInVMfromNative transition(jt);
event.commit();
return;
}
+ // If a thread comes here still _thread_in_Java, which can happen for example
+ // when loading the disassembler library in response to traps in JIT code - all is ok.
+ // Since there is no ljf, an event will be committed without a stacktrace.
}
event.commit();
}
diff --git a/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.hpp b/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.hpp
index fe8431fdc43..30281ad4d05 100644
--- a/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.hpp
+++ b/src/hotspot/share/jfr/support/jfrNativeLibraryLoadEvent.hpp
@@ -52,10 +52,16 @@ class JfrNativeLibraryEventBase : public StackObj {
class NativeLibraryLoadEvent : public JfrNativeLibraryEventBase {
private:
void** _result;
+ bool _fp_env_correction_attempt;
+ bool _fp_env_correction_success;
public:
NativeLibraryLoadEvent(const char* name, void** result);
~NativeLibraryLoadEvent();
bool success() const;
+ bool get_fp_env_correction_attempt() const { return _fp_env_correction_attempt; }
+ bool get_fp_env_correction_success() const { return _fp_env_correction_success; }
+ void set_fp_env_correction_attempt(bool v) { _fp_env_correction_attempt = v; }
+ void set_fp_env_correction_success(bool v) { _fp_env_correction_success = v; }
};
class NativeLibraryUnloadEvent : public JfrNativeLibraryEventBase {
diff --git a/src/hotspot/share/jfr/utilities/jfrVersionSystem.hpp b/src/hotspot/share/jfr/utilities/jfrVersionSystem.hpp
index 5005b5b8f3f..39c40fd7147 100644
--- a/src/hotspot/share/jfr/utilities/jfrVersionSystem.hpp
+++ b/src/hotspot/share/jfr/utilities/jfrVersionSystem.hpp
@@ -105,9 +105,9 @@ class JfrVersionSystem : public JfrCHeapObj {
NodePtr synchronize_with(Type version, NodePtr last) const;
DEBUG_ONLY(void assert_state(const Node* node) const;)
struct PaddedTip {
- DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE, 0);
+ DEFINE_PAD_MINUS_SIZE(0, DEFAULT_PADDING_SIZE, 0);
volatile Type _value;
- DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE, sizeof(volatile Type));
+ DEFINE_PAD_MINUS_SIZE(1, DEFAULT_PADDING_SIZE, sizeof(volatile Type));
};
PaddedTip _tip;
NodePtr _head;
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp
index 421ebd7396f..ff159a490c1 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.hpp
@@ -115,6 +115,13 @@ class CompilerToVM {
// Minimum alignment of an offset into CodeBuffer::SECT_CONSTS
static int data_section_item_alignment;
+ /*
+ * Pointer to JvmtiExport::_should_notify_object_alloc.
+ * Exposed as an int* instead of an address so the
+ * underlying type is part of the JVMCIVMStructs definition.
+ */
+ static int* _should_notify_object_alloc;
+
public:
static void initialize(JVMCI_TRAPS);
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
index a3e2bd525ec..89fa78b10f3 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
@@ -45,6 +45,7 @@
#include "memory/universe.hpp"
#include "oops/compressedOops.hpp"
#include "oops/klass.inline.hpp"
+#include "prims/jvmtiExport.hpp"
#ifdef COMPILER2
#include "opto/c2compiler.hpp"
#endif
@@ -130,6 +131,8 @@ address CompilerToVM::Data::symbol_clinit;
int CompilerToVM::Data::data_section_item_alignment;
+int* CompilerToVM::Data::_should_notify_object_alloc;
+
void CompilerToVM::Data::initialize(JVMCI_TRAPS) {
Klass_vtable_start_offset = in_bytes(Klass::vtable_start_offset());
Klass_vtable_length_offset = in_bytes(Klass::vtable_length_offset());
@@ -196,6 +199,8 @@ void CompilerToVM::Data::initialize(JVMCI_TRAPS) {
data_section_item_alignment = relocInfo::addr_unit();
+ _should_notify_object_alloc = &JvmtiExport::_should_notify_object_alloc;
+
BarrierSet* bs = BarrierSet::barrier_set();
if (bs->is_a(BarrierSet::CardTableBarrierSet)) {
CardTable::CardValue* base = ci_card_table_address();
diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
index 0c389efdd0f..daa3bacf81c 100644
--- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
+++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
@@ -122,6 +122,8 @@
\
static_field(CompilerToVM::Data, data_section_item_alignment, int) \
\
+ static_field(CompilerToVM::Data, _should_notify_object_alloc, int*) \
+ \
static_field(Abstract_VM_Version, _features, uint64_t) \
\
nonstatic_field(Annotations, _class_annotations, AnnotationArray*) \
@@ -220,6 +222,7 @@
nonstatic_field(JavaThread, _lock_stack, LockStack) \
JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_VTMS_transition, bool)) \
JVMTI_ONLY(nonstatic_field(JavaThread, _is_in_tmp_VTMS_transition, bool)) \
+ JVMTI_ONLY(nonstatic_field(JavaThread, _is_disable_suspend, bool)) \
\
nonstatic_field(LockStack, _top, uint32_t) \
\
diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp
index cf6dc58eb46..ed6066bc1db 100644
--- a/src/hotspot/share/logging/logTag.hpp
+++ b/src/hotspot/share/logging/logTag.hpp
@@ -97,6 +97,7 @@ class outputStream;
LOG_TAG(iklass) \
LOG_TAG(indy) \
LOG_TAG(init) \
+ LOG_TAG(inlinecache)\
LOG_TAG(inlining) \
LOG_TAG(install) \
LOG_TAG(interpreter) \
diff --git a/src/hotspot/share/memory/padded.hpp b/src/hotspot/share/memory/padded.hpp
index 1811810c19e..0597dbd9f3c 100644
--- a/src/hotspot/share/memory/padded.hpp
+++ b/src/hotspot/share/memory/padded.hpp
@@ -40,14 +40,14 @@
// effective only when applied to derived-most (leaf) classes.
// When no args are passed to the base ctor.
-template
+template
class Padded : public T {
private:
char _pad_buf_[PADDING_SIZE(T, alignment)];
};
// When either 0 or 1 args may be passed to the base ctor.
-template