diff --git a/.jcheck/conf b/.jcheck/conf
index 18228df5dfe46..b374bde90a538 100644
--- a/.jcheck/conf
+++ b/.jcheck/conf
@@ -15,7 +15,7 @@ version=0
domain=openjdk.org
[checks "whitespace"]
-files=.*\.cpp|.*\.hpp|.*\.c|.*\.h|.*\.java|.*\.cc|.*\.hh|.*\.m|.*\.mm|.*\.md|.*\.gmk|.*\.m4|.*\.ac|Makefile
+files=.*\.cpp|.*\.hpp|.*\.c|.*\.h|.*\.java|.*\.cc|.*\.hh|.*\.m|.*\.mm|.*\.md|.*\.properties|.*\.gmk|.*\.m4|.*\.ac|Makefile
ignore-tabs=.*\.gmk|Makefile
[checks "merge"]
diff --git a/doc/building.html b/doc/building.html
index d51e74d1454be..8a0acada254ac 100644
--- a/doc/building.html
+++ b/doc/building.html
@@ -526,7 +526,7 @@ Linux
The basic tooling is provided as part of the core operating system,
but you will most likely need to install developer packages.
For apt-based distributions (Debian, Ubuntu, etc), try this:
-sudo apt-get install build-essentialsudo apt-get install build-essential autoconfFor rpm-based distributions (Fedora, Red Hat, etc), try this:
sudo yum groupinstall "Development Tools"For Alpine Linux, aside from basic tooling, install the GNU versions
@@ -2166,15 +2166,26 @@
Using Multiple
configure from there, e.g.
mkdir build/<name> && cd build/<name> && bash ../../configure.
Then you can build that configuration using
-make CONF_NAME=<name> or
-make CONF=<pattern>, where
-<pattern> is a substring matching one or several
-configurations, e.g. CONF=debug. The special empty pattern
-(CONF=) will match all available configuration, so
-make CONF= hotspot will build the hotspot
-target for all configurations. Alternatively, you can execute
-make in the configuration directory, e.g.
-cd build/<name> && make.
+make CONF=<selector>, where
+<selector> is interpreted as follows:
+
+If <selector> exacly matches the name of a
+configuration, this and only this configuration will be selected.
+If <selector> matches (i.e. is a substring of)
+the names of several configurations, then all these configurations will
+be selected.
+If <selector> is empty (i.e. CONF=),
+then all configurations will be selected.
+If <selector> begins with !, then
+all configurations not matching the string following
+! will be selected.
+
+A more specialized version, CONF_NAME=<name> also
+exists, which will only match if the given <name>
+exactly matches a single configuration.
+Alternatively, you can execute make in the configuration
+directory, e.g. cd build/<name> && make.
+make CONF_NAME=<name> or
Handling Reconfigurations
If you update the repository and part of the configure script has
changed, the build system will force you to re-run
diff --git a/doc/building.md b/doc/building.md
index 9d928a3924557..ed8a06693551d 100644
--- a/doc/building.md
+++ b/doc/building.md
@@ -349,7 +349,7 @@ will most likely need to install developer packages.
For apt-based distributions (Debian, Ubuntu, etc), try this:
```
-sudo apt-get install build-essential
+sudo apt-get install build-essential autoconf
```
For rpm-based distributions (Fedora, Red Hat, etc), try this:
@@ -1952,12 +1952,25 @@ configuration with the name ``. Alternatively, you can create a directory
under `build` and run `configure` from there, e.g. `mkdir build/ && cd
build/ && bash ../../configure`.
-Then you can build that configuration using `make CONF_NAME=` or `make
-CONF=`, where `` is a substring matching one or several
-configurations, e.g. `CONF=debug`. The special empty pattern (`CONF=`) will
-match *all* available configuration, so `make CONF= hotspot` will build the
-`hotspot` target for all configurations. Alternatively, you can execute `make`
-in the configuration directory, e.g. `cd build/ && make`.
+Then you can build that configuration using `make CONF=`, where
+`` is interpreted as follows:
+
+* If `` exacly matches the name of a configuration, this and only
+ this configuration will be selected.
+* If `` matches (i.e. is a substring of) the names of several
+ configurations, then all these configurations will be selected.
+* If `` is empty (i.e. `CONF=`), then all configurations will be
+ selected.
+* If `` begins with `!`, then all configurations **not** matching the
+ string following `!` will be selected.
+
+A more specialized version, `CONF_NAME=` also exists, which will only
+match if the given `` exactly matches a single configuration.
+
+Alternatively, you can execute `make` in the configuration directory, e.g. `cd
+build/ && make`.
+
+`make CONF_NAME=` or
### Handling Reconfigurations
diff --git a/make/Global.gmk b/make/Global.gmk
index e5e76b475b941..1df6c5fb6bc4b 100644
--- a/make/Global.gmk
+++ b/make/Global.gmk
@@ -87,10 +87,9 @@ help:
$(info $(_) # (gensrc, java, copy, libs, launchers, gendata))
$(info )
$(info Make control variables)
- $(info $(_) CONF= # Build all configurations (note, assignment is empty))
- $(info $(_) CONF= # Build the configuration(s) with a name matching)
- $(info $(_) # )
- $(info $(_) CONF_NAME= # Build the configuration with exactly the )
+ $(info $(_) CONF= # Select which configuration(s) to build)
+ $(info $(_) CONF= # Select all configurations (note, assignment is empty))
+ $(info $(_) CONF_NAME= # Select the configuration with the name )
$(info $(_) SPEC= # Build the configuration given by the spec file)
$(info $(_) LOG= # Change the log level from warn to )
$(info $(_) # Available log levels are:)
diff --git a/make/Hsdis.gmk b/make/Hsdis.gmk
index 7496a3a2cf1b4..6de0e628a5288 100644
--- a/make/Hsdis.gmk
+++ b/make/Hsdis.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -38,7 +38,7 @@ HSDIS_OUTPUT_DIR := $(SUPPORT_OUTPUTDIR)/hsdis
REAL_HSDIS_NAME := hsdis-$(OPENJDK_TARGET_CPU_LEGACY_LIB)$(SHARED_LIBRARY_SUFFIX)
BUILT_HSDIS_LIB := $(HSDIS_OUTPUT_DIR)/$(REAL_HSDIS_NAME)
-HSDIS_TOOLCHAIN := TOOLCHAIN_DEFAULT
+HSDIS_LINK_TYPE := C
HSDIS_TOOLCHAIN_CFLAGS := $(CFLAGS_JDKLIB)
HSDIS_TOOLCHAIN_LDFLAGS := $(LDFLAGS_JDKLIB)
@@ -59,8 +59,8 @@ endif
ifeq ($(HSDIS_BACKEND), llvm)
# Use C++ instead of C
+ HSDIS_LINK_TYPE := C++
HSDIS_TOOLCHAIN_CFLAGS := $(CXXFLAGS_JDKLIB)
- HSDIS_TOOLCHAIN := TOOLCHAIN_LINK_CXX
ifeq ($(call isTargetOs, linux), true)
LLVM_OS := pc-linux-gnu
@@ -91,14 +91,11 @@ ifeq ($(HSDIS_BACKEND), binutils)
endif
endif
- $(eval $(call DefineNativeToolchain, TOOLCHAIN_MINGW, \
- CC := $(MINGW_BASE)-gcc, \
- LD := $(MINGW_BASE)-ld, \
- OBJCOPY := $(MINGW_BASE)-objcopy, \
- RC := $(RC), \
- SYSROOT_CFLAGS := --sysroot=$(MINGW_SYSROOT), \
- SYSROOT_LDFLAGS := --sysroot=$(MINGW_SYSROOT), \
- ))
+ BUILD_HSDIS_CC := $(MINGW_BASE)-gcc
+ BUILD_HSDIS_LD := $(MINGW_BASE)-ld
+ BUILD_HSDIS_OBJCOPY := $(MINGW_BASE)-objcopy
+ BUILD_HSDIS_SYSROOT_CFLAGS := --sysroot=$(MINGW_SYSROOT)
+ BUILD_HSDIS_SYSROOT_LDFLAGS := --sysroot=$(MINGW_SYSROOT)
MINGW_SYSROOT_LIB_PATH := $(MINGW_SYSROOT)/mingw/lib
ifeq ($(wildcard $(MINGW_SYSROOT_LIB_PATH)), )
@@ -122,8 +119,8 @@ ifeq ($(HSDIS_BACKEND), binutils)
TOOLCHAIN_TYPE := gcc
OPENJDK_TARGET_OS := linux
+ OPENJDK_TARGET_OS_TYPE := unix
CC_OUT_OPTION := -o$(SPACE)
- LD_OUT_OPTION := -o$(SPACE)
GENDEPS_FLAGS := -MMD -MF
CFLAGS_DEBUG_SYMBOLS := -g
DISABLED_WARNINGS :=
@@ -131,7 +128,6 @@ ifeq ($(HSDIS_BACKEND), binutils)
CFLAGS_WARNINGS_ARE_ERRORS := -Werror
SHARED_LIBRARY_FLAGS := -shared
- HSDIS_TOOLCHAIN := TOOLCHAIN_MINGW
HSDIS_TOOLCHAIN_CFLAGS :=
HSDIS_TOOLCHAIN_LDFLAGS := -L$(MINGW_GCC_LIB_PATH) -L$(MINGW_SYSROOT_LIB_PATH)
MINGW_DLLCRT := $(MINGW_SYSROOT_LIB_PATH)/dllcrt2.o
@@ -144,9 +140,9 @@ endif
$(eval $(call SetupJdkLibrary, BUILD_HSDIS, \
NAME := hsdis, \
+ LINK_TYPE := $(HSDIS_LINK_TYPE), \
SRC := $(TOPDIR)/src/utils/hsdis/$(HSDIS_BACKEND), \
EXTRA_HEADER_DIRS := $(TOPDIR)/src/utils/hsdis, \
- TOOLCHAIN := $(HSDIS_TOOLCHAIN), \
OUTPUT_DIR := $(HSDIS_OUTPUT_DIR), \
OBJECT_DIR := $(HSDIS_OUTPUT_DIR), \
DISABLED_WARNINGS_gcc := undef format-nonliteral sign-compare, \
diff --git a/make/InitSupport.gmk b/make/InitSupport.gmk
index 31c80e2f7267f..4b14c4f9ad951 100644
--- a/make/InitSupport.gmk
+++ b/make/InitSupport.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -202,8 +202,14 @@ ifeq ($(HAS_SPEC),)
matching_confs := $$(strip $$(all_confs))
else
# Otherwise select those that contain the given CONF string
- matching_confs := $$(strip $$(foreach var, $$(all_confs), \
- $$(if $$(findstring $$(CONF), $$(var)), $$(var))))
+ ifeq ($$(patsubst !%,,$$(CONF)),)
+ # A CONF starting with ! means we should negate the search term
+ matching_confs := $$(strip $$(foreach var, $$(all_confs), \
+ $$(if $$(findstring $$(subst !,,$$(CONF)), $$(var)), ,$$(var))))
+ else
+ matching_confs := $$(strip $$(foreach var, $$(all_confs), \
+ $$(if $$(findstring $$(CONF), $$(var)), $$(var))))
+ endif
ifneq ($$(filter $$(CONF), $$(matching_confs)), )
# If we found an exact match, use that
matching_confs := $$(CONF)
@@ -421,8 +427,9 @@ else # $(HAS_SPEC)=true
# Cleanup after a compare build
define CleanupCompareBuild
- # If running with a COMPARE_BUILD patch, reverse-apply it
- $(if $(COMPARE_BUILD_PATCH), cd $(topdir) && $(PATCH) -R -p1 < $(COMPARE_BUILD_PATCH))
+ # If running with a COMPARE_BUILD patch, reverse-apply it, but continue
+ # even if that fails (can happen with removed files).
+ $(if $(COMPARE_BUILD_PATCH), cd $(topdir) && $(PATCH) -R -p1 < $(COMPARE_BUILD_PATCH) || true)
# Move this build away and restore the original build
$(MKDIR) -p $(topdir)/build/compare-build
$(MV) $(OUTPUTDIR) $(COMPARE_BUILD_OUTPUTDIR)
diff --git a/make/autoconf/build-aux/pkg.m4 b/make/autoconf/build-aux/pkg.m4
index 5f4b22bb27f05..ddb685e9bc35f 100644
--- a/make/autoconf/build-aux/pkg.m4
+++ b/make/autoconf/build-aux/pkg.m4
@@ -25,7 +25,7 @@
# questions.
#
-#
+#
# Copyright © 2004 Scott James Remnant .
#
# This program is free software; you can redistribute it and/or modify
@@ -54,18 +54,18 @@ AC_DEFUN([PKG_PROG_PKG_CONFIG],
m4_pattern_allow([^PKG_CONFIG(_PATH)?$])
AC_ARG_VAR([PKG_CONFIG], [path to pkg-config utility])dnl
if test "x$ac_cv_env_PKG_CONFIG_set" != "xset"; then
- AC_PATH_TOOL([PKG_CONFIG], [pkg-config])
+ AC_PATH_TOOL([PKG_CONFIG], [pkg-config])
fi
if test -n "$PKG_CONFIG"; then
- _pkg_min_version=m4_default([$1], [0.9.0])
- AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version])
- if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
- AC_MSG_RESULT([yes])
- else
- AC_MSG_RESULT([no])
- PKG_CONFIG=""
- fi
-
+ _pkg_min_version=m4_default([$1], [0.9.0])
+ AC_MSG_CHECKING([pkg-config is at least version $_pkg_min_version])
+ if $PKG_CONFIG --atleast-pkgconfig-version $_pkg_min_version; then
+ AC_MSG_RESULT([yes])
+ else
+ AC_MSG_RESULT([no])
+ PKG_CONFIG=""
+ fi
+
fi[]dnl
])# PKG_PROG_PKG_CONFIG
@@ -97,7 +97,7 @@ m4_define([_PKG_CONFIG],
elif test -n "$PKG_CONFIG"; then
PKG_CHECK_EXISTS([$3],
[pkg_cv_[]$1=`$PKG_CONFIG --[]$2 "$3" 2>/dev/null`],
- [pkg_failed=yes])
+ [pkg_failed=yes])
else
pkg_failed=untried
fi[]dnl
@@ -143,14 +143,14 @@ See the pkg-config man page for more details.])
if test $pkg_failed = yes; then
_PKG_SHORT_ERRORS_SUPPORTED
if test $_pkg_short_errors_supported = yes; then
- $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "$2" 2>&1`
- else
- $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors "$2" 2>&1`
+ $1[]_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "$2" 2>&1`
+ else
+ $1[]_PKG_ERRORS=`$PKG_CONFIG --print-errors "$2" 2>&1`
fi
- # Put the nasty error message in config.log where it belongs
- echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD
+ # Put the nasty error message in config.log where it belongs
+ echo "$$1[]_PKG_ERRORS" >&AS_MESSAGE_LOG_FD
- ifelse([$4], , [AC_MSG_ERROR(dnl
+ ifelse([$4], , [AC_MSG_ERROR(dnl
[Package requirements ($2) were not met:
$$1_PKG_ERRORS
@@ -160,10 +160,10 @@ installed software in a non-standard prefix.
_PKG_TEXT
])],
- [AC_MSG_RESULT([no])
+ [AC_MSG_RESULT([no])
$4])
elif test $pkg_failed = untried; then
- ifelse([$4], , [AC_MSG_FAILURE(dnl
+ ifelse([$4], , [AC_MSG_FAILURE(dnl
[The pkg-config script could not be found or is too old. Make sure it
is in your PATH or set the PKG_CONFIG environment variable to the full
path to pkg-config.
@@ -171,11 +171,11 @@ path to pkg-config.
_PKG_TEXT
To get pkg-config, see '
-# and changes all paths in cmd to be relative to the workspace root. This only
-# works properly if the build dir is inside the workspace root. If it's not,
-# relative paths are still calculated, but depending on the distance between the
-# dirs, paths in the build dir may end up as essentially absolute anyway.
-#
-# The fix-deps-file macro is used to adjust the contents of the generated make
-# dependency files to contain paths compatible with make.
-#
-REWRITE_PATHS_RELATIVE = false
-ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT)-$(FILE_MACRO_CFLAGS), false-)
- REWRITE_PATHS_RELATIVE = true
-endif
-
-# CCACHE_BASEDIR needs fix-deps-file as makefiles use absolute filenames for
-# object files while CCACHE_BASEDIR will make ccache relativize all paths for
-# its compiler. The compiler then produces relative dependency files.
-# make does not know a relative and absolute filename is the same so it will
-# ignore such dependencies. This only applies when the OUTPUTDIR is inside
-# the WORKSPACE_ROOT.
-ifneq ($(CCACHE), )
- ifneq ($(filter $(WORKSPACE_ROOT)/%, $(OUTPUTDIR)), )
- REWRITE_PATHS_RELATIVE = true
- endif
-endif
-
-ifeq ($(REWRITE_PATHS_RELATIVE), true)
- # Need to handle -I flags as both '-Ifoo' and '-I foo'.
- MakeCommandRelative = \
- $(CD) $(WORKSPACE_ROOT) && \
- $(foreach o, $1, \
- $(if $(filter $(WORKSPACE_ROOT)/% $(OUTPUTDIR)/%, $o), \
- $(call RelativePath, $o, $(WORKSPACE_ROOT)) \
- , \
- $(if $(filter -I$(WORKSPACE_ROOT)/%, $o), \
- -I$(call RelativePath, $(patsubst -I%, %, $o), $(WORKSPACE_ROOT)) \
- , \
- $o \
- ) \
- ) \
- )
-
- # When compiling with relative paths, the deps file may come out with relative
- # paths, and that path may start with './'. First remove any leading ./, then
- # add WORKSPACE_ROOT to any line not starting with /, while allowing for
- # leading spaces. There may also be multiple entries on the same line, so start
- # with splitting such lines.
- # Non GNU sed (BSD on macosx) cannot substitute in literal \n using regex.
- # Instead use a bash escaped literal newline. To avoid having unmatched quotes
- # ruin the ability for an editor to properly syntax highlight this file, define
- # that newline sequence as a separate variable and add the closing quote behind
- # a comment.
- sed_newline := \'$$'\n''#'
- define fix-deps-file
- $(SED) \
- -e 's|\([^ ]\) \{1,\}\([^\\:]\)|\1 \\$(sed_newline) \2|g' \
- $1.tmp \
- | $(SED) \
- -e 's|^\([ ]*\)\./|\1|' \
- -e '/^[ ]*[^/ ]/s|^\([ ]*\)|\1$(WORKSPACE_ROOT)/|' \
- > $1
- endef
-else
- # By default the MakeCommandRelative macro does nothing.
- MakeCommandRelative = $1
-
- # No adjustment is needed.
- define fix-deps-file
- $(MV) $1.tmp $1
- endef
-endif
-
-################################################################################
-# GetEntitlementsFile
-# Find entitlements file for executable when signing on macosx. If no
-# specialized file is found, returns the default file.
-# $1 Executable to find entitlements file for.
-ENTITLEMENTS_DIR := $(TOPDIR)/make/data/macosxsigning
-ifeq ($(MACOSX_CODESIGN_MODE), debug)
- CODESIGN_PLIST_SUFFIX := -debug
-else
- CODESIGN_PLIST_SUFFIX :=
-endif
-DEFAULT_ENTITLEMENTS_FILE := $(ENTITLEMENTS_DIR)/default$(CODESIGN_PLIST_SUFFIX).plist
-
-GetEntitlementsFile = \
- $(foreach f, $(ENTITLEMENTS_DIR)/$(strip $(notdir $1))$(CODESIGN_PLIST_SUFFIX).plist, \
- $(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
- )
+include native/CompileFile.gmk
+include native/DebugSymbols.gmk
+include native/Flags.gmk
+include native/Link.gmk
+include native/LinkMicrosoft.gmk
+include native/Paths.gmk
################################################################################
-# Create the recipe needed to compile a single native source file.
-#
-# Parameter 1 is the name of the rule, based on the name of the library/
-# program being build and the name of the source code file, e.g.
-# BUILD_LIBFOO_fooMain.cpp.
-#
-# Remaining parameters are named arguments:
-# FILE - The full path of the source file to compiler
-# BASE - The name of the rule for the entire binary to build ($1)
-#
-SetupCompileNativeFile = $(NamedParamsMacroTemplate)
-define SetupCompileNativeFileBody
- $1_FILENAME := $$(notdir $$($1_FILE))
-
- # The target file to be generated.
- $1_OBJ := $$($$($1_BASE)_OBJECT_DIR)/$$(call replace_with_obj_extension, \
- $$($1_FILENAME))
-
- # Generate the corresponding compile_commands.json fragment.
- $1_OBJ_JSON = $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
- $$(OUTPUTDIR)/,,$$($1_OBJ))).json
- $$($1_BASE)_ALL_OBJS_JSON += $$($1_OBJ_JSON)
-
- # Only continue if this object file hasn't been processed already. This lets
- # the first found source file override any other with the same name.
- ifeq ($$($1_OBJ_PROCESSED), )
- $1_OBJ_PROCESSED := true
- # This is the definite source file to use for $1_FILENAME.
- $1_SRC_FILE := $$($1_FILE)
-
- ifeq ($$($1_OPTIMIZATION), )
- $1_OPT_CFLAGS := $$($$($1_BASE)_OPT_CFLAGS)
- $1_OPT_CXXFLAGS := $$($$($1_BASE)_OPT_CXXFLAGS)
- else
- ifeq ($$($1_OPTIMIZATION), NONE)
- $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
- else ifeq ($$($1_OPTIMIZATION), LOW)
- $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
- else ifeq ($$($1_OPTIMIZATION), HIGH)
- $1_OPT_CFLAGS := $(C_O_FLAG_HI)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
- else ifeq ($$($1_OPTIMIZATION), HIGHEST)
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
- else ifeq ($$($1_OPTIMIZATION), HIGHEST_JVM)
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
- else ifeq ($$($1_OPTIMIZATION), SIZE)
- $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
- else
- $$(error Unknown value for file OPTIMIZATION: $$($1_OPTIMIZATION))
- endif
- endif
-
- ifneq ($$($$($1_BASE)_PRECOMPILED_HEADER), )
- ifeq ($$(filter $$($1_FILENAME), $$($$($1_BASE)_PRECOMPILED_HEADER_EXCLUDE)), )
- $1_USE_PCH_FLAGS := $$($$($1_BASE)_USE_PCH_FLAGS)
- endif
- endif
-
- ifneq ($(DISABLE_WARNING_PREFIX), )
- $1_WARNINGS_FLAGS := $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$($$($1_BASE)_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$$($1_FILENAME)) \
- $$($$($1_BASE)_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)_$$($1_FILENAME)))
- endif
-
- $1_BASE_CFLAGS := $$($$($1_BASE)_CFLAGS) $$($$($1_BASE)_EXTRA_CFLAGS) \
- $$($$($1_BASE)_SYSROOT_CFLAGS)
- $1_BASE_CXXFLAGS := $$($$($1_BASE)_CXXFLAGS) $$($$($1_BASE)_EXTRA_CXXFLAGS) \
- $$($$($1_BASE)_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS)
- $1_BASE_ASFLAGS := $$($$($1_BASE)_ASFLAGS) $$($$($1_BASE)_EXTRA_ASFLAGS)
-
- ifneq ($$(filter %.c, $$($1_FILENAME)), )
- # Compile as a C file
- $1_CFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CFLAGS) \
- $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CC)
- else ifneq ($$(filter %.m, $$($1_FILENAME)), )
- # Compile as an Objective-C file
- $1_CFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := -x objective-c $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) \
- $$($1_BASE_CFLAGS) $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CC)
- else ifneq ($$(filter %.S, $$($1_FILENAME)), )
- # Compile as preprocessed assembler file
- $1_FLAGS := $(BASIC_ASFLAGS) $$($1_BASE_ASFLAGS)
- $1_COMPILER := $(AS)
-
- # gcc or clang assembly files must contain an appropriate relative .file
- # path for reproducible builds.
- ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- # If no absolute paths allowed, work out relative source file path
- # for assembly .file substitution, otherwise use full file path
- ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT), false)
- $1_REL_ASM_SRC := $$(call RelativePath, $$($1_FILE), $(WORKSPACE_ROOT))
- else
- $1_REL_ASM_SRC := $$($1_FILE)
- endif
- $1_FLAGS := $$($1_FLAGS) -DASSEMBLY_SRC_FILE='"$$($1_REL_ASM_SRC)"' \
- -include $(TOPDIR)/make/data/autoheaders/assemblyprefix.h
- endif
- else ifneq ($$(filter %.cpp %.cc %.mm, $$($1_FILENAME)), )
- # Compile as a C++ or Objective-C++ file
- $1_CXXFLAGS += $$($1_WARNINGS_FLAGS)
- $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CXXFLAGS) \
- $$($1_OPT_CXXFLAGS) $$($1_CXXFLAGS) -c
- $1_COMPILER := $$($$($1_BASE)_CXX)
- else
- $$(error Internal error in NativeCompilation.gmk: no compiler for file $$($1_FILENAME))
- endif
-
- # And this is the dependency file for this obj file.
- $1_DEPS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d,$$($1_OBJ))
- # The dependency target file lists all dependencies as empty targets to
- # avoid make error "No rule to make target" for removed files
- $1_DEPS_TARGETS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d.targets,$$($1_OBJ))
-
- # Only try to load individual dependency information files if the global
- # file hasn't been loaded (could happen if make was interrupted).
- ifneq ($$($$($1_BASE)_DEPS_FILE_LOADED), true)
- # Include previously generated dependency information. (if it exists)
- -include $$($1_DEPS_FILE)
- -include $$($1_DEPS_TARGETS_FILE)
- endif
-
- ifneq ($$(strip $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)), )
- $1_VARDEPS := $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_OBJ).vardeps)
- endif
-
- $1_OBJ_DEPS := $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
- $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE)
- $1_COMPILE_OPTIONS := $$($1_FLAGS) $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)
- # For reproducible builds with gcc and clang ensure random symbol generation is
- # seeded deterministically
- ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- $1_COMPILE_OPTIONS += -frandom-seed="$$($1_FILENAME)"
- endif
-
- $$($1_OBJ_JSON): $$($1_OBJ_DEPS)
- $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$($1_SRC_FILE), \
- $$($1_COMPILER) $$($1_COMPILE_OPTIONS))
-
- $$($1_OBJ): $$($1_OBJ_DEPS) | $$($$($1_BASE)_BUILD_INFO)
- $$(call LogInfo, Compiling $$($1_FILENAME) (for $$($$($1_BASE)_BASENAME)))
- $$(call MakeDir, $$(@D))
- ifneq ($(TOOLCHAIN_TYPE), microsoft)
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) $$(GENDEPS_FLAGS) \
- $$(addsuffix .tmp, $$($1_DEPS_FILE)) \
- $$($1_COMPILE_OPTIONS)))
- ifneq ($$($1_DEPS_FILE), )
- $$(call fix-deps-file, $$($1_DEPS_FILE))
- # Create a dependency target file from the dependency file.
- # Solution suggested by:
- # http://make.mad-scientist.net/papers/advanced-auto-dependency-generation/
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) \
- > $$($1_DEPS_TARGETS_FILE)
- endif
- else
- # The Visual Studio compiler lacks a feature for generating make
- # dependencies, but by setting -showIncludes, all included files are
- # printed. These are filtered out and parsed into make dependences.
- #
- # Keep as much as possible on one execution line for best performance
- # on Windows. No need to save exit code from compilation since
- # pipefail is always active on Windows.
- ifeq ($$(filter %.S, $$($1_FILENAME)), )
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) -showIncludes $$($1_COMPILE_OPTIONS))) \
- | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
- -e "^$$($1_FILENAME)$$$$" || test "$$$$?" = "1" ; \
- $(ECHO) $$@: \\ > $$($1_DEPS_FILE) ; \
- $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_OBJ).log \
- | $(SORT) -u >> $$($1_DEPS_FILE) ; \
- $(ECHO) >> $$($1_DEPS_FILE) ; \
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) > $$($1_DEPS_TARGETS_FILE)
- else
- # For assembler calls just create empty dependency lists
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_COMPILER) $$($1_FLAGS) \
- $(CC_OUT_OPTION)$$($1_OBJ) -Ta $$($1_SRC_FILE))) \
- | $(TR) -d '\r' | $(GREP) -v -e "Assembling:" || test "$$$$?" = "1" ; \
- $(ECHO) > $$($1_DEPS_FILE) ; \
- $(ECHO) > $$($1_DEPS_TARGETS_FILE)
- endif
- endif
- endif
-endef
-
# Setup make rules for creating a native binary (a shared library or an
# executable).
#
@@ -501,7 +54,8 @@ endef
# NAME The base name for the resulting binary, excluding decorations (like *.exe)
# TYPE Type of binary (EXECUTABLE, LIBRARY or STATIC_LIBRARY). Default is LIBRARY.
# SUFFIX Override the default suffix for the output file
-# TOOLCHAIN Name of toolchain setup to use. Defaults to TOOLCHAIN_DEFAULT.
+# TARGET_TYPE The type to target, BUILD or TARGET. Defaults to TARGET.
+# LINK_TYPE The language to use for the linker, C or C++. Defaults to C.
# SRC one or more directory roots to scan for C/C++ files.
# CFLAGS the compiler flags to be used, used both for C and C++.
# CXXFLAGS the compiler flags to be used for c++, if set overrides CFLAGS.
@@ -519,7 +73,8 @@ endef
# used both for C and C++.
# LIBS__ the libraries to link to for the specified target
# OS and toolchain, used both for C and C++.
-# ARFLAGS the archiver flags to be used
+# ARFLAGS the archiver flags to be used on unix platforms
+# LIBFLAGS the flags for the lib tool used on windows
# OBJECT_DIR the directory where we store the object files
# OUTPUT_DIR the directory where the resulting binary is put
# SYMBOLS_DIR the directory where the debug symbols are put, defaults to OUTPUT_DIR
@@ -534,11 +89,18 @@ endef
# VERSIONINFO_RESOURCE Input file for RC. Setting this implies that RC will be run
# RCFLAGS flags for RC.
# EMBED_MANIFEST if true, embed manifest on Windows.
-# MAPFILE mapfile
-# USE_MAPFILE_FOR_SYMBOLS if true and this is a STATIC_BUILD, just copy the
-# mapfile for the output symbols file
-# CC the compiler to use, default is $(CC)
-# LD the linker to use, default is $(LD)
+# CC the C compiler to use
+# CXX the C++ compiler to use
+# LD the Linker to use
+# AR the static linker to use
+# LIB the Windows lib tool to use for creating static libraries
+# AS the assembler to use
+# MT the Windows MT tool to use
+# RC the Windows RC tool to use
+# OBJCOPY the objcopy tool for debug symbol handling
+# STRIP the tool to use for stripping debug symbols
+# SYSROOT_CFLAGS the compiler flags for using the specific sysroot
+# SYSROOT_LDFLAGS the linker flags for using the specific sysroot
# OPTIMIZATION sets optimization level to NONE, LOW, HIGH, HIGHEST, HIGHEST_JVM, SIZE
# DISABLED_WARNINGS_ Disable the given warnings for the specified toolchain
# DISABLED_WARNINGS__ Disable the given warnings for the specified
@@ -573,9 +135,122 @@ endef
# TARGET_DEPS All prerequisites for the target calculated by the macro
# ALL_OBJS All object files
# IMPORT_LIBRARY The import library created for a shared library on Windows
+#
SetupNativeCompilation = $(NamedParamsMacroTemplate)
define SetupNativeCompilationBody
+ # When reading this code, note that macros named Setup are just setting
+ # variables, and macros called Create are setting up rules to create
+ # files. Macros starting with any other verb are more complicated, and can do
+ # all of the above, and also call directly to the shell.
+
+ ###
+ ### Prepare for compilation and linking
+ ###
+
+ $$(eval $$(call VerifyArguments,$1))
+
+ # Setup variables for the rest of this macro to work with
+ $$(eval $$(call SetupBasicVariables,$1))
+
+ # Setup the toolchain to be used
+ $$(eval $$(call SetupToolchain,$1))
+
+ # Find all source files to compile and determine the output object file names
+ $$(eval $$(call SetupSourceFiles,$1))
+ $$(eval $$(call SetupOutputFiles,$1))
+
+ # Setup CFLAGS/CXXFLAGS based on warnings, optimizations, extra flags etc.
+ $$(eval $$(call SetupCompilerFlags,$1))
+
+ # Machinery needed for the build to function properly
+ $$(eval $$(call SetupBuildSystemSupport,$1))
+
+ $$(eval $$(call RemoveSuperfluousOutputFiles,$1))
+
+ # Need to make sure TARGET is first on list before starting to create files
+ $1 := $$($1_TARGET)
+
+ # Have make print information about the library when we start compiling
+ $$(eval $$(call PrintStartInfo,$1))
+
+ ###
+ ### Compile all native source code files
+ ###
+
+ # Create a PCH, if requested
+ $$(eval $$(call CreatePrecompiledHeader,$1))
+
+ # Now call CreateCompiledNativeFile for each source file we are going to compile.
+ $$(foreach file, $$($1_SRCS), \
+ $$(eval $$(call CreateCompiledNativeFile,$1_$$(notdir $$(file)),\
+ FILE := $$(file), \
+ BASE := $1, \
+ )) \
+ )
+
+ ifeq ($(call isTargetOs, windows), true)
+ # On windows we need to create a resource file
+ $$(eval $$(call CreateWindowsResourceFile,$1))
+ endif
+
+ # Setup a library-wide dependency file from individual object file dependency
+ # files, and import it in the makefile.
+ $$(eval $$(call CreateDependencyFile,$1))
+ $$(eval $$(call ImportDependencyFile,$1))
+
+ ###
+ ### Link the object files into a native output library/executable
+ ###
+
+ # Handle native debug symbols
+ $$(eval $$(call CreateDebugSymbols,$1))
+
+ # Prepare for linking
+ $$(eval $$(call SetupLinkerFlags,$1))
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(eval $$(call SetupLinking,$1))
+ endif
+
+ $$(eval $$(call SetupObjectFileList,$1))
+
+ # Link the individually compiled files into a single unit
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(eval $$(call CreateLinkedResult,$1))
+ else
+ $$(eval $$(call CreateLinkedResultMicrosoft,$1))
+ endif
+
+ ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
+ # Override all targets (this is a hack)
+ $1 := $$($1_ALL_OBJS_JSON)
+ endif
+endef
+
+################################################################################
+# Verify that user passed arguments are valid
+define VerifyArguments
+ ifneq ($$($1_NAME), $(basename $$($1_NAME)))
+ $$(error NAME must not contain any directory path in $1)
+ endif
+ ifneq ($(findstring $$($1_SUFFIX), $$($1_NAME)), )
+ $$(error NAME should be specified without suffix: $$($1_SUFFIX) in $1)
+ endif
+ ifneq ($(findstring $$($1_PREFIX), $$($1_NAME)), )
+ $$(error NAME should be specified without prefix: $$($1_PREFIX) in $1)
+ endif
+ ifeq ($$($1_OUTPUT_DIR), )
+ $$(error OUTPUT_DIR is missing in $1)
+ endif
+ ifneq ($$($1_MANIFEST), )
+ ifeq ($$($1_MANIFEST_VERSION), )
+ $$(error If MANIFEST is provided, then MANIFEST_VERSION is required in $1)
+ endif
+ endif
+endef
+################################################################################
+# Setup basic variables
+define SetupBasicVariables
# If type is unspecified, default to LIBRARY
ifeq ($$($1_TYPE), )
$1_TYPE := LIBRARY
@@ -589,8 +264,6 @@ define SetupNativeCompilationBody
endif
endif
- $$(call SetIfEmpty, $1_COMPILE_WITH_DEBUG_SYMBOLS, $$(COMPILE_WITH_DEBUG_SYMBOLS))
-
# STATIC_LIBS is set from Main.gmk when building static versions of certain
# native libraries.
ifeq ($(STATIC_LIBS), true)
@@ -600,15 +273,6 @@ define SetupNativeCompilationBody
# jmods.
$1_OBJECT_DIR := $$($1_OBJECT_DIR)/static
$1_OUTPUT_DIR := $$($1_OBJECT_DIR)
- # For release builds where debug symbols are configured to be moved to
- # separate debuginfo files, disable debug symbols for static libs instead.
- # We don't currently support this configuration and we don't want symbol
- # information in release builds unless explicitly asked to provide it.
- ifeq ($(DEBUG_LEVEL), release)
- ifeq ($(COPY_DEBUG_SYMBOLS), true)
- $1_COMPILE_WITH_DEBUG_SYMBOLS := false
- endif
- endif
endif
ifeq ($$($1_TYPE), EXECUTABLE)
@@ -629,248 +293,53 @@ define SetupNativeCompilationBody
endif
endif
- ifneq ($$($1_NAME), $(basename $$($1_NAME)))
- $$(error NAME must not contain any directory path in $1)
- endif
- ifneq ($(findstring $$($1_SUFFIX), $$($1_NAME)), )
- $$(error NAME should be specified without suffix: $$($1_SUFFIX) in $1)
- endif
- ifneq ($(findstring $$($1_PREFIX), $$($1_NAME)), )
- $$(error NAME should be specified without prefix: $$($1_PREFIX) in $1)
- endif
- ifeq ($$($1_OUTPUT_DIR), )
- $$(error OUTPUT_DIR is missing in $1)
- endif
- ifneq ($$($1_MANIFEST), )
- ifeq ($$($1_MANIFEST_VERSION), )
- $$(error If MANIFEST is provided, then MANIFEST_VERSION is required in $1)
- endif
- endif
-
$1_BASENAME := $$($1_PREFIX)$$($1_NAME)$$($1_SUFFIX)
$1_TARGET := $$($1_OUTPUT_DIR)/$$($1_BASENAME)
$1_NOSUFFIX := $$($1_PREFIX)$$($1_NAME)
$1_SAFE_NAME := $$(strip $$(subst /,_, $1))
+endef
-# Need to make sure TARGET is first on list
- $1 := $$($1_TARGET)
-
- # Setup the toolchain to be used
- $$(call SetIfEmpty, $1_TOOLCHAIN, TOOLCHAIN_DEFAULT)
- $$(call SetIfEmpty, $1_CC, $$($$($1_TOOLCHAIN)_CC))
- $$(call SetIfEmpty, $1_CXX, $$($$($1_TOOLCHAIN)_CXX))
- $$(call SetIfEmpty, $1_LD, $$($$($1_TOOLCHAIN)_LD))
- $$(call SetIfEmpty, $1_AR, $$($$($1_TOOLCHAIN)_AR))
- $$(call SetIfEmpty, $1_AS, $$($$($1_TOOLCHAIN)_AS))
- $$(call SetIfEmpty, $1_MT, $$($$($1_TOOLCHAIN)_MT))
- $$(call SetIfEmpty, $1_RC, $$($$($1_TOOLCHAIN)_RC))
- $$(call SetIfEmpty, $1_OBJCOPY, $$($$($1_TOOLCHAIN)_OBJCOPY))
- $$(call SetIfEmpty, $1_STRIP, $$($$($1_TOOLCHAIN)_STRIP))
- $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $$($$($1_TOOLCHAIN)_SYSROOT_CFLAGS))
- $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $$($$($1_TOOLCHAIN)_SYSROOT_LDFLAGS))
-
- $$(foreach d, $$($1_SRC), $$(if $$(wildcard $$d), , \
- $$(error SRC specified to SetupNativeCompilation $1 contains missing directory $$d)))
-
- $1_SRCS_RAW := $$(call FindFiles, $$($1_SRC))
- # Order src files according to the order of the src dirs
- $1_SRCS := $$(foreach d, $$($1_SRC), $$(filter $$d%, $$($1_SRCS_RAW)))
- $1_SRCS := $$(filter $$(NATIVE_SOURCE_EXTENSIONS), $$($1_SRCS))
- # Extract the C/C++ files.
- ifneq ($$($1_EXCLUDE_PATTERNS), )
- # We must not match the exclude pattern against the src root(s).
- $1_SRCS_WITHOUT_ROOTS := $$($1_SRCS)
- $$(foreach i, $$($1_SRC), $$(eval $1_SRCS_WITHOUT_ROOTS := $$(patsubst \
- $$i/%,%, $$($1_SRCS_WITHOUT_ROOTS))))
- $1_ALL_EXCLUDE_FILES := $$(call containing, $$($1_EXCLUDE_PATTERNS), \
- $$($1_SRCS_WITHOUT_ROOTS))
- endif
- ifneq ($$($1_EXCLUDE_FILES), )
- $1_ALL_EXCLUDE_FILES += $$($1_EXCLUDE_FILES)
- endif
- ifneq ($$($1_ALL_EXCLUDE_FILES), )
- $1_EXCLUDE_FILES_PAT := $$($1_ALL_EXCLUDE_FILES) \
- $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_ALL_EXCLUDE_FILES)))
- $1_EXCLUDE_FILES_PAT := $$(addprefix %, $$($1_EXCLUDE_FILES_PAT))
- $1_SRCS := $$(filter-out $$($1_EXCLUDE_FILES_PAT), $$($1_SRCS))
- endif
- ifneq ($$($1_INCLUDE_FILES), )
- $1_INCLUDE_FILES_PAT := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_INCLUDE_FILES)))
- $1_SRCS := $$(filter $$($1_INCLUDE_FILES_PAT), $$($1_SRCS))
- endif
- # There can be only a single bin dir root, no need to foreach over the roots.
- $1_BINS := $$(wildcard $$($1_OBJECT_DIR)/*$(OBJ_SUFFIX))
- # Now we have a list of all c/c++ files to compile: $$($1_SRCS)
- # and we have a list of all existing object files: $$($1_BINS)
-
- # Prepend the source/bin path to the filter expressions. Then do the filtering.
- ifneq ($$($1_INCLUDES), )
- $1_SRC_INCLUDES := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_INCLUDES))))
- $1_SRCS := $$(filter $$($1_SRC_INCLUDES), $$($1_SRCS))
- endif
- ifneq ($$($1_EXCLUDES), )
- $1_SRC_EXCLUDES := $$(addsuffix /%, $$($1_EXCLUDES))
- $1_SRC_EXCLUDES += $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_EXCLUDES))))
- $1_SRCS := $$(filter-out $$($1_SRC_EXCLUDES), $$($1_SRCS))
- endif
-
- $1_SRCS += $$($1_EXTRA_FILES)
-
- ifeq ($$($1_SRCS), )
- $$(error No sources found for $1 when looking inside the dirs $$($1_SRC))
- endif
-
- ifeq ($$($1_TYPE), EXECUTABLE)
- ifeq ($(UBSAN_ENABLED), true)
- # We need to set the default options for UBSan. This needs to be included in every executable.
- # Rather than copy and paste code to everything with a main function, we add an additional
- # source file to every executable that exports __ubsan_default_options.
- ifneq ($$(filter %.cpp %.cc, $$($1_SRCS)), )
- $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.cpp
- else
- $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.c
- endif
- endif
- endif
-
- # Calculate the expected output from compiling the sources
- $1_EXPECTED_OBJS_FILENAMES := $$(call replace_with_obj_extension, $$(notdir $$($1_SRCS)))
- $1_EXPECTED_OBJS := $$(addprefix $$($1_OBJECT_DIR)/, $$($1_EXPECTED_OBJS_FILENAMES))
- # Are there too many object files on disk? Perhaps because some source file was removed?
- $1_SUPERFLOUS_OBJS := $$(sort $$(filter-out $$($1_EXPECTED_OBJS), $$($1_BINS)))
- # Clean out the superfluous object files.
- ifneq ($$($1_SUPERFLUOUS_OBJS), )
- $$(shell $(RM) -f $$($1_SUPERFLUOUS_OBJS))
- endif
- # Sort to remove duplicates and provide a reproducible order on the input files to the linker.
- $1_ALL_OBJS := $$(sort $$($1_EXPECTED_OBJS) $$($1_EXTRA_OBJECT_FILES))
- ifeq ($(STATIC_LIBS), true)
- # Exclude the object files that match with $1_STATIC_LIB_EXCLUDE_OBJS.
- ifneq ($$($1_STATIC_LIB_EXCLUDE_OBJS), )
- $1_ALL_OBJS := $$(call not-containing, $$($1_STATIC_LIB_EXCLUDE_OBJS), $$($1_ALL_OBJS))
+################################################################################
+# Setup the toolchain variables
+define SetupToolchain
+ ifeq ($$($1_TARGET_TYPE), BUILD)
+ $$(call SetIfEmpty, $1_CC, $(BUILD_CC))
+ $$(call SetIfEmpty, $1_CXX, $(BUILD_CXX))
+ $$(call SetIfEmpty, $1_AR, $(BUILD_AR))
+ $$(call SetIfEmpty, $1_LIB, $(BUILD_LIB))
+ $$(call SetIfEmpty, $1_AS, $(BUILD_AS))
+ $$(call SetIfEmpty, $1_OBJCOPY, $(BUILD_OBJCOPY))
+ $$(call SetIfEmpty, $1_STRIP, $(BUILD_STRIP))
+ $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $(BUILD_SYSROOT_CFLAGS))
+ $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $(BUILD_SYSROOT_LDFLAGS))
+ ifeq ($$($1_LINK_TYPE), C++)
+ $$(call SetIfEmpty, $1_LD, $(BUILD_LDCXX))
+ else
+ $$(call SetIfEmpty, $1_LD, $(BUILD_LD))
endif
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS, TOOLCHAIN_TYPE and
- # OPENJDK_TARGET_OS plus OPENJDK_TARGET_CPU pair dependent variables for CFLAGS.
- $1_EXTRA_CFLAGS := $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_CFLAGS_$(TOOLCHAIN_TYPE)) \
- $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU))
-
- ifneq ($(DEBUG_LEVEL), release)
- # Pickup extra debug dependent variables for CFLAGS
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_debug)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_debug)
- else
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_release)
- $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_release)
- endif
- ifeq ($(STATIC_LIBS), true)
- $1_EXTRA_CFLAGS += $$(STATIC_LIBS_CFLAGS)
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and/or TOOLCHAIN_TYPE
- # dependent variables for CXXFLAGS.
- $1_EXTRA_CXXFLAGS := $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_CXXFLAGS_$(TOOLCHAIN_TYPE))
-
- ifneq ($(DEBUG_LEVEL), release)
- # Pickup extra debug dependent variables for CXXFLAGS
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_debug)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_debug)
else
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_release)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
- $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_release)
- endif
- ifeq ($(STATIC_LIBS), true)
- $1_EXTRA_CXXFLAGS += $$(STATIC_LIB_CFLAGS)
- endif
-
- # If no C++ flags are explicitly set, default to using the C flags.
- # After that, we can set additional C++ flags that should not interfere
- # with the mechanism for copying the C flags by default.
- ifeq ($$($1_CXXFLAGS), )
- $1_CXXFLAGS := $$($1_CFLAGS)
- endif
- ifeq ($$(strip $$($1_EXTRA_CXXFLAGS)), )
- $1_EXTRA_CXXFLAGS := $$($1_EXTRA_CFLAGS)
- endif
-
- ifeq ($$($1_COMPILE_WITH_DEBUG_SYMBOLS), true)
- $1_EXTRA_CFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
- $1_EXTRA_CXXFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
- $1_EXTRA_ASFLAGS += $$(ASFLAGS_DEBUG_SYMBOLS)
- endif
-
- # Pass the library name for static JNI library naming
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- $1_EXTRA_CFLAGS += -DLIBRARY_NAME=$$($1_NAME)
- $1_EXTRA_CXXFLAGS += -DLIBRARY_NAME=$$($1_NAME)
- endif
-
- # Pick up disabled warnings, if possible on this platform.
- ifneq ($(DISABLE_WARNING_PREFIX), )
- $1_EXTRA_CFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$(DISABLED_WARNINGS) \
- $$(DISABLED_WARNINGS_C) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
- $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
- $1_EXTRA_CXXFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
- $$(DISABLED_WARNINGS) \
- $$(DISABLED_WARNINGS_CXX) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)) \
- $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
- $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
- endif
-
- # Check if warnings should be considered errors.
- # Pick first binary and toolchain specific, then binary specific, then general setting.
- ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), )
- ifeq ($$($1_WARNINGS_AS_ERRORS), )
- $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$(WARNINGS_AS_ERRORS)
+ $$(call SetIfEmpty, $1_CC, $(CC))
+ $$(call SetIfEmpty, $1_CXX, $(CXX))
+ $$(call SetIfEmpty, $1_AR, $(AR))
+ $$(call SetIfEmpty, $1_LIB, $(LIB))
+ $$(call SetIfEmpty, $1_AS, $(AS))
+ $$(call SetIfEmpty, $1_MT, $(MT))
+ $$(call SetIfEmpty, $1_RC, $(RC))
+ $$(call SetIfEmpty, $1_OBJCOPY, $(OBJCOPY))
+ $$(call SetIfEmpty, $1_STRIP, $(STRIP))
+ $$(call SetIfEmpty, $1_SYSROOT_CFLAGS, $(SYSROOT_CFLAGS))
+ $$(call SetIfEmpty, $1_SYSROOT_LDFLAGS, $(SYSROOT_LDFLAGS))
+ ifeq ($$($1_LINK_TYPE), C++)
+ $$(call SetIfEmpty, $1_LD, $(LDCXX))
else
- $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$($1_WARNINGS_AS_ERRORS)
+ $$(call SetIfEmpty, $1_LD, $(LD))
endif
endif
+endef
- ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), true)
- $1_EXTRA_CFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
- $1_EXTRA_CXXFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
- endif
-
- ifeq (NONE, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
- else ifeq (LOW, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
- else ifeq (HIGH, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HI)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
- else ifeq (HIGHEST, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
- else ifeq (HIGHEST_JVM, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
- else ifeq (SIZE, $$($1_OPTIMIZATION))
- $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
- $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
- else ifneq (, $$($1_OPTIMIZATION))
- $$(error Unknown value for OPTIMIZATION: $$($1_OPTIMIZATION))
- endif
-
- $1_BUILD_INFO := $$($1_OBJECT_DIR)/_build-info.marker
-
+################################################################################
+# Setup machinery needed by the build system
+define SetupBuildSystemSupport
# Track variable changes for all variables that affect the compilation command
# lines for all object files in this setup. This includes at least all the
# variables used in the call to add_native_source below.
@@ -879,87 +348,19 @@ define SetupNativeCompilationBody
$$($1_CC) $$($1_CXX) $$($1_AS) $$($1_ASFLAGS)
$1_COMPILE_VARDEPS_FILE := $$(call DependOnVariable, $1_COMPILE_VARDEPS, \
$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).comp.vardeps)
+endef
- ifneq ($$($1_PRECOMPILED_HEADER), )
- ifeq ($(USE_PRECOMPILED_HEADER), true)
- ifeq ($(TOOLCHAIN_TYPE), microsoft)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/$1.pch
- $1_GENERATED_PCH_SRC := $$($1_OBJECT_DIR)/$1_pch.cpp
- $1_GENERATED_PCH_OBJ := $$($1_OBJECT_DIR)/$1_pch$(OBJ_SUFFIX)
-
- $$(eval $$(call SetupCompileNativeFile, $1_$$(notdir $$($1_GENERATED_PCH_SRC)), \
- FILE := $$($1_GENERATED_PCH_SRC), \
- BASE := $1, \
- EXTRA_CXXFLAGS := -Fp$$($1_PCH_FILE) -Yc$$(notdir $$($1_PRECOMPILED_HEADER)), \
- ))
-
- $1_USE_PCH_FLAGS := \
- -Fp$$($1_PCH_FILE) -Yu$$(notdir $$($1_PRECOMPILED_HEADER))
-
- $$($1_ALL_OBJS): $$($1_GENERATED_PCH_OBJ)
-
- # Explicitly add the pch obj file first to ease comparing to old
- # hotspot build.
- $1_ALL_OBJS := $$($1_GENERATED_PCH_OBJ) $$($1_ALL_OBJS)
-
- $$($1_GENERATED_PCH_SRC):
- $(ECHO) "#include \"$$(notdir $$($1_PRECOMPILED_HEADER))\"" > $$@
-
- else ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
- ifeq ($(TOOLCHAIN_TYPE), gcc)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).gch
- $1_USE_PCH_FLAGS := -I$$($1_OBJECT_DIR)/precompiled
- else ifeq ($(TOOLCHAIN_TYPE), clang)
- $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).pch
- $1_USE_PCH_FLAGS := -include-pch $$($1_PCH_FILE)
- endif
- $1_PCH_DEPS_FILE := $$($1_PCH_FILE).d
- $1_PCH_DEPS_TARGETS_FILE := $$($1_PCH_FILE).d.targets
-
- -include $$($1_PCH_DEPS_FILE)
- -include $$($1_PCH_DEPS_TARGETS_FILE)
-
- $1_PCH_COMMAND := $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
- $$($1_OPT_CFLAGS) -x c++-header -c $(GENDEPS_FLAGS) \
- $$(addsuffix .tmp, $$($1_PCH_DEPS_FILE))
-
- $$($1_PCH_FILE): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
- $$(call LogInfo, Generating precompiled header)
- $$(call MakeDir, $$(@D))
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_PCH_COMMAND) $$< -o $$@))
- $$(call fix-deps-file, $$($1_PCH_DEPS_FILE))
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_PCH_DEPS_FILE) \
- > $$($1_PCH_DEPS_TARGETS_FILE)
-
- $$($1_ALL_OBJS): $$($1_PCH_FILE)
-
- # Generate the corresponding compile_commands.json fragment.
- $1_PCH_FILE_JSON := $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
- $$(OUTPUTDIR)/,,$$($1_PCH_FILE))).json
- $1_ALL_OBJS_JSON += $$($1_PCH_FILE_JSON)
-
- $$($1_PCH_FILE_JSON): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
- $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$<, \
- $$($1_PCH_COMMAND) $$< -o $$($1_PCH_FILE))
- endif
- endif
- endif
-
- # Now call SetupCompileNativeFile for each source file we are going to compile.
- $$(foreach file, $$($1_SRCS), \
- $$(eval $$(call SetupCompileNativeFile, $1_$$(notdir $$(file)),\
- FILE := $$(file), \
- BASE := $1, \
- )) \
- )
-
+################################################################################
+# Have make print information about the library when we start compiling
+define PrintStartInfo
# Setup rule for printing progress info when compiling source files.
# This is a rough heuristic and may not always print accurate information.
# The $1_BUILD_INFO and $1_BUILD_INFO_DEPS variables are used in
# TestFilesCompilation.gmk.
$$(call SetIfEmpty, $1_BUILD_INFO_LOG_MACRO, LogWarn)
$1_BUILD_INFO_DEPS := $$($1_SRCS) $$($1_COMPILE_VARDEPS_FILE)
+ $1_BUILD_INFO := $$($1_OBJECT_DIR)/_build-info.marker
+
$$($1_BUILD_INFO): $$($1_BUILD_INFO_DEPS)
ifeq ($$(wildcard $$($1_TARGET)), )
$$(call $$($1_BUILD_INFO_LOG_MACRO), \
@@ -973,47 +374,12 @@ define SetupNativeCompilationBody
$$(if $$(filter %.vardeps, $$?), due to makefile changes))))
endif
$(TOUCH) $$@
+endef
- # On windows we need to create a resource file
- ifeq ($(call isTargetOs, windows), true)
- ifneq ($$($1_VERSIONINFO_RESOURCE), )
- $1_RES := $$($1_OBJECT_DIR)/$$($1_BASENAME).res
- $1_RES_DEPS_FILE := $$($1_RES).d
- $1_RES_DEPS_TARGETS_FILE := $$($1_RES).d.targets
- -include $$($1_RES_DEPS_FILE)
- -include $$($1_RES_DEPS_TARGETS_FILE)
-
- $1_RES_VARDEPS := $$($1_RC) $$($1_RCFLAGS)
- $1_RES_VARDEPS_FILE := $$(call DependOnVariable, $1_RES_VARDEPS, \
- $$($1_RES).vardeps)
-
- $$($1_RES): $$($1_VERSIONINFO_RESOURCE) $$($1_RES_VARDEPS_FILE)
- $$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
- $$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
- $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
- $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
- $$($1_VERSIONINFO_RESOURCE) 2>&1 ))
- # Windows RC compiler does not support -showIncludes, so we mis-use CL
- # for this. Filter out RC specific arguments that are unknown to CL.
- # For some unknown reason, in this case CL actually outputs the show
- # includes to stderr so need to redirect it to hide the output from the
- # main log.
- $$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
- $$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
- $$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
- $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
- $$($1_VERSIONINFO_RESOURCE)) 2>&1 \
- | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
- -e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \
- $(ECHO) $$($1_RES): \\ > $$($1_RES_DEPS_FILE) ; \
- $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX).log \
- >> $$($1_RES_DEPS_FILE) ; \
- $(ECHO) >> $$($1_RES_DEPS_FILE) ;\
- $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_RES_DEPS_FILE) \
- > $$($1_RES_DEPS_TARGETS_FILE)
- endif
- endif
-
+################################################################################
+# Setup a library-wide dependency file from individual object file dependency
+# files
+define CreateDependencyFile
# Create a rule to collect all the individual make dependency files into a
# single makefile.
$1_DEPS_FILE := $$($1_OBJECT_DIR)/$1.d
@@ -1031,7 +397,11 @@ define SetupNativeCompilationBody
$(MV) $$@.tmp $$@
$1 += $$($1_DEPS_FILE)
+endef
+################################################################################
+# Import the dependency file into the makefile
+define ImportDependencyFile
# The include must be on the .old file, which represents the state from the
# previous invocation of make. The file being included must not have a rule
# defined for it as otherwise make will think it has to run the rule before
@@ -1041,328 +411,6 @@ define SetupNativeCompilationBody
$1_DEPS_FILE_LOADED := true
-include $$($1_DEPS_FILE).old
endif
-
- ifneq ($(DISABLE_MAPFILES), true)
- $1_REAL_MAPFILE := $$($1_MAPFILE)
- endif
-
- # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and TOOLCHAIN_TYPE
- # dependent variables for LDFLAGS and LIBS, and additionally the pair dependent
- # TOOLCHAIN_TYPE plus OPENJDK_TARGET_OS
- $1_EXTRA_LDFLAGS += $$($1_LDFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LDFLAGS_$(OPENJDK_TARGET_OS)) \
- $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)) $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
- $1_EXTRA_LIBS += $$($1_LIBS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LIBS_$(OPENJDK_TARGET_OS)) \
- $$($1_LIBS_$(TOOLCHAIN_TYPE)) $$($1_LIBS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
-
- ifneq ($$($1_REAL_MAPFILE), )
- $1_EXTRA_LDFLAGS += $(call SET_SHARED_LIBRARY_MAPFILE,$$($1_REAL_MAPFILE))
- endif
-
- ifneq ($$($1_COPY_DEBUG_SYMBOLS), false)
- $1_COPY_DEBUG_SYMBOLS := $(COPY_DEBUG_SYMBOLS)
- endif
-
- ifneq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), false)
- $1_ZIP_EXTERNAL_DEBUG_SYMBOLS := $(ZIP_EXTERNAL_DEBUG_SYMBOLS)
- endif
-
- ifeq ($$($1_COPY_DEBUG_SYMBOLS), true)
- ifneq ($$($1_DEBUG_SYMBOLS), false)
- $$(call SetIfEmpty, $1_SYMBOLS_DIR, $$($1_OUTPUT_DIR))
- # Only copy debug symbols for dynamic libraries and programs.
- ifneq ($$($1_TYPE), STATIC_LIBRARY)
- # Generate debuginfo files.
- ifeq ($(call isTargetOs, windows), true)
- $1_EXTRA_LDFLAGS += -debug "-pdb:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb" \
- "-map:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).map"
- ifeq ($(SHIP_DEBUG_SYMBOLS), public)
- $1_EXTRA_LDFLAGS += "-pdbstripped:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).stripped.pdb"
- endif
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).map
-
- else ifeq ($(call isTargetOs, linux), true)
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
- # Setup the command line creating debuginfo files, to be run after linking.
- # It cannot be run separately since it updates the original target file
- # Creating the debuglink is done in another command rather than all at once
- # so we can run it after strip is called, since strip can sometimes mangle the
- # embedded debuglink, which we want to avoid.
- $1_CREATE_DEBUGINFO_CMDS := \
- $$($1_OBJCOPY) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES) $$(NEWLINE)
- $1_CREATE_DEBUGLINK_CMDS := $(CD) $$($1_SYMBOLS_DIR) && \
- $$($1_OBJCOPY) --add-gnu-debuglink=$$($1_DEBUGINFO_FILES) $$($1_TARGET)
-
- else ifeq ($(call isTargetOs, aix), true)
- # AIX does not provide the equivalent of OBJCOPY to extract debug symbols,
- # so we copy the compiled object with symbols to the .debuginfo file, which
- # happens prior to the STRIP_CMD on the original target object file.
- $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
- $1_CREATE_DEBUGINFO_CMDS := $(CP) $$($1_TARGET) $$($1_DEBUGINFO_FILES)
-
- else ifeq ($(call isTargetOs, macosx), true)
- $1_DEBUGINFO_FILES := \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Info.plist \
- $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Resources/DWARF/$$($1_BASENAME)
- $1_CREATE_DEBUGINFO_CMDS := \
- $(DSYMUTIL) --out $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM $$($1_TARGET)
- endif
-
- # Since the link rule creates more than one file that we want to track,
- # we have to use some tricks to get make to cooperate. To properly
- # trigger downstream dependants of $$($1_DEBUGINFO_FILES), we must have
- # a recipe in the rule below. To avoid rerunning the recipe every time
- # have it touch the target. If a debuginfo file is deleted by something
- # external, explicitly delete the TARGET to trigger a rebuild of both.
- ifneq ($$(wildcard $$($1_DEBUGINFO_FILES)), $$($1_DEBUGINFO_FILES))
- $$(call LogDebug, Deleting $$($1_BASENAME) because debuginfo files are missing)
- $$(shell $(RM) $$($1_TARGET))
- endif
- $$($1_DEBUGINFO_FILES): $$($1_TARGET)
- $$(if $$(CORRECT_FUNCTION_IN_RECIPE_EVALUATION), \
- $$(if $$(wildcard $$@), , $$(error $$@ was not created for $$<)) \
- )
- $(TOUCH) $$@
-
- $1 += $$($1_DEBUGINFO_FILES)
-
- ifeq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
- ifeq ($(call isTargetOs, windows), true)
- $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).diz
- else
- $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).diz
- endif
- $1 += $$($1_DEBUGINFO_ZIP)
-
- # The dependency on TARGET is needed for debuginfo files
- # to be rebuilt properly.
- $$($1_DEBUGINFO_ZIP): $$($1_DEBUGINFO_FILES) $$($1_TARGET)
- $(CD) $$($1_SYMBOLS_DIR) && \
- $(ZIPEXE) -q -r $$@ $$(subst $$($1_SYMBOLS_DIR)/,, $$($1_DEBUGINFO_FILES))
-
- endif
- endif # !STATIC_LIBRARY
- endif # $1_DEBUG_SYMBOLS != false
- endif # COPY_DEBUG_SYMBOLS
-
- # Unless specifically set, stripping should only happen if symbols are also
- # being copied.
- $$(call SetIfEmpty, $1_STRIP_SYMBOLS, $$($1_COPY_DEBUG_SYMBOLS))
-
- ifneq ($$($1_STRIP_SYMBOLS), false)
- ifneq ($$($1_STRIP), )
- # Default to using the global STRIPFLAGS. Allow for overriding with an empty value
- $1_STRIPFLAGS ?= $(STRIPFLAGS)
- $1_STRIP_CMD := $$($1_STRIP) $$($1_STRIPFLAGS) $$($1_TARGET)
- endif
- endif
-
- $1_LD_OBJ_ARG := $$($1_ALL_OBJS)
-
- # If there are many object files, use an @-file...
- ifneq ($$(word 17, $$($1_ALL_OBJS)), )
- $1_OBJ_FILE_LIST := $$($1_OBJECT_DIR)/_$1_objectfilenames.txt
- ifneq ($(COMPILER_COMMAND_FILE_FLAG), )
- $1_LD_OBJ_ARG := $(COMPILER_COMMAND_FILE_FLAG)$$($1_OBJ_FILE_LIST)
- else
- # ...except for toolchains which don't support them.
- $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
- endif
-
- # If we are building static library, 'AR' on macosx/aix may not support @-file.
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- ifeq ($(call isTargetOs, macosx aix), true)
- $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
- endif
- endif
- endif
-
- # Unfortunately the @-file trick does not work reliably when using clang.
- # Clang does not propagate the @-file parameter to the ld sub process, but
- # instead puts the full content on the command line. At least the llvm ld
- # does not even support an @-file.
- #
- # When linking a large amount of object files, we risk hitting the limit
- # of the command line length even on posix systems if the path length of
- # the output dir is very long due to our use of absolute paths. To
- # mitigate this, use paths relative to the output dir when linking over
- # 500 files with clang and the output dir path is deep.
- ifneq ($$(word 500, $$($1_ALL_OBJS)), )
- ifeq ($$(TOOLCHAIN_TYPE), clang)
- # There is no strlen function in make, but checking path depth is a
- # reasonable approximation.
- ifneq ($$(word 10, $$(subst /, ,$$(OUTPUTDIR))), )
- $1_LINK_OBJS_RELATIVE := true
- $1_ALL_OBJS_RELATIVE := $$(patsubst $$(OUTPUTDIR)/%, %, $$($1_ALL_OBJS))
- endif
- endif
- endif
-
- ifeq ($$($1_TYPE), STATIC_LIBRARY)
- # Include partial linking when building the static library with clang on linux.
- ifeq ($(call isTargetOs, linux), true)
- ifneq ($(findstring $(TOOLCHAIN_TYPE), clang), )
- $1_ENABLE_PARTIAL_LINKING := true
- endif
- endif
-
- $1_VARDEPS := $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $$($1_LIBS) \
- $$($1_EXTRA_LIBS)
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $1_VARDEPS += $$($1_LD) $$($1_SYSROOT_LDFLAGS)
- endif
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
- $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
-
- # Generating a static library, ie object file archive.
- ifeq ($(STATIC_BUILD), true)
- ifeq ($$($1_USE_MAPFILE_FOR_SYMBOLS), true)
- STATIC_MAPFILE_DEP := $$($1_MAPFILE)
- endif
- endif
-
- $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_VARDEPS_FILE) $$(STATIC_MAPFILE_DEP)
-
- $1_AR_OBJ_ARG := $$($1_LD_OBJ_ARG)
- # With clang on linux, partial linking is enabled and 'AR' takes the output
- # object from the partial linking step.
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $1_TARGET_RELOCATABLE := $$($1_OBJECT_DIR)/$$($1_PREFIX)$$($1_NAME)_relocatable$(OBJ_SUFFIX)
- $1_AR_OBJ_ARG := $$($1_TARGET_RELOCATABLE)
- endif
-
- $$($1_TARGET): $$($1_TARGET_DEPS)
- ifneq ($$($1_OBJ_FILE_LIST), )
- ifeq ($$($1_LINK_OBJS_RELATIVE), true)
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
- else
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
- endif
- endif
- $$(call LogInfo, Building static library $$($1_BASENAME))
- $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
- # Do partial linking.
- ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_partial_link, \
- $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_LD) $(LDFLAGS_CXX_PARTIAL_LINKING) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET_RELOCATABLE) \
- $$($1_LD_OBJ_ARG))
- endif
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $(AR_OUT_OPTION)$$($1_TARGET) $$($1_AR_OBJ_ARG) \
- $$($1_RES))
- ifeq ($(STATIC_BUILD), true)
- ifeq ($$($1_USE_MAPFILE_FOR_SYMBOLS), true)
- $(CP) $$($1_MAPFILE) $$(@D)/$$(basename $$(@F)).symbols
- else
- $(GetSymbols)
- endif
- endif
- else
- # A shared dynamic library or an executable binary has been specified
- ifeq ($$($1_TYPE), LIBRARY)
- # Generating a dynamic library.
- $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
-
- # Create loadmap on AIX. Helps in diagnosing some problems.
- ifneq ($(COMPILER_BINDCMD_FILE_FLAG), )
- $1_EXTRA_LDFLAGS += $(COMPILER_BINDCMD_FILE_FLAG)$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).loadmap
- endif
- endif
-
- ifeq ($(call isTargetOs, windows), true)
- ifeq ($$($1_EMBED_MANIFEST), true)
- $1_EXTRA_LDFLAGS += -manifest:embed
- endif
-
- $1_IMPORT_LIBRARY := $$($1_OBJECT_DIR)/$$($1_NAME).lib
- $1_EXTRA_LDFLAGS += "-implib:$$($1_IMPORT_LIBRARY)"
- ifeq ($$($1_TYPE), LIBRARY)
- # To properly trigger downstream dependants of the import library, just as
- # for debug files, we must have a recipe in the rule. To avoid rerunning
- # the recipe every time have it touch the target. If an import library
- # file is deleted by something external, explicitly delete the target to
- # trigger a rebuild of both.
- ifneq ($$(wildcard $$($1_IMPORT_LIBRARY)), $$($1_IMPORT_LIBRARY))
- $$(call LogDebug, Deleting $$($1_BASENAME) because import library is missing)
- $$(shell $(RM) $$($1_TARGET))
- endif
- $$($1_IMPORT_LIBRARY): $$($1_TARGET)
- $(TOUCH) $$@
-
- $1 += $$($1_IMPORT_LIBRARY)
- endif
- endif
-
- $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
- $$($1_LIBS) $$($1_EXTRA_LIBS) $$($1_MT) \
- $$($1_CREATE_DEBUGINFO_CMDS) $$($1_MANIFEST_VERSION) \
- $$($1_STRIP_CMD) $$($1_CREATE_DEBUGLINK_CMDS)
- $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
- $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
-
- $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_MANIFEST) \
- $$($1_REAL_MAPFILE) $$($1_VARDEPS_FILE)
-
- $$($1_TARGET): $$($1_TARGET_DEPS)
- ifneq ($$($1_OBJ_FILE_LIST), )
- ifeq ($$($1_LINK_OBJS_RELATIVE), true)
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
- else
- $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
- endif
- endif
- # Keep as much as possible on one execution line for best performance
- # on Windows
- $$(call LogInfo, Linking $$($1_BASENAME))
- $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
- ifeq ($(call isTargetOs, windows), true)
-
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET) $$($1_LD_OBJ_ARG) $$($1_RES) \
- $$($1_LIBS) $$($1_EXTRA_LIBS)) \
- | $(GREP) -v "^ Creating library .*\.lib and object .*\.exp" || \
- test "$$$$?" = "1" ; \
- $$($1_CREATE_DEBUGINFO_CMDS)
- $$($1_STRIP_CMD)
- $$($1_CREATE_DEBUGLINK_CMDS)
- ifeq ($(call isBuildOsEnv, windows.wsl2), true)
- $$(CHMOD) +x $$($1_TARGET)
- endif
- else
- $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
- $$(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
- $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) $$($1_SYSROOT_LDFLAGS) \
- $(LD_OUT_OPTION)$$($1_TARGET) $$($1_LD_OBJ_ARG) $$($1_RES) \
- $$($1_LIBS) $$($1_EXTRA_LIBS)) ; \
- $$($1_CREATE_DEBUGINFO_CMDS)
- $$($1_STRIP_CMD)
- $$($1_CREATE_DEBUGLINK_CMDS)
- endif
- ifeq ($(call isTargetOs, windows), true)
- ifneq ($$($1_MANIFEST), )
- $$($1_MT) -nologo -manifest $$($1_MANIFEST) -identity:"$$($1_NAME).exe, version=$$($1_MANIFEST_VERSION)" -outputresource:$$@;#1
- endif
- endif
- # On macosx, optionally run codesign on every binary.
- # Remove signature explicitly first to avoid warnings if the linker
- # added a default adhoc signature.
- ifeq ($(MACOSX_CODESIGN_MODE), hardened)
- $(CODESIGN) --remove-signature $$@
- $(CODESIGN) -f -s "$(MACOSX_CODESIGN_IDENTITY)" --timestamp --options runtime \
- --entitlements $$(call GetEntitlementsFile, $$@) $$@
- else ifeq ($(MACOSX_CODESIGN_MODE), debug)
- $(CODESIGN) --remove-signature $$@
- $(CODESIGN) -f -s - --entitlements $$(call GetEntitlementsFile, $$@) $$@
- endif
- endif
-
- ifeq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
- $1 := $$($1_ALL_OBJS_JSON)
- endif
endef
endif # _NATIVE_COMPILATION_GMK
diff --git a/make/common/TestFilesCompilation.gmk b/make/common/TestFilesCompilation.gmk
index d97d0e6c697e7..626eb058f0a1b 100644
--- a/make/common/TestFilesCompilation.gmk
+++ b/make/common/TestFilesCompilation.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -103,6 +103,7 @@ define SetupTestFilesCompilationBody
$$(eval $$(call SetupNativeCompilation, BUILD_TEST_$$(name), \
NAME := $$(unprefixed_name), \
TYPE := $$($1_COMPILATION_TYPE), \
+ LINK_TYPE := $(if $$(filter %.cpp, $$(file)), C++, C), \
EXTRA_FILES := $$(file) $$($1_EXTRA_FILES), \
OBJECT_DIR := $$($1_OUTPUT_DIR)/support/$$(name), \
OUTPUT_DIR := $$($1_OUTPUT_DIR)/$$($1_OUTPUT_SUBDIR), \
@@ -113,7 +114,6 @@ define SetupTestFilesCompilationBody
DISABLED_WARNINGS_clang := undef format-nonliteral \
missing-field-initializers sometimes-uninitialized, \
LIBS := $$($1_LIBS_$$(name)), \
- TOOLCHAIN := $(if $$(filter %.cpp, $$(file)), TOOLCHAIN_LINK_CXX, TOOLCHAIN_DEFAULT), \
OPTIMIZATION := $$(if $$($1_OPTIMIZATION_$$(name)),$$($1_OPTIMIZATION_$$(name)),LOW), \
COPY_DEBUG_SYMBOLS := $$($1_COPY_DEBUG_SYMBOLS), \
STRIP_SYMBOLS := $$(if $$($1_STRIP_SYMBOLS_$$(name)),$$($1_STRIP_SYMBOLS_$$(name)),false), \
diff --git a/make/common/native/CompileFile.gmk b/make/common/native/CompileFile.gmk
new file mode 100644
index 0000000000000..a9384fb0cf509
--- /dev/null
+++ b/make/common/native/CompileFile.gmk
@@ -0,0 +1,351 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to compiling a single native source
+# file (C, C++ or Objective-C) into an object file. It also harbours related
+# functionality for generating PCH (precompiled headers) and Windows resource
+# files.
+
+################################################################################
+# Creates a recipe that creates a compile_commands.json fragment. Remove any
+# occurrences of FIXPATH programs from the command to show the actual invocation.
+#
+# Param 1: Name of file to create
+# Param 2: Working directory
+# Param 3: Source file
+# Param 4: Compile command
+################################################################################
+define WriteCompileCommandsFragment
+ $(call LogInfo, Creating compile commands fragment for $(notdir $3))
+ $(call MakeDir, $(dir $1))
+ $(call WriteFile,{ \
+ "directory": "$(strip $(call FixPath, $2))"$(COMMA) \
+ "file": "$(strip $(call FixPath, $3))"$(COMMA) \
+ "command": "$(strip $(subst $(DQUOTE),\$(DQUOTE),$(subst \,\\,\
+ $(subst $(FIXPATH),,$(call FixPath, $4)))))" \
+ }$(COMMA), \
+ $1)
+endef
+
+################################################################################
+# Extensions of files handled by this macro.
+NATIVE_SOURCE_EXTENSIONS := %.S %.c %.cpp %.cc %.m %.mm
+
+# Replaces native source extensions with the object file extension in a string.
+# Param 1: the string containing source file names with extensions
+# The surrounding strip is needed to keep additional whitespace out
+define replace_with_obj_extension
+$(strip \
+ $(foreach extension, $(NATIVE_SOURCE_EXTENSIONS), \
+ $(patsubst $(extension),%$(OBJ_SUFFIX), $(filter $(extension), $1))) \
+)
+endef
+
+################################################################################
+# This pattern is used to transform the output of the microsoft CL compiler
+# into a make syntax dependency file (.d)
+WINDOWS_SHOWINCLUDE_SED_PATTERN := \
+ -e '/^Note: including file:/!d' \
+ -e 's|Note: including file: *||' \
+ -e 's|\r||g' \
+ -e 's|\\|/|g' \
+ -e 's|^\([a-zA-Z]\):|$(WINENV_PREFIX)/\1|g' \
+ -e '\|$(TOPDIR)|I !d' \
+ -e 's|$$$$| \\|g' \
+ #
+
+################################################################################
+# This pattern is used to transform a dependency file (.d) to a list
+# of make targets for dependent files (.d.targets)
+DEPENDENCY_TARGET_SED_PATTERN := \
+ -e 's/\#.*//' \
+ -e 's/^[^:]*: *//' \
+ -e 's/ *\\$$$$//' \
+ -e 's/^[ ]*//' \
+ -e '/^$$$$/ d' \
+ -e 's/$$$$/ :/' \
+ #
+
+################################################################################
+# Create the recipe needed to compile a single native source file.
+#
+# Parameter 1 is the name of the rule, based on the name of the library/
+# program being build and the name of the source code file, e.g.
+# BUILD_LIBFOO_fooMain.cpp.
+#
+# Remaining parameters are named arguments:
+# FILE - The full path of the source file to compiler
+# BASE - The name of the rule for the entire binary to build ($1)
+#
+CreateCompiledNativeFile = $(NamedParamsMacroTemplate)
+define CreateCompiledNativeFileBody
+ $1_FILENAME := $$(notdir $$($1_FILE))
+
+ # The target file to be generated.
+ $1_OBJ := $$($$($1_BASE)_OBJECT_DIR)/$$(call replace_with_obj_extension, \
+ $$($1_FILENAME))
+
+ # Generate the corresponding compile_commands.json fragment.
+ $1_OBJ_JSON = $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+ $$(OUTPUTDIR)/,,$$($1_OBJ))).json
+ $$($1_BASE)_ALL_OBJS_JSON += $$($1_OBJ_JSON)
+
+ # Only continue if this object file hasn't been processed already. This lets
+ # the first found source file override any other with the same name.
+ ifeq ($$($1_OBJ_PROCESSED), )
+ $1_OBJ_PROCESSED := true
+ # This is the definite source file to use for $1_FILENAME.
+ $1_SRC_FILE := $$($1_FILE)
+
+ $$(eval $$(call SetupCompileFileFlags,$1,$$($1_BASE)))
+
+ ifneq ($$(filter %.c, $$($1_FILENAME)), )
+ # Compile as a C file
+ $1_CFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CFLAGS) \
+ $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CC)
+ else ifneq ($$(filter %.m, $$($1_FILENAME)), )
+ # Compile as an Objective-C file
+ $1_CFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := -x objective-c $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) \
+ $$($1_BASE_CFLAGS) $$($1_OPT_CFLAGS) $$($1_CFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CC)
+ else ifneq ($$(filter %.S, $$($1_FILENAME)), )
+ # Compile as preprocessed assembler file
+ $1_FLAGS := $(BASIC_ASFLAGS) $$($1_BASE_ASFLAGS)
+ $1_COMPILER := $(AS)
+
+ # gcc or clang assembly files must contain an appropriate relative .file
+ # path for reproducible builds.
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ # If no absolute paths allowed, work out relative source file path
+ # for assembly .file substitution, otherwise use full file path
+ ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT), false)
+ $1_REL_ASM_SRC := $$(call RelativePath, $$($1_FILE), $(WORKSPACE_ROOT))
+ else
+ $1_REL_ASM_SRC := $$($1_FILE)
+ endif
+ $1_FLAGS := $$($1_FLAGS) -DASSEMBLY_SRC_FILE='"$$($1_REL_ASM_SRC)"' \
+ -include $(TOPDIR)/make/data/autoheaders/assemblyprefix.h
+ endif
+ else ifneq ($$(filter %.cpp %.cc %.mm, $$($1_FILENAME)), )
+ # Compile as a C++ or Objective-C++ file
+ $1_CXXFLAGS += $$($1_WARNINGS_FLAGS)
+ $1_FLAGS := $(CFLAGS_CCACHE) $$($1_USE_PCH_FLAGS) $$($1_BASE_CXXFLAGS) \
+ $$($1_OPT_CXXFLAGS) $$($1_CXXFLAGS) -c
+ $1_COMPILER := $$($$($1_BASE)_CXX)
+ else
+ $$(error Internal error in NativeCompilation.gmk: no compiler for file $$($1_FILENAME))
+ endif
+
+ # And this is the dependency file for this obj file.
+ $1_DEPS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d,$$($1_OBJ))
+ # The dependency target file lists all dependencies as empty targets to
+ # avoid make error "No rule to make target" for removed files
+ $1_DEPS_TARGETS_FILE := $$(patsubst %$(OBJ_SUFFIX),%.d.targets,$$($1_OBJ))
+
+ # Only try to load individual dependency information files if the global
+ # file hasn't been loaded (could happen if make was interrupted).
+ ifneq ($$($$($1_BASE)_DEPS_FILE_LOADED), true)
+ # Include previously generated dependency information. (if it exists)
+ -include $$($1_DEPS_FILE)
+ -include $$($1_DEPS_TARGETS_FILE)
+ endif
+
+ ifneq ($$(strip $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)), )
+ $1_VARDEPS := $$($1_CFLAGS) $$($1_CXXFLAGS) $$($1_OPTIMIZATION)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, $$($1_OBJ).vardeps)
+ endif
+
+ $1_OBJ_DEPS := $$($1_SRC_FILE) $$($$($1_BASE)_COMPILE_VARDEPS_FILE) \
+ $$($$($1_BASE)_EXTRA_DEPS) $$($1_VARDEPS_FILE)
+ $1_COMPILE_OPTIONS := $$($1_FLAGS) $(CC_OUT_OPTION)$$($1_OBJ) $$($1_SRC_FILE)
+ # For reproducible builds with gcc and clang ensure random symbol generation is
+ # seeded deterministically
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ $1_COMPILE_OPTIONS += -frandom-seed="$$($1_FILENAME)"
+ endif
+
+ $$($1_OBJ_JSON): $$($1_OBJ_DEPS)
+ $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$($1_SRC_FILE), \
+ $$($1_COMPILER) $$($1_COMPILE_OPTIONS))
+
+ $$($1_OBJ): $$($1_OBJ_DEPS) | $$($$($1_BASE)_BUILD_INFO)
+ $$(call LogInfo, Compiling $$($1_FILENAME) (for $$($$($1_BASE)_BASENAME)))
+ $$(call MakeDir, $$(@D))
+ ifneq ($(TOOLCHAIN_TYPE), microsoft)
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) $$(GENDEPS_FLAGS) \
+ $$(addsuffix .tmp, $$($1_DEPS_FILE)) \
+ $$($1_COMPILE_OPTIONS)))
+ ifneq ($$($1_DEPS_FILE), )
+ $$(call fix-deps-file, $$($1_DEPS_FILE))
+ # Create a dependency target file from the dependency file.
+ # Solution suggested by:
+ # http://make.mad-scientist.net/papers/advanced-auto-dependency-generation/
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) \
+ > $$($1_DEPS_TARGETS_FILE)
+ endif
+ else
+ # The Visual Studio compiler lacks a feature for generating make
+ # dependencies, but by setting -showIncludes, all included files are
+ # printed. These are filtered out and parsed into make dependences.
+ #
+ # Keep as much as possible on one execution line for best performance
+ # on Windows. No need to save exit code from compilation since
+ # pipefail is always active on Windows.
+ ifeq ($$(filter %.S, $$($1_FILENAME)), )
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) -showIncludes $$($1_COMPILE_OPTIONS))) \
+ | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
+ -e "^$$($1_FILENAME)$$$$" || test "$$$$?" = "1" ; \
+ $(ECHO) $$@: \\ > $$($1_DEPS_FILE) ; \
+ $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_OBJ).log \
+ | $(SORT) -u >> $$($1_DEPS_FILE) ; \
+ $(ECHO) >> $$($1_DEPS_FILE) ; \
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_DEPS_FILE) > $$($1_DEPS_TARGETS_FILE)
+ else
+ # For assembler calls just create empty dependency lists
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_COMPILER) $$($1_FLAGS) \
+ $(CC_OUT_OPTION)$$($1_OBJ) -Ta $$($1_SRC_FILE))) \
+ | $(TR) -d '\r' | $(GREP) -v -e "Assembling:" || test "$$$$?" = "1" ; \
+ $(ECHO) > $$($1_DEPS_FILE) ; \
+ $(ECHO) > $$($1_DEPS_TARGETS_FILE)
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define CreatePrecompiledHeader
+ ifneq ($$($1_PRECOMPILED_HEADER), )
+ ifeq ($(USE_PRECOMPILED_HEADER), true)
+ ifeq ($(TOOLCHAIN_TYPE), microsoft)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/$1.pch
+ $1_GENERATED_PCH_SRC := $$($1_OBJECT_DIR)/$1_pch.cpp
+ $1_GENERATED_PCH_OBJ := $$($1_OBJECT_DIR)/$1_pch$(OBJ_SUFFIX)
+
+ $$(eval $$(call CreateCompiledNativeFile, $1_$$(notdir $$($1_GENERATED_PCH_SRC)), \
+ FILE := $$($1_GENERATED_PCH_SRC), \
+ BASE := $1, \
+ EXTRA_CXXFLAGS := -Fp$$($1_PCH_FILE) -Yc$$(notdir $$($1_PRECOMPILED_HEADER)), \
+ ))
+
+ $1_USE_PCH_FLAGS := \
+ -Fp$$($1_PCH_FILE) -Yu$$(notdir $$($1_PRECOMPILED_HEADER))
+
+ $$($1_ALL_OBJS): $$($1_GENERATED_PCH_OBJ)
+
+ # Explicitly add the pch obj file first to ease comparing to old
+ # hotspot build.
+ $1_ALL_OBJS := $$($1_GENERATED_PCH_OBJ) $$($1_ALL_OBJS)
+
+ $$($1_GENERATED_PCH_SRC):
+ $(ECHO) "#include \"$$(notdir $$($1_PRECOMPILED_HEADER))\"" > $$@
+
+ else ifneq ($(findstring $(TOOLCHAIN_TYPE), gcc clang), )
+ ifeq ($(TOOLCHAIN_TYPE), gcc)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).gch
+ $1_USE_PCH_FLAGS := -I$$($1_OBJECT_DIR)/precompiled
+ else ifeq ($(TOOLCHAIN_TYPE), clang)
+ $1_PCH_FILE := $$($1_OBJECT_DIR)/precompiled/$$(notdir $$($1_PRECOMPILED_HEADER)).pch
+ $1_USE_PCH_FLAGS := -include-pch $$($1_PCH_FILE)
+ endif
+ $1_PCH_DEPS_FILE := $$($1_PCH_FILE).d
+ $1_PCH_DEPS_TARGETS_FILE := $$($1_PCH_FILE).d.targets
+
+ -include $$($1_PCH_DEPS_FILE)
+ -include $$($1_PCH_DEPS_TARGETS_FILE)
+
+ $1_PCH_COMMAND := $$($1_CC) $$($1_CFLAGS) $$($1_EXTRA_CFLAGS) $$($1_SYSROOT_CFLAGS) \
+ $$($1_OPT_CFLAGS) -x c++-header -c $(GENDEPS_FLAGS) \
+ $$(addsuffix .tmp, $$($1_PCH_DEPS_FILE))
+
+ $$($1_PCH_FILE): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
+ $$(call LogInfo, Generating precompiled header)
+ $$(call MakeDir, $$(@D))
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_PCH_COMMAND) $$< -o $$@))
+ $$(call fix-deps-file, $$($1_PCH_DEPS_FILE))
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_PCH_DEPS_FILE) \
+ > $$($1_PCH_DEPS_TARGETS_FILE)
+
+ $$($1_ALL_OBJS): $$($1_PCH_FILE)
+
+ # Generate the corresponding compile_commands.json fragment.
+ $1_PCH_FILE_JSON := $$(MAKESUPPORT_OUTPUTDIR)/compile-commands/$$(subst /,_,$$(subst \
+ $$(OUTPUTDIR)/,,$$($1_PCH_FILE))).json
+ $1_ALL_OBJS_JSON += $$($1_PCH_FILE_JSON)
+
+ $$($1_PCH_FILE_JSON): $$($1_PRECOMPILED_HEADER) $$($1_COMPILE_VARDEPS_FILE)
+ $$(call WriteCompileCommandsFragment, $$@, $$(PWD), $$<, \
+ $$($1_PCH_COMMAND) $$< -o $$($1_PCH_FILE))
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define CreateWindowsResourceFile
+ ifneq ($$($1_VERSIONINFO_RESOURCE), )
+ $1_RES := $$($1_OBJECT_DIR)/$$($1_BASENAME).res
+ $1_RES_DEPS_FILE := $$($1_RES).d
+ $1_RES_DEPS_TARGETS_FILE := $$($1_RES).d.targets
+ -include $$($1_RES_DEPS_FILE)
+ -include $$($1_RES_DEPS_TARGETS_FILE)
+
+ $1_RES_VARDEPS := $$($1_RC) $$($1_RCFLAGS)
+ $1_RES_VARDEPS_FILE := $$(call DependOnVariable, $1_RES_VARDEPS, \
+ $$($1_RES).vardeps)
+
+ $$($1_RES): $$($1_VERSIONINFO_RESOURCE) $$($1_RES_VARDEPS_FILE)
+ $$(call LogInfo, Compiling resource $$(notdir $$($1_VERSIONINFO_RESOURCE)) (for $$($1_BASENAME)))
+ $$(call MakeDir, $$(@D) $$($1_OBJECT_DIR))
+ $$(call ExecuteWithLog, $$@, $$(call MakeCommandRelative, \
+ $$($1_RC) $$($1_RCFLAGS) $$($1_SYSROOT_CFLAGS) $(CC_OUT_OPTION)$$@ \
+ $$($1_VERSIONINFO_RESOURCE) 2>&1 ))
+ # Windows RC compiler does not support -showIncludes, so we mis-use CL
+ # for this. Filter out RC specific arguments that are unknown to CL.
+ # For some unknown reason, in this case CL actually outputs the show
+ # includes to stderr so need to redirect it to hide the output from the
+ # main log.
+ $$(call ExecuteWithLog, $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX), \
+ $$($1_CC) $$(filter-out -l%, $$($1_RCFLAGS)) \
+ $$($1_SYSROOT_CFLAGS) -showIncludes -nologo -TC \
+ $(CC_OUT_OPTION)$$($1_RES_DEPS_FILE)$(OBJ_SUFFIX) -P -Fi$$($1_RES_DEPS_FILE).pp \
+ $$($1_VERSIONINFO_RESOURCE)) 2>&1 \
+ | $(TR) -d '\r' | $(GREP) -v -e "^Note: including file:" \
+ -e "^$$(notdir $$($1_VERSIONINFO_RESOURCE))$$$$" || test "$$$$?" = "1" ; \
+ $(ECHO) $$($1_RES): \\ > $$($1_RES_DEPS_FILE) ; \
+ $(SED) $(WINDOWS_SHOWINCLUDE_SED_PATTERN) $$($1_RES_DEPS_FILE)$(OBJ_SUFFIX).log \
+ >> $$($1_RES_DEPS_FILE) ; \
+ $(ECHO) >> $$($1_RES_DEPS_FILE) ;\
+ $(SED) $(DEPENDENCY_TARGET_SED_PATTERN) $$($1_RES_DEPS_FILE) \
+ > $$($1_RES_DEPS_TARGETS_FILE)
+ endif
+endef
diff --git a/make/common/native/DebugSymbols.gmk b/make/common/native/DebugSymbols.gmk
new file mode 100644
index 0000000000000..9f49f5e1d5292
--- /dev/null
+++ b/make/common/native/DebugSymbols.gmk
@@ -0,0 +1,118 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to native debug symbol handling.
+
+################################################################################
+define CreateDebugSymbols
+ ifneq ($$($1_COPY_DEBUG_SYMBOLS), false)
+ $1_COPY_DEBUG_SYMBOLS := $(COPY_DEBUG_SYMBOLS)
+ endif
+
+ ifneq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), false)
+ $1_ZIP_EXTERNAL_DEBUG_SYMBOLS := $(ZIP_EXTERNAL_DEBUG_SYMBOLS)
+ endif
+
+ ifeq ($$($1_COPY_DEBUG_SYMBOLS), true)
+ ifneq ($$($1_DEBUG_SYMBOLS), false)
+ $$(call SetIfEmpty, $1_SYMBOLS_DIR, $$($1_OUTPUT_DIR))
+ # Only copy debug symbols for dynamic libraries and programs.
+ ifneq ($$($1_TYPE), STATIC_LIBRARY)
+ # Generate debuginfo files.
+ ifeq ($(call isTargetOs, windows), true)
+ $1_EXTRA_LDFLAGS += -debug "-pdb:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb" \
+ "-map:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).map"
+ ifeq ($(SHIP_DEBUG_SYMBOLS), public)
+ $1_EXTRA_LDFLAGS += "-pdbstripped:$$($1_SYMBOLS_DIR)/$$($1_BASENAME).stripped.pdb"
+ endif
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).pdb \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).map
+
+ else ifeq ($(call isTargetOs, linux), true)
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
+ # Setup the command line creating debuginfo files, to be run after linking.
+ # It cannot be run separately since it updates the original target file
+ # Creating the debuglink is done in another command rather than all at once
+ # so we can run it after strip is called, since strip can sometimes mangle the
+ # embedded debuglink, which we want to avoid.
+ $1_CREATE_DEBUGINFO_CMDS := \
+ $$($1_OBJCOPY) --only-keep-debug $$($1_TARGET) $$($1_DEBUGINFO_FILES) && \
+ $$(CHMOD) -x $$($1_DEBUGINFO_FILES)
+ $1_CREATE_DEBUGLINK_CMDS := $(CD) $$($1_SYMBOLS_DIR) && \
+ $$($1_OBJCOPY) --add-gnu-debuglink=$$($1_DEBUGINFO_FILES) $$($1_TARGET)
+
+ else ifeq ($(call isTargetOs, aix), true)
+ # AIX does not provide the equivalent of OBJCOPY to extract debug symbols,
+ # so we copy the compiled object with symbols to the .debuginfo file, which
+ # happens prior to the STRIP_CMD on the original target object file.
+ $1_DEBUGINFO_FILES := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).debuginfo
+ $1_CREATE_DEBUGINFO_CMDS := $(CP) $$($1_TARGET) $$($1_DEBUGINFO_FILES)
+
+ else ifeq ($(call isTargetOs, macosx), true)
+ $1_DEBUGINFO_FILES := \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Info.plist \
+ $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM/Contents/Resources/DWARF/$$($1_BASENAME)
+ $1_CREATE_DEBUGINFO_CMDS := \
+ $(DSYMUTIL) --out $$($1_SYMBOLS_DIR)/$$($1_BASENAME).dSYM $$($1_TARGET)
+ endif
+
+ # Since the link rule creates more than one file that we want to track,
+ # we have to use some tricks to get make to cooperate. To properly
+ # trigger downstream dependants of $$($1_DEBUGINFO_FILES), we must have
+ # a recipe in the rule below. To avoid rerunning the recipe every time
+ # have it touch the target. If a debuginfo file is deleted by something
+ # external, explicitly delete the TARGET to trigger a rebuild of both.
+ ifneq ($$(wildcard $$($1_DEBUGINFO_FILES)), $$($1_DEBUGINFO_FILES))
+ $$(call LogDebug, Deleting $$($1_BASENAME) because debuginfo files are missing)
+ $$(shell $(RM) $$($1_TARGET))
+ endif
+ $$($1_DEBUGINFO_FILES): $$($1_TARGET)
+ $$(if $$(CORRECT_FUNCTION_IN_RECIPE_EVALUATION), \
+ $$(if $$(wildcard $$@), , $$(error $$@ was not created for $$<)) \
+ )
+ $(TOUCH) $$@
+
+ $1 += $$($1_DEBUGINFO_FILES)
+
+ ifeq ($$($1_ZIP_EXTERNAL_DEBUG_SYMBOLS), true)
+ ifeq ($(call isTargetOs, windows), true)
+ $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_BASENAME).diz
+ else
+ $1_DEBUGINFO_ZIP := $$($1_SYMBOLS_DIR)/$$($1_NOSUFFIX).diz
+ endif
+ $1 += $$($1_DEBUGINFO_ZIP)
+
+ # The dependency on TARGET is needed for debuginfo files
+ # to be rebuilt properly.
+ $$($1_DEBUGINFO_ZIP): $$($1_DEBUGINFO_FILES) $$($1_TARGET)
+ $(CD) $$($1_SYMBOLS_DIR) && \
+ $(ZIPEXE) -q -r $$@ $$(subst $$($1_SYMBOLS_DIR)/,, $$($1_DEBUGINFO_FILES))
+
+ endif
+ endif # !STATIC_LIBRARY
+ endif # $1_DEBUG_SYMBOLS != false
+ endif # COPY_DEBUG_SYMBOLS
+endef
diff --git a/make/common/native/Flags.gmk b/make/common/native/Flags.gmk
new file mode 100644
index 0000000000000..213312047a4ff
--- /dev/null
+++ b/make/common/native/Flags.gmk
@@ -0,0 +1,225 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to setting up compiler and linker
+# flags, based on various more abstract sources of compilation description,
+# like optimization level.
+
+################################################################################
+# $1 is the prefix of the file to be compiled
+# $2 is the prefix of the library, i.e. $$($1_BASE)
+define SetupCompileFileFlags
+ ifeq ($$($1_OPTIMIZATION), )
+ $1_OPT_CFLAGS := $$($2_OPT_CFLAGS)
+ $1_OPT_CXXFLAGS := $$($2_OPT_CXXFLAGS)
+ else
+ ifeq ($$($1_OPTIMIZATION), NONE)
+ $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
+ else ifeq ($$($1_OPTIMIZATION), LOW)
+ $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
+ else ifeq ($$($1_OPTIMIZATION), HIGH)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HI)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
+ else ifeq ($$($1_OPTIMIZATION), HIGHEST)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
+ else ifeq ($$($1_OPTIMIZATION), HIGHEST_JVM)
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
+ else ifeq ($$($1_OPTIMIZATION), SIZE)
+ $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
+ else
+ $$(error Unknown value for file OPTIMIZATION: $$($1_OPTIMIZATION))
+ endif
+ endif
+
+ ifneq ($$($2_PRECOMPILED_HEADER), )
+ ifeq ($$(filter $$($1_FILENAME), $$($2_PRECOMPILED_HEADER_EXCLUDE)), )
+ $1_USE_PCH_FLAGS := $$($2_USE_PCH_FLAGS)
+ endif
+ endif
+
+ ifneq ($(DISABLE_WARNING_PREFIX), )
+ $1_WARNINGS_FLAGS := $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$($2_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$$($1_FILENAME)) \
+ $$($2_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)_$$($1_FILENAME)))
+ endif
+
+ $1_BASE_CFLAGS := $$($2_CFLAGS) $$($2_EXTRA_CFLAGS) \
+ $$($2_SYSROOT_CFLAGS)
+ $1_BASE_CXXFLAGS := $$($2_CXXFLAGS) $$($2_EXTRA_CXXFLAGS) \
+ $$($2_SYSROOT_CFLAGS) $$($1_EXTRA_CXXFLAGS)
+ $1_BASE_ASFLAGS := $$($2_ASFLAGS) $$($2_EXTRA_ASFLAGS)
+endef
+
+################################################################################
+define SetupCompilerFlags
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS, TOOLCHAIN_TYPE and
+ # OPENJDK_TARGET_OS plus OPENJDK_TARGET_CPU pair dependent variables for CFLAGS.
+ $1_EXTRA_CFLAGS := $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_CFLAGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU))
+
+ ifneq ($(DEBUG_LEVEL), release)
+ # Pickup extra debug dependent variables for CFLAGS
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_debug)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_debug)
+ else
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_release)
+ $1_EXTRA_CFLAGS += $$($1_CFLAGS_$(OPENJDK_TARGET_OS)_$(OPENJDK_TARGET_CPU)_release)
+ endif
+ ifeq ($(STATIC_LIBS), true)
+ $1_EXTRA_CFLAGS += $$(STATIC_LIBS_CFLAGS)
+ endif
+
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and/or TOOLCHAIN_TYPE
+ # dependent variables for CXXFLAGS.
+ $1_EXTRA_CXXFLAGS := $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_CXXFLAGS_$(TOOLCHAIN_TYPE))
+
+ ifneq ($(DEBUG_LEVEL), release)
+ # Pickup extra debug dependent variables for CXXFLAGS
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_debug)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_debug)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_debug)
+ else
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_release)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS_TYPE)_release)
+ $1_EXTRA_CXXFLAGS += $$($1_CXXFLAGS_$(OPENJDK_TARGET_OS)_release)
+ endif
+ ifeq ($(STATIC_LIBS), true)
+ $1_EXTRA_CXXFLAGS += $$(STATIC_LIB_CFLAGS)
+ endif
+
+ # If no C++ flags are explicitly set, default to using the C flags.
+ # After that, we can set additional C++ flags that should not interfere
+ # with the mechanism for copying the C flags by default.
+ ifeq ($$($1_CXXFLAGS), )
+ $1_CXXFLAGS := $$($1_CFLAGS)
+ endif
+ ifeq ($$(strip $$($1_EXTRA_CXXFLAGS)), )
+ $1_EXTRA_CXXFLAGS := $$($1_EXTRA_CFLAGS)
+ endif
+
+ $$(call SetIfEmpty, $1_COMPILE_WITH_DEBUG_SYMBOLS, $$(COMPILE_WITH_DEBUG_SYMBOLS))
+
+ ifeq ($(STATIC_LIBS), true)
+ # For release builds where debug symbols are configured to be moved to
+ # separate debuginfo files, disable debug symbols for static libs instead.
+ # We don't currently support this configuration and we don't want symbol
+ # information in release builds unless explicitly asked to provide it.
+ ifeq ($(DEBUG_LEVEL), release)
+ ifeq ($(COPY_DEBUG_SYMBOLS), true)
+ $1_COMPILE_WITH_DEBUG_SYMBOLS := false
+ endif
+ endif
+ endif
+
+ ifeq ($$($1_COMPILE_WITH_DEBUG_SYMBOLS), true)
+ $1_EXTRA_CFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
+ $1_EXTRA_CXXFLAGS += $$(CFLAGS_DEBUG_SYMBOLS)
+ $1_EXTRA_ASFLAGS += $$(ASFLAGS_DEBUG_SYMBOLS)
+ endif
+
+ # Pass the library name for static JNI library naming
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $1_EXTRA_CFLAGS += -DLIBRARY_NAME=$$($1_NAME)
+ $1_EXTRA_CXXFLAGS += -DLIBRARY_NAME=$$($1_NAME)
+ endif
+
+ # Pick up disabled warnings, if possible on this platform.
+ ifneq ($(DISABLE_WARNING_PREFIX), )
+ $1_EXTRA_CFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$(DISABLED_WARNINGS) \
+ $$(DISABLED_WARNINGS_C) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
+ $$($1_DISABLED_WARNINGS_C_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
+ $1_EXTRA_CXXFLAGS += $$(addprefix $(DISABLE_WARNING_PREFIX), \
+ $$(DISABLED_WARNINGS) \
+ $$(DISABLED_WARNINGS_CXX) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)) \
+ $$($1_DISABLED_WARNINGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)) \
+ $$($1_DISABLED_WARNINGS_CXX_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS)))
+ endif
+
+ # Check if warnings should be considered errors.
+ # Pick first binary and toolchain specific, then binary specific, then general setting.
+ ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), )
+ ifeq ($$($1_WARNINGS_AS_ERRORS), )
+ $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$(WARNINGS_AS_ERRORS)
+ else
+ $1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE) := $$($1_WARNINGS_AS_ERRORS)
+ endif
+ endif
+
+ ifeq ($$($1_WARNINGS_AS_ERRORS_$(TOOLCHAIN_TYPE)), true)
+ $1_EXTRA_CFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
+ $1_EXTRA_CXXFLAGS += $(CFLAGS_WARNINGS_ARE_ERRORS)
+ endif
+
+ ifeq (NONE, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_NONE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NONE)
+ else ifeq (LOW, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_NORM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_NORM)
+ else ifeq (HIGH, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HI)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HI)
+ else ifeq (HIGHEST, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST)
+ else ifeq (HIGHEST_JVM, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_HIGHEST_JVM)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_HIGHEST_JVM)
+ else ifeq (SIZE, $$($1_OPTIMIZATION))
+ $1_OPT_CFLAGS := $(C_O_FLAG_SIZE)
+ $1_OPT_CXXFLAGS := $(CXX_O_FLAG_SIZE)
+ else ifneq (, $$($1_OPTIMIZATION))
+ $$(error Unknown value for OPTIMIZATION: $$($1_OPTIMIZATION))
+ endif
+endef
+
+################################################################################
+define SetupLinkerFlags
+ # Pickup extra OPENJDK_TARGET_OS_TYPE, OPENJDK_TARGET_OS and TOOLCHAIN_TYPE
+ # dependent variables for LDFLAGS and LIBS, and additionally the pair dependent
+ # TOOLCHAIN_TYPE plus OPENJDK_TARGET_OS
+ $1_EXTRA_LDFLAGS += $$($1_LDFLAGS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LDFLAGS_$(OPENJDK_TARGET_OS)) \
+ $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)) $$($1_LDFLAGS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
+ $1_EXTRA_LIBS += $$($1_LIBS_$(OPENJDK_TARGET_OS_TYPE)) $$($1_LIBS_$(OPENJDK_TARGET_OS)) \
+ $$($1_LIBS_$(TOOLCHAIN_TYPE)) $$($1_LIBS_$(TOOLCHAIN_TYPE)_$(OPENJDK_TARGET_OS))
+endef
diff --git a/make/common/native/Link.gmk b/make/common/native/Link.gmk
new file mode 100644
index 0000000000000..fb23152d4fb9b
--- /dev/null
+++ b/make/common/native/Link.gmk
@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to linking a native binary;
+# creating either a dynamic library, a static library or an executable.
+
+################################################################################
+# GetEntitlementsFile
+# Find entitlements file for executable when signing on macosx. If no
+# specialized file is found, returns the default file.
+# This macro might be called from custom makefiles.
+# $1 Executable to find entitlements file for.
+ENTITLEMENTS_DIR := $(TOPDIR)/make/data/macosxsigning
+ifeq ($(MACOSX_CODESIGN_MODE), debug)
+ CODESIGN_PLIST_SUFFIX := -debug
+else
+ CODESIGN_PLIST_SUFFIX :=
+endif
+DEFAULT_ENTITLEMENTS_FILE := $(ENTITLEMENTS_DIR)/default$(CODESIGN_PLIST_SUFFIX).plist
+
+GetEntitlementsFile = \
+ $(foreach f, $(ENTITLEMENTS_DIR)/$(strip $(notdir $1))$(CODESIGN_PLIST_SUFFIX).plist, \
+ $(if $(wildcard $f), $f, $(DEFAULT_ENTITLEMENTS_FILE)) \
+ )
+
+################################################################################
+define SetupLinking
+ # Unless specifically set, stripping should only happen if symbols are also
+ # being copied.
+ $$(call SetIfEmpty, $1_STRIP_SYMBOLS, $$($1_COPY_DEBUG_SYMBOLS))
+
+ ifneq ($$($1_STRIP_SYMBOLS), false)
+ # Default to using the global STRIPFLAGS. Allow for overriding with an
+ # empty value
+ $1_STRIPFLAGS ?= $(STRIPFLAGS)
+ $1_STRIP_CMD := $$($1_STRIP) $$($1_STRIPFLAGS) $$($1_TARGET)
+ endif
+endef
+
+################################################################################
+define CreateLinkedResult
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $$(eval $$(call CreateStaticLibrary,$1))
+ else
+ $$(eval $$(call CreateDynamicLibraryOrExecutable,$1))
+ endif
+endef
+
+################################################################################
+define CreateStaticLibrary
+ # Include partial linking when building the static library with clang on linux
+ ifeq ($(call isTargetOs, linux), true)
+ ifneq ($(findstring $(TOOLCHAIN_TYPE), clang), )
+ $1_ENABLE_PARTIAL_LINKING := true
+ endif
+ endif
+
+ $1_VARDEPS := $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) $$($1_LIBS) \
+ $$($1_EXTRA_LIBS)
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $1_VARDEPS += $$($1_LD) $$($1_SYSROOT_LDFLAGS)
+ endif
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_VARDEPS_FILE)
+
+ $1_AR_OBJ_ARG := $$($1_LD_OBJ_ARG)
+ # With clang on linux, partial linking is enabled and 'AR' takes the output
+ # object from the partial linking step.
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $1_TARGET_RELOCATABLE := $$($1_OBJECT_DIR)/$$($1_PREFIX)$$($1_NAME)_relocatable$(OBJ_SUFFIX)
+ $1_AR_OBJ_ARG := $$($1_TARGET_RELOCATABLE)
+ endif
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ ifeq ($$($1_LINK_OBJS_RELATIVE), true)
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
+ else
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ endif
+ $$(call LogInfo, Building static library $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ # Do partial linking.
+ ifeq ($$($1_ENABLE_PARTIAL_LINKING), true)
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_partial_link, \
+ $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_LD) $(LDFLAGS_CXX_PARTIAL_LINKING) $$($1_SYSROOT_LDFLAGS) \
+ -o $$($1_TARGET_RELOCATABLE) \
+ $$($1_LD_OBJ_ARG))
+ endif
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_AR) $$(ARFLAGS) $$($1_ARFLAGS) -r -cs $$($1_TARGET) \
+ $$($1_AR_OBJ_ARG) $$($1_RES))
+ ifeq ($(STATIC_BUILD), true)
+ $(RM) $$(@D)/$$(basename $$(@F)).symbols; \
+ $(ECHO) "Getting symbols from nm"; \
+ $(NM) $(NMFLAGS) -m $$($1_TARGET) | $(GREP) "__TEXT" | \
+ $(EGREP) -v "non-external|private extern|__TEXT,__eh_frame" | \
+ $(SED) -e 's/.* //' > $$(@D)/$$(basename $$(@F)).symbols
+ endif
+endef
+
+################################################################################
+define CreateDynamicLibraryOrExecutable
+ # A shared dynamic library or an executable binary has been specified
+ ifeq ($$($1_TYPE), LIBRARY)
+ # Generating a dynamic library.
+ $1_EXTRA_LDFLAGS += $$(call SET_SHARED_LIBRARY_NAME,$$($1_BASENAME))
+
+ # Create loadmap on AIX. Helps in diagnosing some problems.
+ ifneq ($(COMPILER_BINDCMD_FILE_FLAG), )
+ $1_EXTRA_LDFLAGS += $(COMPILER_BINDCMD_FILE_FLAG)$$($1_OBJECT_DIR)/$$($1_NOSUFFIX).loadmap
+ endif
+ endif
+
+ $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) \
+ $$($1_EXTRA_LDFLAGS) $$($1_LIBS) $$($1_EXTRA_LIBS) \
+ $$($1_CREATE_DEBUGINFO_CMDS) $$($1_STRIP_CMD) $$($1_CREATE_DEBUGLINK_CMDS)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_VARDEPS_FILE)
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ ifeq ($$($1_LINK_OBJS_RELATIVE), true)
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS_RELATIVE, $$($1_OBJ_FILE_LIST)))
+ else
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ endif
+ $$(call LogInfo, Linking $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$(if $$($1_LINK_OBJS_RELATIVE), $$(CD) $$(OUTPUTDIR) ; ) \
+ $$($1_LD) $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
+ $$($1_SYSROOT_LDFLAGS) -o $$($1_TARGET) $$($1_LD_OBJ_ARG) \
+ $$($1_LIBS) $$($1_EXTRA_LIBS))
+ $$($1_CREATE_DEBUGINFO_CMDS)
+ $$($1_STRIP_CMD)
+ $$($1_CREATE_DEBUGLINK_CMDS)
+ # On macosx, optionally run codesign on every binary.
+ # Remove signature explicitly first to avoid warnings if the linker
+ # added a default adhoc signature.
+ ifeq ($(MACOSX_CODESIGN_MODE), hardened)
+ $(CODESIGN) --remove-signature $$@
+ $(CODESIGN) -f -s "$(MACOSX_CODESIGN_IDENTITY)" --timestamp \
+ --options runtime --entitlements \
+ $$(call GetEntitlementsFile, $$@) $$@
+ else ifeq ($(MACOSX_CODESIGN_MODE), debug)
+ $(CODESIGN) --remove-signature $$@
+ $(CODESIGN) -f -s - --entitlements \
+ $$(call GetEntitlementsFile, $$@) $$@
+ endif
+endef
diff --git a/make/common/native/LinkMicrosoft.gmk b/make/common/native/LinkMicrosoft.gmk
new file mode 100644
index 0000000000000..f998bf3d117f1
--- /dev/null
+++ b/make/common/native/LinkMicrosoft.gmk
@@ -0,0 +1,112 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to linking a native binary;
+# creating either a dynamic library, a static library or an executable.
+
+################################################################################
+define CreateLinkedResultMicrosoft
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ $$(eval $$(call CreateStaticLibraryMicrosoft,$1))
+ else
+ $$(eval $$(call CreateDynamicLibraryOrExecutableMicrosoft,$1))
+ endif
+endef
+
+################################################################################
+define CreateStaticLibraryMicrosoft
+ $1_VARDEPS := $$($1_LIB) $$(LIBFLAGS) $$($1_LIBFLAGS) $$($1_LIBS) \
+ $$($1_EXTRA_LIBS)
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $$($1_TARGET): $$($1_ALL_OBJS) $$($1_RES) $$($1_VARDEPS_FILE)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ $$(call LogInfo, Building static library $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$($1_LIB) -nologo $$(LIBFLAGS) $$($1_LIBFLAGS) -out:$$($1_TARGET) \
+ $$($1_LD_OBJ_ARG) $$($1_RES))
+endef
+
+################################################################################
+define CreateDynamicLibraryOrExecutableMicrosoft
+ ifeq ($$($1_EMBED_MANIFEST), true)
+ $1_EXTRA_LDFLAGS += -manifest:embed
+ endif
+
+ $1_IMPORT_LIBRARY := $$($1_OBJECT_DIR)/$$($1_NAME).lib
+ $1_EXTRA_LDFLAGS += "-implib:$$($1_IMPORT_LIBRARY)"
+
+ ifeq ($$($1_TYPE), LIBRARY)
+ # To properly trigger downstream dependants of the import library, just as
+ # for debug files, we must have a recipe in the rule. To avoid rerunning
+ # the recipe every time have it touch the target. If an import library
+ # file is deleted by something external, explicitly delete the target to
+ # trigger a rebuild of both.
+ ifneq ($$(wildcard $$($1_IMPORT_LIBRARY)), $$($1_IMPORT_LIBRARY))
+ $$(call LogDebug, Deleting $$($1_BASENAME) because import library is missing)
+ $$(shell $(RM) $$($1_TARGET))
+ endif
+ $$($1_IMPORT_LIBRARY): $$($1_TARGET)
+ $(TOUCH) $$@
+
+ $1 += $$($1_IMPORT_LIBRARY)
+ endif
+
+ $1_VARDEPS := $$($1_LD) $$($1_SYSROOT_LDFLAGS) $$($1_LDFLAGS) \
+ $$($1_EXTRA_LDFLAGS) $$($1_LIBS) $$($1_EXTRA_LIBS) $$($1_MT) \
+ $$($1_MANIFEST_VERSION)
+
+ $1_VARDEPS_FILE := $$(call DependOnVariable, $1_VARDEPS, \
+ $$($1_OBJECT_DIR)/$$($1_NOSUFFIX).vardeps)
+
+ $1_TARGET_DEPS := $$($1_ALL_OBJS) $$($1_RES) $$($1_MANIFEST) \
+ $$($1_VARDEPS_FILE)
+
+ $$($1_TARGET): $$($1_TARGET_DEPS)
+ ifneq ($$($1_OBJ_FILE_LIST), )
+ $$(eval $$(call ListPathsSafely, $1_ALL_OBJS, $$($1_OBJ_FILE_LIST)))
+ endif
+ $$(call LogInfo, Linking $$($1_BASENAME))
+ $$(call MakeDir, $$($1_OUTPUT_DIR) $$($1_SYMBOLS_DIR))
+ $$(call ExecuteWithLog, $$($1_OBJECT_DIR)/$$($1_SAFE_NAME)_link, \
+ $$($1_LD) -nologo $$($1_LDFLAGS) $$($1_EXTRA_LDFLAGS) \
+ $$($1_SYSROOT_LDFLAGS) -out:$$($1_TARGET) $$($1_LD_OBJ_ARG) \
+ $$($1_RES) $$($1_LIBS) $$($1_EXTRA_LIBS)) \
+ | $(GREP) -v "^ Creating library .*\.lib and object .*\.exp" || \
+ test "$$$$?" = "1"
+ ifeq ($(call isBuildOsEnv, windows.wsl2), true)
+ $$(CHMOD) +x $$($1_TARGET)
+ endif
+ ifneq ($$($1_MANIFEST), )
+ $$($1_MT) -nologo -manifest $$($1_MANIFEST) \
+ -identity:"$$($1_NAME).exe, version=$$($1_MANIFEST_VERSION)" \
+ -outputresource:$$@;#1
+ endif
+endef
diff --git a/make/common/native/Paths.gmk b/make/common/native/Paths.gmk
new file mode 100644
index 0000000000000..67aa61d86e968
--- /dev/null
+++ b/make/common/native/Paths.gmk
@@ -0,0 +1,247 @@
+#
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation. Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# This file contains functionality related to handling paths for source files
+# and object files. This is complicated by the fact that we usually, but not
+# always, use absolute instead of relative paths. It is further complicated
+# by the fact that not all tools allow inputting large lists of files as
+# "@-files", which we normally use to avoid hitting command line length limits.
+# Finally this file contains functionality for locating all source code files
+# that should be included in the compilation.
+
+################################################################################
+# When absolute paths are not allowed in the output, and the compiler does not
+# support any options to avoid it, we need to rewrite compile commands to use
+# relative paths. By doing this, the __FILE__ macro will resolve to relative
+# paths. The relevant input paths on the command line are the -I flags and the
+# path to the source file itself.
+#
+# The macro MakeCommandRelative is used to rewrite the command line like this:
+# 'CD $(WORKSPACE_ROOT) && '
+# and changes all paths in cmd to be relative to the workspace root. This only
+# works properly if the build dir is inside the workspace root. If it's not,
+# relative paths are still calculated, but depending on the distance between the
+# dirs, paths in the build dir may end up as essentially absolute anyway.
+#
+# The fix-deps-file macro is used to adjust the contents of the generated make
+# dependency files to contain paths compatible with make.
+#
+REWRITE_PATHS_RELATIVE = false
+ifeq ($(ALLOW_ABSOLUTE_PATHS_IN_OUTPUT)-$(FILE_MACRO_CFLAGS), false-)
+ REWRITE_PATHS_RELATIVE = true
+endif
+
+# CCACHE_BASEDIR needs fix-deps-file as makefiles use absolute filenames for
+# object files while CCACHE_BASEDIR will make ccache relativize all paths for
+# its compiler. The compiler then produces relative dependency files.
+# make does not know a relative and absolute filename is the same so it will
+# ignore such dependencies. This only applies when the OUTPUTDIR is inside
+# the WORKSPACE_ROOT.
+ifneq ($(CCACHE), )
+ ifneq ($(filter $(WORKSPACE_ROOT)/%, $(OUTPUTDIR)), )
+ REWRITE_PATHS_RELATIVE = true
+ endif
+endif
+
+ifeq ($(REWRITE_PATHS_RELATIVE), true)
+ # Need to handle -I flags as both '-Ifoo' and '-I foo'.
+ MakeCommandRelative = \
+ $(CD) $(WORKSPACE_ROOT) && \
+ $(foreach o, $1, \
+ $(if $(filter $(WORKSPACE_ROOT)/% $(OUTPUTDIR)/%, $o), \
+ $(call RelativePath, $o, $(WORKSPACE_ROOT)) \
+ , \
+ $(if $(filter -I$(WORKSPACE_ROOT)/%, $o), \
+ -I$(call RelativePath, $(patsubst -I%, %, $o), $(WORKSPACE_ROOT)) \
+ , \
+ $o \
+ ) \
+ ) \
+ )
+
+ # When compiling with relative paths, the deps file may come out with relative
+ # paths, and that path may start with './'. First remove any leading ./, then
+ # add WORKSPACE_ROOT to any line not starting with /, while allowing for
+ # leading spaces. There may also be multiple entries on the same line, so start
+ # with splitting such lines.
+ # Non GNU sed (BSD on macosx) cannot substitute in literal \n using regex.
+ # Instead use a bash escaped literal newline. To avoid having unmatched quotes
+ # ruin the ability for an editor to properly syntax highlight this file, define
+ # that newline sequence as a separate variable and add the closing quote behind
+ # a comment.
+ sed_newline := \'$$'\n''#'
+ define fix-deps-file
+ $(SED) \
+ -e 's|\([^ ]\) \{1,\}\([^\\:]\)|\1 \\$(sed_newline) \2|g' \
+ $1.tmp \
+ | $(SED) \
+ -e 's|^\([ ]*\)\./|\1|' \
+ -e '/^[ ]*[^/ ]/s|^\([ ]*\)|\1$(WORKSPACE_ROOT)/|' \
+ > $1
+ endef
+else
+ # By default the MakeCommandRelative macro does nothing.
+ MakeCommandRelative = $1
+
+ # No adjustment is needed.
+ define fix-deps-file
+ $(MV) $1.tmp $1
+ endef
+endif
+
+################################################################################
+define SetupSourceFiles
+ $$(foreach d, $$($1_SRC), $$(if $$(wildcard $$d), , \
+ $$(error SRC specified to SetupNativeCompilation $1 contains missing directory $$d)))
+
+ $1_SRCS_RAW := $$(call FindFiles, $$($1_SRC))
+ # Order src files according to the order of the src dirs
+ $1_SRCS := $$(foreach d, $$($1_SRC), $$(filter $$d%, $$($1_SRCS_RAW)))
+ $1_SRCS := $$(filter $$(NATIVE_SOURCE_EXTENSIONS), $$($1_SRCS))
+ # Extract the C/C++ files.
+ ifneq ($$($1_EXCLUDE_PATTERNS), )
+ # We must not match the exclude pattern against the src root(s).
+ $1_SRCS_WITHOUT_ROOTS := $$($1_SRCS)
+ $$(foreach i, $$($1_SRC), $$(eval $1_SRCS_WITHOUT_ROOTS := $$(patsubst \
+ $$i/%,%, $$($1_SRCS_WITHOUT_ROOTS))))
+ $1_ALL_EXCLUDE_FILES := $$(call containing, $$($1_EXCLUDE_PATTERNS), \
+ $$($1_SRCS_WITHOUT_ROOTS))
+ endif
+ ifneq ($$($1_EXCLUDE_FILES), )
+ $1_ALL_EXCLUDE_FILES += $$($1_EXCLUDE_FILES)
+ endif
+ ifneq ($$($1_ALL_EXCLUDE_FILES), )
+ $1_EXCLUDE_FILES_PAT := $$($1_ALL_EXCLUDE_FILES) \
+ $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_ALL_EXCLUDE_FILES)))
+ $1_EXCLUDE_FILES_PAT := $$(addprefix %, $$($1_EXCLUDE_FILES_PAT))
+ $1_SRCS := $$(filter-out $$($1_EXCLUDE_FILES_PAT), $$($1_SRCS))
+ endif
+ ifneq ($$($1_INCLUDE_FILES), )
+ $1_INCLUDE_FILES_PAT := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$($1_INCLUDE_FILES)))
+ $1_SRCS := $$(filter $$($1_INCLUDE_FILES_PAT), $$($1_SRCS))
+ endif
+ # Now we have a list of all c/c++ files to compile: $$($1_SRCS)
+
+ # Prepend the source/bin path to the filter expressions. Then do the filtering.
+ ifneq ($$($1_INCLUDES), )
+ $1_SRC_INCLUDES := $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_INCLUDES))))
+ $1_SRCS := $$(filter $$($1_SRC_INCLUDES), $$($1_SRCS))
+ endif
+ ifneq ($$($1_EXCLUDES), )
+ $1_SRC_EXCLUDES := $$(addsuffix /%, $$($1_EXCLUDES))
+ $1_SRC_EXCLUDES += $$(foreach i, $$($1_SRC), $$(addprefix $$i/, $$(addsuffix /%, $$($1_EXCLUDES))))
+ $1_SRCS := $$(filter-out $$($1_SRC_EXCLUDES), $$($1_SRCS))
+ endif
+
+ $1_SRCS += $$($1_EXTRA_FILES)
+
+ ifeq ($$($1_SRCS), )
+ $$(error No sources found for $1 when looking inside the dirs $$($1_SRC))
+ endif
+
+ ifeq ($$($1_TYPE), EXECUTABLE)
+ ifeq ($(UBSAN_ENABLED), true)
+ # We need to set the default options for UBSan. This needs to be included in every executable.
+ # Rather than copy and paste code to everything with a main function, we add an additional
+ # source file to every executable that exports __ubsan_default_options.
+ ifneq ($$(filter %.cpp %.cc, $$($1_SRCS)), )
+ $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.cpp
+ else
+ $1_SRCS += $(TOPDIR)/make/data/ubsan/ubsan_default_options.c
+ endif
+ endif
+ endif
+endef
+
+################################################################################
+define SetupOutputFiles
+ # Calculate the expected output from compiling the sources
+ $1_EXPECTED_OBJS_FILENAMES := $$(call replace_with_obj_extension, $$(notdir $$($1_SRCS)))
+ $1_EXPECTED_OBJS := $$(addprefix $$($1_OBJECT_DIR)/, $$($1_EXPECTED_OBJS_FILENAMES))
+ # Sort to remove duplicates and provide a reproducible order on the input files to the linker.
+ $1_ALL_OBJS := $$(sort $$($1_EXPECTED_OBJS) $$($1_EXTRA_OBJECT_FILES))
+ ifeq ($(STATIC_LIBS), true)
+ # Exclude the object files that match with $1_STATIC_LIB_EXCLUDE_OBJS.
+ ifneq ($$($1_STATIC_LIB_EXCLUDE_OBJS), )
+ $1_ALL_OBJS := $$(call not-containing, $$($1_STATIC_LIB_EXCLUDE_OBJS), $$($1_ALL_OBJS))
+ endif
+ endif
+endef
+
+################################################################################
+define RemoveSuperfluousOutputFiles
+ # Are there too many object files on disk? Perhaps because some source file was removed?
+ $1_BINS := $$(wildcard $$($1_OBJECT_DIR)/*$(OBJ_SUFFIX))
+ $1_SUPERFLOUS_OBJS := $$(sort $$(filter-out $$($1_EXPECTED_OBJS), $$($1_BINS)))
+ # Clean out the superfluous object files.
+ ifneq ($$($1_SUPERFLUOUS_OBJS), )
+ $$(shell $(RM) -f $$($1_SUPERFLUOUS_OBJS))
+ endif
+endef
+
+################################################################################
+define SetupObjectFileList
+ $1_LD_OBJ_ARG := $$($1_ALL_OBJS)
+
+ # If there are many object files, use an @-file...
+ ifneq ($$(word 17, $$($1_ALL_OBJS)), )
+ $1_OBJ_FILE_LIST := $$($1_OBJECT_DIR)/_$1_objectfilenames.txt
+ ifneq ($(COMPILER_COMMAND_FILE_FLAG), )
+ $1_LD_OBJ_ARG := $(COMPILER_COMMAND_FILE_FLAG)$$($1_OBJ_FILE_LIST)
+ else
+ # ...except for toolchains which don't support them.
+ $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
+ endif
+
+ # If we are building static library, 'AR' on macosx/aix may not support @-file.
+ ifeq ($$($1_TYPE), STATIC_LIBRARY)
+ ifeq ($(call isTargetOs, macosx aix), true)
+ $1_LD_OBJ_ARG := `cat $$($1_OBJ_FILE_LIST)`
+ endif
+ endif
+ endif
+
+ # Unfortunately the @-file trick does not work reliably when using clang.
+ # Clang does not propagate the @-file parameter to the ld sub process, but
+ # instead puts the full content on the command line. At least the llvm ld
+ # does not even support an @-file.
+ #
+ # When linking a large amount of object files, we risk hitting the limit
+ # of the command line length even on posix systems if the path length of
+ # the output dir is very long due to our use of absolute paths. To
+ # mitigate this, use paths relative to the output dir when linking over
+ # 500 files with clang and the output dir path is deep.
+ ifneq ($$(word 500, $$($1_ALL_OBJS)), )
+ ifeq ($$(TOOLCHAIN_TYPE), clang)
+ # There is no strlen function in make, but checking path depth is a
+ # reasonable approximation.
+ ifneq ($$(word 10, $$(subst /, ,$$(OUTPUTDIR))), )
+ $1_LINK_OBJS_RELATIVE := true
+ $1_ALL_OBJS_RELATIVE := $$(patsubst $$(OUTPUTDIR)/%, %, $$($1_ALL_OBJS))
+ endif
+ endif
+ endif
+endef
diff --git a/make/conf/jib-profiles.js b/make/conf/jib-profiles.js
index 45a1e6528b358..af164624877a4 100644
--- a/make/conf/jib-profiles.js
+++ b/make/conf/jib-profiles.js
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1181,12 +1181,6 @@ var getJibProfilesDependencies = function (input, common) {
revision: (input.build_cpu == "x64" ? "Xcode11.3.1-MacOSX10.15+1.2" : devkit_platform_revisions[devkit_platform])
},
- cups: {
- organization: common.organization,
- ext: "tar.gz",
- revision: "1.0118+1.0"
- },
-
jtreg: {
server: "jpg",
product: "jtreg",
diff --git a/make/data/hotspot-symbols/symbols-aix b/make/data/hotspot-symbols/symbols-aix
deleted file mode 100644
index 1d32104e8a1ed..0000000000000
--- a/make/data/hotspot-symbols/symbols-aix
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_aix_signal
-numa_error
-numa_warn
diff --git a/make/data/hotspot-symbols/symbols-aix-debug b/make/data/hotspot-symbols/symbols-aix-debug
deleted file mode 100644
index 10887ab2b61fb..0000000000000
--- a/make/data/hotspot-symbols/symbols-aix-debug
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_AccessVMBooleanFlag
-JVM_AccessVMIntFlag
-JVM_VMBreakPoint
diff --git a/make/data/hotspot-symbols/symbols-linux b/make/data/hotspot-symbols/symbols-linux
deleted file mode 100644
index d1f258297d82d..0000000000000
--- a/make/data/hotspot-symbols/symbols-linux
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_linux_signal
-JVM_IsUseContainerSupport
-numa_error
-numa_warn
diff --git a/make/data/hotspot-symbols/symbols-macosx b/make/data/hotspot-symbols/symbols-macosx
deleted file mode 100644
index d0243562b67f9..0000000000000
--- a/make/data/hotspot-symbols/symbols-macosx
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_handle_bsd_signal
diff --git a/make/data/hotspot-symbols/symbols-shared b/make/data/hotspot-symbols/symbols-shared
deleted file mode 100644
index c5b13ef1ee867..0000000000000
--- a/make/data/hotspot-symbols/symbols-shared
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-AsyncGetCallTrace
-jio_fprintf
-jio_printf
-jio_snprintf
-jio_vfprintf
-jio_vsnprintf
-JNI_CreateJavaVM
-JNI_GetCreatedJavaVMs
-JNI_GetDefaultJavaVMInitArgs
-JVM_IsForeignLinkerSupported
-JVM_FindClassFromBootLoader
-JVM_InitAgentProperties
diff --git a/make/data/hotspot-symbols/symbols-unix b/make/data/hotspot-symbols/symbols-unix
deleted file mode 100644
index fbb82a11facb7..0000000000000
--- a/make/data/hotspot-symbols/symbols-unix
+++ /dev/null
@@ -1,233 +0,0 @@
-#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-JVM_ActiveProcessorCount
-JVM_AreNestMates
-JVM_ArrayCopy
-JVM_AssertionStatusDirectives
-JVM_BeforeHalt
-JVM_CallStackWalk
-JVM_Clone
-JVM_ConstantPoolGetClassAt
-JVM_ConstantPoolGetClassAtIfLoaded
-JVM_ConstantPoolGetClassRefIndexAt
-JVM_ConstantPoolGetDoubleAt
-JVM_ConstantPoolGetFieldAt
-JVM_ConstantPoolGetFieldAtIfLoaded
-JVM_ConstantPoolGetFloatAt
-JVM_ConstantPoolGetIntAt
-JVM_ConstantPoolGetLongAt
-JVM_ConstantPoolGetMemberRefInfoAt
-JVM_ConstantPoolGetMethodAt
-JVM_ConstantPoolGetMethodAtIfLoaded
-JVM_ConstantPoolGetNameAndTypeRefIndexAt
-JVM_ConstantPoolGetNameAndTypeRefInfoAt
-JVM_ConstantPoolGetSize
-JVM_ConstantPoolGetStringAt
-JVM_ConstantPoolGetTagAt
-JVM_ConstantPoolGetUTF8At
-JVM_CurrentCarrierThread
-JVM_CurrentThread
-JVM_SetCurrentThread
-JVM_CurrentTimeMillis
-JVM_DefineClass
-JVM_DefineClassWithSource
-JVM_DesiredAssertionStatus
-JVM_DumpAllStacks
-JVM_DumpClassListToFile
-JVM_DumpDynamicArchive
-JVM_DumpThreads
-JVM_ExpandStackFrameInfo
-JVM_FillInStackTrace
-JVM_FindClassFromCaller
-JVM_FindClassFromClass
-JVM_FindLibraryEntry
-JVM_FindLoadedClass
-JVM_FindPrimitiveClass
-JVM_FindSignal
-JVM_FreeMemory
-JVM_GC
-JVM_GetAllThreads
-JVM_GetAndClearReferencePendingList
-JVM_GetArrayElement
-JVM_GetArrayLength
-JVM_GetCallerClass
-JVM_GetClassAccessFlags
-JVM_GetClassAnnotations
-JVM_GetClassConstantPool
-JVM_GetClassContext
-JVM_GetClassCPEntriesCount
-JVM_GetClassCPTypes
-JVM_GetClassDeclaredConstructors
-JVM_GetClassDeclaredFields
-JVM_GetClassDeclaredMethods
-JVM_GetClassFieldsCount
-JVM_GetClassFileVersion
-JVM_GetClassInterfaces
-JVM_GetClassMethodsCount
-JVM_GetClassModifiers
-JVM_GetClassNameUTF
-JVM_GetClassSignature
-JVM_GetClassSigners
-JVM_GetClassTypeAnnotations
-JVM_GetCPClassNameUTF
-JVM_GetCPFieldClassNameUTF
-JVM_GetCPFieldModifiers
-JVM_GetCPFieldNameUTF
-JVM_GetCPFieldSignatureUTF
-JVM_GetCPMethodClassNameUTF
-JVM_GetCPMethodModifiers
-JVM_GetCPMethodNameUTF
-JVM_GetCPMethodSignatureUTF
-JVM_GetDeclaredClasses
-JVM_GetDeclaringClass
-JVM_GetEnclosingMethodInfo
-JVM_GetExtendedNPEMessage
-JVM_GetFieldIxModifiers
-JVM_GetFieldTypeAnnotations
-JVM_GetInheritedAccessControlContext
-JVM_GetManagement
-JVM_GetMethodIxArgsSize
-JVM_GetMethodIxByteCode
-JVM_GetMethodIxByteCodeLength
-JVM_GetMethodIxExceptionIndexes
-JVM_GetMethodIxExceptionsCount
-JVM_GetMethodIxExceptionTableEntry
-JVM_GetMethodIxExceptionTableLength
-JVM_GetMethodIxLocalsCount
-JVM_GetMethodIxMaxStack
-JVM_GetMethodIxModifiers
-JVM_GetMethodIxNameUTF
-JVM_GetMethodIxSignatureUTF
-JVM_GetMethodParameters
-JVM_GetMethodTypeAnnotations
-JVM_GetNanoTimeAdjustment
-JVM_GetNestHost
-JVM_GetNestMembers
-JVM_GetNextThreadIdOffset
-JVM_GetPermittedSubclasses
-JVM_GetPrimitiveArrayElement
-JVM_GetProperties
-JVM_GetProtectionDomain
-JVM_GetRandomSeedForDumping
-JVM_GetRecordComponents
-JVM_GetSimpleBinaryName
-JVM_GetStackAccessControlContext
-JVM_GetSystemPackage
-JVM_GetSystemPackages
-JVM_GetTemporaryDirectory
-JVM_GetVmArguments
-JVM_Halt
-JVM_HasReferencePendingList
-JVM_HoldsLock
-JVM_GetStackTrace
-JVM_IHashCode
-JVM_InitClassName
-JVM_InitStackTraceElement
-JVM_InitStackTraceElementArray
-JVM_InitializeFromArchive
-JVM_InternString
-JVM_Interrupt
-JVM_InvokeMethod
-JVM_IsArrayClass
-JVM_IsCDSDumpingEnabled
-JVM_IsConstructorIx
-JVM_IsDumpingClassList
-JVM_IsFinalizationEnabled
-JVM_IsHiddenClass
-JVM_IsInterface
-JVM_IsPreviewEnabled
-JVM_IsContinuationsSupported
-JVM_IsPrimitiveClass
-JVM_IsRecord
-JVM_IsSameClassPackage
-JVM_IsSharingEnabled
-JVM_IsSupportedJNIVersion
-JVM_IsVMGeneratedMethodIx
-JVM_LatestUserDefinedLoader
-JVM_LoadZipLibrary
-JVM_LoadLibrary
-JVM_LookupDefineClass
-JVM_LookupLambdaProxyClassFromArchive
-JVM_LogLambdaFormInvoker
-JVM_MaxMemory
-JVM_MaxObjectInspectionAge
-JVM_MonitorNotify
-JVM_MonitorNotifyAll
-JVM_MonitorWait
-JVM_MoreStackWalk
-JVM_NanoTime
-JVM_NativePath
-JVM_NewArray
-JVM_NewInstanceFromConstructor
-JVM_NewMultiArray
-JVM_PhantomReferenceRefersTo
-JVM_PrintWarningAtDynamicAgentLoad
-JVM_RaiseSignal
-JVM_RawMonitorCreate
-JVM_RawMonitorDestroy
-JVM_RawMonitorEnter
-JVM_RawMonitorExit
-JVM_ReferenceClear
-JVM_ReferenceRefersTo
-JVM_RegisterContinuationMethods
-JVM_RegisterLambdaProxyClassForArchiving
-JVM_RegisterSignal
-JVM_ReleaseUTF
-JVM_ReportFinalizationComplete
-JVM_SetArrayElement
-JVM_SetClassSigners
-JVM_SetNativeThreadName
-JVM_SetPrimitiveArrayElement
-JVM_SetStackWalkContinuation
-JVM_SetThreadPriority
-JVM_SleepNanos
-JVM_StartThread
-JVM_TotalMemory
-JVM_UnloadLibrary
-JVM_WaitForReferencePendingList
-JVM_Yield
-
-# Module related API's
-JVM_AddModuleExports
-JVM_AddModuleExportsToAll
-JVM_AddModuleExportsToAllUnnamed
-JVM_AddReadsModule
-JVM_DefineArchivedModules
-JVM_DefineModule
-JVM_SetBootLoaderUnnamedModule
-
-# Virtual thread notifications for JVMTI
-JVM_VirtualThreadStart
-JVM_VirtualThreadEnd
-JVM_VirtualThreadMount
-JVM_VirtualThreadUnmount
-JVM_VirtualThreadHideFrames
-JVM_VirtualThreadDisableSuspend
-
-# Scoped values
-JVM_EnsureMaterializedForStackWalk_func
-JVM_FindScopedValueBindings
-JVM_ScopedValueCache
-JVM_SetScopedValueCache
-#
diff --git a/make/data/hotspot-symbols/version-script.txt b/make/data/hotspot-symbols/version-script.txt
new file mode 100644
index 0000000000000..29578bf7cb2e1
--- /dev/null
+++ b/make/data/hotspot-symbols/version-script.txt
@@ -0,0 +1,11 @@
+SUNWprivate_1.1 {
+ global:
+ *;
+
+ local:
+ __bss_start;
+ _edata;
+ _end;
+ _fini;
+ _init;
+};
diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk
index bb356476847ac..f9e09706141fd 100644
--- a/make/hotspot/gensrc/GensrcAdlc.gmk
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -45,7 +45,6 @@ ifeq ($(call check-jvm-feature, compiler2), true)
ADLC_CFLAGS := -qnortti -qeh -q64 -DAIX
endif
else ifeq ($(call isBuildOs, windows), true)
- ADLC_LDFLAGS += -nologo
ADLC_CFLAGS := -nologo -EHsc
ADLC_CFLAGS_WARNINGS := -W3 -D_CRT_SECURE_NO_WARNINGS
endif
@@ -72,7 +71,8 @@ ifeq ($(call check-jvm-feature, compiler2), true)
$(eval $(call SetupNativeCompilation, BUILD_ADLC, \
NAME := adlc, \
TYPE := EXECUTABLE, \
- TOOLCHAIN := TOOLCHAIN_BUILD_LINK_CXX, \
+ TARGET_TYPE := BUILD, \
+ LINK_TYPE := C++, \
SRC := $(TOPDIR)/src/hotspot/share/adlc, \
EXTRA_FILES := $(TOPDIR)/src/hotspot/share/opto/opcodes.cpp, \
CFLAGS := $(ADLC_CFLAGS) $(ADLC_CFLAGS_WARNINGS), \
diff --git a/make/hotspot/lib/CompileGtest.gmk b/make/hotspot/lib/CompileGtest.gmk
index 0d17f7a3be562..a50d1ffac9eb4 100644
--- a/make/hotspot/lib/CompileGtest.gmk
+++ b/make/hotspot/lib/CompileGtest.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -50,7 +50,7 @@ endif
$(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
NAME := gtest, \
TYPE := STATIC_LIBRARY, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/libgtest, \
OBJECT_DIR := $(JVM_OUTPUTDIR)/libgtest/objs, \
SRC := \
@@ -75,25 +75,11 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBGTEST, \
TARGETS += $(BUILD_GTEST_LIBGTEST)
################################################################################
-
-ifeq ($(call isTargetOs, windows), true)
- GTEST_JVM_MAPFILE := $(JVM_MAPFILE)
-else
- GTEST_JVM_MAPFILE := $(JVM_OUTPUTDIR)/gtest/mapfile
-
- $(JVM_OUTPUTDIR)/gtest/symbols: $(JVM_OUTPUTDIR)/symbols
- $(call MakeDir, $(@D))
- ( $(CAT) $< ; echo "runUnitTests" ) > $@
-
- $(GTEST_JVM_MAPFILE): $(JVM_OUTPUTDIR)/gtest/symbols
- $(call create-mapfile)
-endif
-
# Additional disabled warnings are due to code in the test source.
$(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
NAME := jvm, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/gtest, \
OBJECT_DIR := $(JVM_OUTPUTDIR)/gtest/objs, \
SRC := $(GTEST_TEST_SRC), \
@@ -123,8 +109,6 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
LIBS_unix := -lgtest, \
LIBS_windows := $(JVM_OUTPUTDIR)/libgtest/gtest.lib, \
OPTIMIZATION := $(JVM_OPTIMIZATION), \
- MAPFILE := $(GTEST_JVM_MAPFILE), \
- USE_MAPFILE_FOR_SYMBOLS := true, \
COPY_DEBUG_SYMBOLS := $(GTEST_COPY_DEBUG_SYMBOLS), \
ZIP_EXTERNAL_DEBUG_SYMBOLS := false, \
STRIP_SYMBOLS := false, \
@@ -134,14 +118,19 @@ $(eval $(call SetupJdkLibrary, BUILD_GTEST_LIBJVM, \
$(BUILD_GTEST_LIBJVM) : $(BUILD_GTEST_LIBGTEST)
+ifeq ($(call isTargetOs, windows), true)
+ $(BUILD_GTEST_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
+endif
+
+
TARGETS += $(BUILD_GTEST_LIBJVM)
################################################################################
$(eval $(call SetupJdkExecutable, BUILD_GTEST_LAUNCHER, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
NAME := gtestLauncher, \
TYPE := EXECUTABLE, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_OUTPUTDIR)/gtest, \
EXTRA_FILES := $(GTEST_LAUNCHER_SRC), \
OBJECT_DIR := $(JVM_OUTPUTDIR)/gtest/launcher-objs, \
diff --git a/make/hotspot/lib/CompileJvm.gmk b/make/hotspot/lib/CompileJvm.gmk
index 3393d9e00aa25..69cd80f5171c8 100644
--- a/make/hotspot/lib/CompileJvm.gmk
+++ b/make/hotspot/lib/CompileJvm.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,6 @@ include lib/JvmFlags.gmk
# Setup compilation of the main Hotspot native library (libjvm).
JVM_OUTPUTDIR := $(JVM_VARIANT_OUTPUTDIR)/libjvm
-JVM_MAPFILE := $(JVM_OUTPUTDIR)/mapfile
################################################################################
# Platform independent setup
@@ -146,12 +145,28 @@ $(call FillFindCache, $(JVM_SRC_DIRS))
# operator new.
LIBJVM_STATIC_EXCLUDE_OBJS := operator_new.o
+ifeq ($(call isTargetOs, windows), true)
+ ifeq ($(STATIC_LIBS), true)
+ WIN_EXPORT_FILE := $(JVM_OUTPUTDIR)/static-win-exports.def
+ else
+ WIN_EXPORT_FILE := $(JVM_OUTPUTDIR)/win-exports.def
+ endif
+
+ JVM_LDFLAGS += -def:$(WIN_EXPORT_FILE)
+endif
+
+ifeq ($(call isTargetOs, linux), true)
+ HOTSPOT_VERSION_SCRIPT := $(TOPDIR)/make/data/hotspot-symbols/version-script.txt
+
+ JVM_LDFLAGS += -Wl,--exclude-libs,ALL -Wl,-version-script=$(HOTSPOT_VERSION_SCRIPT)
+endif
+
################################################################################
# Now set up the actual compilation of the main hotspot native library
$(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
NAME := jvm, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OUTPUT_DIR := $(JVM_LIB_OUTPUTDIR), \
SRC := $(JVM_SRC_DIRS), \
EXCLUDES := $(JVM_EXCLUDES), \
@@ -169,6 +184,7 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
DISABLED_WARNINGS_gcc_jvmciCodeInstaller.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_jvmtiTagMap.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_postaloc.cpp := address, \
+ DISABLED_WARNINGS_gcc_shenandoahLock.cpp := stringop-overflow, \
DISABLED_WARNINGS_gcc_synchronizer.cpp := stringop-overflow, \
DISABLED_WARNINGS_clang := $(DISABLED_WARNINGS_clang), \
DISABLED_WARNINGS_clang_arguments.cpp := missing-field-initializers, \
@@ -194,8 +210,6 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
LIBS := $(JVM_LIBS), \
OPTIMIZATION := $(JVM_OPTIMIZATION), \
OBJECT_DIR := $(JVM_OUTPUTDIR)/objs, \
- MAPFILE := $(JVM_MAPFILE), \
- USE_MAPFILE_FOR_SYMBOLS := true, \
STRIPFLAGS := $(JVM_STRIPFLAGS), \
EMBED_MANIFEST := true, \
RC_FILEDESC := $(HOTSPOT_VM_DISTRO) $(OPENJDK_TARGET_CPU_BITS)-Bit $(JVM_VARIANT) VM, \
@@ -204,11 +218,47 @@ $(eval $(call SetupJdkLibrary, BUILD_LIBJVM, \
STATIC_LIB_EXCLUDE_OBJS := $(LIBJVM_STATIC_EXCLUDE_OBJS), \
))
+ifeq ($(call isTargetOs, windows), true)
+ # The following lines create a list of vftable symbols to be filtered out of
+ # the symbol file. Removing this line causes the linker to complain about too
+ # many (> 64K) symbols, so the _guess_ is that this line is here to keep down
+ # the number of exported symbols below that limit.
+ #
+ # Some usages of C++ lambdas require the vftable symbol of classes that use
+ # the lambda type as a template parameter. The usage of those classes won't
+ # link if their vftable symbols are removed. That's why there's an exception
+ # for vftable symbols containing the string 'lambda'.
+ #
+ # A very simple example of a lambda usage that fails if the lambda vftable
+ # symbols are missing in the symbol file:
+ #
+ # #include
+ # std::function f = [](){}
+ FILTER_SYMBOLS_AWK_SCRIPT := \
+ '{ \
+ if ($$7 ~ /\?\?_7.*@@6B@/ && $$7 !~ /type_info/ && $$7 !~ /lambda/) print " " $$7; \
+ }'
+
+ # A more correct solution would be to send BUILD_LIBJVM_ALL_OBJS instead of
+ # cd && *.obj, but this will result in very long command lines, which could be
+ # problematic.
+ $(WIN_EXPORT_FILE): $(BUILD_LIBJVM_ALL_OBJS)
+ $(call LogInfo, Generating list of symbols to export from object files)
+ $(call MakeDir, $(@D))
+ $(ECHO) "EXPORTS" > $@.tmp
+ $(CD) $(BUILD_LIBJVM_OBJECT_DIR) && \
+ $(DUMPBIN) -symbols *$(OBJ_SUFFIX) | $(AWK) $(FILTER_SYMBOLS_AWK_SCRIPT) | $(SORT) -u >> $@.tmp
+ $(RM) $@
+ $(MV) $@.tmp $@
+
+ $(BUILD_LIBJVM_TARGET): $(WIN_EXPORT_FILE)
+endif
+
# Always recompile abstract_vm_version.cpp if libjvm needs to be relinked. This ensures
# that the internal vm version is updated as it relies on __DATE__ and __TIME__
# macros.
ABSTRACT_VM_VERSION_OBJ := $(JVM_OUTPUTDIR)/objs/abstract_vm_version$(OBJ_SUFFIX)
-$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ) $(JVM_MAPFILE), \
+$(ABSTRACT_VM_VERSION_OBJ): $(filter-out $(ABSTRACT_VM_VERSION_OBJ), \
$(BUILD_LIBJVM_TARGET_DEPS))
ifneq ($(GENERATE_COMPILE_COMMANDS_ONLY), true)
@@ -236,11 +286,6 @@ endif
# 1540-1090 : (I) The destructor of "..." might not be called.
# 1540-1639 : (I) The behavior of long type bit fields has changed ...
-# Include mapfile generation. It relies on BUILD_LIBJVM_ALL_OBJS which is only
-# defined after the above call to BUILD_LIBJVM. Mapfile will be generated
-# after all object files are built, but before the jvm library is linked.
-include lib/JvmMapfile.gmk
-
TARGETS += $(BUILD_LIBJVM)
################################################################################
diff --git a/make/hotspot/lib/JvmMapfile.gmk b/make/hotspot/lib/JvmMapfile.gmk
deleted file mode 100644
index b2199e7d17c6f..0000000000000
--- a/make/hotspot/lib/JvmMapfile.gmk
+++ /dev/null
@@ -1,176 +0,0 @@
-#
-# Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation. Oracle designates this
-# particular file as subject to the "Classpath" exception as provided
-# by Oracle in the LICENSE file that accompanied this code.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-$(eval $(call IncludeCustomExtension, hotspot/lib/JvmMapfile.gmk))
-
-################################################################################
-# Combine a list of static symbols
-
-ifeq ($(call And, $(call isTargetOs, windows) $(call isTargetCpu, x86_64)), false)
- # On Windows x86_64, we should not have any symbols at all, since that
- # results in duplicate warnings from the linker (JDK-8043491).
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-shared
-endif
-
-ifeq ($(call isTargetOsType, unix), true)
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-unix
-endif
-
-ifneq ($(wildcard $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)), )
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)
-endif
-
-ifneq ($(findstring debug, $(DEBUG_LEVEL)), )
- ifneq ($(wildcard $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)-debug), )
- SYMBOLS_SRC += $(TOPDIR)/make/data/hotspot-symbols/symbols-$(OPENJDK_TARGET_OS)-debug
- endif
-endif
-
-################################################################################
-# Create a dynamic list of symbols from the built object files. This is highly
-# platform dependent.
-
-ifeq ($(call isTargetOs, linux), true)
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) --defined-only *$(OBJ_SUFFIX)
- ifneq ($(FILTER_SYMBOLS_PATTERN), )
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
- endif
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)^_ZTV|^gHotSpotVM|^UseSharedSpaces$$
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|^_ZN9Arguments17SharedArchivePathE$$
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$3 ~ /$(FILTER_SYMBOLS_PATTERN)/) print $$3; \
- }'
-
-else ifeq ($(call isTargetOs, macosx), true)
- # nm on macosx prints out "warning: nm: no name list" to stderr for
- # files without symbols. Hide this, even at the expense of hiding real errors.
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -Uj *$(OBJ_SUFFIX) 2> /dev/null
- ifneq ($(FILTER_SYMBOLS_PATTERN), )
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)|
- endif
- FILTER_SYMBOLS_PATTERN := $(FILTER_SYMBOLS_PATTERN)^_ZTV|^gHotSpotVM
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$3 ~ /$(FILTER_SYMBOLS_PATTERN)/) print $$3; \
- }'
-
-# NOTE: The script is from the old build. It is broken and finds no symbols.
-# The script below might be what was intended, but it fails to link with tons
-# of 'cannot export hidden symbol vtable for X'.
-# '{ if ($$1 ~ /^__ZTV/ || $$1 ~ /^_gHotSpotVM/) print substr($$1, 2) }'
-else ifeq ($(call isTargetOs, aix), true)
- # NOTE: The old build had the solution below. This should to be fixed in
- # configure instead.
-
- # On AIX we have to prevent that we pick up the 'nm' version from the GNU binutils
- # which may be installed under /opt/freeware/bin. So better use an absolute path here!
- # NM=/usr/bin/nm
-
- DUMP_SYMBOLS_CMD := $(NM) $(NMFLAGS) -B -C *$(OBJ_SUFFIX)
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if (($$2="d" || $$2="D") && ($$3 ~ /^__vft/ || $$3 ~ /^gHotSpotVM/)) print $$3; \
- if ($$3 ~ /^UseSharedSpaces$$/) print $$3; \
- if ($$3 ~ /^SharedArchivePath__9Arguments$$/) print $$3; \
- }'
-
-else ifeq ($(call isTargetOs, windows), true)
- DUMP_SYMBOLS_CMD := $(DUMPBIN) -symbols *$(OBJ_SUFFIX)
-
- # The following lines create a list of vftable symbols to be filtered out of
- # the mapfile. Removing this line causes the linker to complain about too many
- # (> 64K) symbols, so the _guess_ is that this line is here to keep down the
- # number of exported symbols below that limit.
- #
- # Some usages of C++ lambdas require the vftable symbol of classes that use
- # the lambda type as a template parameter. The usage of those classes won't
- # link if their vftable symbols are removed. That's why there's an exception
- # for vftable symbols containing the string 'lambda'.
- #
- # A very simple example of a lambda usage that fails if the lambda vftable
- # symbols are missing in the mapfile:
- #
- # #include
- # std::function f = [](){}
-
- FILTER_SYMBOLS_AWK_SCRIPT := \
- '{ \
- if ($$7 ~ /\?\?_7.*@@6B@/ && $$7 !~ /type_info/ && $$7 !~ /lambda/) print $$7; \
- }'
-
-else
- $(error Unknown target OS $(OPENJDK_TARGET_OS) in JvmMapfile.gmk)
-endif
-
-# A more correct solution would be to send BUILD_LIBJVM_ALL_OBJS instead of
-# cd && *.o, but this will result in very long command lines, which is
-# problematic on some platforms.
-$(JVM_OUTPUTDIR)/symbols-objects: $(BUILD_LIBJVM_ALL_OBJS)
- $(call LogInfo, Generating symbol list from object files)
- $(CD) $(JVM_OUTPUTDIR)/objs && \
- $(DUMP_SYMBOLS_CMD) | $(AWK) $(FILTER_SYMBOLS_AWK_SCRIPT) | $(SORT) -u > $@
-
-SYMBOLS_SRC += $(JVM_OUTPUTDIR)/symbols-objects
-
-################################################################################
-# Now concatenate all symbol lists into a single file and remove comments.
-
-$(JVM_OUTPUTDIR)/symbols: $(SYMBOLS_SRC)
- $(SED) -e '/^#/d' $^ > $@
-
-################################################################################
-# Finally convert the symbol list into a platform-specific mapfile
-
-ifeq ($(call isTargetOs, macosx), true)
- # On macosx, we need to add a leading underscore
- define create-mapfile-work
- $(AWK) '{ if ($$0 ~ ".") { print " _" $$0 } }' < $^ > $@.tmp
- endef
-else ifeq ($(call isTargetOs, windows), true)
- # On windows, add an 'EXPORTS' header
- define create-mapfile-work
- $(ECHO) "EXPORTS" > $@.tmp
- $(AWK) '{ if ($$0 ~ ".") { print " " $$0 } }' < $^ >> $@.tmp
- endef
-else
- # Assume standard linker script
- define create-mapfile-work
- $(PRINTF) "SUNWprivate_1.1 { \n global: \n" > $@.tmp
- $(AWK) '{ if ($$0 ~ ".") { print " " $$0 ";" } }' < $^ >> $@.tmp
- $(PRINTF) " local: \n *; \n }; \n" >> $@.tmp
- endef
-endif
-
-define create-mapfile
- $(call LogInfo, Creating mapfile)
- $(call MakeDir, $(@D))
- $(call create-mapfile-work)
- $(RM) $@
- $(MV) $@.tmp $@
-endef
-
-$(JVM_MAPFILE): $(JVM_OUTPUTDIR)/symbols
- $(call create-mapfile)
diff --git a/make/modules/java.base/Lib.gmk b/make/modules/java.base/Lib.gmk
index 924cb8aae268b..54050d0798626 100644
--- a/make/modules/java.base/Lib.gmk
+++ b/make/modules/java.base/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -242,7 +242,7 @@ endif
ifeq ($(call isTargetOs, linux)+$(call isTargetCpu, x86_64)+$(INCLUDE_COMPILER2)+$(filter $(TOOLCHAIN_TYPE), gcc), true+true+true+gcc)
$(eval $(call SetupJdkLibrary, BUILD_LIB_SIMD_SORT, \
NAME := simdsort, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := HIGH, \
CFLAGS := $(CFLAGS_JDKLIB), \
CXXFLAGS := $(CXXFLAGS_JDKLIB) -std=c++17, \
diff --git a/make/modules/java.base/lib/CoreLibraries.gmk b/make/modules/java.base/lib/CoreLibraries.gmk
index 8904c39449e29..b27013536f8e3 100644
--- a/make/modules/java.base/lib/CoreLibraries.gmk
+++ b/make/modules/java.base/lib/CoreLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -115,7 +115,7 @@ TARGETS += $(BUILD_LIBZIP)
$(eval $(call SetupJdkLibrary, BUILD_LIBJIMAGE, \
NAME := jimage, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
CFLAGS := $(CFLAGS_JDKLIB), \
CXXFLAGS := $(CXXFLAGS_JDKLIB), \
diff --git a/make/modules/java.desktop/Lib.gmk b/make/modules/java.desktop/Lib.gmk
index be1ac3f1fb8a2..cb831faebf62c 100644
--- a/make/modules/java.desktop/Lib.gmk
+++ b/make/modules/java.desktop/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -51,13 +51,14 @@ ifeq ($(call isTargetOs, aix), false)
-DUSE_PLATFORM_MIDI_IN=TRUE \
#
+ LIBJSOUND_LINK_TYPE := C
ifeq ($(call isTargetOs, macosx), true)
- LIBJSOUND_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ LIBJSOUND_LINK_TYPE := C++
endif
$(eval $(call SetupJdkLibrary, BUILD_LIBJSOUND, \
NAME := jsound, \
- TOOLCHAIN := $(LIBJSOUND_TOOLCHAIN), \
+ LINK_TYPE := $(LIBJSOUND_LINK_TYPE), \
OPTIMIZATION := LOW, \
CFLAGS := $(CFLAGS_JDKLIB) \
$(LIBJSOUND_CFLAGS), \
diff --git a/make/modules/java.desktop/lib/Awt2dLibraries.gmk b/make/modules/java.desktop/lib/Awt2dLibraries.gmk
index e274005e60741..aaf98d088fdad 100644
--- a/make/modules/java.desktop/lib/Awt2dLibraries.gmk
+++ b/make/modules/java.desktop/lib/Awt2dLibraries.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -506,8 +506,10 @@ else
# noexcept-type required for GCC 7 builds. Not required for GCC 8+.
# expansion-to-defined required for GCC 9 builds. Not required for GCC 10+.
# maybe-uninitialized required for GCC 8 builds. Not required for GCC 9+.
+ # calloc-transposed-args required for GCC 14 builds. (fixed upstream in Harfbuzz 032c931e1c0cfb20f18e5acb8ba005775242bd92)
HARFBUZZ_DISABLED_WARNINGS_CXX_gcc := class-memaccess noexcept-type \
- expansion-to-defined dangling-reference maybe-uninitialized
+ expansion-to-defined dangling-reference maybe-uninitialized \
+ calloc-transposed-args
HARFBUZZ_DISABLED_WARNINGS_clang := missing-field-initializers range-loop-analysis
HARFBUZZ_DISABLED_WARNINGS_microsoft := 4267 4244
@@ -560,9 +562,9 @@ LIBFONTMANAGER_CFLAGS += $(X_CFLAGS) -DLE_STANDALONE -DHEADLESS
# libawt_xawt). See JDK-8196516 for details.
$(eval $(call SetupJdkLibrary, BUILD_LIBFONTMANAGER, \
NAME := fontmanager, \
+ LINK_TYPE := C++, \
EXCLUDE_FILES := $(LIBFONTMANAGER_EXCLUDE_FILES) \
AccelGlyphCache.c, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
CFLAGS := $(CFLAGS_JDKLIB) $(LIBFONTMANAGER_CFLAGS), \
CXXFLAGS := $(CXXFLAGS_JDKLIB) $(LIBFONTMANAGER_CFLAGS), \
OPTIMIZATION := $(LIBFONTMANAGER_OPTIMIZATION), \
diff --git a/make/modules/jdk.hotspot.agent/Lib.gmk b/make/modules/jdk.hotspot.agent/Lib.gmk
index ebdbfecd461f9..6d85061583781 100644
--- a/make/modules/jdk.hotspot.agent/Lib.gmk
+++ b/make/modules/jdk.hotspot.agent/Lib.gmk
@@ -45,16 +45,16 @@ else ifeq ($(call isTargetOs, windows), true)
endif
endif
-SA_TOOLCHAIN := $(TOOLCHAIN_DEFAULT)
+SA_LINK_TYPE := C
ifeq ($(call isTargetOs, linux), true)
- SA_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ SA_LINK_TYPE := C++
endif
################################################################################
$(eval $(call SetupJdkLibrary, BUILD_LIBSA, \
NAME := saproc, \
- TOOLCHAIN := $(SA_TOOLCHAIN), \
+ LINK_TYPE := $(SA_LINK_TYPE), \
OPTIMIZATION := HIGH, \
DISABLED_WARNINGS_gcc := sign-compare, \
DISABLED_WARNINGS_gcc_ps_core.c := pointer-arith, \
diff --git a/make/modules/jdk.internal.le/Lib.gmk b/make/modules/jdk.internal.le/Lib.gmk
index 75a2446cc5a18..85550e3cc1d8e 100644
--- a/make/modules/jdk.internal.le/Lib.gmk
+++ b/make/modules/jdk.internal.le/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,7 @@ ifeq ($(call isTargetOs, linux macosx windows), true)
$(eval $(call SetupJdkLibrary, BUILD_LIBLE, \
NAME := le, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
CFLAGS := $(CXXFLAGS_JDKLIB), \
LDFLAGS := $(LDFLAGS_JDKLIB), \
diff --git a/make/modules/jdk.jpackage/Lib.gmk b/make/modules/jdk.jpackage/Lib.gmk
index 1d3e27e8a6b77..58e40d772e135 100644
--- a/make/modules/jdk.jpackage/Lib.gmk
+++ b/make/modules/jdk.jpackage/Lib.gmk
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -32,13 +32,13 @@ include LauncherCommon.gmk
ifeq ($(call isTargetOs, linux), true)
JPACKAGE_APPLAUNCHER_SRC := \
$(call FindSrcDirsForComponent, jdk.jpackage, applauncher)
- JPACKAGE_APPLAUNCHER_TOOLCHAIN := TOOLCHAIN_DEFAULT
+ JPACKAGE_APPLAUNCHER_LINK_TYPE := C
JPACKAGE_APPLAUNCHER_INCLUDE_FILES := %.c
else
JPACKAGE_APPLAUNCHER_SRC := \
$(call FindSrcDirsForComponent, jdk.jpackage, applauncher) \
$(call FindSrcDirsForComponent, jdk.jpackage, common)
- JPACKAGE_APPLAUNCHER_TOOLCHAIN := TOOLCHAIN_LINK_CXX
+ JPACKAGE_APPLAUNCHER_LINK_TYPE := C++
endif
@@ -59,11 +59,11 @@ JPACKAGE_APPLAUNCHER_INCLUDES := $(addprefix -I, $(JPACKAGE_APPLAUNCHER_SRC))
# Output app launcher executable in resources dir, and symbols in the object dir
$(eval $(call SetupJdkExecutable, BUILD_JPACKAGE_APPLAUNCHEREXE, \
NAME := jpackageapplauncher, \
+ LINK_TYPE := $(JPACKAGE_APPLAUNCHER_LINK_TYPE), \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncher, \
SRC := $(JPACKAGE_APPLAUNCHER_SRC), \
INCLUDE_FILES := $(JPACKAGE_APPLAUNCHER_INCLUDE_FILES), \
- TOOLCHAIN := $(JPACKAGE_APPLAUNCHER_TOOLCHAIN), \
OPTIMIZATION := LOW, \
DISABLED_WARNINGS_clang_LinuxPackage.c := format-nonliteral, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
@@ -103,7 +103,7 @@ ifeq ($(call isTargetOs, linux), true)
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/libjpackageapplauncheraux, \
SRC := $(JPACKAGE_LIBAPPLAUNCHER_SRC), \
EXCLUDE_FILES := LinuxLauncher.c LinuxPackage.c, \
- TOOLCHAIN := TOOLCHAIN_LINK_CXX, \
+ LINK_TYPE := C++, \
OPTIMIZATION := LOW, \
DISABLED_WARNINGS_clang_JvmLauncherLib.c := format-nonliteral, \
DISABLED_WARNINGS_clang_tstrings.cpp := format-nonliteral, \
@@ -177,10 +177,10 @@ ifeq ($(call isTargetOs, windows), true)
# Build non-console version of launcher
$(eval $(call SetupJdkExecutable, BUILD_JPACKAGE_APPLAUNCHERWEXE, \
NAME := jpackageapplauncherw, \
+ LINK_TYPE := $(BUILD_JPACKAGE_APPLAUNCHEREXE_LINK_TYPE), \
OUTPUT_DIR := $(JPACKAGE_OUTPUT_DIR), \
SYMBOLS_DIR := $(SUPPORT_OUTPUTDIR)/native/$(MODULE)/jpackageapplauncherw, \
SRC := $(BUILD_JPACKAGE_APPLAUNCHEREXE_SRC), \
- TOOLCHAIN := $(BUILD_JPACKAGE_APPLAUNCHEREXE_TOOLCHAIN), \
OPTIMIZATION := $(BUILD_JPACKAGE_APPLAUNCHEREXE_OPTIMIZATION), \
CXXFLAGS := $(BUILD_JPACKAGE_APPLAUNCHEREXE_CXXFLAGS), \
CXXFLAGS_windows := $(BUILD_JPACKAGE_APPLAUNCHEREXE_CXXFLAGS_windows) -DJP_LAUNCHERW, \
diff --git a/make/scripts/compare.sh b/make/scripts/compare.sh
index a395e0cd850ca..44c700c48957e 100644
--- a/make/scripts/compare.sh
+++ b/make/scripts/compare.sh
@@ -1,6 +1,6 @@
#!/bin/bash
#
-# Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -60,13 +60,15 @@ else
STAT_PRINT_SIZE="-c %s"
fi
-COMPARE_EXCEPTIONS_INCLUDE="$TOPDIR/make/scripts/compare_exceptions.sh.incl"
-if [ ! -e "$COMPARE_EXCEPTIONS_INCLUDE" ]; then
- echo "Error: Cannot locate the exceptions file, it should have been here: $COMPARE_EXCEPTIONS_INCLUDE"
- exit 1
+
+if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
+ # We ship a pdb file inside a published zip. Such files can never be built
+ # reproducibly, so ignore it.
+ ACCEPTED_JARZIP_CONTENTS="/modules_libs/java.security.jgss/w2k_lsa_auth.dll.pdb"
+elif [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
+ # Due to signing, we can never get a byte-by-byte identical build on macOS
+ STRIP_TESTS_BEFORE_COMPARE="true"
fi
-# Include exception definitions
-. "$COMPARE_EXCEPTIONS_INCLUDE"
################################################################################
#
@@ -117,35 +119,6 @@ diff_text() {
TMP=$($DIFF $THIS_FILE $OTHER_FILE)
- if test "x$SUFFIX" = "xclass"; then
- if [ "$NAME" = "SystemModules\$all.class" ] \
- || [ "$NAME" = "SystemModules\$default.class" ]; then
- # The SystemModules\$*.classes are not comparable as they contain the
- # module hashes which would require a whole other level of
- # reproducible builds to get reproducible. There is also random
- # order of map initialization.
- TMP=""
- elif [ "$NAME" = "module-info.class" ]; then
- # The module-info.class have several issues with random ordering of
- # elements in HashSets.
- MODULES_CLASS_FILTER="$SED \
- -e 's/,$//' \
- -e 's/;$//' \
- -e 's/^ *[0-9]*://' \
- -e 's/#[0-9]* */#/' \
- -e 's/ *\/\// \/\//' \
- -e 's/aload *[0-9]*/aload X/' \
- -e 's/ldc_w/ldc /' \
- | $SORT \
- "
- $JAVAP -c -constants -l -p "${OTHER_FILE}" \
- | eval "$MODULES_CLASS_FILTER" > ${OTHER_FILE}.javap &
- $JAVAP -c -constants -l -p "${THIS_FILE}" \
- | eval "$MODULES_CLASS_FILTER" > ${THIS_FILE}.javap &
- wait
- TMP=$($DIFF ${OTHER_FILE}.javap ${THIS_FILE}.javap)
- fi
- fi
if test -n "$TMP"; then
echo Files $OTHER_FILE and $THIS_FILE differ
@@ -312,75 +285,60 @@ compare_file_types() {
}
################################################################################
-# Compare the rest of the files
+# Find all files to compare and separate them into different categories
-compare_general_files() {
+locate_files() {
THIS_DIR=$1
- OTHER_DIR=$2
- WORK_DIR=$3
+ TEMP_DIR=$COMPARE_ROOT/support
+ $MKDIR -p $TEMP_DIR
- GENERAL_FILES=$(cd $THIS_DIR && $FIND . -type f ! -name "*.so" ! -name "*.jar" \
- ! -name "*.zip" ! -name "*.debuginfo" ! -name "*.dylib" ! -name "jexec" \
- ! -name "modules" ! -name "ct.sym" ! -name "*.diz" ! -name "*.dll" \
- ! -name "*.cpl" ! -name "*.pdb" ! -name "*.exp" ! -name "*.ilk" \
- ! -name "*.lib" ! -name "*.jmod" ! -name "*.exe" \
- ! -name "*.obj" ! -name "*.o" ! -name "jspawnhelper" ! -name "*.a" \
- ! -name "*.tar.gz" ! -name "gtestLauncher" \
- ! -name "*.map" \
- | $GREP -v "./bin/" | $SORT | $FILTER)
+ ALL_FILES_PATH=$TEMP_DIR/all_files.txt
+ cd $THIS_DIR && $FIND . -type f | $SORT | $FILTER > $ALL_FILES_PATH
- echo Other files with binary differences...
- for f in $GENERAL_FILES
- do
- # Skip all files in test/*/native
- if [[ "$f" == */native/* ]]; then
- continue
- fi
- if [ -e $OTHER_DIR/$f ]; then
- SUFFIX="${f##*.}"
- if [ "$(basename $f)" = "release" ]; then
- # In release file, ignore differences in source rev numbers
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE)
- $MKDIR -p $(dirname $THIS_FILE)
- RELEASE_FILTER="$SED -e 's/SOURCE=".*"/SOURCE=/g'"
- $CAT $OTHER_DIR/$f | eval "$RELEASE_FILTER" > $OTHER_FILE
- $CAT $THIS_DIR/$f | eval "$RELEASE_FILTER" > $THIS_FILE
- elif [ "$SUFFIX" = "svg" ]; then
- # GraphViz has non-determinism when generating svg files
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
- SVG_FILTER="$SED \
- -e 's/edge[0-9][0-9]*/edgeX/g'
- "
- $CAT $OTHER_DIR/$f | eval "$SVG_FILTER" > $OTHER_FILE
- $CAT $THIS_DIR/$f | eval "$SVG_FILTER" > $THIS_FILE
- elif [ "$SUFFIX" = "jar_contents" ]; then
- # The jar_contents files may have some lines in random order
- OTHER_FILE=$WORK_DIR/$f.other
- THIS_FILE=$WORK_DIR/$f.this
- $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
- $RM $OTHER_FILE $THIS_FILE
- $CAT $OTHER_DIR/$f | $SORT > $OTHER_FILE
- $CAT $THIS_DIR/$f | $SORT > $THIS_FILE
- else
- OTHER_FILE=$OTHER_DIR/$f
- THIS_FILE=$THIS_DIR/$f
- fi
- DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
- if [ -n "$DIFF_OUT" ]; then
- echo $f
- REGRESSIONS=true
- if [ "$SHOW_DIFFS" = "true" ]; then
- echo "$DIFF_OUT"
- fi
- fi
- fi
- done
+ ZIP_FILES_PATH=$TEMP_DIR/zip_files.txt
+ ZIP_FILTER="-e '\.zip$' -e '\.tar.gz$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $ZIP_FILTER > $ZIP_FILES_PATH
+
+ JMOD_FILES_PATH=$TEMP_DIR/jmod_files.txt
+ JMOD_FILTER="-e '\.jmod$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $JMOD_FILTER > $JMOD_FILES_PATH
+
+ JAR_FILES_PATH=$TEMP_DIR/jar_files.txt
+ JAR_FILTER="-e '\.jar$' -e '\.war$' -e '/module$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $JAR_FILTER > $JAR_FILES_PATH
+
+ LIB_FILES_PATH=$TEMP_DIR/lib_files.txt
+ LIB_FILTER="-e '\.dylib$' -e '/lib.*\.so$' -e '\.dll$' -e '\.obj$' -e '\.o$' -e '\.a$' -e '\.cpl$'"
+ # On macos, filter out the dSYM debug symbols files. They are identically named .dylib files that reside
+ # under a *.dSYM directory
+ LIB_EXCLUDE="-e '/lib.*\.dSYM/'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $LIB_FILTER | eval $GREP -v $LIB_EXCLUDE > $LIB_FILES_PATH
+ DEBUG_FILES_PATH=$TEMP_DIR/debug_files.txt
+ DEBUG_FILTER="-e '\.dSYM/' -e '\.debuginfo$' -e '\.diz$' -e '\.pdb$' -e '\.map$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $DEBUG_FILTER > $DEBUG_FILES_PATH
+ EXEC_FILES_PATH=$TEMP_DIR/exec_files.txt
+ if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
+ EXEC_FILTER="-e '\.exe$'"
+ $CAT "$ALL_FILES_PATH" | eval $GREP $EXEC_FILTER > $EXEC_FILES_PATH
+ else
+ # Find all files with the executable bit set
+ cd $THIS_DIR && $FIND . -type f -perm -100 | $SORT | $FILTER > $EXEC_FILES_PATH
+ fi
+
+ OTHER_FILES_PATH=$TEMP_DIR/other_files.txt
+ ACCOUNTED_FILES_PATH=$TEMP_DIR/accounted_files.txt
+ $CAT $ZIP_FILES_PATH $JMOD_FILES_PATH $JAR_FILES_PATH $LIB_FILES_PATH $DEBUG_FILES_PATH $EXEC_FILES_PATH > $ACCOUNTED_FILES_PATH
+ $CAT $ACCOUNTED_FILES_PATH $ALL_FILES_PATH | $SORT | $UNIQ -u > $OTHER_FILES_PATH
+
+ ALL_ZIP_FILES=$($CAT $ZIP_FILES_PATH)
+ ALL_JMOD_FILES=$($CAT $JMOD_FILES_PATH)
+ ALL_JAR_FILES=$($CAT $JAR_FILES_PATH)
+ ALL_LIB_FILES=$($CAT $LIB_FILES_PATH)
+ ALL_DEBUG_FILES=$($CAT $DEBUG_FILES_PATH)
+ ALL_EXEC_FILES=$($CAT $EXEC_FILES_PATH)
+ ALL_OTHER_FILES=$($CAT $OTHER_FILES_PATH)
}
################################################################################
@@ -450,12 +408,14 @@ compare_zip_file() {
if [ -n "$ONLY_OTHER" ]; then
echo " Only OTHER $ZIP_FILE contains:"
echo "$ONLY_OTHER" | sed "s|Only in $OTHER_UNZIPDIR| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
if [ -n "$ONLY_THIS" ]; then
echo " Only THIS $ZIP_FILE contains:"
echo "$ONLY_THIS" | sed "s|Only in $THIS_UNZIPDIR| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
@@ -484,6 +444,7 @@ compare_zip_file() {
done
if [ -s "$WORK_DIR/$ZIP_FILE.diffs" ]; then
+ REGRESSIONS=true
return_value=1
echo " Differing files in $ZIP_FILE"
$CAT $WORK_DIR/$ZIP_FILE.diffs | $GREP 'differ$' | cut -f 2 -d ' ' | \
@@ -508,6 +469,7 @@ compare_zip_file() {
compare_bin_file $THIS_UNZIPDIR $OTHER_UNZIPDIR $WORK_DIR/$ZIP_FILE.bin \
$file
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
done
@@ -547,12 +509,14 @@ compare_jmod_file() {
if [ -n "$ONLY_OTHER" ]; then
echo " Only OTHER $JMOD_FILE contains:"
echo "$ONLY_OTHER" | sed "s|^>| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
if [ -n "$ONLY_THIS" ]; then
echo " Only THIS $JMOD_FILE contains:"
echo "$ONLY_THIS" | sed "s|^<| |"g | sed 's|: |/|g'
+ REGRESSIONS=true
return_value=1
fi
@@ -567,19 +531,18 @@ compare_all_zip_files() {
OTHER_DIR=$2
WORK_DIR=$3
- ZIPS=$(cd $THIS_DIR && $FIND . -type f -name "*.zip" -o -name "*.tar.gz" \
- | $SORT | $FILTER )
+ locate_files $THIS_DIR
- if [ -n "$ZIPS" ]; then
+ if [ -n "$ALL_ZIP_FILES" ]; then
echo Zip/tar.gz files...
return_value=0
- for f in $ZIPS; do
+ for f in $ALL_ZIP_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_zip_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -596,18 +559,18 @@ compare_all_jmod_files() {
OTHER_DIR=$2
WORK_DIR=$3
- JMODS=$(cd $THIS_DIR && $FIND . -type f -name "*.jmod" | $SORT | $FILTER )
+ locate_files $THIS_DIR
- if [ -n "$JMODS" ]; then
+ if [ -n "$ALL_JMOD_FILES" ]; then
echo Jmod files...
return_value=0
- for f in $JMODS; do
+ for f in $ALL_JMOD_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_jmod_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -624,20 +587,18 @@ compare_all_jar_files() {
OTHER_DIR=$2
WORK_DIR=$3
- # TODO filter?
- ZIPS=$(cd $THIS_DIR && $FIND . -type f -name "*.jar" -o -name "*.war" \
- -o -name "modules" | $SORT | $FILTER)
+ locate_files $THIS_DIR
- if [ -n "$ZIPS" ]; then
+ if [ -n "$ALL_JAR_FILES" ]; then
echo Jar files...
return_value=0
- for f in $ZIPS; do
+ for f in $ALL_JAR_FILES; do
if [ -f "$OTHER_DIR/$f" ]; then
compare_zip_file $THIS_DIR $OTHER_DIR $WORK_DIR $f
if [ "$?" != "0" ]; then
- return_value=1
REGRESSIONS=true
+ return_value=1
fi
fi
done
@@ -699,14 +660,16 @@ compare_bin_file() {
unset _NT_SYMBOL_PATH
if [ "$(uname -o)" = "Cygwin" ]; then
THIS=$(cygpath -msa $THIS)
- OTHER=$(cygpath -msa $OTHER)
+ if [ -n "$OTHER" ]; then
+ OTHER=$(cygpath -msa $OTHER)
+ fi
fi
# Build an _NT_SYMBOL_PATH that contains all known locations for
# pdb files.
PDB_DIRS="$(ls -d \
{$OTHER,$THIS}/support/modules_{cmds,libs}/{*,*/*} \
{$OTHER,$THIS}/support/native/jdk.jpackage/* \
- )"
+ 2> /dev/null )"
export _NT_SYMBOL_PATH="$(echo $PDB_DIRS | tr ' ' ';')"
fi
@@ -1047,23 +1010,16 @@ compare_all_libs() {
OTHER_DIR=$2
WORK_DIR=$3
- LIBS=$(cd $THIS_DIR && $FIND . -type f \( -name 'lib*.so' -o -name '*.dylib' \
- -o -name '*.dll' -o -name '*.obj' -o -name '*.o' -o -name '*.a' \
- -o -name '*.cpl' \) | $SORT | $FILTER)
-
- # On macos, filter out the dSYM debug symbols files as they are also
- # named *.dylib.
- if [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
- LIBS=$(echo "$LIBS" | $GREP -v '\.dSYM/')
- fi
+ locate_files $THIS_DIR
- if [ -n "$LIBS" ]; then
+ if [ -n "$ALL_LIB_FILES" ]; then
echo Libraries...
print_binary_diff_header
- for l in $LIBS; do
+ for l in $ALL_LIB_FILES; do
if [ -f "$OTHER_DIR/$l" ]; then
compare_bin_file $THIS_DIR $OTHER_DIR $WORK_DIR $l
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
fi
@@ -1081,33 +1037,16 @@ compare_all_execs() {
OTHER_DIR=$2
WORK_DIR=$3
- if [ "$OPENJDK_TARGET_OS" = "windows" ]; then
- EXECS=$(cd $THIS_DIR && $FIND . -type f -name '*.exe' | $SORT | $FILTER)
- else
- EXECS=$(cd $THIS_DIR && $FIND . -name db -prune -o -type f -perm -100 \! \
- \( -name '*.so' -o -name '*.dylib' -o -name '*.dll' -o -name '*.cgi' \
- -o -name '*.jar' -o -name '*.diz' -o -name 'jcontrol' -o -name '*.properties' \
- -o -name '*.data' -o -name '*.bfc' -o -name '*.src' -o -name '*.txt' \
- -o -name '*.cfg' -o -name 'meta-index' -o -name '*.properties.ja' \
- -o -name '*.xml' -o -name '*.html' -o -name '*.png' -o -name 'README' \
- -o -name '*.zip' -o -name '*.jimage' -o -name '*.java' -o -name '*.mf' \
- -o -name '*.jpg' -o -name '*.wsdl' -o -name '*.js' -o -name '*.sh' \
- -o -name '*.bat' -o -name '*LICENSE' -o -name '*.d' -o -name '*store' \
- -o -name 'blocked' -o -name '*certs' -o -name '*.ttf' \
- -o -name '*.jfc' -o -name '*.dat' -o -name 'release' -o -name '*.dir'\
- -o -name '*.sym' -o -name '*.idl' -o -name '*.h' -o -name '*.access' \
- -o -name '*.template' -o -name '*.policy' -o -name '*.security' \
- -o -name 'COPYRIGHT' -o -name '*.1' -o -name '*.debuginfo' \
- -o -name 'classlist' \) | $SORT | $FILTER)
- fi
-
- if [ -n "$EXECS" ]; then
+ locate_files $THIS_DIR
+
+ if [ -n "$ALL_EXEC_FILES" ]; then
echo Executables...
print_binary_diff_header
- for e in $EXECS; do
+ for e in $ALL_EXEC_FILES; do
if [ -f "$OTHER_DIR/$e" ]; then
compare_bin_file $THIS_DIR $OTHER_DIR $WORK_DIR $e
if [ "$?" != "0" ]; then
+ REGRESSIONS=true
return_value=1
fi
fi
@@ -1117,6 +1056,95 @@ compare_all_execs() {
return $return_value
}
+################################################################################
+# Compare native debug symbol files
+
+compare_all_debug_files() {
+ THIS_DIR=$1
+ OTHER_DIR=$2
+ WORK_DIR=$3
+
+ locate_files $THIS_DIR
+
+ echo Debug symbol files with binary differences...
+ for f in $ALL_DEBUG_FILES
+ do
+ if [ -e $OTHER_DIR/$f ]; then
+ SUFFIX="${f##*.}"
+ if [ "$SUFFIX" = "pdb" ]; then
+ # pdb files are never reproducible
+ DIFF_OUT=""
+ else
+ OTHER_FILE=$OTHER_DIR/$f
+ THIS_FILE=$THIS_DIR/$f
+ DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
+ fi
+
+ if [ -n "$DIFF_OUT" ]; then
+ echo $f
+ REGRESSIONS=true
+ if [ "$SHOW_DIFFS" = "true" ]; then
+ echo "$DIFF_OUT"
+ fi
+ fi
+ fi
+ done
+}
+
+################################################################################
+# Compare the rest of the files
+
+compare_all_other_files() {
+ THIS_DIR=$1
+ OTHER_DIR=$2
+ WORK_DIR=$3
+
+ locate_files $THIS_DIR
+
+ echo Other files with binary differences...
+ for f in $ALL_OTHER_FILES
+ do
+ # Skip all files in test/*/native
+ if [[ "$f" == */native/* ]]; then
+ continue
+ fi
+ if [ -e $OTHER_DIR/$f ]; then
+ SUFFIX="${f##*.}"
+ if [ "$(basename $f)" = "release" ]; then
+ # In release file, ignore differences in source rev numbers
+ OTHER_FILE=$WORK_DIR/$f.other
+ THIS_FILE=$WORK_DIR/$f.this
+ $MKDIR -p $(dirname $OTHER_FILE)
+ $MKDIR -p $(dirname $THIS_FILE)
+ RELEASE_FILTER="$SED -e 's/SOURCE=".*"/SOURCE=/g'"
+ $CAT $OTHER_DIR/$f | eval "$RELEASE_FILTER" > $OTHER_FILE
+ $CAT $THIS_DIR/$f | eval "$RELEASE_FILTER" > $THIS_FILE
+ elif [ "$SUFFIX" = "jar_contents" ]; then
+ # The jar_contents files are generated by the build and may have
+ # some lines in random order. They are only included for demos,
+ # which they shouldn't really...
+ OTHER_FILE=$WORK_DIR/$f.other
+ THIS_FILE=$WORK_DIR/$f.this
+ $MKDIR -p $(dirname $OTHER_FILE) $(dirname $THIS_FILE)
+ $RM $OTHER_FILE $THIS_FILE
+ $CAT $OTHER_DIR/$f | $SORT > $OTHER_FILE
+ $CAT $THIS_DIR/$f | $SORT > $THIS_FILE
+ else
+ OTHER_FILE=$OTHER_DIR/$f
+ THIS_FILE=$THIS_DIR/$f
+ fi
+ DIFF_OUT=$($DIFF $OTHER_FILE $THIS_FILE 2>&1)
+ if [ -n "$DIFF_OUT" ]; then
+ echo $f
+ REGRESSIONS=true
+ if [ "$SHOW_DIFFS" = "true" ]; then
+ echo "$DIFF_OUT"
+ fi
+ fi
+ fi
+ done
+}
+
################################################################################
# Initiate configuration
@@ -1515,22 +1543,31 @@ fi
if [ "$CMP_GENERAL" = "true" ]; then
if [ -n "$THIS_JDK" ] && [ -n "$OTHER_JDK" ]; then
echo -n "JDK "
- compare_general_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
+ compare_all_other_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
+ echo -n "JDK "
+ compare_all_debug_files $THIS_JDK $OTHER_JDK $COMPARE_ROOT/jdk
fi
if [ -n "$THIS_JDK_BUNDLE" ] && [ -n "$OTHER_JDK_BUNDLE" ]; then
echo -n "JDK Bundle "
- compare_general_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
+ compare_all_other_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
+ echo -n "JDK Bundle "
+ compare_all_debug_files $THIS_JDK_BUNDLE $OTHER_JDK_BUNDLE $COMPARE_ROOT/jdk-bundle
fi
if [ -n "$THIS_DOCS" ] && [ -n "$OTHER_DOCS" ]; then
echo -n "Docs "
- compare_general_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
+ compare_all_other_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
+ echo -n "Docs "
+ compare_all_debug_files $THIS_DOCS $OTHER_DOCS $COMPARE_ROOT/docs
fi
if [ -n "$THIS_TEST" ] && [ -n "$OTHER_TEST" ]; then
echo -n "Test "
- compare_general_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
+ compare_all_other_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
+ echo -n "Test "
+ compare_all_debug_files $THIS_TEST $OTHER_TEST $COMPARE_ROOT/test
fi
if [ -n "$THIS_BASE_DIR" ] && [ -n "$OTHER_BASE_DIR" ]; then
- compare_general_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
+ compare_all_other_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
+ compare_all_debug_files $THIS_BASE_DIR $OTHER_BASE_DIR $COMPARE_ROOT/base_dir
fi
fi
diff --git a/make/scripts/compare_exceptions.sh.incl b/make/scripts/compare_exceptions.sh.incl
index cfbfeeb5be4f9..e69de29bb2d1d 100644
--- a/make/scripts/compare_exceptions.sh.incl
+++ b/make/scripts/compare_exceptions.sh.incl
@@ -1,65 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-# This script is not to be run as stand-alone, it should be included from
-# compare.sh.
-
-##########################################################################################
-# Check that we are run via inclusion from compare.sh and not as stand-alone.
-if [ -z "$COMPARE_EXCEPTIONS_INCLUDE" ]; then
- echo "Error: This script should not be run as stand-alone. It is included by compare.sh"
- exit 1
-fi
-
-##########################################################################################
-# Diff exceptions
-
-if [ "$OPENJDK_TARGET_OS" = "linux" ]; then
- if [ "$USE_PRECOMPILED_HEADER" = "true" ]; then
- ACCEPTED_BIN_DIFF="
- ./lib/server/libjvm.so
- ./hotspot/gtest/server/libjvm.so
- "
- STRIP_BEFORE_COMPARE="
- ./hotspot/gtest/server/libjvm.so
- "
- fi
-elif [ "$OPENJDK_TARGET_OS" = "windows" ]; then
- SKIP_BIN_DIFF="true"
- SKIP_FULLDUMP_DIFF="true"
- ACCEPTED_JARZIP_CONTENTS="
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll.pdb
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll.map
- /modules_libs/java.security.jgss/w2k_lsa_auth.dll
- "
-elif [ "$OPENJDK_TARGET_OS" = "macosx" ]; then
- ACCEPTED_BIN_DIFF="
- ./lib/libawt_lwawt.dylib
- ./lib/libosxapp.dylib
- ./lib/libosxui.dylib
- ./lib/server/libjvm.dylib
- ./hotspot/gtest/server/libjvm.dylib
- "
- STRIP_TESTS_BEFORE_COMPARE="true"
-fi
diff --git a/src/demo/share/java2d/J2DBench/Makefile b/src/demo/share/java2d/J2DBench/Makefile
index 04b0818a2c35b..edc4494e131de 100644
--- a/src/demo/share/java2d/J2DBench/Makefile
+++ b/src/demo/share/java2d/J2DBench/Makefile
@@ -29,6 +29,23 @@
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
+
+ifndef SOURCE
+export SOURCE := 7
+endif
+ifndef TARGET
+export TARGET := 7
+endif
+ifndef JAVAC
+export JAVAC := javac
+endif
+ifndef JAVA
+export JAVA := java
+endif
+ifndef JAR
+export JAR := jar
+endif
+
SOURCEPATH=src
CLASSES=build
DIST=dist
@@ -80,18 +97,18 @@ SCM_DIRs = .hg .svn CVS RCS SCCS Codemgr_wsdata deleted_files
all: mkdirs J2DBench.jar J2DAnalyzer.jar
run: mkdirs J2DBench.jar
- java -jar $(DIST)/J2DBench.jar
+ $(JAVA) -jar $(DIST)/J2DBench.jar
analyze: mkdirs J2DAnalyzer.jar
- java -jar $(DIST)/J2DAnalyzer.jar
+ $(JAVA) -jar $(DIST)/J2DAnalyzer.jar
J2DBench.jar: \
$(J2DBENCH_CLASSES) $(J2DBENCH_RESOURCES) \
$(CLASSES)/j2dbench.manifest
- jar cvmf $(CLASSES)/j2dbench.manifest $(DIST)/J2DBench.jar -C $(CLASSES) j2dbench
+ $(JAR) cvmf $(CLASSES)/j2dbench.manifest $(DIST)/J2DBench.jar -C $(CLASSES) j2dbench
J2DAnalyzer.jar: $(J2DANALYZER_CLASSES) $(CLASSES)/j2danalyzer.manifest
- jar cvmf $(CLASSES)/j2danalyzer.manifest \
+ $(JAR) cvmf $(CLASSES)/j2danalyzer.manifest \
$(DIST)/J2DAnalyzer.jar -C $(CLASSES) j2dbench/report
$(CLASSES)/j2dbench/tests/iio/images: $(RESOURCES)/images
@@ -120,7 +137,7 @@ $(CLASSES):
mkdirs: $(DIST) $(CLASSES)
$(CLASSES)/j2dbench/%.class: $(SOURCEPATH)/j2dbench/%.java
- javac -g:none -source 1.7 -target 1.7 -d $(CLASSES) -sourcepath $(SOURCEPATH) $<
+ $(JAVAC) -g:none -source $(SOURCE) -target $(TARGET) -d $(CLASSES) -sourcepath $(SOURCEPATH) $<
clean:
rm -rf $(CLASSES)
diff --git a/src/demo/share/java2d/J2DBench/README b/src/demo/share/java2d/J2DBench/README
index 3b9f25c13f14e..513c984a6555f 100644
--- a/src/demo/share/java2d/J2DBench/README
+++ b/src/demo/share/java2d/J2DBench/README
@@ -23,6 +23,9 @@ The benchmark requires at least jdk1.4 to compile and run. Note that
source/target is set to 1.7 in the makefile and build.xml, because of
support in jdk 14 compiler. To check compatibility with jdk1.4 you can
use "-source 1.4 -target 1.4" options and jdk1.7.
+Yo can use TARGET/SOURCE of makefile and -Dtarget/surce to set them up for your convinience.
+Similarly you can set JAVA/JAVAC/JAR and -Djava/javac to select diffferent java/javac then is on yoru PATH
+Unluckily in ant, you can not set jar, but ant should honor JAVA_HOME
-----------------------------------------------------------------------
How To Compile
diff --git a/src/demo/share/java2d/J2DBench/build.xml b/src/demo/share/java2d/J2DBench/build.xml
index 7b202946cf145..415c315899eac 100644
--- a/src/demo/share/java2d/J2DBench/build.xml
+++ b/src/demo/share/java2d/J2DBench/build.xml
@@ -39,6 +39,27 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
@@ -64,6 +86,7 @@
description="run J2DAnalyzer" >
diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad
index f22af58f40ab5..a07ff041c487a 100644
--- a/src/hotspot/cpu/aarch64/aarch64.ad
+++ b/src/hotspot/cpu/aarch64/aarch64.ad
@@ -2205,14 +2205,14 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
st->print_cr("# MachUEPNode");
if (UseCompressedClassPointers) {
- st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- if (CompressedKlassPointers::shift() != 0) {
- st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
- }
+ st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmpw rscratch1, r10");
} else {
- st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
+ st->print_cr("\tcmp rscratch1, r10");
}
- st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
}
#endif
@@ -2221,14 +2221,7 @@ void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
-
- __ cmp_klass(j_rarg0, rscratch2, rscratch1);
- Label skip;
- // TODO
- // can we avoid this skip and still use a reloc?
- __ br(Assembler::EQ, skip);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(skip);
+ __ ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
@@ -2582,7 +2575,7 @@ Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
}
// Binary src (Replicate con)
-bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
+static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
if (n == nullptr || m == nullptr) {
return false;
}
@@ -2623,7 +2616,7 @@ bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
// (XorV src (Replicate m1))
// (XorVMask src (MaskAll m1))
-bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
+static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
if (n != nullptr && m != nullptr) {
return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
VectorNode::is_all_ones_vector(m);
@@ -3715,7 +3708,7 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
} else {
// Emit stub for static call
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, call);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
@@ -8289,7 +8282,7 @@ instruct membar_acquire() %{
ins_cost(VOLATILE_REF_COST);
format %{ "membar_acquire\n\t"
- "dmb ishld" %}
+ "dmb ish" %}
ins_encode %{
__ block_comment("membar_acquire");
@@ -8343,12 +8336,11 @@ instruct membar_release() %{
ins_cost(VOLATILE_REF_COST);
format %{ "membar_release\n\t"
- "dmb ishst\n\tdmb ishld" %}
+ "dmb ish" %}
ins_encode %{
__ block_comment("membar_release");
- __ membar(Assembler::StoreStore);
- __ membar(Assembler::LoadStore);
+ __ membar(Assembler::LoadStore|Assembler::StoreStore);
%}
ins_pipe(pipe_serial);
%}
@@ -16441,13 +16433,12 @@ instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
- // TODO
- // identify correct cost
ins_cost(5 * INSN_COST);
- format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
ins_encode %{
__ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
@@ -16458,6 +16449,7 @@ instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegP
instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
%{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, TEMP tmp2);
@@ -16471,6 +16463,37 @@ instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRe
ins_pipe(pipe_serial);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
+
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
+%{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object box));
+ effect(TEMP tmp, TEMP tmp2);
+
+ ins_cost(5 * INSN_COST);
+ format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
+
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
+ %}
+
+ ins_pipe(pipe_serial);
+%}
// ============================================================================
// Safepoint Instructions
diff --git a/src/hotspot/cpu/aarch64/ad_encode.m4 b/src/hotspot/cpu/aarch64/ad_encode.m4
index e6c87cf5b0559..4897998d8709e 100644
--- a/src/hotspot/cpu/aarch64/ad_encode.m4
+++ b/src/hotspot/cpu/aarch64/ad_encode.m4
@@ -19,7 +19,7 @@ dnl Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
dnl or visit www.oracle.com if you need additional information or have any
dnl questions.
dnl
-dnl
+dnl
dnl Process this file with m4 ad_encode.m4 to generate the load/store
dnl patterns used in aarch64.ad.
dnl
@@ -90,4 +90,3 @@ STORE(vRegD,strd,Float,,8)
loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
%}
-
diff --git a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
index c7b867a4207d9..76f88764416e3 100644
--- a/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/assembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020 Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -118,10 +118,6 @@ extern "C" {
else
Disassembler::decode((address)start, (address)start + len);
}
-
- JNIEXPORT void das1(uintptr_t insn) {
- das(insn, 1);
- }
}
#define __ as->
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index b83d618506298..9bd8c6b8e9f88 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -53,7 +53,6 @@
#endif
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = rscratch2; // where the IC klass is cached
const Register SYNC_header = r0; // synchronization header
const Register SHIFT_count = r0; // where count for shift operations must be
@@ -293,27 +292,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- int start_offset = __ offset();
- __ inline_cache_check(receiver, ic_klass);
-
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- Label dont;
- __ br(Assembler::EQ, dont);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // We align the verified entry point unless the method body
- // (including its inline cache check) will fit in a single 64-byte
- // icache line.
- if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
- // force alignment after the cache check.
- __ align(CodeEntryAlignment);
- }
-
- __ bind(dont);
- return start_offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -1230,7 +1209,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -2042,7 +2021,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
__ emit_static_call_stub();
- assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
+ assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
<= call_stub_size(), "stub too big");
__ end_a_stub();
}
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
index 43ec189255f9c..ef1b5fe2703e6 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.hpp
@@ -71,8 +71,8 @@ friend class ArrayCopyStub;
void deoptimize_trap(CodeEmitInfo *info);
enum {
- // call stub: CompiledStaticCall::to_interp_stub_size() +
- // CompiledStaticCall::to_trampoline_stub_size()
+ // call stub: CompiledDirectCall::to_interp_stub_size() +
+ // CompiledDirectCall::to_trampoline_stub_size()
_call_stub_size = 13 * NativeInstruction::instruction_size,
_exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(175),
_deopt_handler_size = 7 * NativeInstruction::instruction_size
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
index d3a746178f14e..e48d64d90696c 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -80,12 +80,12 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
br(Assembler::NE, slow_case);
}
- // Load object header
- ldr(hdr, Address(obj, hdr_offset));
if (LockingMode == LM_LIGHTWEIGHT) {
lightweight_lock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ ldr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orr(hdr, hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -144,11 +144,6 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(hdr, Address(obj, oopDesc::mark_offset_in_bytes()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(hdr, markWord::monitor_value);
- br(Assembler::NE, slow_case);
lightweight_unlock(obj, hdr, temp, rscratch2, slow_case);
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
@@ -193,6 +188,12 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
strw(len, Address(obj, arrayOopDesc::length_offset_in_bytes()));
+ int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ // Clear gap/first 4 bytes following the length field.
+ strw(zr, Address(obj, base_offset));
+ }
} else if (UseCompressedClassPointers) {
store_klass_gap(obj, zr);
}
@@ -271,7 +272,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, t1, t2, klass);
// determine alignment mask
@@ -284,7 +285,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- mov(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mov(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
add(arr_size, arr_size, len, ext::uxtw, f);
andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
@@ -292,8 +293,11 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
initialize_header(obj, klass, len, t1, t2);
+ // Align-up to word boundary, because we clear the 4 bytes potentially
+ // following the length field in initialize_header().
+ int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
// clear rest of allocated space
- initialize_body(obj, arr_size, header_size * BytesPerWord, t1, t2);
+ initialize_body(obj, arr_size, base_offset, t1, t2);
if (Compilation::current()->bailed_out()) {
return;
}
@@ -308,17 +312,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
verify_oop(obj);
}
-
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
-
- cmp_klass(receiver, iCache, rscratch1);
-}
-
-
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
index 4aa6206aa6073..d210c21d12b8f 100644
--- a/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c1_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -100,7 +100,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
void set_rsp_offset(int n) { _rsp_offset = n; }
diff --git a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
index d2f4744a04914..63a32e714e365 100644
--- a/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_Runtime1_aarch64.cpp
@@ -38,7 +38,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_aarch64.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_aarch64.hpp"
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
index 8910fba97a558..7e3ceb1f02029 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp
@@ -32,6 +32,7 @@
#include "opto/output.hpp"
#include "opto/subnode.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -55,6 +56,7 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
// Load markWord from object into displaced_header.
@@ -73,7 +75,8 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Set tmp to be (markWord of object | UNLOCK_VALUE).
orr(tmp, disp_hdr, markWord::unlocked_value);
@@ -102,10 +105,6 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_lock(oop, disp_hdr, tmp, tmp3Reg, no_count);
- b(count);
}
// Handle existing monitor.
@@ -119,14 +118,13 @@ void C2_MacroAssembler::fast_lock(Register objectReg, Register boxReg, Register
cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
/*release*/ true, /*weak*/ false, tmp3Reg); // Sets flags for result
- if (LockingMode != LM_LIGHTWEIGHT) {
- // Store a non-null value into the box to avoid looking like a re-entrant
- // lock. The fast-path monitor unlock code checks for
- // markWord::monitor_value so use markWord::unused_mark which has the
- // relevant bit set, and also matches ObjectSynchronizer::enter.
- mov(tmp, (address)markWord::unused_mark().value());
- str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
- }
+ // Store a non-null value into the box to avoid looking like a re-entrant
+ // lock. The fast-path monitor unlock code checks for
+ // markWord::monitor_value so use markWord::unused_mark which has the
+ // relevant bit set, and also matches ObjectSynchronizer::enter.
+ mov(tmp, (address)markWord::unused_mark().value());
+ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
+
br(Assembler::EQ, cont); // CAS success means locking succeeded
cmp(tmp3Reg, rthread);
@@ -157,6 +155,7 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
Label object_has_monitor;
Label count, no_count;
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert_different_registers(oop, box, tmp, disp_hdr);
if (LockingMode == LM_LEGACY) {
@@ -175,7 +174,8 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
if (LockingMode == LM_MONITOR) {
tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
b(cont);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Check if it is still a light weight lock, this is is true if we
// see the stack address of the basicLock in the markWord of the
// object.
@@ -183,10 +183,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
/*release*/ true, /*weak*/ false, tmp);
b(cont);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- lightweight_unlock(oop, tmp, box, disp_hdr, no_count);
- b(count);
}
assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
@@ -196,19 +192,6 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
add(tmp, tmp, -(int)markWord::monitor_value); // monitor
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is anonymous, we need to fix it -- in an outline stub.
- Register tmp2 = disp_hdr;
- ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset()));
- // We cannot use tbnz here, the target might be too far away and cannot
- // be encoded.
- tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
- Compile::current()->output()->add_stub(stub);
- br(Assembler::NE, stub->entry());
- bind(stub->continuation());
- }
-
ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset()));
Label notRecursive;
@@ -241,6 +224,262 @@ void C2_MacroAssembler::fast_unlock(Register objectReg, Register boxReg, Registe
bind(no_count);
}
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
+ Register t2, Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. MUST branch to with flag == EQ
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(t1, obj);
+ ldrw(t1, Address(t1, Klass::access_flags_offset()));
+ tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
+ br(Assembler::NE, slow_path);
+ }
+
+ const Register t1_mark = t1;
+
+ { // Lightweight locking
+
+ // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
+ Label push;
+
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ // Check if lock-stack is full.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
+ br(Assembler::GT, slow_path);
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, push);
+
+ // Relaxed normal load to check for monitor. Optimization for monitor case.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Not inflated
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
+
+ // Try to lock. Transition lock-bits 0b01 => 0b00
+ orr(t1_mark, t1_mark, markWord::unlocked_value);
+ eor(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
+ br(Assembler::NE, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, t2_top));
+ addw(t2_top, t2_top, oopSize);
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_tagged_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+ const Register t2_owner_addr = t2;
+ const Register t3_owner = t3;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
+
+ // CAS owner (null => current thread).
+ cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
+ /*release*/ false, /*weak*/ false, t3_owner);
+ br(Assembler::EQ, locked);
+
+ // Check if recursive.
+ cmp(t3_owner, rthread);
+ br(Assembler::NE, slow_path);
+
+ // Recursive.
+ increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
+ }
+
+ bind(locked);
+ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that locked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Lock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Lock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
+ Register t3) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert_different_registers(obj, t1, t2, t3);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_load_monitor;
+ // Finish fast unlock successfully. MUST branch to with flag == EQ
+ Label unlocked;
+ // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
+ Label slow_path;
+
+ const Register t1_mark = t1;
+ const Register t2_top = t2;
+ const Register t3_t = t3;
+
+ { // Lightweight unlock
+
+ // Check if obj is top of lock-stack.
+ ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(t2_top, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ // Top of lock stack was not obj. Must be monitor.
+ br(Assembler::NE, inflated_load_monitor);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
+ strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t3_t, t2_top, oopSize);
+ ldr(t3_t, Address(rthread, t3_t));
+ cmp(obj, t3_t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive.
+ // Load Mark.
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check header for monitor (0b10).
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t3_t, t1_mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ // Compare and exchange failed.
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
+ addw(t2_top, t2_top, oopSize);
+ str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow_path);
+ }
+
+
+ { // Handle inflated monitor.
+ bind(inflated_load_monitor);
+ ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+#ifdef ASSERT
+ tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+#ifdef ASSERT
+ Label check_done;
+ subw(t2_top, t2_top, oopSize);
+ cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
+ br(Assembler::LT, check_done);
+ ldr(t3_t, Address(rthread, t2_top));
+ cmp(obj, t3_t);
+ br(Assembler::NE, inflated);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+#endif
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register t1_monitor = t1_mark;
+ const uintptr_t monitor_tag = markWord::monitor_value;
+
+ // Untag the monitor.
+ sub(t1_monitor, t1_mark, monitor_tag);
+
+ const Register t2_recursions = t2;
+ Label not_recursive;
+
+ // Check if recursive.
+ ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ cbz(t2_recursions, not_recursive);
+
+ // Recursive unlock.
+ sub(t2_recursions, t2_recursions, 1u);
+ str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
+ // Set flag == EQ
+ cmp(t2_recursions, t2_recursions);
+ b(unlocked);
+
+ bind(not_recursive);
+
+ Label release;
+ const Register t2_owner_addr = t2;
+
+ // Compute owner address.
+ lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
+
+ // Check if the entry lists are empty.
+ ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
+ ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
+ orr(rscratch1, rscratch1, t3_t);
+ cmp(rscratch1, zr);
+ br(Assembler::EQ, release);
+
+ // The owner may be anonymous and we removed the last obj entry in
+ // the lock-stack. This loses the information about the owner.
+ // Write the thread to the owner field so the runtime knows the owner.
+ str(rthread, Address(t2_owner_addr));
+ b(slow_path);
+
+ bind(release);
+ // Set owner to null.
+ // Release to satisfy the JMM
+ stlr(zr, t2_owner_addr);
+ }
+
+ bind(unlocked);
+ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with Flags == EQ.
+ Label flag_correct;
+ br(Assembler::EQ, flag_correct);
+ stop("Fast Unlock Flag != EQ");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with Flags == NE.
+ br(Assembler::NE, flag_correct);
+ stop("Fast Unlock Flag != NE");
+ bind(flag_correct);
+#endif
+ // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
+}
+
// Search for str1 in str2 and return index or -1
// Clobbers: rscratch1, rscratch2, rflags. May also clobber v0-v1, when icnt1==-1.
void C2_MacroAssembler::string_indexof(Register str2, Register str1,
diff --git a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
index dfa7d88cb93fe..1481f975020c9 100644
--- a/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,9 +36,11 @@
public:
// Code used by cmpFastLock and cmpFastUnlock mach instructions in .ad file.
- // See full description in macroAssembler_aarch64.cpp.
void fast_lock(Register object, Register box, Register tmp, Register tmp2, Register tmp3);
void fast_unlock(Register object, Register box, Register tmp, Register tmp2);
+ // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in .ad file.
+ void fast_lock_lightweight(Register object, Register t1, Register t2, Register t3);
+ void fast_unlock_lightweight(Register object, Register t1, Register t2, Register t3);
void string_compare(Register str1, Register str2,
Register cnt1, Register cnt2, Register result,
diff --git a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
index c58ff8828bce6..23c08f11d1a8b 100644
--- a/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/compiledIC_aarch64.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -36,7 +35,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
precond(cbuf.stubs()->start() != badAddress);
precond(cbuf.stubs()->end() != badAddress);
@@ -71,11 +70,11 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return MacroAssembler::static_call_stub_size();
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// Somewhat pessimistically, we count 3 instructions here (although
// there are only two) because we sometimes emit an alignment nop.
// Trampoline stubs are always word aligned.
@@ -83,21 +82,14 @@ int CompiledStaticCall::to_trampoline_stub_size() {
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub + NativeInstruction::instruction_size);
@@ -115,7 +107,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -132,7 +124,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/aarch64/frame_aarch64.cpp b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
index c5b2ff8a4c01c..8d0fa8895d15c 100644
--- a/src/hotspot/cpu/aarch64/frame_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/frame_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -678,7 +678,7 @@ static void printbc(Method *m, intptr_t bcx) {
printf("%s : %s ==> %s\n", m->name_and_sig_as_C_string(), buf, name);
}
-void internal_pf(uintptr_t sp, uintptr_t fp, uintptr_t pc, uintptr_t bcx) {
+static void internal_pf(uintptr_t sp, uintptr_t fp, uintptr_t pc, uintptr_t bcx) {
if (! fp)
return;
diff --git a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
index 42081d422c869..427987da97141 100644
--- a/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/g1/g1BarrierSetAssembler_aarch64.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/cpu/aarch64/globals_aarch64.hpp b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
index 13f2e4b61b9a4..293cc6eb0d0c6 100644
--- a/src/hotspot/cpu/aarch64/globals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/globals_aarch64.hpp
@@ -127,8 +127,6 @@ define_pd_global(intx, InlineSmallCode, 1000);
range(1, 99) \
product(ccstr, UseBranchProtection, "none", \
"Branch Protection to use: none, standard, pac-ret") \
- product(bool, AlwaysMergeDMB, false, DIAGNOSTIC, \
- "Always merge DMB instructions in code emission") \
// end of ARCH_FLAGS
diff --git a/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp b/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp
deleted file mode 100644
index bd8cfc42600e2..0000000000000
--- a/src/hotspot/cpu/aarch64/icBuffer_aarch64.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_aarch64.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return (MacroAssembler::far_branches() ? 6 : 4) * NativeInstruction::instruction_size;
-}
-
-#define __ masm->
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
- // assert(cached_value == nullptr || cached_oop->is_perm(), "must be perm oop");
-
- address start = __ pc();
- Label l;
- __ ldr(rscratch2, l);
- int jump_code_size = __ far_jump(ExternalAddress(entry_point));
- // IC stub code size is not expected to vary depending on target address.
- // We use NOPs to make the [ldr + far_jump + nops + int64] stub size equal to ic_stub_code_size.
- for (int size = NativeInstruction::instruction_size + jump_code_size + 8;
- size < ic_stub_code_size(); size += NativeInstruction::instruction_size) {
- __ nop();
- }
- __ bind(l);
- assert((uintptr_t)__ pc() % wordSize == 0, "");
- __ emit_int64((int64_t)cached_value);
- // Only need to invalidate the 1st two instructions - not the whole ic stub
- ICache::invalidate_range(code_begin, InlineCacheBuffer::ic_stub_code_size());
- assert(__ pc() - start == ic_stub_code_size(), "must be");
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(code_begin + 4);
- return jump->jump_destination();
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // The word containing the cached value is at the end of this IC buffer
- uintptr_t *p = (uintptr_t *)(code_begin + ic_stub_code_size() - wordSize);
- void* o = (void*)*p;
- return o;
-}
diff --git a/src/hotspot/cpu/aarch64/immediate_aarch64.cpp b/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
index 3d87fde2b5bcd..7caafc19fbd31 100644
--- a/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/immediate_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -53,7 +53,7 @@ struct li_pair {
static struct li_pair InverseLITable[LI_TABLE_SIZE];
// comparator to sort entries in the inverse table
-int compare_immediate_pair(const void *i1, const void *i2)
+static int compare_immediate_pair(const void *i1, const void *i2)
{
struct li_pair *li1 = (struct li_pair *)i1;
struct li_pair *li2 = (struct li_pair *)i2;
@@ -142,7 +142,7 @@ static inline uint32_t uimm(uint32_t val, int hi, int lo)
// result
// a bit string containing count copies of input bit string
//
-uint64_t replicate(uint64_t bits, int nbits, int count)
+static uint64_t replicate(uint64_t bits, int nbits, int count)
{
assert(count > 0, "must be");
assert(nbits > 0, "must be");
@@ -231,8 +231,8 @@ uint64_t replicate(uint64_t bits, int nbits, int count)
// For historical reasons the implementation of this function is much
// more convoluted than is really necessary.
-int expandLogicalImmediate(uint32_t immN, uint32_t immr,
- uint32_t imms, uint64_t &bimm)
+static int expandLogicalImmediate(uint32_t immN, uint32_t immr,
+ uint32_t imms, uint64_t &bimm)
{
int len; // ought to be <= 6
uint32_t levels; // 6 bits
@@ -446,4 +446,3 @@ uint32_t encoding_for_fp_immediate(float immediate)
res = (s << 7) | (r << 4) | f;
return res;
}
-
diff --git a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
index 69a61e281f352..b5625b7fc6134 100644
--- a/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/interp_masm_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -701,7 +701,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg)
}
if (LockingMode == LM_LIGHTWEIGHT) {
- ldr(tmp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, tmp, tmp2, tmp3, slow_case);
b(count);
} else if (LockingMode == LM_LEGACY) {
@@ -818,22 +817,6 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg)
if (LockingMode == LM_LIGHTWEIGHT) {
Label slow_case;
-
- // Check for non-symmetric locking. This is allowed by the spec and the interpreter
- // must handle it.
- Register tmp = rscratch1;
- // First check for lock-stack underflow.
- ldrw(tmp, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(tmp, (unsigned)LockStack::start_offset());
- br(Assembler::LE, slow_case);
- // Then check if the top of the lock-stack matches the unlocked object.
- subw(tmp, tmp, oopSize);
- ldr(tmp, Address(rthread, tmp));
- cmpoop(tmp, obj_reg);
- br(Assembler::NE, slow_case);
-
- ldr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- tbnz(header_reg, exact_log2(markWord::monitor_value), slow_case);
lightweight_unlock(obj_reg, header_reg, swap_reg, tmp_reg, slow_case);
b(count);
bind(slow_case);
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
index 124af3bafbe3a..e88e7ff1f6273 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -23,12 +23,11 @@
*
*/
-#include
-
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "ci/ciEnv.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compileTask.hpp"
#include "compiler/disassembler.hpp"
#include "compiler/oopMap.hpp"
@@ -55,6 +54,7 @@
#include "runtime/jniHandles.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
+#include "utilities/globalDefinitions.hpp"
#include "utilities/powerOfTwo.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
@@ -66,6 +66,8 @@
#include "opto/output.hpp"
#endif
+#include
+
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
#else
@@ -965,7 +967,7 @@ int MacroAssembler::max_trampoline_stub_size() {
}
void MacroAssembler::emit_static_call_stub() {
- // CompiledDirectStaticCall::set_to_interpreted knows the
+ // CompiledDirectCall::set_to_interpreted knows the
// exact layout of this stub.
isb();
@@ -995,10 +997,51 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
// address const_ptr = long_constant((jlong)Universe::non_oop_word());
// uintptr_t offset;
// ldr_constant(rscratch2, const_ptr);
- movptr(rscratch2, (uintptr_t)Universe::non_oop_word());
+ movptr(rscratch2, (intptr_t)Universe::non_oop_word());
return trampoline_call(Address(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ if (target_needs_far_branch(CAST_FROM_FN_PTR(address, SharedRuntime::get_ic_miss_stub()))) {
+ return NativeInstruction::instruction_size * 7;
+ } else {
+ return NativeInstruction::instruction_size * 5;
+ }
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = j_rarg0;
+ Register data = rscratch2;
+ Register tmp1 = rscratch1;
+ Register tmp2 = r10;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ ldrw(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldrw(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ cmpw(tmp1, tmp2);
+ } else {
+ ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldr(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ cmp(tmp1, tmp2);
+ }
+
+ Label dont;
+ br(Assembler::EQ, dont);
+ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ bind(dont);
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
+}
+
// Implementation of call_VM versions
void MacroAssembler::call_VM(Register oop_result,
@@ -1100,7 +1143,14 @@ void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thr
}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) nop();
+ align(modulus, offset());
+}
+
+// Ensure that the code at target bytes offset from the current offset() is aligned
+// according to modulus.
+void MacroAssembler::align(int modulus, int target) {
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) nop();
}
void MacroAssembler::post_call_nop() {
@@ -1197,7 +1247,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -2066,21 +2116,14 @@ void MacroAssembler::membar(Membar_mask_bits order_constraint) {
address last = code()->last_insn();
if (last != nullptr && nativeInstruction_at(last)->is_Membar() && prev == last) {
NativeMembar *bar = NativeMembar_at(prev);
- // Don't promote DMB ST|DMB LD to DMB (a full barrier) because
- // doing so would introduce a StoreLoad which the caller did not
- // intend
- if (AlwaysMergeDMB || bar->get_kind() == order_constraint
- || bar->get_kind() == AnyAny
- || order_constraint == AnyAny) {
- // We are merging two memory barrier instructions. On AArch64 we
- // can do this simply by ORing them together.
- bar->set_kind(bar->get_kind() | order_constraint);
- BLOCK_COMMENT("merged membar");
- return;
- }
+ // We are merging two memory barrier instructions. On AArch64 we
+ // can do this simply by ORing them together.
+ bar->set_kind(bar->get_kind() | order_constraint);
+ BLOCK_COMMENT("merged membar");
+ } else {
+ code()->set_last_insn(pc());
+ dmb(Assembler::barrier(order_constraint));
}
- code()->set_last_insn(pc());
- dmb(Assembler::barrier(order_constraint));
}
bool MacroAssembler::try_merge_ldst(Register rt, const Address &adr, size_t size_in_bytes, bool is_store) {
@@ -6339,97 +6382,122 @@ void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with ZF cleared.
-// Falls through upon success with ZF set.
//
// - obj: the object to be locked
-// - hdr: the header, already loaded from obj, will be destroyed
-// - t1, t2: temporary registers, will be destroyed
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers, will be destroyed
+// - slow: branched to if locking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
-
- // Check if we would have space on lock-stack for the object.
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- cmpw(t1, (unsigned)LockStack::end_offset() - 1);
- br(Assembler::GT, slow);
-
- // Load (object->mark() | 1) into hdr
- orr(hdr, hdr, markWord::unlocked_value);
- // Clear lock-bits, into t2
- eor(t2, hdr, markWord::unlocked_value);
- // Try to swing header from unlocked to locked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
+
+ Label push;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if the lock-stack is full.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ cmpw(top, (unsigned)LockStack::end_offset());
+ br(Assembler::GE, slow);
+
+ // Check for recursion.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, push);
+
+ // Check header for monitor (0b10).
+ tst(mark, markWord::monitor_value);
+ br(Assembler::NE, slow);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(mark, mark, markWord::unlocked_value);
+ eor(t, mark, markWord::unlocked_value);
+ cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::xword,
+ /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
br(Assembler::NE, slow);
- // After successful lock, push object on lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- str(obj, Address(rthread, t1));
- addw(t1, t1, oopSize);
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ str(obj, Address(rthread, top));
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with ZF cleared.
-// Falls through upon success, with ZF set.
//
// - obj: the object to be unlocked
-// - hdr: the (pre-loaded) header of the object
-// - t1, t2: temporary registers
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
+// - t1, t2, t3: temporary registers
+// - slow: branched to if unlocking fails, absolute offset may larger than 32KB (imm14 encoding).
+void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow) {
assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
- assert_different_registers(obj, hdr, t1, t2, rscratch1);
+ // cmpxchg clobbers rscratch1.
+ assert_different_registers(obj, t1, t2, t3, rscratch1);
#ifdef ASSERT
{
- // The following checks rely on the fact that LockStack is only ever modified by
- // its owning thread, even if the lock got inflated concurrently; removal of LockStack
- // entries after inflation will happen delayed in that case.
-
// Check for lock-stack underflow.
Label stack_ok;
ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
cmpw(t1, (unsigned)LockStack::start_offset());
- br(Assembler::GT, stack_ok);
+ br(Assembler::GE, stack_ok);
STOP("Lock-stack underflow");
bind(stack_ok);
}
- {
- // Check if the top of the lock-stack matches the unlocked object.
- Label tos_ok;
- subw(t1, t1, oopSize);
- ldr(t1, Address(rthread, t1));
- cmpoop(t1, obj);
- br(Assembler::EQ, tos_ok);
- STOP("Top of lock-stack does not match the unlocked object");
- bind(tos_ok);
- }
- {
- // Check that hdr is fast-locked.
- Label hdr_ok;
- tst(hdr, markWord::lock_mask_in_place);
- br(Assembler::EQ, hdr_ok);
- STOP("Header is not fast-locked");
- bind(hdr_ok);
- }
#endif
- // Load the new header (unlocked) into t1
- orr(t1, hdr, markWord::unlocked_value);
+ Label unlocked, push_and_slow;
+ const Register top = t1;
+ const Register mark = t2;
+ const Register t = t3;
- // Try to swing header from locked to unlocked
- // Clobbers rscratch1 when UseLSE is false
- cmpxchg(obj, hdr, t1, Assembler::xword,
- /*acquire*/ true, /*release*/ true, /*weak*/ false, t2);
+ // Check if obj is top of lock-stack.
+ ldrw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ subw(top, top, oopSize);
+ ldr(t, Address(rthread, top));
+ cmp(obj, t);
br(Assembler::NE, slow);
- // After successful unlock, pop object from lock-stack
- ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
- subw(t1, t1, oopSize);
+ // Pop lock-stack.
+ DEBUG_ONLY(str(zr, Address(rthread, top));)
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Check if recursive.
+ subw(t, top, oopSize);
+ ldr(t, Address(rthread, t));
+ cmp(obj, t);
+ br(Assembler::EQ, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ ldr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ tbnz(mark, log2i_exact(markWord::monitor_value), push_and_slow);
+
#ifdef ASSERT
- str(zr, Address(rthread, t1));
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ tbz(mark, log2i_exact(markWord::unlocked_value), not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
#endif
- strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
+ orr(t, mark, markWord::unlocked_value);
+ cmpxchg(obj, mark, t, Assembler::xword,
+ /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
+ br(Assembler::EQ, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ DEBUG_ONLY(str(obj, Address(rthread, top));)
+ addw(top, top, oopSize);
+ strw(top, Address(rthread, JavaThread::lock_stack_top_offset()));
+ b(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
index e92e0ee6aa934..dad7ec4d4975e 100644
--- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -720,6 +720,7 @@ class MacroAssembler: public Assembler {
// Alignment
void align(int modulus);
+ void align(int modulus, int target);
// nop
void post_call_nop();
@@ -1247,6 +1248,8 @@ class MacroAssembler: public Assembler {
// Emit the CompiledIC call idiom
address ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment);
public:
@@ -1599,8 +1602,8 @@ class MacroAssembler: public Assembler {
// Code for java.lang.Thread::onSpinWait() intrinsic.
void spin_wait();
- void lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register t1, Register t2, Label& slow);
+ void lightweight_lock(Register obj, Register t1, Register t2, Register t3, Label& slow);
+ void lightweight_unlock(Register obj, Register t1, Register t2, Register t3, Label& slow);
private:
// Check the current thread doesn't need a cross modify fence.
diff --git a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
index 82da734611693..97a10afde7ab2 100644
--- a/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/sharedRuntime_aarch64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
* Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -30,7 +30,6 @@
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -39,7 +38,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_aarch64.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -740,9 +738,7 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
-
- Register holder = rscratch2;
+ Register data = rscratch2;
Register receiver = j_rarg0;
Register tmp = r10; // A call-clobbered register not used for arg passing
@@ -757,17 +753,12 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
{
__ block_comment("c2i_unverified_entry {");
- __ load_klass(rscratch1, receiver);
- __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ cmp(rscratch1, tmp);
- __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ br(Assembler::EQ, ok);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- __ bind(ok);
// Method might have been compiled since the call site was patched to
// interpreted; if that is the case treat it as a miss so we can get
// the call site corrected.
+ __ ic_check(1 /* end_alignment */);
+ __ ldr(rmethod, Address(data, CompiledICData::speculated_method_offset()));
+
__ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
__ cbz(rscratch1, skip_fixup);
__ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
@@ -1118,7 +1109,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ b(exit);
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1183,7 +1174,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
}
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1539,25 +1530,15 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// restoring them except rfp. rfp is the only callee save register
// as far as the interpreter and the compiler(s) are concerned.
-
- const Register ic_reg = rscratch2;
const Register receiver = j_rarg0;
- Label hit;
Label exception_pending;
- assert_different_registers(ic_reg, receiver, rscratch1);
+ assert_different_registers(receiver, rscratch1);
__ verify_oop(receiver);
- __ cmp_klass(receiver, ic_reg, rscratch1);
- __ br(Assembler::EQ, hit);
-
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ __ ic_check(8 /* end_alignment */);
// Verified entry point must be aligned
- __ align(8);
-
- __ bind(hit);
-
int vep_offset = ((intptr_t)__ pc()) - start;
// If we have to make this method not-entrant we'll overwrite its
@@ -1815,7 +1796,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ br(Assembler::NE, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ ldr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, tmp, lock_tmp, slow_path_lock);
}
__ bind(count);
@@ -1958,8 +1938,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "");
- __ ldr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ tbnz(old_hdr, exact_log2(markWord::monitor_value), slow_path_unlock);
__ lightweight_unlock(obj_reg, old_hdr, swap_reg, lock_tmp, slow_path_unlock);
__ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
}
diff --git a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
index 14e9764457508..28ec07815be5c 100644
--- a/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/upcallLinker_aarch64.cpp
@@ -245,9 +245,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(rmethod, entry);
__ str(rmethod, Address(rthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ push_cont_fastpath(rthread);
+
__ ldr(rscratch1, Address(rmethod, Method::from_compiled_offset()));
__ blr(rscratch1);
+ __ pop_cont_fastpath(rthread);
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
index b53e427649781..18f310c746cd4 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp
@@ -191,9 +191,6 @@ void VM_Version::initialize() {
if (FLAG_IS_DEFAULT(UseSIMDForMemoryOps)) {
FLAG_SET_DEFAULT(UseSIMDForMemoryOps, true);
}
- if (FLAG_IS_DEFAULT(AlwaysMergeDMB)) {
- FLAG_SET_DEFAULT(AlwaysMergeDMB, true);
- }
}
// Cortex A53
diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
index 0a85d339a552f..6883dc0d93e16 100644
--- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -169,6 +169,7 @@ enum Ampere_CPU_Model {
// Aarch64 supports fast class initialization checks
static bool supports_fast_class_init_checks() { return true; }
constexpr static bool supports_stack_watermark_barrier() { return true; }
+ constexpr static bool supports_recursive_lightweight_locking() { return true; }
static void get_compatible_board(char *buf, int buflen);
diff --git a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
index c895ff5cc0ec1..2bb53d16a3c97 100644
--- a/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/vtableStubs_aarch64.cpp
@@ -26,10 +26,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_aarch64.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -168,22 +168,22 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// Entry arguments:
- // rscratch2: CompiledICHolder
+ // rscratch2: CompiledICData
// j_rarg0: Receiver
// This stub is called from compiled code which has no callee-saved registers,
// so all registers except arguments are free at this point.
const Register recv_klass_reg = r10;
- const Register holder_klass_reg = r16; // declaring interface klass (DECC)
+ const Register holder_klass_reg = r16; // declaring interface klass (DEFC)
const Register resolved_klass_reg = r17; // resolved interface klass (REFC)
const Register temp_reg = r11;
const Register temp_reg2 = r15;
- const Register icholder_reg = rscratch2;
+ const Register icdata_reg = rscratch2;
Label L_no_such_interface;
- __ ldr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ ldr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ ldr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ ldr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
start_pc = __ pc();
diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad
index 6b18e76e6d7eb..1a833b08c4cf4 100644
--- a/src/hotspot/cpu/arm/arm.ad
+++ b/src/hotspot/cpu/arm/arm.ad
@@ -869,12 +869,7 @@ uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
#define R_RTEMP "R_R12"
void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
st->print_cr("\nUEP:");
- if (UseCompressedClassPointers) {
- st->print_cr("\tLDR_w " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
- st->print_cr("\tdecode_klass " R_RTEMP);
- } else {
- st->print_cr("\tLDR " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
- }
+ st->print_cr("\tLDR " R_RTEMP ",[R_R0 + oopDesc::klass_offset_in_bytes]\t! Inline cache check");
st->print_cr("\tCMP " R_RTEMP ",R_R8" );
st->print ("\tB.NE SharedRuntime::handle_ic_miss_stub");
}
@@ -882,13 +877,7 @@ void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream *st ) const {
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
C2_MacroAssembler _masm(&cbuf);
- Register iCache = reg_to_register_object(Matcher::inline_cache_reg_encode());
- assert(iCache == Ricklass, "should be");
- Register receiver = R0;
-
- __ load_klass(Rtemp, receiver);
- __ cmp(Rtemp, iCache);
- __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
+ __ ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
@@ -1241,7 +1230,7 @@ encode %{
emit_call_reloc(cbuf, as_MachCall(), $meth, rspec);
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
diff --git a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
index 999309c02258d..688790f07e548 100644
--- a/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_LIRAssembler_arm.cpp
@@ -161,10 +161,7 @@ void LIR_Assembler::osr_entry() {
int LIR_Assembler::check_icache() {
- Register receiver = LIR_Assembler::receiverOpr()->as_register();
- int offset = __ offset();
- __ inline_cache_check(receiver, Ricklass);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -971,7 +968,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -1950,7 +1947,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
// If not a single instruction, NativeMovConstReg::next_instruction_address()
// must jump over the whole following ldr_literal.
- // (See CompiledStaticCall::set_to_interpreted())
+ // (See CompiledDirectCall::set_to_interpreted())
#ifdef ASSERT
address ldr_site = __ pc();
#endif
diff --git a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
index c09e54e0e57ad..d9d042bb2e4e7 100644
--- a/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_MacroAssembler_arm.cpp
@@ -43,16 +43,6 @@
// arm [macro]assembler) and used with care in the other C1 specific
// files.
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- Label verified;
- load_klass(Rtemp, receiver);
- cmp(Rtemp, iCache);
- b(verified, eq); // jump over alignment no-ops
- jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
- align(CodeEntryAlignment);
- bind(verified);
-}
-
void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
assert((frame_size_in_bytes % StackAlignmentInBytes) == 0, "frame size should be aligned");
diff --git a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
index 62faa6170833b..9862a074a687f 100644
--- a/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
+++ b/src/hotspot/cpu/arm/c1_Runtime1_arm.cpp
@@ -37,7 +37,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_arm.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_arm.hpp"
diff --git a/src/hotspot/cpu/arm/compiledIC_arm.cpp b/src/hotspot/cpu/arm/compiledIC_arm.cpp
index 2d4187b7d6c6a..71389d2353d66 100644
--- a/src/hotspot/cpu/arm/compiledIC_arm.cpp
+++ b/src/hotspot/cpu/arm/compiledIC_arm.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
@@ -37,7 +36,7 @@
#if COMPILER2_OR_JVMCI
#define __ _masm.
// emit call stub, compiled java to interpreter
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
// Stub is fixed up when the corresponding call is converted from calling
// compiled code to calling interpreted code.
// set (empty), R9
@@ -59,7 +58,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
InlinedMetadata object_literal(nullptr);
// single instruction, see NativeMovConstReg::next_instruction_address() in
- // CompiledStaticCall::set_to_interpreted()
+ // CompiledDirectCall::set_to_interpreted()
__ ldr_literal(Rmethod, object_literal);
__ set_inst_mark(); // Who uses this?
@@ -87,32 +86,25 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
#undef __
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 10; // 4 in emit_to_interp_stub + 1 in Java_Static_Call
}
#endif // COMPILER2_OR_JVMCI
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// ARM doesn't use trampolines.
return 0;
}
// size of C2 call stub, compiled java to interpreter
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 8 * NativeInstruction::instruction_size;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -128,7 +120,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -144,7 +136,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
index 3aa71dca8cbf0..6d724c750aa34 100644
--- a/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/gc/g1/g1BarrierSetAssembler_arm.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/arm/icBuffer_arm.cpp b/src/hotspot/cpu/arm/icBuffer_arm.cpp
deleted file mode 100644
index e3a1c148ec6a0..0000000000000
--- a/src/hotspot/cpu/arm/icBuffer_arm.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2008, 2016, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_arm.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm->
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return (4 * Assembler::InstructionSize);
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
-
- InlinedAddress oop_literal((address) cached_value);
- __ ldr_literal(Ricklass, oop_literal);
- // FIXME: OK to remove reloc here?
- __ patchable_jump(entry_point, relocInfo::runtime_call_type, Rtemp);
- __ bind_literal(oop_literal);
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- address jump_address;
- jump_address = code_begin + NativeInstruction::instruction_size;
- NativeJump* jump = nativeJump_at(jump_address);
- return jump->jump_destination();
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin);
- return (void*)move->data();
-}
-
-#undef __
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.cpp b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
index b827e69d02233..99d619bddb55a 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.cpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.cpp
@@ -28,6 +28,7 @@
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.hpp"
#include "ci/ciEnv.hpp"
+#include "code/compiledIC.hpp"
#include "code/nativeInst.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
@@ -297,11 +298,13 @@ Address MacroAssembler::receiver_argument_address(Register params_base, Register
return Address(tmp, -Interpreter::stackElementSize);
}
+void MacroAssembler::align(int modulus, int target) {
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) nop();
+}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) {
- nop();
- }
+ align(modulus, offset());
}
int MacroAssembler::set_last_Java_frame(Register last_java_sp,
@@ -1860,3 +1863,31 @@ void MacroAssembler::lightweight_unlock(Register obj, Register t1, Register t2,
// Fallthrough: success
}
+
+int MacroAssembler::ic_check_size() {
+ return NativeInstruction::instruction_size * 7;
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = j_rarg0;
+ Register tmp1 = R4;
+ Register tmp2 = R5;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ ldr(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ldr(tmp2, Address(Ricklass, CompiledICData::speculated_klass_offset()));
+ cmp(tmp1, tmp2);
+
+ Label dont;
+ b(dont, eq);
+ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type);
+ bind(dont);
+ return uep_offset;
+}
diff --git a/src/hotspot/cpu/arm/macroAssembler_arm.hpp b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
index d9e49ab986c3a..691c8fa70ee8b 100644
--- a/src/hotspot/cpu/arm/macroAssembler_arm.hpp
+++ b/src/hotspot/cpu/arm/macroAssembler_arm.hpp
@@ -221,6 +221,7 @@ class MacroAssembler: public Assembler {
inline bool ignore_non_patchable_relocations() { return true; }
void align(int modulus);
+ void align(int modulus, int target);
// Support for VM calls
//
@@ -1077,6 +1078,9 @@ class MacroAssembler: public Assembler {
void safepoint_poll(Register tmp1, Label& slow_path);
void get_polling_page(Register dest);
void read_polling_page(Register dest, relocInfo::relocType rtype);
+
+ static int ic_check_size();
+ int ic_check(int end_alignment);
};
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
index 23ee01d335264..6a4062f29b3ba 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_arm.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
index 7006d7709813a..15b57188730df 100644
--- a/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
+++ b/src/hotspot/cpu/arm/nativeInst_arm_32.hpp
@@ -385,7 +385,7 @@ class NativeMovConstReg: public NativeInstruction {
}
void set_pc_relative_offset(address addr, address pc);
address next_instruction_address() const {
- // NOTE: CompiledStaticCall::set_to_interpreted() calls this but
+ // NOTE: CompiledDirectCall::set_to_interpreted() calls this but
// are restricted to single-instruction ldr. No need to jump over
// several instructions.
assert(is_ldr_literal(), "Should only use single-instructions load");
diff --git a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
index 716c7b7575e9c..3792fab082ba6 100644
--- a/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
+++ b/src/hotspot/cpu/arm/sharedRuntime_arm.cpp
@@ -24,15 +24,14 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "interpreter/interpreter.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/jniHandles.hpp"
@@ -626,12 +625,9 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label skip_fixup;
const Register receiver = R0;
const Register holder_klass = Rtemp; // XXX should be OK for C2 but not 100% sure
- const Register receiver_klass = R4;
- __ load_klass(receiver_klass, receiver);
- __ ldr(holder_klass, Address(Ricklass, CompiledICHolder::holder_klass_offset()));
- __ ldr(Rmethod, Address(Ricklass, CompiledICHolder::holder_metadata_offset()));
- __ cmp(receiver_klass, holder_klass);
+ __ ic_check(1 /* end_alignment */);
+ __ ldr(Rmethod, Address(Ricklass, CompiledICData::speculated_method_offset()));
__ ldr(Rtemp, Address(Rmethod, Method::code_offset()), eq);
__ cmp(Rtemp, 0, eq);
@@ -819,21 +815,14 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// Unverified entry point
address start = __ pc();
- // Inline cache check, same as in C1_MacroAssembler::inline_cache_check()
const Register receiver = R0; // see receiverOpr()
- __ load_klass(Rtemp, receiver);
- __ cmp(Rtemp, Ricklass);
- Label verified;
-
- __ b(verified, eq); // jump over alignment no-ops too
- __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
- __ align(CodeEntryAlignment);
+ __ verify_oop(receiver);
+ // Inline cache check
+ __ ic_check(CodeEntryAlignment /* end_alignment */);
// Verified entry point
- __ bind(verified);
int vep_offset = __ pc() - start;
-
if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
// Object.hashCode, System.identityHashCode can pull the hashCode from the header word
// instead of doing a full VM transition once it's been computed.
diff --git a/src/hotspot/cpu/arm/vtableStubs_arm.cpp b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
index 539e288f63fb2..1229b5073f506 100644
--- a/src/hotspot/cpu/arm/vtableStubs_arm.cpp
+++ b/src/hotspot/cpu/arm/vtableStubs_arm.cpp
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_arm.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "oops/klass.inline.hpp"
@@ -160,7 +160,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass(Rclass, R0);
// Receiver subtype check against REFC.
- __ ldr(Rintf, Address(Ricklass, CompiledICHolder::holder_klass_offset()));
+ __ ldr(Rintf, Address(Ricklass, CompiledICData::itable_refc_klass_offset()));
__ lookup_interface_method(// inputs: rec. class, interface, itable index
Rclass, Rintf, noreg,
// outputs: temp reg1, temp reg2
@@ -171,7 +171,7 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
start_pc = __ pc();
// Get Method* and entry point for compiler
- __ ldr(Rintf, Address(Ricklass, CompiledICHolder::holder_metadata_offset()));
+ __ ldr(Rintf, Address(Ricklass, CompiledICData::itable_defc_klass_offset()));
__ lookup_interface_method(// inputs: rec. class, interface, itable index
Rclass, Rintf, itable_index,
// outputs: temp reg1, temp reg2, temp reg3
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
index 47b681ce26be4..d78dec964cbb0 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.inline.hpp
@@ -451,7 +451,7 @@ inline void Assembler::bcctrl(int boint, int biint, int bhint, relocInfo::relocT
// helper function for b
inline bool Assembler::is_within_range_of_b(address a, address pc) {
- // Guard against illegal branch targets, e.g. -1 (see CompiledStaticCall and ad-file).
+ // Guard against illegal branch targets, e.g. -1 (see CompiledDirectCall and ad-file).
if ((((uint64_t)a) & 0x3) != 0) return false;
const int range = 1 << (29-6); // li field is from bit 6 to bit 29.
@@ -465,7 +465,7 @@ inline bool Assembler::is_within_range_of_b(address a, address pc) {
// helper functions for bcxx.
inline bool Assembler::is_within_range_of_bcxx(address a, address pc) {
- // Guard against illegal branch targets, e.g. -1 (see CompiledStaticCall and ad-file).
+ // Guard against illegal branch targets, e.g. -1 (see CompiledDirectCall and ad-file).
if ((((uint64_t)a) & 0x3) != 0) return false;
const int range = 1 << (29-16); // bd field is from bit 16 to bit 29.
diff --git a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
index 1a00c9ad268c7..dc70c73d4b330 100644
--- a/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_CodeStubs_ppc.cpp
@@ -456,6 +456,9 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
__ extsw(R7_ARG5, length()->as_register());
ce->emit_static_call_stub();
+ if (ce->compilation()->bailed_out()) {
+ return; // CodeCache is full
+ }
bool success = ce->emit_trampoline_stub_for_call(SharedRuntime::get_resolve_static_call_stub());
if (!success) { return; }
diff --git a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
index d316c2b3db2be..3ae35949b2148 100644
--- a/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_LIRAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -77,9 +77,7 @@ int LIR_Assembler::initial_frame_size_in_bytes() const {
// we fetch the class of the receiver and compare it with the cached class.
// If they do not match we jump to slow case.
int LIR_Assembler::check_icache() {
- int offset = __ offset();
- __ inline_cache_check(R3_ARG1, R19_inline_cache_reg);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -2300,7 +2298,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->tmp1()->as_register(),
op->tmp2()->as_register(),
op->tmp3()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
index 577dcae25f4bc..ba0187d0363ca 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2018 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -40,29 +40,6 @@
#include "utilities/macros.hpp"
#include "utilities/powerOfTwo.hpp"
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- const Register temp_reg = R12_scratch2;
- Label Lmiss;
-
- verify_oop(receiver, FILE_AND_LINE);
- load_klass_check_null(temp_reg, receiver, &Lmiss);
-
- if (TrapBasedICMissChecks && TrapBasedNullChecks) {
- trap_ic_miss_check(temp_reg, iCache);
- } else {
- Label Lok;
- cmpd(CCR0, temp_reg, iCache);
- beq(CCR0, Lok);
- bind(Lmiss);
- //load_const_optimized(temp_reg, SharedRuntime::get_ic_miss_stub(), R0);
- calculate_address_from_global_toc(temp_reg, SharedRuntime::get_ic_miss_stub(), true, true, false);
- mtctr(temp_reg);
- bctr();
- align(32, 12);
- bind(Lok);
- }
-}
-
void C1_MacroAssembler::explicit_null_check(Register base) {
Unimplemented();
@@ -333,7 +310,7 @@ void C1_MacroAssembler::allocate_array(
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
@@ -365,7 +342,7 @@ void C1_MacroAssembler::allocate_array(
sldi(t1, len, log2_elt_size);
arr_len_in_bytes = t1;
}
- addi(arr_size, arr_len_in_bytes, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ addi(arr_size, arr_len_in_bytes, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
clrrdi(arr_size, arr_size, LogMinObjAlignmentInBytes); // Align array size.
// Allocate space & initialize header.
@@ -375,8 +352,18 @@ void C1_MacroAssembler::allocate_array(
// Initialize body.
const Register base = t2;
const Register index = t3;
- addi(base, obj, hdr_size * wordSize); // compute address of first element
- addi(index, arr_size, -(hdr_size * wordSize)); // compute index = number of bytes to clear
+ addi(base, obj, base_offset_in_bytes); // compute address of first element
+ addi(index, arr_size, -(base_offset_in_bytes)); // compute index = number of bytes to clear
+
+ // Zero first 4 bytes, if start offset is not word aligned.
+ if (!is_aligned(base_offset_in_bytes, BytesPerWord)) {
+ assert(is_aligned(base_offset_in_bytes, BytesPerInt), "must be 4-byte aligned");
+ li(t1, 0);
+ stw(t1, 0, base);
+ addi(base, base, BytesPerInt);
+ // Note: initialize_body will align index down, no need to correct it here.
+ }
+
initialize_body(base, index);
if (CURRENT_ENV->dtrace_alloc_probes()) {
diff --git a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
index 5fa19d5fd5dad..ab31431e67d9c 100644
--- a/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/c1_MacroAssembler_ppc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2015 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -80,7 +80,7 @@
Register t1, // temp register
Register t2, // temp register
Register t3, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // continuation point if fast allocation fails
diff --git a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
index 2ba6a6bca4e03..63914c5d1cb93 100644
--- a/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
+++ b/src/hotspot/cpu/ppc/c1_Runtime1_ppc.cpp
@@ -34,7 +34,6 @@
#include "gc/shared/cardTableBarrierSet.hpp"
#include "interpreter/interpreter.hpp"
#include "nativeInst_ppc.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_ppc.hpp"
diff --git a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
index 54f9cfa936797..355ac4815d551 100644
--- a/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
+++ b/src/hotspot/cpu/ppc/compiledIC_ppc.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
@@ -37,7 +36,7 @@
// ----------------------------------------------------------------------------
-// A PPC CompiledDirectStaticCall looks like this:
+// A PPC CompiledDirectCall looks like this:
//
// >>>> consts
//
@@ -79,7 +78,7 @@
const int IC_pos_in_java_to_interp_stub = 8;
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
#ifdef COMPILER2
if (mark == nullptr) {
// Get the mark within main instrs section which is set to the address of the call.
@@ -91,7 +90,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
MacroAssembler _masm(&cbuf);
// Start the stub.
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return nullptr; // CodeCache is full
}
@@ -135,7 +134,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// FIXME: Assert that the stub can be identified and patched.
// Java_to_interp_stub_size should be good.
- assert((__ offset() - stub_start_offset) <= CompiledStaticCall::to_interp_stub_size(),
+ assert((__ offset() - stub_start_offset) <= CompiledDirectCall::to_interp_stub_size(),
"should be good size");
assert(!is_NativeCallTrampolineStub_at(__ addr_at(stub_start_offset)),
"must not confuse java_to_interp with trampoline stubs");
@@ -153,27 +152,20 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// Size of java_to_interp stub, this doesn't need to be accurate but it must
// be larger or equal to the real size of the stub.
// Used for optimization in Compile::Shorten_branches.
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 12 * BytesPerInstWord;
}
// Relocation entries for call stub, compiled java to interpreter.
// Used for optimization in Compile::Shorten_branches.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 5;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub + IC_pos_in_java_to_interp_stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -188,7 +180,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -204,7 +196,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
index cd2fd355bbb97..ab520162d350e 100644
--- a/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/g1/g1BarrierSetAssembler_ppc.cpp
@@ -30,9 +30,9 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/ppc/globals_ppc.hpp b/src/hotspot/cpu/ppc/globals_ppc.hpp
index f46ca5db3b7df..a2a94c178fb8b 100644
--- a/src/hotspot/cpu/ppc/globals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/globals_ppc.hpp
@@ -60,7 +60,7 @@ define_pd_global(bool, VMContinuations, true);
// Use large code-entry alignment.
define_pd_global(uintx, CodeCacheSegmentSize, 128);
-define_pd_global(intx, CodeEntryAlignment, 128);
+define_pd_global(intx, CodeEntryAlignment, 64);
define_pd_global(intx, OptoLoopAlignment, 16);
define_pd_global(intx, InlineSmallCode, 1500);
diff --git a/src/hotspot/cpu/ppc/icBuffer_ppc.cpp b/src/hotspot/cpu/ppc/icBuffer_ppc.cpp
deleted file mode 100644
index 4157a5b0fd788..0000000000000
--- a/src/hotspot/cpu/ppc/icBuffer_ppc.cpp
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_ppc.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm.
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return MacroAssembler::load_const_size + MacroAssembler::b64_patchable_size;
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler masm(&code);
- // Note: even though the code contains an embedded metadata, we do not need reloc info
- // because
- // (1) the metadata is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear.
-
- // Load the oop ...
- __ load_const(R19_method, (address) cached_value, R0);
- // ... and jump to entry point.
- __ b64_patchable((address) entry_point, relocInfo::none);
-
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(move->next_instruction_address());
- return jump->jump_destination();
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- void* o = (void*)move->data();
- return o;
-}
-
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
index b9d1cdb19ac9d..fe19cf0350020 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.cpp
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
#include "gc/shared/barrierSet.hpp"
@@ -1195,6 +1196,81 @@ void MacroAssembler::post_call_nop() {
assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found");
}
+int MacroAssembler::ic_check_size() {
+ bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
+ use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
+ use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
+
+ int num_ins;
+ if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
+ num_ins = 3;
+ if (use_trap_based_null_check) num_ins += 1;
+ } else {
+ num_ins = 7;
+ if (!implicit_null_checks_available) num_ins += 2;
+ }
+ return num_ins * BytesPerInstWord;
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
+ use_fast_receiver_null_check = implicit_null_checks_available || TrapBasedNullChecks,
+ use_trap_based_null_check = !implicit_null_checks_available && TrapBasedNullChecks;
+
+ Register receiver = R3_ARG1;
+ Register data = R19_inline_cache_reg;
+ Register tmp1 = R11_scratch1;
+ Register tmp2 = R12_scratch2;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, end_alignment, end_alignment - ic_check_size());
+
+ int uep_offset = offset();
+
+ if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
+ // Fast version which uses SIGTRAP
+
+ if (use_trap_based_null_check) {
+ trap_null_check(receiver);
+ }
+ if (UseCompressedClassPointers) {
+ lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ } else {
+ ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ }
+ ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
+ trap_ic_miss_check(tmp1, tmp2);
+
+ } else {
+ // Slower version which doesn't use SIGTRAP
+
+ // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
+ calculate_address_from_global_toc(tmp1, SharedRuntime::get_ic_miss_stub(),
+ true, true, false); // 2 instructions
+ mtctr(tmp1);
+
+ if (!implicit_null_checks_available) {
+ cmpdi(CCR0, receiver, 0);
+ beqctr(CCR0);
+ }
+ if (UseCompressedClassPointers) {
+ lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ } else {
+ ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
+ }
+ ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
+ cmpd(CCR0, tmp1, tmp2);
+ bnectr(CCR0);
+ }
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
+}
+
void MacroAssembler::call_VM_base(Register oop_result,
Register last_java_sp,
address entry_point,
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
index cddc8b92fa09a..ec370a450ac35 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.hpp
@@ -367,6 +367,9 @@ class MacroAssembler: public Assembler {
Register toc);
#endif
+ static int ic_check_size();
+ int ic_check(int end_alignment);
+
protected:
// It is imperative that all calls into the VM are handled via the
diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad
index 783edf727b3bc..110c6b9b66860 100644
--- a/src/hotspot/cpu/ppc/ppc.ad
+++ b/src/hotspot/cpu/ppc/ppc.ad
@@ -1978,42 +1978,7 @@ void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
- // Inline_cache contains a klass.
- Register ic_klass = as_Register(Matcher::inline_cache_reg_encode());
- Register receiver_klass = R12_scratch2; // tmp
-
- assert_different_registers(ic_klass, receiver_klass, R11_scratch1, R3_ARG1);
- assert(R11_scratch1 == R11, "need prologue scratch register");
-
- // Check for nullptr argument if we don't have implicit null checks.
- if (!ImplicitNullChecks || !os::zero_page_read_protected()) {
- if (TrapBasedNullChecks) {
- __ trap_null_check(R3_ARG1);
- } else {
- Label valid;
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ bne_predict_taken(CCR0, valid);
- // We have a null argument, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ bind(valid);
- }
- }
- // Assume argument is not nullptr, load klass from receiver.
- __ load_klass(receiver_klass, R3_ARG1);
-
- if (TrapBasedICMissChecks) {
- __ trap_ic_miss_check(receiver_klass, ic_klass);
- } else {
- Label valid;
- __ cmpd(CCR0, receiver_klass, ic_klass);
- __ beq_predict_taken(CCR0, valid);
- // We have an unexpected klass, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ bind(valid);
- }
-
+ __ ic_check(CodeEntryAlignment);
// Argument is valid and klass is as expected, continue.
}
@@ -2062,7 +2027,10 @@ int HandlerImpl::emit_exception_handler(CodeBuffer &cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_exception_handler());
- if (base == nullptr) return 0; // CodeBuffer::expand failed
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return 0; // CodeBuffer::expand failed
+ }
int offset = __ offset();
__ b64_patchable((address)OptoRuntime::exception_blob()->content_begin(),
@@ -2079,7 +2047,10 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
C2_MacroAssembler _masm(&cbuf);
address base = __ start_a_stub(size_deopt_handler());
- if (base == nullptr) return 0; // CodeBuffer::expand failed
+ if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
+ return 0; // CodeBuffer::expand failed
+ }
int offset = __ offset();
__ bl64_patchable((address)SharedRuntime::deopt_blob()->unpack(),
@@ -2801,15 +2772,16 @@ encode %{
intptr_t val = $src$$constant;
relocInfo::relocType constant_reloc = $src->constant_reloc(); // src
address const_toc_addr;
+ RelocationHolder r; // Initializes type to none.
if (constant_reloc == relocInfo::oop_type) {
// Create an oop constant and a corresponding relocation.
- AddressLiteral a = __ allocate_oop_address((jobject)val);
+ AddressLiteral a = __ constant_oop_address((jobject)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
+ r = a.rspec();
} else if (constant_reloc == relocInfo::metadata_type) {
+ // Notify OOP recorder (don't need the relocation)
AddressLiteral a = __ constant_metadata_address((Metadata *)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
} else {
// Create a non-oop constant, no relocation needed.
const_toc_addr = __ long_constant((jlong)$src$$constant);
@@ -2819,6 +2791,7 @@ encode %{
ciEnv::current()->record_out_of_memory_failure();
return;
}
+ __ relocate(r); // If set above.
// Get the constant's TOC offset.
toc_offset = __ offset_to_method_toc(const_toc_addr);
@@ -2832,15 +2805,16 @@ encode %{
intptr_t val = $src$$constant;
relocInfo::relocType constant_reloc = $src->constant_reloc(); // src
address const_toc_addr;
+ RelocationHolder r; // Initializes type to none.
if (constant_reloc == relocInfo::oop_type) {
// Create an oop constant and a corresponding relocation.
- AddressLiteral a = __ allocate_oop_address((jobject)val);
+ AddressLiteral a = __ constant_oop_address((jobject)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
+ r = a.rspec();
} else if (constant_reloc == relocInfo::metadata_type) {
+ // Notify OOP recorder (don't need the relocation)
AddressLiteral a = __ constant_metadata_address((Metadata *)val);
const_toc_addr = __ address_constant((address)a.value(), RelocationHolder::none);
- __ relocate(a.rspec());
} else { // non-oop pointers, e.g. card mark base, heap top
// Create a non-oop constant, no relocation needed.
const_toc_addr = __ long_constant((jlong)$src$$constant);
@@ -2850,6 +2824,7 @@ encode %{
ciEnv::current()->record_out_of_memory_failure();
return;
}
+ __ relocate(r); // If set above.
// Get the constant's TOC offset.
const int toc_offset = __ offset_to_method_toc(const_toc_addr);
// Store the toc offset of the constant.
@@ -3452,7 +3427,7 @@ encode %{
__ bl(__ pc()); // Emits a relocation.
// The stub for call to interpreter.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
@@ -3507,7 +3482,7 @@ encode %{
// Create the nodes for loading the IC from the TOC.
loadConLNodesTuple loadConLNodes_IC =
- loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong)Universe::non_oop_word()),
+ loadConLNodesTuple_create(ra_, n_toc, new immLOper((jlong) Universe::non_oop_word()),
OptoReg::Name(R19_H_num), OptoReg::Name(R19_num));
// Create the call node.
diff --git a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
index ebe918785edc0..5a080adc7a9fa 100644
--- a/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
+++ b/src/hotspot/cpu/ppc/sharedRuntime_ppc.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/debugInfoRec.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "frame_ppc.hpp"
#include "compiler/oopMap.hpp"
@@ -35,7 +34,6 @@
#include "interpreter/interpreter.hpp"
#include "interpreter/interp_masm.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/continuation.hpp"
@@ -1174,8 +1172,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
BLOCK_COMMENT("c2i unverified entry");
c2i_unverified_entry = __ pc();
- // inline_cache contains a compiledICHolder
- const Register ic = R19_method;
+ // inline_cache contains a CompiledICData
+ const Register ic = R19_inline_cache_reg;
const Register ic_klass = R11_scratch1;
const Register receiver_klass = R12_scratch2;
const Register code = R21_tmp1;
@@ -1186,45 +1184,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label call_interpreter;
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
- "klass offset should reach into any page");
- // Check for null argument if we don't have implicit null checks.
- if (!ImplicitNullChecks || !os::zero_page_read_protected()) {
- if (TrapBasedNullChecks) {
- __ trap_null_check(R3_ARG1);
- } else {
- Label valid;
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ bne_predict_taken(CCR0, valid);
- // We have a null argument, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ BIND(valid);
- }
- }
- // Assume argument is not null, load klass from receiver.
- __ load_klass(receiver_klass, R3_ARG1);
-
- __ ld(ic_klass, CompiledICHolder::holder_klass_offset(), ic);
-
- if (TrapBasedICMissChecks) {
- __ trap_ic_miss_check(receiver_klass, ic_klass);
- } else {
- Label valid;
- __ cmpd(CCR0, receiver_klass, ic_klass);
- __ beq_predict_taken(CCR0, valid);
- // We have an unexpected klass, branch to ic_miss_stub.
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- __ BIND(valid);
- }
-
+ __ ic_check(4 /* end_alignment */);
+ __ ld(R19_method, CompiledICData::speculated_method_offset(), ic);
// Argument is valid and klass is as expected, continue.
- // Extract method from inline cache, verified entry point needs it.
- __ ld(R19_method, CompiledICHolder::holder_metadata_offset(), ic);
- assert(R19_method == ic, "the inline cache register is dead here");
-
__ ld(code, method_(code));
__ cmpdi(CCR0, code, 0);
__ ld(ientry, method_(interpreter_entry)); // preloaded
@@ -1798,7 +1761,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// static stub for the call above
CodeBuffer* cbuf = masm->code_section()->outer();
- stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, c2i_call_pc);
+ stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, c2i_call_pc);
guarantee(stub != nullptr, "no space for static stub");
}
@@ -1891,7 +1854,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// static stub for the call above
CodeBuffer* cbuf = masm->code_section()->outer();
- stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, call_pc);
+ stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, call_pc);
guarantee(stub != nullptr, "no space for static stub");
}
@@ -2188,7 +2151,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
intptr_t frame_done_pc;
intptr_t oopmap_pc;
- Label ic_miss;
Label handle_pending_exception;
Register r_callers_sp = R21;
@@ -2212,19 +2174,9 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
// Check ic: object class == cached class?
if (!method_is_static) {
- Register ic = R19_inline_cache_reg;
- Register receiver_klass = r_temp_1;
-
- __ cmpdi(CCR0, R3_ARG1, 0);
- __ beq(CCR0, ic_miss);
- __ verify_oop(R3_ARG1, FILE_AND_LINE);
- __ load_klass(receiver_klass, R3_ARG1);
-
- __ cmpd(CCR0, receiver_klass, ic);
- __ bne(CCR0, ic_miss);
+ __ ic_check(4 /* end_alignment */);
}
-
// Generate the Verified Entry Point (VEP).
// --------------------------------------------------------------------------
vep_start_pc = (intptr_t)__ pc();
@@ -2704,16 +2656,6 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ b64_patchable((address)StubRoutines::forward_exception_entry(),
relocInfo::runtime_call_type);
- // Handler for a cache miss (out-of-line).
- // --------------------------------------------------------------------------
-
- if (!method_is_static) {
- __ bind(ic_miss);
-
- __ b64_patchable((address)SharedRuntime::get_ic_miss_stub(),
- relocInfo::runtime_call_type);
- }
-
// Done.
// --------------------------------------------------------------------------
diff --git a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
index a1c8f46ce2d57..b60fd4f16d163 100644
--- a/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
+++ b/src/hotspot/cpu/ppc/upcallLinker_ppc.cpp
@@ -243,10 +243,14 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ load_const_optimized(R19_method, (intptr_t)entry);
__ std(R19_method, in_bytes(JavaThread::callee_target_offset()), R16_thread);
+ __ push_cont_fastpath();
+
__ ld(call_target_address, in_bytes(Method::from_compiled_offset()), R19_method);
__ mtctr(call_target_address);
__ bctrl();
+ __ pop_cont_fastpath();
+
// return value shuffle
if (!needs_return_buffer) {
// CallArranger can pick a return type that goes in the same reg for both CCs.
diff --git a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
index fe4eb3df8f12f..28ba04d833bed 100644
--- a/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
+++ b/src/hotspot/cpu/ppc/vtableStubs_ppc_64.cpp
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_ppc.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/klassVtable.hpp"
@@ -181,13 +181,13 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass_check_null(rcvr_klass, R3_ARG1);
// Receiver subtype check against REFC.
- __ ld(interface, CompiledICHolder::holder_klass_offset(), R19_method);
+ __ ld(interface, CompiledICData::itable_refc_klass_offset(), R19_method);
__ lookup_interface_method(rcvr_klass, interface, noreg,
R0, tmp1, tmp2,
L_no_such_interface, /*return_method=*/ false);
// Get Method* and entrypoint for compiler
- __ ld(interface, CompiledICHolder::holder_metadata_offset(), R19_method);
+ __ ld(interface, CompiledICData::itable_defc_klass_offset(), R19_method);
__ lookup_interface_method(rcvr_klass, interface, itable_index,
R19_method, tmp1, tmp2,
L_no_such_interface, /*return_method=*/ true);
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 0bbf3771a04bc..5d0fa3fad3cec 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -51,7 +51,6 @@
#endif
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = t1; // where the IC klass is cached
const Register SYNC_header = x10; // synchronization header
const Register SHIFT_count = x10; // where count for shift operations must be
@@ -265,26 +264,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- int start_offset = __ offset();
- Label dont;
- __ inline_cache_check(receiver, ic_klass, dont);
-
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // We align the verified entry point unless the method body
- // (including its inline cache check) will fit in a single 64-byte
- // icache line.
- if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
- // force alignment after the cache check.
- __ align(CodeEntryAlignment);
- }
-
- __ bind(dont);
- return start_offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::jobject2reg(jobject o, Register reg) {
@@ -1040,7 +1020,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
@@ -1398,7 +1378,7 @@ void LIR_Assembler::emit_static_call_stub() {
__ relocate(static_stub_Relocation::spec(call_pc));
__ emit_static_call_stub();
- assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
+ assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
<= call_stub_size(), "stub too big");
__ end_a_stub();
}
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
index b088498e6fc08..ce23213776c08 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.hpp
@@ -68,7 +68,7 @@ friend class ArrayCopyStub;
enum {
// See emit_static_call_stub for detail
- // CompiledStaticCall::to_interp_stub_size() (14) + CompiledStaticCall::to_trampoline_stub_size() (1 + 3 + address)
+ // CompiledDirectCall::to_interp_stub_size() (14) + CompiledDirectCall::to_trampoline_stub_size() (1 + 3 + address)
_call_stub_size = 14 * NativeInstruction::instruction_size +
(NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size),
// See emit_exception_handler for detail
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
index 6c1dce0de1598..770dd6a9d0f37 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -280,7 +280,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case) {
assert_different_registers(obj, len, tmp1, tmp2, klass);
// determine alignment mask
@@ -292,7 +292,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
const Register arr_size = tmp2; // okay to be the same
// align object end
- mv(arr_size, (int32_t)header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ mv(arr_size, (int32_t)base_offset_in_bytes + MinObjAlignmentInBytesMask);
shadd(arr_size, len, arr_size, t0, f);
andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask);
@@ -300,9 +300,20 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
initialize_header(obj, klass, len, tmp1, tmp2);
+ // Clear leading 4 bytes, if necessary.
+ // TODO: This could perhaps go into initialize_body() and also clear the leading 4 bytes
+ // for non-array objects, thereby replacing the klass-gap clearing code in initialize_header().
+ int base_offset = base_offset_in_bytes;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ sw(zr, Address(obj, base_offset));
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, BytesPerWord), "must be word-aligned");
+
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ initialize_body(obj, arr_size, base_offset, len_zero);
membar(MacroAssembler::StoreStore);
@@ -314,15 +325,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register tmp1
verify_oop(obj);
}
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache, Label &L) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
- assert_different_registers(receiver, iCache, t0, t2);
- cmp_klass(receiver, iCache, t0, t2 /* call-clobbered t2 as a tmp */, L);
-}
-
void C1_MacroAssembler::build_frame(int framesize, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= framesize, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
index b737a438511c8..2d7f8d7485d4f 100644
--- a/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/c1_MacroAssembler_riscv.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014, 2015, Red Hat Inc. All rights reserved.
* Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -101,7 +101,7 @@ using MacroAssembler::null_check;
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int header_size, int f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register tmp1, Register tmp2, int base_offset_in_bytes, int f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
diff --git a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
index b76163a30841d..9fa8939837a85 100644
--- a/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_Runtime1_riscv.cpp
@@ -37,7 +37,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_riscv.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_riscv.hpp"
diff --git a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
index e29dee56de8d8..fdb2bcb06ff97 100644
--- a/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
+++ b/src/hotspot/cpu/riscv/compiledIC_riscv.cpp
@@ -27,7 +27,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -37,7 +36,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
precond(cbuf.stubs()->start() != badAddress);
precond(cbuf.stubs()->end() != badAddress);
// Stub is fixed up when the corresponding call is converted from
@@ -69,11 +68,11 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return MacroAssembler::static_call_stub_size();
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// Somewhat pessimistically, we count 4 instructions here (although
// there are only 3) because we sometimes emit an alignment nop.
// Trampoline stubs are always word aligned.
@@ -81,21 +80,14 @@ int CompiledStaticCall::to_trampoline_stub_size() {
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder
= nativeMovConstReg_at(stub);
@@ -112,7 +104,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -129,7 +121,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
index 5d0b0fb472934..fa7df32d7e944 100644
--- a/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/g1/g1BarrierSetAssembler_riscv.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
index e368bbdc9141f..68cd51ece5f70 100644
--- a/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
+++ b/src/hotspot/cpu/riscv/globalDefinitions_riscv.hpp
@@ -50,6 +50,9 @@ const bool CCallingConventionRequiresIntsAsLongs = false;
#define USE_POINTERS_TO_REGISTER_IMPL_ARRAY
+// auipc useable for all cc -> cc calls and jumps
+#define CODE_CACHE_SIZE_LIMIT ((2*G)-(2*K))
+
// The expected size in bytes of a cache line.
#define DEFAULT_CACHE_LINE_SIZE 64
diff --git a/src/hotspot/cpu/riscv/icBuffer_riscv.cpp b/src/hotspot/cpu/riscv/icBuffer_riscv.cpp
deleted file mode 100644
index ab904817816fc..0000000000000
--- a/src/hotspot/cpu/riscv/icBuffer_riscv.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_riscv.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // 6: auipc + ld + auipc + jalr + address(2 * instruction_size)
- return 6 * NativeInstruction::instruction_size;
-}
-
-#define __ masm->
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- assert_cond(code_begin != nullptr && entry_point != nullptr);
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // Note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
-
- address start = __ pc();
- Label l;
- __ ld(t1, l);
- __ far_jump(ExternalAddress(entry_point));
- __ align(wordSize);
- __ bind(l);
- __ emit_int64((intptr_t)cached_value);
- // Only need to invalidate the 1st two instructions - not the whole ic stub
- ICache::invalidate_range(code_begin, InlineCacheBuffer::ic_stub_code_size());
- assert(__ pc() - start == ic_stub_code_size(), "must be");
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- NativeJump* jump = nativeJump_at(move->next_instruction_address());
- return jump->jump_destination();
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // The word containing the cached value is at the end of this IC buffer
- uintptr_t *p = (uintptr_t *)(code_begin + ic_stub_code_size() - wordSize);
- void* o = (void*)*p;
- return o;
-}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index ce336c16aa718..96e07319e843f 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -27,6 +27,7 @@
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -634,8 +635,8 @@ void MacroAssembler::unimplemented(const char* what) {
}
void MacroAssembler::emit_static_call_stub() {
- IncompressibleRegion ir(this); // Fixed length: see CompiledStaticCall::to_interp_stub_size().
- // CompiledDirectStaticCall::set_to_interpreted knows the
+ IncompressibleRegion ir(this); // Fixed length: see CompiledDirectCall::to_interp_stub_size().
+ // CompiledDirectCall::set_to_interpreted knows the
// exact layout of this stub.
mov_metadata(xmethod, (Metadata*)nullptr);
@@ -2542,7 +2543,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -3542,6 +3543,48 @@ address MacroAssembler::ic_call(address entry, jint method_index) {
return trampoline_call(Address(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ // No compressed
+ return (NativeInstruction::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) +
+ far_branch_size();
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ IncompressibleRegion ir(this);
+ Register receiver = j_rarg0;
+ Register data = t1;
+
+ Register tmp1 = t0; // t0 always scratch
+ // t2 is saved on call, thus should have been saved before this check.
+ // Hence we can clobber it.
+ Register tmp2 = t2;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, ic_check_size());
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ } else {
+ ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ ld(tmp2, Address(data, CompiledICData::speculated_klass_offset()));
+ }
+
+ Label ic_hit;
+ beq(tmp1, tmp2, ic_hit);
+ // Note, far_jump is not fixed size.
+ // Is this ever generates a movptr alignment/size will be off.
+ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ bind(ic_hit);
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point.");
+ return uep_offset;
+}
+
// Emit a trampoline stub for a call to a target which is too far away.
//
// code sequences:
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index d283654e6e179..63cfb22855180 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -1193,7 +1193,10 @@ class MacroAssembler: public Assembler {
//
// Return: the call PC or null if CodeCache is full.
address trampoline_call(Address entry);
+
address ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment = NativeInstruction::instruction_size);
// Support for memory inc/dec
// n.b. increment/decrement calls with an Address destination will
diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad
index a6f0959942414..10a80cd094024 100644
--- a/src/hotspot/cpu/riscv/riscv.ad
+++ b/src/hotspot/cpu/riscv/riscv.ad
@@ -1808,14 +1808,13 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
assert_cond(st != nullptr);
st->print_cr("# MachUEPNode");
if (UseCompressedClassPointers) {
- st->print_cr("\tlwu t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- if (CompressedKlassPointers::shift() != 0) {
- st->print_cr("\tdecode_klass_not_null t0, t0");
- }
+ st->print_cr("\tlwu t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tlwu t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
} else {
- st->print_cr("\tld t0, [j_rarg0, oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tld t0, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tld t2, [t1 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
}
- st->print_cr("\tbeq t0, t1, ic_hit");
+ st->print_cr("\tbeq t0, t2, ic_hit");
st->print_cr("\tj, SharedRuntime::_ic_miss_stub\t # Inline cache check");
st->print_cr("\tic_hit:");
}
@@ -1825,15 +1824,11 @@ void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
// This is the unverified entry point.
C2_MacroAssembler _masm(&cbuf);
+ __ ic_check(CodeEntryAlignment);
- Label skip;
- __ cmp_klass(j_rarg0, t1, t0, t2 /* call-clobbered t2 as a tmp */, skip);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(skip);
-
- // These NOPs are critical so that verified entry point is properly
- // 4 bytes aligned for patching by NativeJump::patch_verified_entry()
- __ align(NativeInstruction::instruction_size);
+ // Verified entry point must be properly 4 bytes aligned for patching by NativeJump::patch_verified_entry().
+ // ic_check() aligns to CodeEntryAlignment >= InteriorEntryAlignment(min 16) > NativeInstruction::instruction_size(4).
+ assert(((__ offset()) % CodeEntryAlignment) == 0, "Misaligned verified entry point");
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
@@ -2402,7 +2397,7 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
} else {
// Emit stub for static call
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, call);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
diff --git a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
index 9f04e20ea3b73..7435b552d15de 100644
--- a/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
+++ b/src/hotspot/cpu/riscv/sharedRuntime_riscv.cpp
@@ -29,7 +29,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -38,7 +37,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_riscv.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -622,10 +620,8 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
-
- const Register holder = t1;
const Register receiver = j_rarg0;
+ const Register data = t1;
const Register tmp = t2; // A call-clobbered register not used for arg passing
// -------------------------------------------------------------------------
@@ -639,16 +635,10 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
{
__ block_comment("c2i_unverified_entry {");
- __ load_klass(t0, receiver, tmp);
- __ ld(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ ld(xmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ beq(t0, tmp, ok);
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- __ bind(ok);
- // Method might have been compiled since the call site was patched to
- // interpreted; if that is the case treat it as a miss so we can get
- // the call site corrected.
+ __ ic_check();
+ __ ld(xmethod, Address(data, CompiledICData::speculated_method_offset()));
+
__ ld(t0, Address(xmethod, in_bytes(Method::code_offset())));
__ beqz(t0, skip_fixup);
__ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
@@ -985,7 +975,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ j(exit);
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1051,7 +1041,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
}
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, tr_call);
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1425,19 +1415,10 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
const Register ic_reg = t1;
const Register receiver = j_rarg0;
- Label hit;
- Label exception_pending;
-
__ verify_oop(receiver);
- assert_different_registers(ic_reg, receiver, t0, t2);
- __ cmp_klass(receiver, ic_reg, t0, t2 /* call-clobbered t2 as a tmp */, hit);
-
- __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ assert_different_registers(receiver, t0, t1);
- // Verified entry point must be aligned
- __ align(8);
-
- __ bind(hit);
+ __ ic_check();
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -1872,6 +1853,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ leave();
// Any exception pending?
+ Label exception_pending;
__ ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset())));
__ bnez(t0, exception_pending);
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index 4bd33d08f8928..bbdafb922cc46 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -4809,6 +4809,348 @@ class StubGenerator: public StubCodeGenerator {
return (address) start;
}
+
+ // ------------------------ SHA-1 intrinsic ------------------------
+
+ // K't =
+ // 5a827999, 0 <= t <= 19
+ // 6ed9eba1, 20 <= t <= 39
+ // 8f1bbcdc, 40 <= t <= 59
+ // ca62c1d6, 60 <= t <= 79
+ void sha1_prepare_k(Register cur_k, int round) {
+ assert(round >= 0 && round < 80, "must be");
+
+ static const int64_t ks[] = {0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xca62c1d6};
+ if ((round % 20) == 0) {
+ __ mv(cur_k, ks[round/20]);
+ }
+ }
+
+ // W't =
+ // M't, 0 <= t <= 15
+ // ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ void sha1_prepare_w(Register cur_w, Register ws[], Register buf, int round) {
+ assert(round >= 0 && round < 80, "must be");
+
+ if (round < 16) {
+ // in the first 16 rounds, in ws[], every register contains 2 W't, e.g.
+ // in ws[0], high part contains W't-0, low part contains W't-1,
+ // in ws[1], high part contains W't-2, low part contains W't-3,
+ // ...
+ // in ws[7], high part contains W't-14, low part contains W't-15.
+
+ if ((round % 2) == 0) {
+ __ ld(ws[round/2], Address(buf, (round/2) * 8));
+ // reverse bytes, as SHA-1 is defined in big-endian.
+ __ revb(ws[round/2], ws[round/2]);
+ __ srli(cur_w, ws[round/2], 32);
+ } else {
+ __ mv(cur_w, ws[round/2]);
+ }
+
+ return;
+ }
+
+ if ((round % 2) == 0) {
+ int idx = 16;
+ // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ __ srli(t1, ws[(idx-8)/2], 32);
+ __ xorr(t0, ws[(idx-3)/2], t1);
+
+ __ srli(t1, ws[(idx-14)/2], 32);
+ __ srli(cur_w, ws[(idx-16)/2], 32);
+ __ xorr(cur_w, cur_w, t1);
+
+ __ xorr(cur_w, cur_w, t0);
+ __ rolw_imm(cur_w, cur_w, 1, t0);
+
+ // copy the cur_w value to ws[8].
+ // now, valid w't values are at:
+ // w0: ws[0]'s lower 32 bits
+ // w1 ~ w14: ws[1] ~ ws[7]
+ // w15: ws[8]'s higher 32 bits
+ __ slli(ws[idx/2], cur_w, 32);
+
+ return;
+ }
+
+ int idx = 17;
+ // W't = ROTL'1(W't-3 ^ W't-8 ^ W't-14 ^ W't-16), 16 <= t <= 79
+ __ srli(t1, ws[(idx-3)/2], 32);
+ __ xorr(t0, t1, ws[(idx-8)/2]);
+
+ __ xorr(cur_w, ws[(idx-16)/2], ws[(idx-14)/2]);
+
+ __ xorr(cur_w, cur_w, t0);
+ __ rolw_imm(cur_w, cur_w, 1, t0);
+
+ // copy the cur_w value to ws[8]
+ __ zero_extend(cur_w, cur_w, 32);
+ __ orr(ws[idx/2], ws[idx/2], cur_w);
+
+ // shift the w't registers, so they start from ws[0] again.
+ // now, valid w't values are at:
+ // w0 ~ w15: ws[0] ~ ws[7]
+ Register ws_0 = ws[0];
+ for (int i = 0; i < 16/2; i++) {
+ ws[i] = ws[i+1];
+ }
+ ws[8] = ws_0;
+ }
+
+ // f't(x, y, z) =
+ // Ch(x, y, z) = (x & y) ^ (~x & z) , 0 <= t <= 19
+ // Parity(x, y, z) = x ^ y ^ z , 20 <= t <= 39
+ // Maj(x, y, z) = (x & y) ^ (x & z) ^ (y & z) , 40 <= t <= 59
+ // Parity(x, y, z) = x ^ y ^ z , 60 <= t <= 79
+ void sha1_f(Register dst, Register x, Register y, Register z, int round) {
+ assert(round >= 0 && round < 80, "must be");
+ assert_different_registers(dst, x, y, z, t0, t1);
+
+ if (round < 20) {
+ // (x & y) ^ (~x & z)
+ __ andr(t0, x, y);
+ __ andn(dst, z, x);
+ __ xorr(dst, dst, t0);
+ } else if (round >= 40 && round < 60) {
+ // (x & y) ^ (x & z) ^ (y & z)
+ __ andr(t0, x, y);
+ __ andr(t1, x, z);
+ __ andr(dst, y, z);
+ __ xorr(dst, dst, t0);
+ __ xorr(dst, dst, t1);
+ } else {
+ // x ^ y ^ z
+ __ xorr(dst, x, y);
+ __ xorr(dst, dst, z);
+ }
+ }
+
+ // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
+ // e = d
+ // d = c
+ // c = ROTL'30(b)
+ // b = a
+ // a = T
+ void sha1_process_round(Register a, Register b, Register c, Register d, Register e,
+ Register cur_k, Register cur_w, Register tmp, int round) {
+ assert(round >= 0 && round < 80, "must be");
+ assert_different_registers(a, b, c, d, e, cur_w, cur_k, tmp, t0);
+
+ // T = ROTL'5(a) + f't(b, c, d) + e + K't + W't
+
+ // cur_w will be recalculated at the beginning of each round,
+ // so, we can reuse it as a temp register here.
+ Register tmp2 = cur_w;
+
+ // reuse e as a temporary register, as we will mv new value into it later
+ Register tmp3 = e;
+ __ add(tmp2, cur_k, tmp2);
+ __ add(tmp3, tmp3, tmp2);
+ __ rolw_imm(tmp2, a, 5, t0);
+
+ sha1_f(tmp, b, c, d, round);
+
+ __ add(tmp2, tmp2, tmp);
+ __ add(tmp2, tmp2, tmp3);
+
+ // e = d
+ // d = c
+ // c = ROTL'30(b)
+ // b = a
+ // a = T
+ __ mv(e, d);
+ __ mv(d, c);
+
+ __ rolw_imm(c, b, 30);
+ __ mv(b, a);
+ __ mv(a, tmp2);
+ }
+
+ // H(i)0 = a + H(i-1)0
+ // H(i)1 = b + H(i-1)1
+ // H(i)2 = c + H(i-1)2
+ // H(i)3 = d + H(i-1)3
+ // H(i)4 = e + H(i-1)4
+ void sha1_calculate_im_hash(Register a, Register b, Register c, Register d, Register e,
+ Register prev_ab, Register prev_cd, Register prev_e) {
+ assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ __ add(a, a, prev_ab);
+ __ srli(prev_ab, prev_ab, 32);
+ __ add(b, b, prev_ab);
+
+ __ add(c, c, prev_cd);
+ __ srli(prev_cd, prev_cd, 32);
+ __ add(d, d, prev_cd);
+
+ __ add(e, e, prev_e);
+ }
+
+ void sha1_preserve_prev_abcde(Register a, Register b, Register c, Register d, Register e,
+ Register prev_ab, Register prev_cd, Register prev_e) {
+ assert_different_registers(a, b, c, d, e, prev_ab, prev_cd, prev_e, t0);
+
+ __ slli(t0, b, 32);
+ __ zero_extend(prev_ab, a, 32);
+ __ orr(prev_ab, prev_ab, t0);
+
+ __ slli(t0, d, 32);
+ __ zero_extend(prev_cd, c, 32);
+ __ orr(prev_cd, prev_cd, t0);
+
+ __ mv(prev_e, e);
+ }
+
+ // Intrinsic for:
+ // void sun.security.provider.SHA.implCompress0(byte[] buf, int ofs)
+ // void sun.security.provider.DigestBase.implCompressMultiBlock0(byte[] b, int ofs, int limit)
+ //
+ // Arguments:
+ //
+ // Inputs:
+ // c_rarg0: byte[] src array + offset
+ // c_rarg1: int[] SHA.state
+ // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
+ // c_rarg2: int offset
+ // c_rarg3: int limit
+ //
+ // Outputs:
+ // - - - - - - below are only for implCompressMultiBlock0 - - - - - -
+ // c_rarg0: int offset, when (multi_block == true)
+ //
+ address generate_sha1_implCompress(bool multi_block, const char *name) {
+ __ align(CodeEntryAlignment);
+ StubCodeMark mark(this, "StubRoutines", name);
+
+ address start = __ pc();
+ __ enter();
+
+ RegSet saved_regs = RegSet::range(x18, x27);
+ if (multi_block) {
+ // use x9 as src below.
+ saved_regs += RegSet::of(x9);
+ }
+ __ push_reg(saved_regs, sp);
+
+ // c_rarg0 - c_rarg3: x10 - x13
+ Register buf = c_rarg0;
+ Register state = c_rarg1;
+ Register offset = c_rarg2;
+ Register limit = c_rarg3;
+ // use src to contain the original start point of the array.
+ Register src = x9;
+
+ if (multi_block) {
+ __ sub(limit, limit, offset);
+ __ add(limit, limit, buf);
+ __ sub(src, buf, offset);
+ }
+
+ // [args-reg]: x14 - x17
+ // [temp-reg]: x28 - x31
+ // [saved-reg]: x18 - x27
+
+ // h0/1/2/3/4
+ const Register a = x14, b = x15, c = x16, d = x17, e = x28;
+ // w0, w1, ... w15
+ // put two adjecent w's in one register:
+ // one at high word part, another at low word part
+ // at different round (even or odd), w't value reside in different items in ws[].
+ // w0 ~ w15, either reside in
+ // ws[0] ~ ws[7], where
+ // w0 at higher 32 bits of ws[0],
+ // w1 at lower 32 bits of ws[0],
+ // ...
+ // w14 at higher 32 bits of ws[7],
+ // w15 at lower 32 bits of ws[7].
+ // or, reside in
+ // w0: ws[0]'s lower 32 bits
+ // w1 ~ w14: ws[1] ~ ws[7]
+ // w15: ws[8]'s higher 32 bits
+ Register ws[9] = {x29, x30, x31, x18,
+ x19, x20, x21, x22,
+ x23}; // auxiliary register for calculating w's value
+ // current k't's value
+ const Register cur_k = x24;
+ // current w't's value
+ const Register cur_w = x25;
+ // values of a, b, c, d, e in the previous round
+ const Register prev_ab = x26, prev_cd = x27;
+ const Register prev_e = offset; // reuse offset/c_rarg2
+
+ // load 5 words state into a, b, c, d, e.
+ //
+ // To minimize the number of memory operations, we apply following
+ // optimization: read the states (a/b/c/d) of 4-byte values in pairs,
+ // with a single ld, and split them into 2 registers.
+ //
+ // And, as the core algorithm of SHA-1 works on 32-bits words, so
+ // in the following code, it does not care about the content of
+ // higher 32-bits in a/b/c/d/e. Based on this observation,
+ // we can apply further optimization, which is to just ignore the
+ // higher 32-bits in a/c/e, rather than set the higher
+ // 32-bits of a/c/e to zero explicitly with extra instructions.
+ __ ld(a, Address(state, 0));
+ __ srli(b, a, 32);
+ __ ld(c, Address(state, 8));
+ __ srli(d, c, 32);
+ __ lw(e, Address(state, 16));
+
+ Label L_sha1_loop;
+ if (multi_block) {
+ __ BIND(L_sha1_loop);
+ }
+
+ sha1_preserve_prev_abcde(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ for (int round = 0; round < 80; round++) {
+ // prepare K't value
+ sha1_prepare_k(cur_k, round);
+
+ // prepare W't value
+ sha1_prepare_w(cur_w, ws, buf, round);
+
+ // one round process
+ sha1_process_round(a, b, c, d, e, cur_k, cur_w, t2, round);
+ }
+
+ // compute the intermediate hash value
+ sha1_calculate_im_hash(a, b, c, d, e, prev_ab, prev_cd, prev_e);
+
+ if (multi_block) {
+ int64_t block_bytes = 16 * 4;
+ __ addi(buf, buf, block_bytes);
+
+ __ bge(limit, buf, L_sha1_loop, true);
+ }
+
+ // store back the state.
+ __ zero_extend(a, a, 32);
+ __ slli(b, b, 32);
+ __ orr(a, a, b);
+ __ sd(a, Address(state, 0));
+ __ zero_extend(c, c, 32);
+ __ slli(d, d, 32);
+ __ orr(c, c, d);
+ __ sd(c, Address(state, 8));
+ __ sw(e, Address(state, 16));
+
+ // return offset
+ if (multi_block) {
+ __ sub(c_rarg0, buf, src);
+ }
+
+ __ pop_reg(saved_regs, sp);
+
+ __ leave();
+ __ ret();
+
+ return (address) start;
+ }
+
+
+
#ifdef COMPILER2
static const int64_t right_2_bits = right_n_bits(2);
@@ -5273,6 +5615,11 @@ static const int64_t right_3_bits = right_n_bits(3);
StubRoutines::_chacha20Block = generate_chacha20Block();
}
+ if (UseSHA1Intrinsics) {
+ StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
+ StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
+ }
+
#endif // COMPILER2_OR_JVMCI
}
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
index 7c604e8c11cc2..90a7e0967b240 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
@@ -39,7 +39,7 @@ enum platform_dependent_constants {
// simply increase sizes if too small (assembler will crash if too small)
_initial_stubs_code_size = 10000,
_continuation_stubs_code_size = 2000,
- _compiler_stubs_code_size = 15000 ZGC_ONLY(+5000),
+ _compiler_stubs_code_size = 25000 ZGC_ONLY(+5000),
_final_stubs_code_size = 20000 ZGC_ONLY(+10000)
};
diff --git a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
index 142d9d636938d..8a6557dde93a1 100644
--- a/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
+++ b/src/hotspot/cpu/riscv/upcallLinker_riscv.cpp
@@ -267,9 +267,13 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(xmethod, entry);
__ sd(xmethod, Address(xthread, JavaThread::callee_target_offset())); // just in case callee is deoptimized
+ __ push_cont_fastpath(xthread);
+
__ ld(t0, Address(xmethod, Method::from_compiled_offset()));
__ jalr(t0);
+ __ pop_cont_fastpath(xthread);
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/riscv/vm_version_riscv.cpp b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
index 390ba51ee4f34..e1711dc5592f9 100644
--- a/src/hotspot/cpu/riscv/vm_version_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vm_version_riscv.cpp
@@ -149,16 +149,6 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseAESCTRIntrinsics, false);
}
- if (UseSHA1Intrinsics) {
- warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
- }
-
- if (UseSHA3Intrinsics) {
- warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
- FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
- }
-
if (UseCRC32Intrinsics) {
warning("CRC32 intrinsics are not available on this CPU.");
FLAG_SET_DEFAULT(UseCRC32Intrinsics, false);
@@ -260,11 +250,8 @@ void VM_Version::initialize() {
// NOTE: Make sure codes dependent on UseRVV are put after c2_initialize(),
// as there are extra checks inside it which could disable UseRVV
// in some situations.
- if (UseZvkn && !UseRVV) {
- FLAG_SET_DEFAULT(UseZvkn, false);
- warning("Cannot enable Zvkn on cpu without RVV support.");
- }
+ // ChaCha20
if (UseRVV) {
if (FLAG_IS_DEFAULT(UseChaCha20Intrinsics)) {
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, true);
@@ -276,29 +263,65 @@ void VM_Version::initialize() {
FLAG_SET_DEFAULT(UseChaCha20Intrinsics, false);
}
- if (!UseZvkn && UseSHA) {
- warning("SHA instructions are not available on this CPU");
- FLAG_SET_DEFAULT(UseSHA, false);
- } else if (UseZvkn && FLAG_IS_DEFAULT(UseSHA)) {
+ // SHA's
+ if (FLAG_IS_DEFAULT(UseSHA)) {
FLAG_SET_DEFAULT(UseSHA, true);
}
- if (!UseSHA) {
+ // SHA-1, no RVV required though.
+ if (UseSHA) {
+ if (FLAG_IS_DEFAULT(UseSHA1Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA1Intrinsics, true);
+ }
+ } else if (UseSHA1Intrinsics) {
+ warning("Intrinsics for SHA-1 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA1Intrinsics, false);
+ }
+
+ // UseZvkn (depends on RVV) and SHA-2.
+ if (UseZvkn && !UseRVV) {
+ FLAG_SET_DEFAULT(UseZvkn, false);
+ warning("Cannot enable Zvkn on cpu without RVV support.");
+ }
+ // SHA-2, depends on Zvkn.
+ if (UseSHA) {
+ if (UseZvkn) {
+ if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
+ }
+ if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
+ }
+ } else {
+ if (UseSHA256Intrinsics) {
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
+ }
+ if (UseSHA512Intrinsics) {
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, UseZvkn needed.");
+ FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
+ }
+ }
+ } else {
if (UseSHA256Intrinsics) {
- warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, UseZvkn needed.");
+ warning("Intrinsics for SHA-224 and SHA-256 crypto hash functions not available on this CPU, as UseSHA disabled.");
FLAG_SET_DEFAULT(UseSHA256Intrinsics, false);
}
if (UseSHA512Intrinsics) {
- warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, UseZvkn needed.");
+ warning("Intrinsics for SHA-384 and SHA-512 crypto hash functions not available on this CPU, as UseSHA disabled.");
FLAG_SET_DEFAULT(UseSHA512Intrinsics, false);
}
- } else {
- if (FLAG_IS_DEFAULT(UseSHA256Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA256Intrinsics, true);
- }
- if (FLAG_IS_DEFAULT(UseSHA512Intrinsics)) {
- FLAG_SET_DEFAULT(UseSHA512Intrinsics, true);
- }
+ }
+
+ // SHA-3
+ if (UseSHA3Intrinsics) {
+ warning("Intrinsics for SHA3-224, SHA3-256, SHA3-384 and SHA3-512 crypto hash functions not available on this CPU.");
+ FLAG_SET_DEFAULT(UseSHA3Intrinsics, false);
+ }
+
+ // UseSHA
+ if (!(UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA3Intrinsics || UseSHA512Intrinsics)) {
+ FLAG_SET_DEFAULT(UseSHA, false);
}
}
diff --git a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
index 9d08796681f3f..5d945dbc32309 100644
--- a/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
+++ b/src/hotspot/cpu/riscv/vtableStubs_riscv.cpp
@@ -27,10 +27,10 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_riscv.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -171,22 +171,22 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
assert(VtableStub::receiver_location() == j_rarg0->as_VMReg(), "receiver expected in j_rarg0");
// Entry arguments:
- // t1: CompiledICHolder
+ // t1: CompiledICData
// j_rarg0: Receiver
// This stub is called from compiled code which has no callee-saved registers,
// so all registers except arguments are free at this point.
const Register recv_klass_reg = x18;
- const Register holder_klass_reg = x19; // declaring interface klass (DECC)
+ const Register holder_klass_reg = x19; // declaring interface klass (DEFC)
const Register resolved_klass_reg = x30; // resolved interface klass (REFC)
const Register temp_reg = x28;
const Register temp_reg2 = x29;
- const Register icholder_reg = t1;
+ const Register icdata_reg = t1;
Label L_no_such_interface;
- __ ld(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ ld(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ ld(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ ld(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
start_pc = __ pc();
diff --git a/src/hotspot/cpu/s390/assembler_s390.hpp b/src/hotspot/cpu/s390/assembler_s390.hpp
index 9bb143001b944..91cc7e611bfd1 100644
--- a/src/hotspot/cpu/s390/assembler_s390.hpp
+++ b/src/hotspot/cpu/s390/assembler_s390.hpp
@@ -107,7 +107,7 @@ class RelAddr {
static bool is_in_range_of_RelAddr(address target, address pc, bool shortForm) {
// Guard against illegal branch targets, e.g. -1. Occurrences in
- // CompiledStaticCall and ad-file. Do not assert (it's a test
+ // CompiledDirectCall and ad-file. Do not assert (it's a test
// function!). Just return false in case of illegal operands.
if ((((uint64_t)target) & 0x0001L) != 0) return false;
if ((((uint64_t)pc) & 0x0001L) != 0) return false;
diff --git a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
index 200f7ee978dcb..b7f1d3605681a 100644
--- a/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_CodeStubs_s390.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2018 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -428,6 +428,7 @@ void ArrayCopyStub::emit_code(LIR_Assembler* ce) {
"must be aligned");
ce->emit_static_call_stub();
+ CHECK_BAILOUT();
// Prepend each BRASL with a nop.
__ relocate(relocInfo::static_call_type);
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
index 355c66047c1b2..503440a5fcc01 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp
@@ -76,10 +76,7 @@ int LIR_Assembler::initial_frame_size_in_bytes() const {
// We fetch the class of the receiver and compare it with the cached class.
// If they do not match we jump to the slow case.
int LIR_Assembler::check_icache() {
- Register receiver = receiverOpr()->as_register();
- int offset = __ offset();
- __ inline_cache_check(receiver, Z_inline_cache);
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -2385,7 +2382,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
op->len()->as_register(),
op->tmp1()->as_register(),
op->tmp2()->as_register(),
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
type2aelembytes(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
index 229216ef20d44..c8815f3a729a4 100644
--- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.hpp
@@ -45,7 +45,7 @@
}
enum {
- _call_stub_size = 512, // See Compile::MAX_stubs_size and CompiledStaticCall::emit_to_interp_stub.
+ _call_stub_size = 512, // See Compile::MAX_stubs_size and CompiledDirectCall::emit_to_interp_stub.
_exception_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(128),
_deopt_handler_size = DEBUG_ONLY(1*K) NOT_DEBUG(64)
};
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
index 40edca6559aa4..58bdcee5d5f8f 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -40,31 +40,6 @@
#include "runtime/stubRoutines.hpp"
#include "utilities/macros.hpp"
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- Label ic_miss, ic_hit;
- verify_oop(receiver, FILE_AND_LINE);
- int klass_offset = oopDesc::klass_offset_in_bytes();
-
- if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
- if (VM_Version::has_CompareBranch()) {
- z_cgij(receiver, 0, Assembler::bcondEqual, ic_miss);
- } else {
- z_ltgr(receiver, receiver);
- z_bre(ic_miss);
- }
- }
-
- compare_klass_ptr(iCache, klass_offset, receiver, false);
- z_bre(ic_hit);
-
- // If icache check fails, then jump to runtime routine.
- // Note: RECEIVER must still contain the receiver!
- load_const_optimized(Z_R1_scratch, AddressLiteral(SharedRuntime::get_ic_miss_stub()));
- z_br(Z_R1_scratch);
- align(CodeEntryAlignment);
- bind(ic_hit);
-}
-
void C1_MacroAssembler::explicit_null_check(Register base) {
ShouldNotCallThis(); // unused
}
@@ -296,7 +271,7 @@ void C1_MacroAssembler::allocate_array(
Register len, // array length
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
@@ -322,8 +297,8 @@ void C1_MacroAssembler::allocate_array(
case 8: z_sllg(arr_size, len, 3); break;
default: ShouldNotReachHere();
}
- add2reg(arr_size, hdr_size * wordSize + MinObjAlignmentInBytesMask); // Add space for header & alignment.
- z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
+ add2reg(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask); // Add space for header & alignment.
+ z_nill(arr_size, (~MinObjAlignmentInBytesMask) & 0xffff); // Align array size.
try_allocate(obj, arr_size, 0, t1, slow_case);
@@ -333,9 +308,9 @@ void C1_MacroAssembler::allocate_array(
Label done;
Register object_fields = t1;
Register Rzero = Z_R1_scratch;
- z_aghi(arr_size, -(hdr_size * BytesPerWord));
+ z_aghi(arr_size, -base_offset_in_bytes);
z_bre(done); // Jump if size of fields is zero.
- z_la(object_fields, hdr_size * BytesPerWord, obj);
+ z_la(object_fields, base_offset_in_bytes, obj);
z_xgr(Rzero, Rzero);
initialize_body(object_fields, arr_size, Rzero);
bind(done);
diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
index 7a4f76af1546e..c77258509e1a5 100644
--- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -86,7 +86,7 @@
Register len, // array length
Register t1, // temp register
Register t2, // temp register
- int hdr_size, // object header size in words
+ int base_offset_in_bytes, // elements offset in bytes
int elt_size, // element size in bytes
Register klass, // object klass
Label& slow_case // Continuation point if fast allocation fails.
diff --git a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
index 257148827be4e..decb3a1cafc31 100644
--- a/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
+++ b/src/hotspot/cpu/s390/c1_Runtime1_s390.cpp
@@ -35,7 +35,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_s390.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_s390.hpp"
diff --git a/src/hotspot/cpu/s390/compiledIC_s390.cpp b/src/hotspot/cpu/s390/compiledIC_s390.cpp
index 7ea90c1de7c69..3adcfbc85f185 100644
--- a/src/hotspot/cpu/s390/compiledIC_s390.cpp
+++ b/src/hotspot/cpu/s390/compiledIC_s390.cpp
@@ -26,7 +26,6 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/mutexLocker.hpp"
@@ -40,7 +39,7 @@
#undef __
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/* = nullptr*/) {
#ifdef COMPILER2
// Stub is fixed up when the corresponding call is converted from calling
// compiled code to calling interpreted code.
@@ -54,7 +53,7 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
// That's why we must use the macroassembler to generate a stub.
MacroAssembler _masm(&cbuf);
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return nullptr; // CodeBuffer::expand failed.
}
@@ -81,27 +80,20 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark/*
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return 2 * MacroAssembler::load_const_from_toc_size() +
2; // branch
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 5; // 4 in emit_java_to_interp + 1 in Java_Static_Call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub + NativeCall::get_IC_pos_in_java_to_interp_stub());
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -115,7 +107,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
// Reset stub.
address stub = static_stub->addr();
assert(stub != nullptr, "stub not found");
@@ -131,7 +123,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
index 3ed99f68c475c..8ce9305a865e5 100644
--- a/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/gc/g1/g1BarrierSetAssembler_s390.cpp
@@ -31,9 +31,9 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/jniHandles.hpp"
#include "runtime/sharedRuntime.hpp"
diff --git a/src/hotspot/cpu/s390/icBuffer_s390.cpp b/src/hotspot/cpu/s390/icBuffer_s390.cpp
deleted file mode 100644
index 0dc936d6fad0c..0000000000000
--- a/src/hotspot/cpu/s390/icBuffer_s390.cpp
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_s390.hpp"
-#include "oops/oop.inline.hpp"
-
-#define __ masm.
-
-int InlineCacheBuffer::ic_stub_code_size() {
- return MacroAssembler::load_const_size() + Assembler::z_brul_size();
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_oop, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler masm(&code);
- // Note: even though the code contains an embedded oop, we do not need reloc info
- // because
- // (1) the oop is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear.
-
- // Load the oop,
- __ load_const(Z_method, (address) cached_oop); // inline cache reg = Z_method
- // and do a tail-call (pc-relative).
- __ z_brul((address) entry_point);
- __ flush();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // Creation also verifies the object.
- return MacroAssembler::get_target_addr_pcrel(move->next_instruction_address());
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // Creation also verifies the object.
- return (void*)move->data();
-}
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
index 14fc07ec00794..0226d494c8958 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp
@@ -26,6 +26,7 @@
#include "precompiled.hpp"
#include "asm/codeBuffer.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -1097,7 +1098,13 @@ void MacroAssembler::clear_mem(const Address& addr, unsigned int size) {
}
void MacroAssembler::align(int modulus) {
- while (offset() % modulus != 0) z_nop();
+ align(modulus, offset());
+}
+
+void MacroAssembler::align(int modulus, int target) {
+ assert(((modulus % 2 == 0) && (target % 2 == 0)), "needs to be even");
+ int delta = target - offset();
+ while ((offset() + delta) % modulus != 0) z_nop();
}
// Special version for non-relocateable code if required alignment
@@ -2150,6 +2157,45 @@ void MacroAssembler::call_VM_leaf_base(address entry_point) {
call_VM_leaf_base(entry_point, allow_relocation);
}
+int MacroAssembler::ic_check_size() {
+ return 30 + (ImplicitNullChecks ? 0 : 6);
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register R2_receiver = Z_ARG1;
+ Register R0_scratch = Z_R0_scratch;
+ Register R1_scratch = Z_R1_scratch;
+ Register R9_data = Z_inline_cache;
+ Label success, failure;
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+ if (!ImplicitNullChecks) {
+ z_cgij(R2_receiver, 0, Assembler::bcondEqual, failure);
+ }
+
+ if (UseCompressedClassPointers) {
+ z_llgf(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ } else {
+ z_lg(R1_scratch, Address(R2_receiver, oopDesc::klass_offset_in_bytes()));
+ }
+ z_cg(R1_scratch, Address(R9_data, in_bytes(CompiledICData::speculated_klass_offset())));
+ z_bre(success);
+
+ bind(failure);
+ load_const(R1_scratch, AddressLiteral(SharedRuntime::get_ic_miss_stub()));
+ z_br(R1_scratch);
+ bind(success);
+
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point, offset() = %d, end_alignment = %d", offset(), end_alignment);
+ return uep_offset;
+}
+
void MacroAssembler::call_VM_base(Register oop_result,
Register last_java_sp,
address entry_point,
diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
index bf14b42e2d1b3..924583abdf563 100644
--- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp
+++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp
@@ -257,6 +257,7 @@ class MacroAssembler: public Assembler {
// nop padding
void align(int modulus);
+ void align(int modulus, int target);
void align_address(int modulus);
//
@@ -566,6 +567,9 @@ class MacroAssembler: public Assembler {
// Get the pc where the last call will return to. Returns _last_calls_return_pc.
inline address last_calls_return_pc();
+ static int ic_check_size();
+ int ic_check(int end_alignment);
+
private:
static bool is_call_far_patchable_variant0_at(address instruction_addr); // Dynamic TOC: load target addr from CP and call.
static bool is_call_far_patchable_variant2_at(address instruction_addr); // PC-relative call, prefixed with NOPs.
diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad
index fa53c73269196..5fcb885cdc3ab 100644
--- a/src/hotspot/cpu/s390/s390.ad
+++ b/src/hotspot/cpu/s390/s390.ad
@@ -1,6 +1,6 @@
//
// Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2017, 2022 SAP SE. All rights reserved.
+// Copyright (c) 2017, 2024 SAP SE. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -1341,51 +1341,9 @@ void MachUEPNode::format(PhaseRegAlloc *ra_, outputStream *os) const {
#endif
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
+ // This is Unverified Entry Point
C2_MacroAssembler _masm(&cbuf);
- const int ic_miss_offset = 2;
-
- // Inline_cache contains a klass.
- Register ic_klass = as_Register(Matcher::inline_cache_reg_encode());
- // ARG1 is the receiver oop.
- Register R2_receiver = Z_ARG1;
- int klass_offset = oopDesc::klass_offset_in_bytes();
- AddressLiteral icmiss(SharedRuntime::get_ic_miss_stub());
- Register R1_ic_miss_stub_addr = Z_R1_scratch;
-
- // Null check of receiver.
- // This is the null check of the receiver that actually should be
- // done in the caller. It's here because in case of implicit null
- // checks we get it for free.
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()),
- "second word in oop should not require explicit null check.");
- if (!ImplicitNullChecks) {
- Label valid;
- if (VM_Version::has_CompareBranch()) {
- __ z_cgij(R2_receiver, 0, Assembler::bcondNotEqual, valid);
- } else {
- __ z_ltgr(R2_receiver, R2_receiver);
- __ z_bre(valid);
- }
- // The ic_miss_stub will handle the null pointer exception.
- __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
- __ z_br(R1_ic_miss_stub_addr);
- __ bind(valid);
- }
-
- // Check whether this method is the proper implementation for the class of
- // the receiver (ic miss check).
- {
- Label valid;
- // Compare cached class against klass from receiver.
- // This also does an implicit null check!
- __ compare_klass_ptr(ic_klass, klass_offset, R2_receiver, false);
- __ z_bre(valid);
- // The inline cache points to the wrong method. Call the
- // ic_miss_stub to find the proper method.
- __ load_const_optimized(R1_ic_miss_stub_addr, icmiss);
- __ z_br(R1_ic_miss_stub_addr);
- __ bind(valid);
- }
+ __ ic_check(CodeEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
@@ -1447,6 +1405,7 @@ int HandlerImpl::emit_exception_handler(CodeBuffer &cbuf) {
address base = __ start_a_stub(size_exception_handler());
if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -1468,6 +1427,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
address base = __ start_a_stub(size_deopt_handler());
if (base == nullptr) {
+ ciEnv::current()->record_failure("CodeCache is full");
return 0; // CodeBuffer::expand failed
}
@@ -2146,7 +2106,7 @@ encode %{
assert(__ inst_mark() != nullptr, "emit_call_reloc must set_inst_mark()");
if (_method) { // Emit stub for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
index ed1795cfa339f..11e1e617d8e3a 100644
--- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
+++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp
@@ -26,8 +26,8 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/oopMap.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
#include "gc/shared/gcLocker.hpp"
@@ -35,7 +35,6 @@
#include "interpreter/interp_masm.hpp"
#include "memory/resourceArea.hpp"
#include "nativeInst_s390.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "registerSaver_s390.hpp"
@@ -1500,17 +1499,15 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
unsigned int wrapper_FrameDone;
unsigned int wrapper_CRegsSet;
Label handle_pending_exception;
- Label ic_miss;
//---------------------------------------------------------------------
// Unverified entry point (UEP)
//---------------------------------------------------------------------
- wrapper_UEPStart = __ offset();
// check ic: object class <-> cached class
- if (!method_is_static) __ nmethod_UEP(ic_miss);
- // Fill with nops (alignment of verified entry point).
- __ align(CodeEntryAlignment);
+ if (!method_is_static) {
+ wrapper_UEPStart = __ ic_check(CodeEntryAlignment /* end_alignment */);
+ }
//---------------------------------------------------------------------
// Verified entry point (VEP)
@@ -2026,13 +2023,7 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm,
__ restore_return_pc();
__ z_br(Z_R1_scratch);
- //---------------------------------------------------------------------
- // Handler for a cache miss (out-of-line)
- //---------------------------------------------------------------------
- __ call_ic_miss_handler(ic_miss, 0x77, 0, Z_R1_scratch);
__ flush();
-
-
//////////////////////////////////////////////////////////////////////
// end of code generation
//////////////////////////////////////////////////////////////////////
@@ -2318,9 +2309,6 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
Label skip_fixup;
{
Label ic_miss;
- const int klass_offset = oopDesc::klass_offset_in_bytes();
- const int holder_klass_offset = in_bytes(CompiledICHolder::holder_klass_offset());
- const int holder_metadata_offset = in_bytes(CompiledICHolder::holder_metadata_offset());
// Out-of-line call to ic_miss handler.
__ call_ic_miss_handler(ic_miss, 0x11, 0, Z_R1_scratch);
@@ -2329,27 +2317,11 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
__ align(CodeEntryAlignment);
c2i_unverified_entry = __ pc();
- // Check the pointers.
- if (!ImplicitNullChecks || MacroAssembler::needs_explicit_null_check(klass_offset)) {
- __ z_ltgr(Z_ARG1, Z_ARG1);
- __ z_bre(ic_miss);
- }
- __ verify_oop(Z_ARG1, FILE_AND_LINE);
-
- // Check ic: object class <-> cached class
- // Compress cached class for comparison. That's more efficient.
- if (UseCompressedClassPointers) {
- __ z_lg(Z_R11, holder_klass_offset, Z_method); // Z_R11 is overwritten a few instructions down anyway.
- __ compare_klass_ptr(Z_R11, klass_offset, Z_ARG1, false); // Cached class can't be zero.
- } else {
- __ z_clc(klass_offset, sizeof(void *)-1, Z_ARG1, holder_klass_offset, Z_method);
- }
- __ z_brne(ic_miss); // Cache miss: call runtime to handle this.
-
+ __ ic_check(2);
+ __ z_lg(Z_method, Address(Z_inline_cache, CompiledICData::speculated_method_offset()));
// This def MUST MATCH code in gen_c2i_adapter!
const Register code = Z_R11;
- __ z_lg(Z_method, holder_metadata_offset, Z_method);
__ load_and_test_long(Z_R0, method_(code));
__ z_brne(ic_miss); // Cache miss: call runtime to handle this.
diff --git a/src/hotspot/cpu/s390/vtableStubs_s390.cpp b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
index 5a79369ceab47..573c23d796708 100644
--- a/src/hotspot/cpu/s390/vtableStubs_s390.cpp
+++ b/src/hotspot/cpu/s390/vtableStubs_s390.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016, 2021 SAP SE. All rights reserved.
+ * Copyright (c) 2016, 2023 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_s390.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/klassVtable.hpp"
@@ -197,12 +197,12 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
__ load_klass(rcvr_klass, Z_ARG1);
// Receiver subtype check against REFC.
- __ z_lg(interface, Address(Z_method, CompiledICHolder::holder_klass_offset()));
+ __ z_lg(interface, Address(Z_method, CompiledICData::itable_refc_klass_offset()));
__ lookup_interface_method(rcvr_klass, interface, noreg,
noreg, Z_R1, no_such_interface, /*return_method=*/ false);
// Get Method* and entrypoint for compiler
- __ z_lg(interface, Address(Z_method, CompiledICHolder::holder_metadata_offset()));
+ __ z_lg(interface, Address(Z_method, CompiledICData::itable_defc_klass_offset()));
__ lookup_interface_method(rcvr_klass, interface, itable_index,
Z_method, Z_R1, no_such_interface, /*return_method=*/ true);
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index ff0726840d30a..c279e3073af87 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -72,7 +72,6 @@ static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jl
NEEDS_CLEANUP // remove this definitions ?
-const Register IC_Klass = rax; // where the IC klass is cached
const Register SYNC_header = rax; // synchronization header
const Register SHIFT_count = rcx; // where count for shift operations must be
@@ -336,23 +335,7 @@ void LIR_Assembler::osr_entry() {
// inline cache check; done before the frame is built.
int LIR_Assembler::check_icache() {
- Register receiver = FrameMap::receiver_opr->as_register();
- Register ic_klass = IC_Klass;
- const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
- const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
- if (!do_post_padding) {
- // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
- __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
- }
- int offset = __ offset();
- __ inline_cache_check(receiver, IC_Klass);
- assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
- if (do_post_padding) {
- // force alignment after the cache check.
- // It's been verified to be aligned if !VerifyOops
- __ align(CodeEntryAlignment);
- }
- return offset;
+ return __ ic_check(CodeEntryAlignment);
}
void LIR_Assembler::clinit_barrier(ciMethod* method) {
@@ -1635,7 +1618,7 @@ void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
len,
tmp1,
tmp2,
- arrayOopDesc::header_size(op->type()),
+ arrayOopDesc::base_offset_in_bytes(op->type()),
array_element_size(op->type()),
op->klass()->as_register(),
*op->stub()->entry());
diff --git a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
index b6a27abf0f37e..7088cf33cf646 100644
--- a/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRGenerator_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1207,9 +1207,10 @@ void LIRGenerator::do_vectorizedMismatch(Intrinsic* x) {
__ move(result_reg, result);
}
+#ifndef _LP64
// _i2l, _i2f, _i2d, _l2i, _l2f, _l2d, _f2i, _f2l, _f2d, _d2i, _d2l, _d2f
// _i2b, _i2c, _i2s
-LIR_Opr fixed_register_for(BasicType type) {
+static LIR_Opr fixed_register_for(BasicType type) {
switch (type) {
case T_FLOAT: return FrameMap::fpu0_float_opr;
case T_DOUBLE: return FrameMap::fpu0_double_opr;
@@ -1218,6 +1219,7 @@ LIR_Opr fixed_register_for(BasicType type) {
default: ShouldNotReachHere(); return LIR_OprFact::illegalOpr;
}
}
+#endif
void LIRGenerator::do_Convert(Convert* x) {
#ifdef _LP64
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
index 78361a305aeeb..caca3a1528261 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "c1/c1_Runtime1.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compilerDefinitions.inline.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetAssembler.hpp"
@@ -34,10 +35,12 @@
#include "oops/arrayOop.hpp"
#include "oops/markWord.hpp"
#include "runtime/basicLock.hpp"
+#include "runtime/globals.hpp"
#include "runtime/os.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/checkedCast.hpp"
+#include "utilities/globalDefinitions.hpp"
int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr, Register tmp, Label& slow_case) {
const int aligned_mask = BytesPerWord -1;
@@ -60,9 +63,6 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
jcc(Assembler::notZero, slow_case);
}
- // Load object header
- movptr(hdr, Address(obj, hdr_offset));
-
if (LockingMode == LM_LIGHTWEIGHT) {
#ifdef _LP64
const Register thread = r15_thread;
@@ -73,6 +73,8 @@ int C1_MacroAssembler::lock_object(Register hdr, Register obj, Register disp_hdr
lightweight_lock(obj, hdr, thread, tmp, slow_case);
} else if (LockingMode == LM_LEGACY) {
Label done;
+ // Load object header
+ movptr(hdr, Address(obj, hdr_offset));
// and mark it as unlocked
orptr(hdr, markWord::unlocked_value);
// save unlocked object header into the displaced header location on the stack
@@ -134,9 +136,14 @@ void C1_MacroAssembler::unlock_object(Register hdr, Register obj, Register disp_
verify_oop(obj);
if (LockingMode == LM_LIGHTWEIGHT) {
- movptr(disp_hdr, Address(obj, hdr_offset));
- andptr(disp_hdr, ~(int32_t)markWord::lock_mask_in_place);
- lightweight_unlock(obj, disp_hdr, hdr, slow_case);
+#ifdef _LP64
+ lightweight_unlock(obj, disp_hdr, r15_thread, hdr, slow_case);
+#else
+ // This relies on the implementation of lightweight_unlock being able to handle
+ // that the reg_rax and thread Register parameters may alias each other.
+ get_thread(disp_hdr);
+ lightweight_unlock(obj, disp_hdr, disp_hdr, hdr, slow_case);
+#endif
} else if (LockingMode == LM_LEGACY) {
// test if object header is pointing to the displaced header, and if so, restore
// the displaced header in the object - if the object header is not pointing to
@@ -179,6 +186,15 @@ void C1_MacroAssembler::initialize_header(Register obj, Register klass, Register
if (len->is_valid()) {
movl(Address(obj, arrayOopDesc::length_offset_in_bytes()), len);
+#ifdef _LP64
+ int base_offset = arrayOopDesc::length_offset_in_bytes() + BytesPerInt;
+ if (!is_aligned(base_offset, BytesPerWord)) {
+ assert(is_aligned(base_offset, BytesPerInt), "must be 4-byte aligned");
+ // Clear gap/first 4 bytes following the length field.
+ xorl(t1, t1);
+ movl(Address(obj, base_offset), t1);
+ }
+#endif
}
#ifdef _LP64
else if (UseCompressedClassPointers) {
@@ -262,7 +278,7 @@ void C1_MacroAssembler::initialize_object(Register obj, Register klass, Register
verify_oop(obj);
}
-void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case) {
+void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case) {
assert(obj == rax, "obj must be in rax, for cmpxchg");
assert_different_registers(obj, len, t1, t2, klass);
@@ -275,7 +291,7 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
const Register arr_size = t2; // okay to be the same
// align object end
- movptr(arr_size, header_size * BytesPerWord + MinObjAlignmentInBytesMask);
+ movptr(arr_size, base_offset_in_bytes + MinObjAlignmentInBytesMask);
lea(arr_size, Address(arr_size, len, f));
andptr(arr_size, ~MinObjAlignmentInBytesMask);
@@ -285,7 +301,10 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
// clear rest of allocated space
const Register len_zero = len;
- initialize_body(obj, arr_size, header_size * BytesPerWord, len_zero);
+ // Align-up to word boundary, because we clear the 4 bytes potentially
+ // following the length field in initialize_header().
+ int base_offset = align_up(base_offset_in_bytes, BytesPerWord);
+ initialize_body(obj, arr_size, base_offset, len_zero);
if (CURRENT_ENV->dtrace_alloc_probes()) {
assert(obj == rax, "must be");
@@ -295,30 +314,6 @@ void C1_MacroAssembler::allocate_array(Register obj, Register len, Register t1,
verify_oop(obj);
}
-
-
-void C1_MacroAssembler::inline_cache_check(Register receiver, Register iCache) {
- verify_oop(receiver);
- // explicit null check not needed since load from [klass_offset] causes a trap
- // check against inline cache
- assert(!MacroAssembler::needs_explicit_null_check(oopDesc::klass_offset_in_bytes()), "must add explicit null check");
- int start_offset = offset();
-
- if (UseCompressedClassPointers) {
- load_klass(rscratch1, receiver, rscratch2);
- cmpptr(rscratch1, iCache);
- } else {
- cmpptr(iCache, Address(receiver, oopDesc::klass_offset_in_bytes()));
- }
- // if icache check fails, then jump to runtime routine
- // Note: RECEIVER must still contain the receiver!
- jump_cc(Assembler::notEqual,
- RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
- assert(UseCompressedClassPointers || offset() - start_offset == ic_cmp_size, "check alignment in emit_method_entry");
-}
-
-
void C1_MacroAssembler::build_frame(int frame_size_in_bytes, int bang_size_in_bytes) {
assert(bang_size_in_bytes >= frame_size_in_bytes, "stack bang size incorrect");
// Make sure there is enough stack space for this method's activation.
diff --git a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
index b3593feb05640..ae340e64fb737 100644
--- a/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c1_MacroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -89,7 +89,7 @@
// header_size: size of object header in words
// f : element scale factor
// slow_case : exit to slow case implementation if fast allocation fails
- void allocate_array(Register obj, Register len, Register t, Register t2, int header_size, Address::ScaleFactor f, Register klass, Label& slow_case);
+ void allocate_array(Register obj, Register len, Register t, Register t2, int base_offset_in_bytes, Address::ScaleFactor f, Register klass, Label& slow_case);
int rsp_offset() const { return _rsp_offset; }
void set_rsp_offset(int n) { _rsp_offset = n; }
diff --git a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
index 8b56f464f2739..2c24c0c2cfb17 100644
--- a/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_Runtime1_x86.cpp
@@ -38,7 +38,6 @@
#include "interpreter/interpreter.hpp"
#include "memory/universe.hpp"
#include "nativeInst_x86.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "register_x86.hpp"
diff --git a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
index b9b4e8af02c5f..6dc8d14064ad2 100644
--- a/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_CodeStubs_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -73,26 +73,74 @@ void C2EntryBarrierStub::emit(C2_MacroAssembler& masm) {
__ jmp(continuation(), false /* maybe_short */);
}
-#ifdef _LP64
-int C2HandleAnonOMOwnerStub::max_size() const {
- // Max size of stub has been determined by testing with 0, in which case
- // C2CodeStubList::emit() will throw an assertion and report the actual size that
- // is needed.
- return DEBUG_ONLY(36) NOT_DEBUG(21);
+int C2FastUnlockLightweightStub::max_size() const {
+ return 128;
}
-void C2HandleAnonOMOwnerStub::emit(C2_MacroAssembler& masm) {
- __ bind(entry());
- Register mon = monitor();
- Register t = tmp();
- __ movptr(Address(mon, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), r15_thread);
- __ subl(Address(r15_thread, JavaThread::lock_stack_top_offset()), oopSize);
+void C2FastUnlockLightweightStub::emit(C2_MacroAssembler& masm) {
+ assert(_t == rax, "must be");
+
+ Label restore_held_monitor_count_and_slow_path;
+
+ { // Restore lock-stack and handle the unlock in runtime.
+
+ __ bind(_push_and_slow_path);
#ifdef ASSERT
- __ movl(t, Address(r15_thread, JavaThread::lock_stack_top_offset()));
- __ movptr(Address(r15_thread, t), 0);
+ // The obj was only cleared in debug.
+ __ movl(_t, Address(_thread, JavaThread::lock_stack_top_offset()));
+ __ movptr(Address(_thread, _t), _obj);
#endif
- __ jmp(continuation());
-}
+ __ addl(Address(_thread, JavaThread::lock_stack_top_offset()), oopSize);
+ }
+
+ { // Restore held monitor count and slow path.
+
+ __ bind(restore_held_monitor_count_and_slow_path);
+ // Restore held monitor count.
+ __ increment(Address(_thread, JavaThread::held_monitor_count_offset()));
+ // increment will always result in ZF = 0 (no overflows).
+ __ jmp(slow_path_continuation());
+ }
+
+ { // Handle monitor medium path.
+
+ __ bind(_check_successor);
+
+ Label fix_zf_and_unlocked;
+ const Register monitor = _mark;
+
+#ifndef _LP64
+ __ jmpb(restore_held_monitor_count_and_slow_path);
+#else // _LP64
+ // successor null check.
+ __ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), NULL_WORD);
+ __ jccb(Assembler::equal, restore_held_monitor_count_and_slow_path);
+
+ // Release lock.
+ __ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+
+ // Fence.
+ // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
+ __ lock(); __ addl(Address(rsp, 0), 0);
+
+ // Recheck successor.
+ __ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), NULL_WORD);
+ // Observed a successor after the release -> fence we have handed off the monitor
+ __ jccb(Assembler::notEqual, fix_zf_and_unlocked);
+
+ // Try to relock, if it fails the monitor has been handed over
+ // TODO: Caveat, this may fail due to deflation, which does
+ // not handle the monitor handoff. Currently only works
+ // due to the responsible thread.
+ __ xorptr(rax, rax);
+ __ lock(); __ cmpxchgptr(_thread, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ __ jccb (Assembler::equal, restore_held_monitor_count_and_slow_path);
#endif
+ __ bind(fix_zf_and_unlocked);
+ __ xorl(rax, rax);
+ __ jmp(unlocked_continuation());
+ }
+}
+
#undef __
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
index 7512a366e7ea0..b6ecde62af655 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp
@@ -33,9 +33,13 @@
#include "opto/output.hpp"
#include "opto/opcodes.hpp"
#include "opto/subnode.hpp"
+#include "runtime/globals.hpp"
#include "runtime/objectMonitor.hpp"
#include "runtime/stubRoutines.hpp"
#include "utilities/checkedCast.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+#include "utilities/sizes.hpp"
#ifdef PRODUCT
#define BLOCK_COMMENT(str) /* nothing */
@@ -554,6 +558,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
RTMLockingCounters* stack_rtm_counters,
Metadata* method_data,
bool use_rtm, bool profile_rtm) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_lock_lightweight");
// Ensure the register assignments are disjoint
assert(tmpReg == rax, "");
@@ -605,7 +610,8 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
if (LockingMode == LM_MONITOR) {
// Clear ZF so that we take the slow path at the DONE label. objReg is known to be not 0.
testptr(objReg, objReg);
- } else if (LockingMode == LM_LEGACY) {
+ } else {
+ assert(LockingMode == LM_LEGACY, "must be");
// Attempt stack-locking ...
orptr (tmpReg, markWord::unlocked_value);
movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS
@@ -620,10 +626,6 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
// Next instruction set ZFlag == 1 (Success) if difference is less then one page.
andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - (int)os::vm_page_size())) );
movptr(Address(boxReg, 0), tmpReg);
- } else {
- assert(LockingMode == LM_LIGHTWEIGHT, "");
- lightweight_lock(objReg, tmpReg, thread, scrReg, NO_COUNT);
- jmp(COUNT);
}
jmp(DONE_LABEL);
@@ -754,6 +756,7 @@ void C2_MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmp
// Xcheck:jni is enabled.
void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
+ assert(LockingMode != LM_LIGHTWEIGHT, "lightweight locking should use fast_unlock_lightweight");
assert(boxReg == rax, "");
assert_different_registers(objReg, boxReg, tmpReg);
@@ -784,23 +787,6 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
}
// It's inflated.
- if (LockingMode == LM_LIGHTWEIGHT) {
- // If the owner is ANONYMOUS, we need to fix it - in an outline stub.
- testb(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t) ObjectMonitor::ANONYMOUS_OWNER);
-#ifdef _LP64
- if (!Compile::current()->output()->in_scratch_emit_size()) {
- C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmpReg, boxReg);
- Compile::current()->output()->add_stub(stub);
- jcc(Assembler::notEqual, stub->entry());
- bind(stub->continuation());
- } else
-#endif
- {
- // We can't easily implement this optimization on 32 bit because we don't have a thread register.
- // Call the slow-path instead.
- jcc(Assembler::notEqual, NO_COUNT);
- }
- }
#if INCLUDE_RTM_OPT
if (use_rtm) {
@@ -922,19 +908,14 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
jmpb (DONE_LABEL);
#endif
- if (LockingMode != LM_MONITOR) {
+ if (LockingMode == LM_LEGACY) {
bind (Stacked);
- if (LockingMode == LM_LIGHTWEIGHT) {
- mov(boxReg, tmpReg);
- lightweight_unlock(objReg, boxReg, tmpReg, NO_COUNT);
- jmp(COUNT);
- } else if (LockingMode == LM_LEGACY) {
- movptr(tmpReg, Address (boxReg, 0)); // re-fetch
- lock();
- cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
- }
+ movptr(tmpReg, Address (boxReg, 0)); // re-fetch
+ lock();
+ cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
// Intentional fall-thru into DONE_LABEL
}
+
bind(DONE_LABEL);
// ZFlag == 1 count in fast path
@@ -955,6 +936,247 @@ void C2_MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register t
bind(NO_COUNT);
}
+void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
+ Register t, Register thread) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert(rax_reg == rax, "Used for CAS");
+ assert_different_registers(obj, box, rax_reg, t, thread);
+
+ // Handle inflated monitor.
+ Label inflated;
+ // Finish fast lock successfully. ZF value is irrelevant.
+ Label locked;
+ // Finish fast lock unsuccessfully. MUST jump with ZF == 0
+ Label slow_path;
+
+ if (DiagnoseSyncOnValueBasedClasses != 0) {
+ load_klass(rax_reg, obj, t);
+ movl(rax_reg, Address(rax_reg, Klass::access_flags_offset()));
+ testl(rax_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
+ jcc(Assembler::notZero, slow_path);
+ }
+
+ const Register mark = t;
+
+ { // Lightweight Lock
+
+ Label push;
+
+ const Register top = box;
+
+ // Load the mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Prefetch top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Check for monitor (0b10).
+ testptr(mark, markWord::monitor_value);
+ jcc(Assembler::notZero, inflated);
+
+ // Check if lock-stack is full.
+ cmpl(top, LockStack::end_offset() - 1);
+ jcc(Assembler::greater, slow_path);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ jccb(Assembler::equal, push);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ movptr(rax_reg, mark);
+ orptr(rax_reg, markWord::unlocked_value);
+ andptr(mark, ~(int32_t)markWord::unlocked_value);
+ lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::notEqual, slow_path);
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ movptr(Address(thread, top), obj);
+ addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+ jmpb(locked);
+ }
+
+ { // Handle inflated monitor.
+ bind(inflated);
+
+ const Register tagged_monitor = mark;
+
+ // CAS owner (null => current thread).
+ xorptr(rax_reg, rax_reg);
+ lock(); cmpxchgptr(thread, Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
+ jccb(Assembler::equal, locked);
+
+ // Check if recursive.
+ cmpptr(thread, rax_reg);
+ jccb(Assembler::notEqual, slow_path);
+
+ // Recursive.
+ increment(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ }
+
+ bind(locked);
+ increment(Address(thread, JavaThread::held_monitor_count_offset()));
+ // Set ZF = 1
+ xorl(rax_reg, rax_reg);
+
+#ifdef ASSERT
+ // Check that locked label is reached with ZF set.
+ Label zf_correct;
+ jccb(Assembler::zero, zf_correct);
+ stop("Fast Lock ZF != 1");
+#endif
+
+ bind(slow_path);
+#ifdef ASSERT
+ // Check that slow_path label is reached with ZF not set.
+ jccb(Assembler::notZero, zf_correct);
+ stop("Fast Lock ZF != 0");
+ bind(zf_correct);
+#endif
+ // C2 uses the value of ZF to determine the continuation.
+}
+
+void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread) {
+ assert(LockingMode == LM_LIGHTWEIGHT, "must be");
+ assert(reg_rax == rax, "Used for CAS");
+ assert_different_registers(obj, reg_rax, t);
+
+ // Handle inflated monitor.
+ Label inflated, inflated_check_lock_stack;
+ // Finish fast unlock successfully. MUST jump with ZF == 1
+ Label unlocked;
+
+ // Assume success.
+ decrement(Address(thread, JavaThread::held_monitor_count_offset()));
+
+ const Register mark = t;
+ const Register top = reg_rax;
+
+ Label dummy;
+ C2FastUnlockLightweightStub* stub = nullptr;
+
+ if (!Compile::current()->output()->in_scratch_emit_size()) {
+ stub = new (Compile::current()->comp_arena()) C2FastUnlockLightweightStub(obj, mark, reg_rax, thread);
+ Compile::current()->output()->add_stub(stub);
+ }
+
+ Label& push_and_slow_path = stub == nullptr ? dummy : stub->push_and_slow_path();
+ Label& check_successor = stub == nullptr ? dummy : stub->check_successor();
+
+ { // Lightweight Unlock
+
+ // Load top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Prefetch mark.
+ movptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Check if obj is top of lock-stack.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ // Top of lock stack was not obj. Must be monitor.
+ jcc(Assembler::notEqual, inflated_check_lock_stack);
+
+ // Pop lock-stack.
+ DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
+ subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
+ jcc(Assembler::equal, unlocked);
+
+ // We elide the monitor check, let the CAS fail instead.
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ movptr(reg_rax, mark);
+ andptr(reg_rax, ~(int32_t)markWord::lock_mask);
+ orptr(mark, markWord::unlocked_value);
+ lock(); cmpxchgptr(mark, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::notEqual, push_and_slow_path);
+ jmp(unlocked);
+ }
+
+
+ { // Handle inflated monitor.
+ bind(inflated_check_lock_stack);
+#ifdef ASSERT
+ Label check_done;
+ subl(top, oopSize);
+ cmpl(top, in_bytes(JavaThread::lock_stack_base_offset()));
+ jcc(Assembler::below, check_done);
+ cmpptr(obj, Address(thread, top));
+ jccb(Assembler::notEqual, inflated_check_lock_stack);
+ stop("Fast Unlock lock on stack");
+ bind(check_done);
+ testptr(mark, markWord::monitor_value);
+ jccb(Assembler::notZero, inflated);
+ stop("Fast Unlock not monitor");
+#endif
+
+ bind(inflated);
+
+ // mark contains the tagged ObjectMonitor*.
+ const Register monitor = mark;
+
+#ifndef _LP64
+ // Check if recursive.
+ xorptr(reg_rax, reg_rax);
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Check if the entry lists are empty.
+ movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Release lock.
+ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+#else // _LP64
+ Label recursive;
+
+ // Check if recursive.
+ cmpptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 0);
+ jccb(Assembler::notEqual, recursive);
+
+ // Check if the entry lists are empty.
+ movptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
+ orptr(reg_rax, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
+ jcc(Assembler::notZero, check_successor);
+
+ // Release lock.
+ movptr(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
+ jmpb(unlocked);
+
+ // Recursive unlock.
+ bind(recursive);
+ decrement(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
+ xorl(t, t);
+#endif
+ }
+
+ bind(unlocked);
+ if (stub != nullptr) {
+ bind(stub->unlocked_continuation());
+ }
+
+#ifdef ASSERT
+ // Check that unlocked label is reached with ZF set.
+ Label zf_correct;
+ jccb(Assembler::zero, zf_correct);
+ stop("Fast Unlock ZF != 1");
+#endif
+
+ if (stub != nullptr) {
+ bind(stub->slow_path_continuation());
+ }
+#ifdef ASSERT
+ // Check that stub->continuation() label is reached with ZF not set.
+ jccb(Assembler::notZero, zf_correct);
+ stop("Fast Unlock ZF != 0");
+ bind(zf_correct);
+#endif
+ // C2 uses the value of ZF to determine the continuation.
+}
+
//-------------------------------------------------------------------------------------------
// Generic instructions support for use in .ad files C2 code generation
diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
index 151f2148372d5..26f7fb44aa939 100644
--- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp
@@ -43,6 +43,10 @@
bool use_rtm, bool profile_rtm);
void fast_unlock(Register obj, Register box, Register tmp, bool use_rtm);
+ void fast_lock_lightweight(Register obj, Register box, Register rax_reg,
+ Register t, Register thread);
+ void fast_unlock_lightweight(Register obj, Register reg_rax, Register t, Register thread);
+
#if INCLUDE_RTM_OPT
void rtm_counters_update(Register abort_status, Register rtm_counters);
void branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel);
diff --git a/src/hotspot/cpu/x86/compiledIC_x86.cpp b/src/hotspot/cpu/x86/compiledIC_x86.cpp
index 8fc001039fbd3..95b41f62b6aab 100644
--- a/src/hotspot/cpu/x86/compiledIC_x86.cpp
+++ b/src/hotspot/cpu/x86/compiledIC_x86.cpp
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
@@ -36,7 +35,7 @@
// ----------------------------------------------------------------------------
#define __ _masm.
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
// Stub is fixed up when the corresponding call is converted from
// calling compiled code to calling interpreted code.
// movq rbx, 0
@@ -66,32 +65,25 @@ address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark)
}
#undef __
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
return NOT_LP64(10) // movl; jmp
LP64_ONLY(15); // movq (1+1+8); jmp (1+4)
}
-int CompiledStaticCall::to_trampoline_stub_size() {
+int CompiledDirectCall::to_trampoline_stub_size() {
// x86 doesn't use trampolines.
return 0;
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
return 4; // 3 in emit_to_interp_stub + 1 in emit_call
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
address stub = find_stub();
guarantee(stub != nullptr, "stub not found");
- {
- ResourceMark rm;
- log_trace(inlinecache)("CompiledDirectStaticCall@" INTPTR_FORMAT ": set_to_interpreted %s",
- p2i(instruction_address()),
- callee->name_and_sig_as_C_string());
- }
-
// Creation also verifies the object.
NativeMovConstReg* method_holder = nativeMovConstReg_at(stub);
NativeJump* jump = nativeJump_at(method_holder->next_instruction_address());
@@ -105,7 +97,7 @@ void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, ad
set_destination_mt_safe(stub);
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
assert(CompiledICLocker::is_safe(static_stub->addr()), "mt unsafe call");
// Reset stub.
address stub = static_stub->addr();
@@ -122,7 +114,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
// Verify call.
_call->verify();
_call->verify_alignment();
diff --git a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
index f609846f00d6c..f9f77c23f14ca 100644
--- a/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/g1/g1BarrierSetAssembler_x86.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "interpreter/interp_masm.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/cpu/x86/icBuffer_x86.cpp b/src/hotspot/cpu/x86/icBuffer_x86.cpp
deleted file mode 100644
index af374b5741659..0000000000000
--- a/src/hotspot/cpu/x86/icBuffer_x86.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/macroAssembler.hpp"
-#include "asm/macroAssembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_x86.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // Worst case, if destination is not a near call:
- // lea rax, lit1
- // lea scratch, lit2
- // jmp scratch
-
- // Best case
- // lea rax, lit1
- // jmp lit2
-
- int best = NativeMovConstReg::instruction_size + NativeJump::instruction_size;
- int worst = 2 * NativeMovConstReg::instruction_size + 3;
- return MAX2(best, worst);
-}
-
-
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point) {
- ResourceMark rm;
- CodeBuffer code(code_begin, ic_stub_code_size());
- MacroAssembler* masm = new MacroAssembler(&code);
- // note: even though the code contains an embedded value, we do not need reloc info
- // because
- // (1) the value is old (i.e., doesn't matter for scavenges)
- // (2) these ICStubs are removed *before* a GC happens, so the roots disappear
- // assert(cached_value == nullptr || cached_oop->is_perm(), "must be perm oop");
- masm->lea(rax, AddressLiteral((address) cached_value, relocInfo::metadata_type));
- masm->jump(ExternalAddress(entry_point));
-}
-
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin); // creation also verifies the object
- address jmp = move->next_instruction_address();
- NativeInstruction* ni = nativeInstruction_at(jmp);
- if (ni->is_jump()) {
- NativeJump* jump = nativeJump_at(jmp);
- return jump->jump_destination();
- } else {
- assert(ni->is_far_jump(), "unexpected instruction");
- NativeFarJump* jump = nativeFarJump_at(jmp);
- return jump->jump_destination();
- }
-}
-
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- // creation also verifies the object
- NativeMovConstReg* move = nativeMovConstReg_at(code_begin);
- // Verifies the jump
- address jmp = move->next_instruction_address();
- NativeInstruction* ni = nativeInstruction_at(jmp);
- if (ni->is_jump()) {
- NativeJump* jump = nativeJump_at(jmp);
- } else {
- assert(ni->is_far_jump(), "unexpected instruction");
- NativeFarJump* jump = nativeFarJump_at(jmp);
- }
- void* o = (void*)move->data();
- return o;
-}
diff --git a/src/hotspot/cpu/x86/interp_masm_x86.cpp b/src/hotspot/cpu/x86/interp_masm_x86.cpp
index f5f83ae21f475..33570f3155b15 100644
--- a/src/hotspot/cpu/x86/interp_masm_x86.cpp
+++ b/src/hotspot/cpu/x86/interp_masm_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1192,8 +1192,6 @@ void InterpreterMacroAssembler::lock_object(Register lock_reg) {
const Register thread = lock_reg;
get_thread(thread);
#endif
- // Load object header, prepare for CAS from unlocked to locked.
- movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
lightweight_lock(obj_reg, swap_reg, thread, tmp_reg, slow_case);
} else if (LockingMode == LM_LEGACY) {
// Load immediate 1 into swap_reg %rax
@@ -1311,20 +1309,13 @@ void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
if (LockingMode == LM_LIGHTWEIGHT) {
#ifdef _LP64
- const Register thread = r15_thread;
+ lightweight_unlock(obj_reg, swap_reg, r15_thread, header_reg, slow_case);
#else
- const Register thread = header_reg;
- get_thread(thread);
+ // This relies on the implementation of lightweight_unlock being able to handle
+ // that the reg_rax and thread Register parameters may alias each other.
+ get_thread(swap_reg);
+ lightweight_unlock(obj_reg, swap_reg, swap_reg, header_reg, slow_case);
#endif
- // Handle unstructured locking.
- Register tmp = swap_reg;
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- cmpptr(obj_reg, Address(thread, tmp, Address::times_1, -oopSize));
- jcc(Assembler::notEqual, slow_case);
- // Try to swing header from locked to unlocked.
- movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- lightweight_unlock(obj_reg, swap_reg, header_reg, slow_case);
} else if (LockingMode == LM_LEGACY) {
// Load the old header from BasicLock structure
movptr(header_reg, Address(swap_reg,
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.cpp b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
index ba4b089c7aa6e..f0e7a08dd5f2a 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compiler_globals.hpp"
#include "compiler/disassembler.hpp"
#include "crc32c.h"
@@ -1341,13 +1342,45 @@ void MacroAssembler::ic_call(address entry, jint method_index) {
RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
#ifdef _LP64
// Needs full 64-bit immediate for later patching.
- mov64(rax, (intptr_t)Universe::non_oop_word());
+ mov64(rax, (int64_t)Universe::non_oop_word());
#else
movptr(rax, (intptr_t)Universe::non_oop_word());
#endif
call(AddressLiteral(entry, rh));
}
+int MacroAssembler::ic_check_size() {
+ return LP64_ONLY(14) NOT_LP64(12);
+}
+
+int MacroAssembler::ic_check(int end_alignment) {
+ Register receiver = LP64_ONLY(j_rarg0) NOT_LP64(rcx);
+ Register data = rax;
+ Register temp = LP64_ONLY(rscratch1) NOT_LP64(rbx);
+
+ // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
+ // before the inline cache check here, and not after
+ align(end_alignment, offset() + ic_check_size());
+
+ int uep_offset = offset();
+
+ if (UseCompressedClassPointers) {
+ movl(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ cmpl(temp, Address(data, CompiledICData::speculated_klass_offset()));
+ } else {
+ movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
+ cmpptr(temp, Address(data, CompiledICData::speculated_klass_offset()));
+ }
+
+ // if inline cache check fails, then jump to runtime routine
+ jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
+ assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
+
+ return uep_offset;
+}
+
void MacroAssembler::emit_static_call_stub() {
// Static stub relocation also tags the Method* in the code-stream.
mov_metadata(rbx, (Metadata*) nullptr); // Method is zapped till fixup time.
@@ -4087,8 +4120,9 @@ static void restore_xmm_register(MacroAssembler* masm, int offset, XMMRegister r
}
}
-int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers, bool save_fpu,
- int& gp_area_size, int& fp_area_size, int& xmm_area_size) {
+static int register_section_sizes(RegSet gp_registers, XMMRegSet xmm_registers,
+ bool save_fpu, int& gp_area_size,
+ int& fp_area_size, int& xmm_area_size) {
gp_area_size = align_up(gp_registers.size() * Register::max_slots_per_register * VMRegImpl::stack_slot_size,
StackAlignmentInBytes);
@@ -4354,7 +4388,7 @@ void MacroAssembler::lookup_interface_method(Register recv_klass,
}
// Look up the method for a megamorphic invokeinterface call in a single pass over itable:
-// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICHolder
+// - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData
// - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index
// The target method is determined by .
// The receiver klass is in recv_klass.
@@ -9877,68 +9911,116 @@ void MacroAssembler::check_stack_alignment(Register sp, const char* msg, unsigne
}
// Implements lightweight-locking.
-// Branches to slow upon failure to lock the object, with ZF cleared.
-// Falls through upon success with unspecified ZF.
//
// obj: the object to be locked
-// hdr: the (pre-loaded) header of the object, must be rax
+// reg_rax: rax
// thread: the thread which attempts to lock obj
// tmp: a temporary register
-void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
- assert(hdr == rax, "header must be in rax for cmpxchg");
- assert_different_registers(obj, hdr, thread, tmp);
-
- // First we need to check if the lock-stack has room for pushing the object reference.
- // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
- // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
- // avoids one branch.
- cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
- jcc(Assembler::greater, slow);
-
- // Now we attempt to take the fast-lock.
- // Clear lock_mask bits (locked state).
- andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
- movptr(tmp, hdr);
- // Set unlocked_value bit.
- orptr(hdr, markWord::unlocked_value);
- lock();
- cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
+ assert(reg_rax == rax, "");
+ assert_different_registers(obj, reg_rax, thread, tmp);
+
+ Label push;
+ const Register top = tmp;
+
+ // Preload the markWord. It is important that this is the first
+ // instruction emitted as it is part of C1's null check semantics.
+ movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
+
+ // Load top.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ // Check if the lock-stack is full.
+ cmpl(top, LockStack::end_offset());
+ jcc(Assembler::greaterEqual, slow);
+
+ // Check for recursion.
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
+ jcc(Assembler::equal, push);
+
+ // Check header for monitor (0b10).
+ testptr(reg_rax, markWord::monitor_value);
+ jcc(Assembler::notZero, slow);
+
+ // Try to lock. Transition lock bits 0b01 => 0b00
+ movptr(tmp, reg_rax);
+ andptr(tmp, ~(int32_t)markWord::unlocked_value);
+ orptr(reg_rax, markWord::unlocked_value);
+ lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
jcc(Assembler::notEqual, slow);
- // If successful, push object to lock-stack.
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- movptr(Address(thread, tmp), obj);
- incrementl(tmp, oopSize);
- movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
+ // Restore top, CAS clobbers register.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+
+ bind(push);
+ // After successful lock, push object on lock-stack.
+ movptr(Address(thread, top), obj);
+ incrementl(top, oopSize);
+ movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
}
// Implements lightweight-unlocking.
-// Branches to slow upon failure, with ZF cleared.
-// Falls through upon success, with unspecified ZF.
//
// obj: the object to be unlocked
-// hdr: the (pre-loaded) header of the object, must be rax
+// reg_rax: rax
+// thread: the thread
// tmp: a temporary register
-void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) {
- assert(hdr == rax, "header must be in rax for cmpxchg");
- assert_different_registers(obj, hdr, tmp);
-
- // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
- movptr(tmp, hdr); // The expected old value
- orptr(tmp, markWord::unlocked_value);
- lock();
- cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+//
+// x86_32 Note: reg_rax and thread may alias each other due to limited register
+// availiability.
+void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
+ assert(reg_rax == rax, "");
+ assert_different_registers(obj, reg_rax, tmp);
+ LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
+
+ Label unlocked, push_and_slow;
+ const Register top = tmp;
+
+ // Check if obj is top of lock-stack.
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+ cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
jcc(Assembler::notEqual, slow);
- // Pop the lock object from the lock-stack.
-#ifdef _LP64
- const Register thread = r15_thread;
-#else
- const Register thread = rax;
- get_thread(thread);
-#endif
+
+ // Pop lock-stack.
+ DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+
+ // Check if recursive.
+ cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
+ jcc(Assembler::equal, unlocked);
+
+ // Not recursive. Check header for monitor (0b10).
+ movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
+ testptr(reg_rax, markWord::monitor_value);
+ jcc(Assembler::notZero, push_and_slow);
+
+#ifdef ASSERT
+ // Check header not unlocked (0b01).
+ Label not_unlocked;
+ testptr(reg_rax, markWord::unlocked_value);
+ jcc(Assembler::zero, not_unlocked);
+ stop("lightweight_unlock already unlocked");
+ bind(not_unlocked);
+#endif
+
+ // Try to unlock. Transition lock bits 0b00 => 0b01
+ movptr(tmp, reg_rax);
+ orptr(tmp, markWord::unlocked_value);
+ lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
+ jcc(Assembler::equal, unlocked);
+
+ bind(push_and_slow);
+ // Restore lock-stack and handle the unlock in runtime.
+ if (thread == reg_rax) {
+ // On x86_32 we may lose the thread.
+ get_thread(thread);
+ }
#ifdef ASSERT
- movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
- movptr(Address(thread, tmp), 0);
+ movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
+ movptr(Address(thread, top), obj);
#endif
+ addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
+ jmp(slow);
+
+ bind(unlocked);
}
diff --git a/src/hotspot/cpu/x86/macroAssembler_x86.hpp b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
index 4b30168452796..4789b63decc6c 100644
--- a/src/hotspot/cpu/x86/macroAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/macroAssembler_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -896,6 +896,8 @@ class MacroAssembler: public Assembler {
// Emit the CompiledIC call idiom
void ic_call(address entry, jint method_index = 0);
+ static int ic_check_size();
+ int ic_check(int end_alignment);
void emit_static_call_stub();
@@ -2031,8 +2033,8 @@ class MacroAssembler: public Assembler {
void check_stack_alignment(Register sp, const char* msg, unsigned bias = 0, Register tmp = noreg);
- void lightweight_lock(Register obj, Register hdr, Register thread, Register tmp, Label& slow);
- void lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow);
+ void lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow);
+ void lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow);
};
/**
diff --git a/src/hotspot/cpu/x86/peephole_x86_64.cpp b/src/hotspot/cpu/x86/peephole_x86_64.cpp
index 8c956aeb05393..92a29490edaf8 100644
--- a/src/hotspot/cpu/x86/peephole_x86_64.cpp
+++ b/src/hotspot/cpu/x86/peephole_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,8 +33,8 @@
// lea d, [s1 + s2] and
// mov d, s1; shl d, s2 into
// lea d, [s1 << s2] with s2 = 1, 2, 3
-bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_,
- MachNode* (*new_root)(), uint inst0_rule, bool imm) {
+static bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseRegAlloc* ra_,
+ MachNode* (*new_root)(), uint inst0_rule, bool imm) {
MachNode* inst0 = block->get_node(block_index)->as_Mach();
assert(inst0->rule() == inst0_rule, "sanity");
@@ -136,7 +136,7 @@ bool lea_coalesce_helper(Block* block, int block_index, PhaseCFG* cfg_, PhaseReg
// This helper func takes a condition and returns the flags that need to be set for the condition
// It uses the same flags as the test instruction, so if the e.g. the overflow bit is required,
// this func returns clears_overflow, as that is what the test instruction does and what the downstream path expects
-juint map_condition_to_required_test_flags(Assembler::Condition condition) {
+static juint map_condition_to_required_test_flags(Assembler::Condition condition) {
switch (condition) {
case Assembler::Condition::zero: // Same value as equal
case Assembler::Condition::notZero: // Same value as notEqual
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
index 571160523cbe4..febc1b2c3b143 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,8 +25,8 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
#include "asm/macroAssembler.inline.hpp"
+#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
@@ -36,7 +36,6 @@
#include "interpreter/interpreter.hpp"
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "prims/methodHandles.hpp"
#include "runtime/jniHandles.hpp"
@@ -944,25 +943,18 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Register holder = rax;
+ Register data = rax;
Register receiver = rcx;
Register temp = rbx;
{
-
- Label missed;
- __ movptr(temp, Address(receiver, oopDesc::klass_offset_in_bytes()));
- __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ jcc(Assembler::notEqual, missed);
+ __ ic_check(1 /* end_alignment */);
+ __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
// Method might have been compiled since the call site was patched to
// interpreted if that is the case treat it as a miss so we can get
// the call site corrected.
__ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
__ jcc(Assembler::equal, skip_fixup);
-
- __ bind(missed);
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
}
address c2i_entry = __ pc();
@@ -1449,23 +1441,12 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// as far as the interpreter and the compiler(s) are concerned.
- const Register ic_reg = rax;
const Register receiver = rcx;
- Label hit;
Label exception_pending;
__ verify_oop(receiver);
- __ cmpptr(ic_reg, Address(receiver, oopDesc::klass_offset_in_bytes()));
- __ jcc(Assembler::equal, hit);
-
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
// verified entry must be aligned for code patching.
- // and the first 5 bytes must be in the same cache line
- // if we align at 8 then we will be sure 5 bytes are in the same line
- __ align(8);
-
- __ bind(hit);
+ __ ic_check(8 /* end_alignment */);
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -1713,8 +1694,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ jcc(Assembler::notEqual, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- // Load object header
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, thread, lock_reg, slow_path_lock);
}
__ bind(count_mon);
@@ -1872,9 +1851,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ dec_held_monitor_count();
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+ __ lightweight_unlock(obj_reg, swap_reg, thread, lock_reg, slow_path_unlock);
__ dec_held_monitor_count();
}
diff --git a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
index cab50e85ec51c..c666f982d0f52 100644
--- a/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
+++ b/src/hotspot/cpu/x86/sharedRuntime_x86_64.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,6 @@
#include "asm/macroAssembler.inline.hpp"
#include "code/compiledIC.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/oopMap.hpp"
@@ -42,7 +41,6 @@
#include "logging/log.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
#include "prims/methodHandles.hpp"
@@ -1000,20 +998,14 @@ AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm
address c2i_unverified_entry = __ pc();
Label skip_fixup;
- Label ok;
- Register holder = rax;
+ Register data = rax;
Register receiver = j_rarg0;
Register temp = rbx;
{
- __ load_klass(temp, receiver, rscratch1);
- __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
- __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
- __ jcc(Assembler::equal, ok);
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- __ bind(ok);
+ __ ic_check(1 /* end_alignment */);
+ __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
// Method might have been compiled since the call site was patched to
// interpreted if that is the case treat it as a miss so we can get
// the call site corrected.
@@ -1450,7 +1442,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
__ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
// Emit stub for static call
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc());
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, __ pc());
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1487,7 +1479,7 @@ static void gen_continuation_enter(MacroAssembler* masm,
// Emit stub for static call
CodeBuffer* cbuf = masm->code_section()->outer();
- address stub = CompiledStaticCall::emit_to_interp_stub(*cbuf, __ pc());
+ address stub = CompiledDirectCall::emit_to_interp_stub(*cbuf, __ pc());
if (stub == nullptr) {
fatal("CodeCache is full at gen_continuation_enter");
}
@@ -1883,25 +1875,13 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
// restoring them except rbp. rbp is the only callee save register
// as far as the interpreter and the compiler(s) are concerned.
-
- const Register ic_reg = rax;
const Register receiver = j_rarg0;
- Label hit;
Label exception_pending;
- assert_different_registers(ic_reg, receiver, rscratch1, rscratch2);
+ assert_different_registers(receiver, rscratch1, rscratch2);
__ verify_oop(receiver);
- __ load_klass(rscratch1, receiver, rscratch2);
- __ cmpq(ic_reg, rscratch1);
- __ jcc(Assembler::equal, hit);
-
- __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- // Verified entry point must be aligned
- __ align(8);
-
- __ bind(hit);
+ __ ic_check(8 /* end_alignment */);
int vep_offset = ((intptr_t)__ pc()) - start;
@@ -2190,8 +2170,6 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ jcc(Assembler::notEqual, slow_path_lock);
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- // Load object header
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
__ lightweight_lock(obj_reg, swap_reg, r15_thread, rscratch1, slow_path_lock);
}
__ bind(count_mon);
@@ -2334,9 +2312,7 @@ nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
__ dec_held_monitor_count();
} else {
assert(LockingMode == LM_LIGHTWEIGHT, "must be");
- __ movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
- __ andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
- __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock);
+ __ lightweight_unlock(obj_reg, swap_reg, r15_thread, lock_reg, slow_path_unlock);
__ dec_held_monitor_count();
}
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.cpp b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
index 3be83eed9d22f..bc1cbdbba26b5 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.cpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.cpp
@@ -279,7 +279,7 @@ uint32_t _crc32c_pow_2k_table[TILL_CYCLE]; // because _crc32c_pow_2k_table[TILL_
// A. Kadatch and B. Jenkins / Everything we know about CRC but afraid to forget September 3, 2010 8
// Listing 1: Multiplication of normalized polynomials
// "a" and "b" occupy D least significant bits.
-uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
+static uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
uint32_t product = 0;
uint32_t b_pow_x_table[D + 1]; // b_pow_x_table[k] = (b * x**k) mod P
b_pow_x_table[0] = b;
@@ -303,7 +303,7 @@ uint32_t crc32c_multiply(uint32_t a, uint32_t b) {
#undef P
// A. Kadatch and B. Jenkins / Everything we know about CRC but afraid to forget September 3, 2010 9
-void crc32c_init_pow_2k(void) {
+static void crc32c_init_pow_2k(void) {
// _crc32c_pow_2k_table(0) =
// x^(2^k) mod P(x) = x mod P(x) = x
// Since we are operating on a reflected values
@@ -318,7 +318,7 @@ void crc32c_init_pow_2k(void) {
}
// x^N mod P(x)
-uint32_t crc32c_f_pow_n(uint32_t n) {
+static uint32_t crc32c_f_pow_n(uint32_t n) {
// result = 1 (polynomial)
uint32_t one, result = 0x80000000, i = 0;
diff --git a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
index 89e7a466264b9..7b9d49dd46140 100644
--- a/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
+++ b/src/hotspot/cpu/x86/upcallLinker_x86_64.cpp
@@ -300,8 +300,12 @@ address UpcallLinker::make_upcall_stub(jobject receiver, Method* entry,
__ mov_metadata(rbx, entry);
__ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx); // just in case callee is deoptimized
+ __ push_cont_fastpath();
+
__ call(Address(rbx, Method::from_compiled_offset()));
+ __ pop_cont_fastpath();
+
// return value shuffle
if (!needs_return_buffer) {
#ifdef ASSERT
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
index 2412c053106e7..f8213a2539f04 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.cpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -809,7 +809,7 @@ void VM_Version::get_processor_features() {
_stepping = cpu_stepping();
if (cpu_family() > 4) { // it supports CPUID
- _features = feature_flags(); // These can be changed by VM settings
+ _features = _cpuid_info.feature_flags(); // These can be changed by VM settings
_cpu_features = _features; // Preserve features
// Logical processors are only available on P4s and above,
// and only if hyperthreading is available.
@@ -2891,13 +2891,13 @@ int64_t VM_Version::maximum_qualified_cpu_frequency(void) {
return _max_qualified_cpu_frequency;
}
-uint64_t VM_Version::feature_flags() {
+uint64_t VM_Version::CpuidInfo::feature_flags() const {
uint64_t result = 0;
- if (_cpuid_info.std_cpuid1_edx.bits.cmpxchg8 != 0)
+ if (std_cpuid1_edx.bits.cmpxchg8 != 0)
result |= CPU_CX8;
- if (_cpuid_info.std_cpuid1_edx.bits.cmov != 0)
+ if (std_cpuid1_edx.bits.cmov != 0)
result |= CPU_CMOV;
- if (_cpuid_info.std_cpuid1_edx.bits.clflush != 0)
+ if (std_cpuid1_edx.bits.clflush != 0)
result |= CPU_FLUSH;
#ifdef _LP64
// clflush should always be available on x86_64
@@ -2905,158 +2905,158 @@ uint64_t VM_Version::feature_flags() {
// to flush the code cache.
assert ((result & CPU_FLUSH) != 0, "clflush should be available");
#endif
- if (_cpuid_info.std_cpuid1_edx.bits.fxsr != 0 || (is_amd_family() &&
- _cpuid_info.ext_cpuid1_edx.bits.fxsr != 0))
+ if (std_cpuid1_edx.bits.fxsr != 0 || (is_amd_family() &&
+ ext_cpuid1_edx.bits.fxsr != 0))
result |= CPU_FXSR;
// HT flag is set for multi-core processors also.
if (threads_per_core() > 1)
result |= CPU_HT;
- if (_cpuid_info.std_cpuid1_edx.bits.mmx != 0 || (is_amd_family() &&
- _cpuid_info.ext_cpuid1_edx.bits.mmx != 0))
+ if (std_cpuid1_edx.bits.mmx != 0 || (is_amd_family() &&
+ ext_cpuid1_edx.bits.mmx != 0))
result |= CPU_MMX;
- if (_cpuid_info.std_cpuid1_edx.bits.sse != 0)
+ if (std_cpuid1_edx.bits.sse != 0)
result |= CPU_SSE;
- if (_cpuid_info.std_cpuid1_edx.bits.sse2 != 0)
+ if (std_cpuid1_edx.bits.sse2 != 0)
result |= CPU_SSE2;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse3 != 0)
+ if (std_cpuid1_ecx.bits.sse3 != 0)
result |= CPU_SSE3;
- if (_cpuid_info.std_cpuid1_ecx.bits.ssse3 != 0)
+ if (std_cpuid1_ecx.bits.ssse3 != 0)
result |= CPU_SSSE3;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse4_1 != 0)
+ if (std_cpuid1_ecx.bits.sse4_1 != 0)
result |= CPU_SSE4_1;
- if (_cpuid_info.std_cpuid1_ecx.bits.sse4_2 != 0)
+ if (std_cpuid1_ecx.bits.sse4_2 != 0)
result |= CPU_SSE4_2;
- if (_cpuid_info.std_cpuid1_ecx.bits.popcnt != 0)
+ if (std_cpuid1_ecx.bits.popcnt != 0)
result |= CPU_POPCNT;
- if (_cpuid_info.std_cpuid1_ecx.bits.avx != 0 &&
- _cpuid_info.std_cpuid1_ecx.bits.osxsave != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.sse != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.ymm != 0) {
+ if (std_cpuid1_ecx.bits.avx != 0 &&
+ std_cpuid1_ecx.bits.osxsave != 0 &&
+ xem_xcr0_eax.bits.sse != 0 &&
+ xem_xcr0_eax.bits.ymm != 0) {
result |= CPU_AVX;
result |= CPU_VZEROUPPER;
- if (_cpuid_info.std_cpuid1_ecx.bits.f16c != 0)
+ if (std_cpuid1_ecx.bits.f16c != 0)
result |= CPU_F16C;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx2 != 0)
+ if (sef_cpuid7_ebx.bits.avx2 != 0)
result |= CPU_AVX2;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512f != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.opmask != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.zmm512 != 0 &&
- _cpuid_info.xem_xcr0_eax.bits.zmm32 != 0) {
+ if (sef_cpuid7_ebx.bits.avx512f != 0 &&
+ xem_xcr0_eax.bits.opmask != 0 &&
+ xem_xcr0_eax.bits.zmm512 != 0 &&
+ xem_xcr0_eax.bits.zmm32 != 0) {
result |= CPU_AVX512F;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512cd != 0)
+ if (sef_cpuid7_ebx.bits.avx512cd != 0)
result |= CPU_AVX512CD;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512dq != 0)
+ if (sef_cpuid7_ebx.bits.avx512dq != 0)
result |= CPU_AVX512DQ;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512ifma != 0)
+ if (sef_cpuid7_ebx.bits.avx512ifma != 0)
result |= CPU_AVX512_IFMA;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512pf != 0)
+ if (sef_cpuid7_ebx.bits.avx512pf != 0)
result |= CPU_AVX512PF;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512er != 0)
+ if (sef_cpuid7_ebx.bits.avx512er != 0)
result |= CPU_AVX512ER;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512bw != 0)
+ if (sef_cpuid7_ebx.bits.avx512bw != 0)
result |= CPU_AVX512BW;
- if (_cpuid_info.sef_cpuid7_ebx.bits.avx512vl != 0)
+ if (sef_cpuid7_ebx.bits.avx512vl != 0)
result |= CPU_AVX512VL;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vpopcntdq != 0)
result |= CPU_AVX512_VPOPCNTDQ;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vpclmulqdq != 0)
result |= CPU_AVX512_VPCLMULQDQ;
- if (_cpuid_info.sef_cpuid7_ecx.bits.vaes != 0)
+ if (sef_cpuid7_ecx.bits.vaes != 0)
result |= CPU_AVX512_VAES;
- if (_cpuid_info.sef_cpuid7_ecx.bits.gfni != 0)
+ if (sef_cpuid7_ecx.bits.gfni != 0)
result |= CPU_GFNI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vnni != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vnni != 0)
result |= CPU_AVX512_VNNI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_bitalg != 0)
+ if (sef_cpuid7_ecx.bits.avx512_bitalg != 0)
result |= CPU_AVX512_BITALG;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vbmi != 0)
result |= CPU_AVX512_VBMI;
- if (_cpuid_info.sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
+ if (sef_cpuid7_ecx.bits.avx512_vbmi2 != 0)
result |= CPU_AVX512_VBMI2;
}
}
- if (_cpuid_info.std_cpuid1_ecx.bits.hv != 0)
+ if (std_cpuid1_ecx.bits.hv != 0)
result |= CPU_HV;
- if (_cpuid_info.sef_cpuid7_ebx.bits.bmi1 != 0)
+ if (sef_cpuid7_ebx.bits.bmi1 != 0)
result |= CPU_BMI1;
- if (_cpuid_info.std_cpuid1_edx.bits.tsc != 0)
+ if (std_cpuid1_edx.bits.tsc != 0)
result |= CPU_TSC;
- if (_cpuid_info.ext_cpuid7_edx.bits.tsc_invariance != 0)
+ if (ext_cpuid7_edx.bits.tsc_invariance != 0)
result |= CPU_TSCINV_BIT;
- if (_cpuid_info.std_cpuid1_ecx.bits.aes != 0)
+ if (std_cpuid1_ecx.bits.aes != 0)
result |= CPU_AES;
- if (_cpuid_info.sef_cpuid7_ebx.bits.erms != 0)
+ if (sef_cpuid7_ebx.bits.erms != 0)
result |= CPU_ERMS;
- if (_cpuid_info.sef_cpuid7_edx.bits.fast_short_rep_mov != 0)
+ if (sef_cpuid7_edx.bits.fast_short_rep_mov != 0)
result |= CPU_FSRM;
- if (_cpuid_info.std_cpuid1_ecx.bits.clmul != 0)
+ if (std_cpuid1_ecx.bits.clmul != 0)
result |= CPU_CLMUL;
- if (_cpuid_info.sef_cpuid7_ebx.bits.rtm != 0)
+ if (sef_cpuid7_ebx.bits.rtm != 0)
result |= CPU_RTM;
- if (_cpuid_info.sef_cpuid7_ebx.bits.adx != 0)
+ if (sef_cpuid7_ebx.bits.adx != 0)
result |= CPU_ADX;
- if (_cpuid_info.sef_cpuid7_ebx.bits.bmi2 != 0)
+ if (sef_cpuid7_ebx.bits.bmi2 != 0)
result |= CPU_BMI2;
- if (_cpuid_info.sef_cpuid7_ebx.bits.sha != 0)
+ if (sef_cpuid7_ebx.bits.sha != 0)
result |= CPU_SHA;
- if (_cpuid_info.std_cpuid1_ecx.bits.fma != 0)
+ if (std_cpuid1_ecx.bits.fma != 0)
result |= CPU_FMA;
- if (_cpuid_info.sef_cpuid7_ebx.bits.clflushopt != 0)
+ if (sef_cpuid7_ebx.bits.clflushopt != 0)
result |= CPU_FLUSHOPT;
- if (_cpuid_info.ext_cpuid1_edx.bits.rdtscp != 0)
+ if (ext_cpuid1_edx.bits.rdtscp != 0)
result |= CPU_RDTSCP;
- if (_cpuid_info.sef_cpuid7_ecx.bits.rdpid != 0)
+ if (sef_cpuid7_ecx.bits.rdpid != 0)
result |= CPU_RDPID;
// AMD|Hygon features.
if (is_amd_family()) {
- if ((_cpuid_info.ext_cpuid1_edx.bits.tdnow != 0) ||
- (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0))
+ if ((ext_cpuid1_edx.bits.tdnow != 0) ||
+ (ext_cpuid1_ecx.bits.prefetchw != 0))
result |= CPU_3DNOW_PREFETCH;
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0)
+ if (ext_cpuid1_ecx.bits.lzcnt != 0)
result |= CPU_LZCNT;
- if (_cpuid_info.ext_cpuid1_ecx.bits.sse4a != 0)
+ if (ext_cpuid1_ecx.bits.sse4a != 0)
result |= CPU_SSE4A;
}
// Intel features.
if (is_intel()) {
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0) {
+ if (ext_cpuid1_ecx.bits.lzcnt != 0) {
result |= CPU_LZCNT;
}
- if (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0) {
+ if (ext_cpuid1_ecx.bits.prefetchw != 0) {
result |= CPU_3DNOW_PREFETCH;
}
- if (_cpuid_info.sef_cpuid7_ebx.bits.clwb != 0) {
+ if (sef_cpuid7_ebx.bits.clwb != 0) {
result |= CPU_CLWB;
}
- if (_cpuid_info.sef_cpuid7_edx.bits.serialize != 0)
+ if (sef_cpuid7_edx.bits.serialize != 0)
result |= CPU_SERIALIZE;
}
// ZX features.
if (is_zx()) {
- if (_cpuid_info.ext_cpuid1_ecx.bits.lzcnt != 0) {
+ if (ext_cpuid1_ecx.bits.lzcnt != 0) {
result |= CPU_LZCNT;
}
- if (_cpuid_info.ext_cpuid1_ecx.bits.prefetchw != 0) {
+ if (ext_cpuid1_ecx.bits.prefetchw != 0) {
result |= CPU_3DNOW_PREFETCH;
}
}
// Protection key features.
- if (_cpuid_info.sef_cpuid7_ecx.bits.pku != 0) {
+ if (sef_cpuid7_ecx.bits.pku != 0) {
result |= CPU_PKU;
}
- if (_cpuid_info.sef_cpuid7_ecx.bits.ospke != 0) {
+ if (sef_cpuid7_ecx.bits.ospke != 0) {
result |= CPU_OSPKE;
}
// Control flow enforcement (CET) features.
- if (_cpuid_info.sef_cpuid7_ecx.bits.cet_ss != 0) {
+ if (sef_cpuid7_ecx.bits.cet_ss != 0) {
result |= CPU_CET_SS;
}
- if (_cpuid_info.sef_cpuid7_edx.bits.cet_ibt != 0) {
+ if (sef_cpuid7_edx.bits.cet_ibt != 0) {
result |= CPU_CET_IBT;
}
diff --git a/src/hotspot/cpu/x86/vm_version_x86.hpp b/src/hotspot/cpu/x86/vm_version_x86.hpp
index cfc16acabc674..03596a6e4468c 100644
--- a/src/hotspot/cpu/x86/vm_version_x86.hpp
+++ b/src/hotspot/cpu/x86/vm_version_x86.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -428,7 +428,8 @@ class VM_Version : public Abstract_VM_Version {
//
// The info block is laid out in subblocks of 4 dwords corresponding to
// eax, ebx, ecx and edx, whether or not they contain anything useful.
- struct CpuidInfo {
+ class CpuidInfo {
+ public:
// cpuid function 0
uint32_t std_max_function;
uint32_t std_vendor_name_0;
@@ -522,6 +523,31 @@ class VM_Version : public Abstract_VM_Version {
// Space to save zmm registers after signal handle
int zmm_save[16*4]; // Save zmm0, zmm7, zmm8, zmm31
+
+ uint64_t feature_flags() const;
+
+ // Asserts
+ void assert_is_initialized() const {
+ assert(std_cpuid1_eax.bits.family != 0, "VM_Version not initialized");
+ }
+
+ // Extractors
+ uint32_t extended_cpu_family() const {
+ uint32_t result = std_cpuid1_eax.bits.family;
+ result += std_cpuid1_eax.bits.ext_family;
+ return result;
+ }
+
+ uint32_t extended_cpu_model() const {
+ uint32_t result = std_cpuid1_eax.bits.model;
+ result |= std_cpuid1_eax.bits.ext_model << 4;
+ return result;
+ }
+
+ uint32_t cpu_stepping() const {
+ uint32_t result = std_cpuid1_eax.bits.stepping;
+ return result;
+ }
};
private:
@@ -529,23 +555,6 @@ class VM_Version : public Abstract_VM_Version {
static CpuidInfo _cpuid_info;
// Extractors and predicates
- static uint32_t extended_cpu_family() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.family;
- result += _cpuid_info.std_cpuid1_eax.bits.ext_family;
- return result;
- }
-
- static uint32_t extended_cpu_model() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.model;
- result |= _cpuid_info.std_cpuid1_eax.bits.ext_model << 4;
- return result;
- }
-
- static uint32_t cpu_stepping() {
- uint32_t result = _cpuid_info.std_cpuid1_eax.bits.stepping;
- return result;
- }
-
static uint logical_processor_count() {
uint result = threads_per_core();
return result;
@@ -553,7 +562,6 @@ class VM_Version : public Abstract_VM_Version {
static bool compute_has_intel_jcc_erratum();
- static uint64_t feature_flags();
static bool os_supports_avx_vectors();
static void get_processor_features();
@@ -594,11 +602,6 @@ class VM_Version : public Abstract_VM_Version {
// Override Abstract_VM_Version implementation
static void print_platform_virtualization_info(outputStream*);
- // Asserts
- static void assert_is_initialized() {
- assert(_cpuid_info.std_cpuid1_eax.bits.family != 0, "VM_Version not initialized");
- }
-
//
// Processor family:
// 3 - 386
@@ -614,6 +617,10 @@ class VM_Version : public Abstract_VM_Version {
// processors. Use the feature test functions below to
// determine whether a particular instruction is supported.
//
+ static void assert_is_initialized() { _cpuid_info.assert_is_initialized(); }
+ static uint32_t extended_cpu_family() { return _cpuid_info.extended_cpu_family(); }
+ static uint32_t extended_cpu_model() { return _cpuid_info.extended_cpu_model(); }
+ static uint32_t cpu_stepping() { return _cpuid_info.cpu_stepping(); }
static int cpu_family() { return _cpu;}
static bool is_P6() { return cpu_family() >= 6; }
static bool is_amd() { assert_is_initialized(); return _cpuid_info.std_vendor_name_0 == 0x68747541; } // 'htuA'
@@ -770,6 +777,10 @@ class VM_Version : public Abstract_VM_Version {
return true;
}
+ constexpr static bool supports_recursive_lightweight_locking() {
+ return true;
+ }
+
// For AVX CPUs only. f16c support is disabled if UseAVX == 0.
static bool supports_float16() {
return supports_f16c() || supports_avx512vl();
diff --git a/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp b/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
index 0e78e0274d7f2..398f2e37eb5cc 100644
--- a/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
+++ b/src/hotspot/cpu/x86/vtableStubs_x86_32.cpp
@@ -24,10 +24,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_x86.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -176,21 +176,21 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
#endif /* PRODUCT */
// Entry arguments:
- // rax: CompiledICHolder
+ // rax: CompiledICData
// rcx: Receiver
// Most registers are in use; we'll use rax, rbx, rcx, rdx, rsi, rdi
// (If we need to make rsi, rdi callee-save, do a push/pop here.)
const Register recv_klass_reg = rsi;
- const Register holder_klass_reg = rax; // declaring interface klass (DECC)
+ const Register holder_klass_reg = rax; // declaring interface klass (DEFC)
const Register resolved_klass_reg = rdi; // resolved interface klass (REFC)
const Register temp_reg = rdx;
const Register method = rbx;
- const Register icholder_reg = rax;
+ const Register icdata_reg = rax;
const Register receiver = rcx;
- __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ movptr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ movptr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
Label L_no_such_interface;
diff --git a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
index f162a651183f9..158d6f9c6922b 100644
--- a/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
+++ b/src/hotspot/cpu/x86/vtableStubs_x86_64.cpp
@@ -24,10 +24,10 @@
#include "precompiled.hpp"
#include "asm/macroAssembler.hpp"
+#include "code/compiledIC.hpp"
#include "code/vtableStubs.hpp"
#include "interp_masm_x86.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.hpp"
#include "oops/instanceKlass.hpp"
#include "oops/klassVtable.hpp"
#include "runtime/sharedRuntime.hpp"
@@ -168,21 +168,21 @@ VtableStub* VtableStubs::create_itable_stub(int itable_index) {
#endif // PRODUCT
// Entry arguments:
- // rax: CompiledICHolder
+ // rax: CompiledICData
// j_rarg0: Receiver
// Most registers are in use; we'll use rax, rbx, r10, r11
// (various calling sequences use r[cd]x, r[sd]i, r[89]; stay away from them)
const Register recv_klass_reg = r10;
- const Register holder_klass_reg = rax; // declaring interface klass (DECC)
+ const Register holder_klass_reg = rax; // declaring interface klass (DEFC)
const Register resolved_klass_reg = r14; // resolved interface klass (REFC)
const Register temp_reg = r11;
const Register temp_reg2 = r13;
const Register method = rbx;
- const Register icholder_reg = rax;
+ const Register icdata_reg = rax;
- __ movptr(resolved_klass_reg, Address(icholder_reg, CompiledICHolder::holder_klass_offset()));
- __ movptr(holder_klass_reg, Address(icholder_reg, CompiledICHolder::holder_metadata_offset()));
+ __ movptr(resolved_klass_reg, Address(icdata_reg, CompiledICData::itable_refc_klass_offset()));
+ __ movptr(holder_klass_reg, Address(icdata_reg, CompiledICData::itable_defc_klass_offset()));
Label L_no_such_interface;
diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad
index a31548eb8c3f9..6df02d280bcef 100644
--- a/src/hotspot/cpu/x86/x86.ad
+++ b/src/hotspot/cpu/x86/x86.ad
@@ -1358,7 +1358,7 @@ int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf) {
return offset;
}
-Assembler::Width widthForType(BasicType bt) {
+static Assembler::Width widthForType(BasicType bt) {
if (bt == T_BYTE) {
return Assembler::B;
} else if (bt == T_SHORT) {
diff --git a/src/hotspot/cpu/x86/x86_32.ad b/src/hotspot/cpu/x86/x86_32.ad
index 9aa0051043575..2fe655a576778 100644
--- a/src/hotspot/cpu/x86/x86_32.ad
+++ b/src/hotspot/cpu/x86/x86_32.ad
@@ -504,7 +504,7 @@ void emit_cmpfp_fixup(MacroAssembler& _masm) {
__ bind(exit);
}
-void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
+static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
Label done;
__ movl(dst, -1);
__ jcc(Assembler::parity, done);
@@ -1383,24 +1383,12 @@ void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
MacroAssembler masm(&cbuf);
-#ifdef ASSERT
- uint insts_size = cbuf.insts_size();
-#endif
- masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes()));
- masm.jump_cc(Assembler::notEqual,
- RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
- /* WARNING these NOPs are critical so that verified entry point is properly
- aligned for patching by NativeJump::patch_verified_entry() */
- int nops_cnt = 2;
- if( !OptoBreakpoint ) // Leave space for int3
- nops_cnt += 1;
- masm.nop(nops_cnt);
-
- assert(cbuf.insts_size() - insts_size == size(ra_), "checking code size of inline cache node");
+ masm.ic_check(CodeEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
- return OptoBreakpoint ? 11 : 12;
+ return MachNode::size(ra_); // too many variables; just compute it
+ // the hard way
}
@@ -1842,7 +1830,7 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, cbuf.insts()->mark_off());
} else {
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
@@ -13776,7 +13764,7 @@ instruct cmpFastLockRTM(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eD
%}
instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr, eRegP thread) %{
- predicate(!Compile::current()->use_rtm());
+ predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP scr, USE_KILL box, TEMP thread);
ins_cost(300);
@@ -13790,6 +13778,7 @@ instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP
%}
instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, USE_KILL box);
ins_cost(300);
@@ -13800,6 +13789,32 @@ instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
ins_pipe(pipe_slow);
%}
+instruct cmpFastLockLightweight(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI eax_reg, eRegP tmp, eRegP thread) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP eax_reg, TEMP tmp, USE_KILL box, TEMP thread);
+ ins_cost(300);
+ format %{ "FASTLOCK $object,$box\t! kills $box,$eax_reg,$tmp" %}
+ ins_encode %{
+ __ get_thread($thread$$Register);
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct cmpFastUnlockLightweight(eFlagsReg cr, eRegP object, eAXRegP eax_reg, eRegP tmp, eRegP thread) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object eax_reg));
+ effect(TEMP tmp, USE_KILL eax_reg, TEMP thread);
+ ins_cost(300);
+ format %{ "FASTUNLOCK $object,$eax_reg\t! kills $eax_reg,$tmp" %}
+ ins_encode %{
+ __ get_thread($thread$$Register);
+ __ fast_unlock_lightweight($object$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
instruct mask_all_evexL_LT32(kReg dst, eRegL src) %{
predicate(Matcher::vector_length(n) <= 32);
match(Set dst (MaskAll src));
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index a248daaa1917b..d43929efd3ec5 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -519,7 +519,7 @@ int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
}
// This could be in MacroAssembler but it's fairly C2 specific
-void emit_cmpfp_fixup(MacroAssembler& _masm) {
+static void emit_cmpfp_fixup(MacroAssembler& _masm) {
Label exit;
__ jccb(Assembler::noParity, exit);
__ pushf();
@@ -539,7 +539,7 @@ void emit_cmpfp_fixup(MacroAssembler& _masm) {
__ bind(exit);
}
-void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
+static void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
Label done;
__ movl(dst, -1);
__ jcc(Assembler::parity, done);
@@ -558,10 +558,10 @@ void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
// je #
// |-jz -> a | b # a & b
// | -> a #
-void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
- XMMRegister a, XMMRegister b,
- XMMRegister xmmt, Register rt,
- bool min, bool single) {
+static void emit_fp_min_max(MacroAssembler& _masm, XMMRegister dst,
+ XMMRegister a, XMMRegister b,
+ XMMRegister xmmt, Register rt,
+ bool min, bool single) {
Label nan, zero, below, above, done;
@@ -1472,40 +1472,19 @@ void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
{
if (UseCompressedClassPointers) {
st->print_cr("movl rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
- st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
- st->print_cr("\tcmpq rax, rscratch1\t # Inline cache check");
+ st->print_cr("\tcmpl rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
} else {
- st->print_cr("\tcmpq rax, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t"
- "# Inline cache check");
+ st->print_cr("movq rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
+ st->print_cr("\tcmpq rscratch1, [rax + CompiledICData::speculated_klass_offset()]\t # Inline cache check");
}
st->print_cr("\tjne SharedRuntime::_ic_miss_stub");
- st->print_cr("\tnop\t# nops to align entry point");
}
#endif
void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
{
MacroAssembler masm(&cbuf);
- uint insts_size = cbuf.insts_size();
- if (UseCompressedClassPointers) {
- masm.load_klass(rscratch1, j_rarg0, rscratch2);
- masm.cmpptr(rax, rscratch1);
- } else {
- masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
- }
-
- masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
-
- /* WARNING these NOPs are critical so that verified entry point is properly
- 4 bytes aligned for patching by NativeJump::patch_verified_entry() */
- int nops_cnt = 4 - ((cbuf.insts_size() - insts_size) & 0x3);
- if (OptoBreakpoint) {
- // Leave space for int3
- nops_cnt -= 1;
- }
- nops_cnt &= 0x3; // Do not add nops if code is aligned.
- if (nops_cnt > 0)
- masm.nop(nops_cnt);
+ masm.ic_check(InteriorEntryAlignment);
}
uint MachUEPNode::size(PhaseRegAlloc* ra_) const
@@ -1840,7 +1819,7 @@ encode %{
cbuf.shared_stub_to_interp_for(_method, call_offset);
} else {
// Emit stubs for static call.
- address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
+ address stub = CompiledDirectCall::emit_to_interp_stub(cbuf, mark);
if (stub == nullptr) {
ciEnv::current()->record_failure("CodeCache is full");
return;
@@ -4480,7 +4459,7 @@ instruct loadD(regD dst, memory mem)
// max = java.lang.Math.max(float a, float b)
instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "maxF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4491,7 +4470,7 @@ instruct maxF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
%}
instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxF a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4505,7 +4484,7 @@ instruct maxF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRe
// max = java.lang.Math.max(double a, double b)
instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
effect(USE a, USE b, TEMP atmp, TEMP btmp, TEMP tmp);
format %{ "maxD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4516,7 +4495,7 @@ instruct maxD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
%}
instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MaxD a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4530,7 +4509,7 @@ instruct maxD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRe
// min = java.lang.Math.min(float a, float b)
instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp, legRegF btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "minF $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4541,7 +4520,7 @@ instruct minF_reg(legRegF dst, legRegF a, legRegF b, legRegF tmp, legRegF atmp,
%}
instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRegI tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinF a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -4555,7 +4534,7 @@ instruct minF_reduction_reg(legRegF dst, legRegF a, legRegF b, legRegF xmmt, rRe
// min = java.lang.Math.min(double a, double b)
instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp, legRegD btmp) %{
- predicate(UseAVX > 0 && !SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && !VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP tmp, TEMP atmp, TEMP btmp);
format %{ "minD $dst, $a, $b \t! using tmp, atmp and btmp as TEMP" %}
@@ -4566,7 +4545,7 @@ instruct minD_reg(legRegD dst, legRegD a, legRegD b, legRegD tmp, legRegD atmp,
%}
instruct minD_reduction_reg(legRegD dst, legRegD a, legRegD b, legRegD xmmt, rRegL tmp, rFlagsReg cr) %{
- predicate(UseAVX > 0 && SuperWord::is_reduction(n));
+ predicate(UseAVX > 0 && VLoopReductions::is_reduction(n));
match(Set dst (MinD a b));
effect(USE a, USE b, TEMP xmmt, TEMP tmp, KILL cr);
@@ -12404,7 +12383,7 @@ instruct cmpFastLockRTM(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp,
%}
instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
- predicate(!Compile::current()->use_rtm());
+ predicate(LockingMode != LM_LIGHTWEIGHT && !Compile::current()->use_rtm());
match(Set cr (FastLock object box));
effect(TEMP tmp, TEMP scr, USE_KILL box);
ins_cost(300);
@@ -12417,6 +12396,7 @@ instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRe
%}
instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
+ predicate(LockingMode != LM_LIGHTWEIGHT);
match(Set cr (FastUnlock object box));
effect(TEMP tmp, USE_KILL box);
ins_cost(300);
@@ -12427,6 +12407,30 @@ instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
ins_pipe(pipe_slow);
%}
+instruct cmpFastLockLightweight(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI rax_reg, rRegP tmp) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastLock object box));
+ effect(TEMP rax_reg, TEMP tmp, USE_KILL box);
+ ins_cost(300);
+ format %{ "fastlock $object,$box\t! kills $box,$rax_reg,$tmp" %}
+ ins_encode %{
+ __ fast_lock_lightweight($object$$Register, $box$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
+instruct cmpFastUnlockLightweight(rFlagsReg cr, rRegP object, rax_RegP rax_reg, rRegP tmp) %{
+ predicate(LockingMode == LM_LIGHTWEIGHT);
+ match(Set cr (FastUnlock object rax_reg));
+ effect(TEMP tmp, USE_KILL rax_reg);
+ ins_cost(300);
+ format %{ "fastunlock $object,$rax_reg\t! kills $rax_reg,$tmp" %}
+ ins_encode %{
+ __ fast_unlock_lightweight($object$$Register, $rax_reg$$Register, $tmp$$Register, r15_thread);
+ %}
+ ins_pipe(pipe_slow);
+%}
+
// ============================================================================
// Safepoint Instructions
diff --git a/src/hotspot/cpu/zero/compiledIC_zero.cpp b/src/hotspot/cpu/zero/compiledIC_zero.cpp
index b0564643af080..24153aeacc5e1 100644
--- a/src/hotspot/cpu/zero/compiledIC_zero.cpp
+++ b/src/hotspot/cpu/zero/compiledIC_zero.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
@@ -43,27 +42,27 @@
// ----------------------------------------------------------------------------
-address CompiledStaticCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
+address CompiledDirectCall::emit_to_interp_stub(CodeBuffer &cbuf, address mark) {
ShouldNotReachHere(); // Only needed for COMPILER2.
return nullptr;
}
-int CompiledStaticCall::to_interp_stub_size() {
+int CompiledDirectCall::to_interp_stub_size() {
ShouldNotReachHere(); // Only needed for COMPILER2.
return 0;
}
// Relocation entries for call stub, compiled java to interpreter.
-int CompiledStaticCall::reloc_to_interp_stub() {
+int CompiledDirectCall::reloc_to_interp_stub() {
ShouldNotReachHere(); // Only needed for COMPILER2.
return 0;
}
-void CompiledDirectStaticCall::set_to_interpreted(const methodHandle& callee, address entry) {
+void CompiledDirectCall::set_to_interpreted(const methodHandle& callee, address entry) {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
-void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
+void CompiledDirectCall::set_stub_to_clean(static_stub_Relocation* static_stub) {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
@@ -71,7 +70,7 @@ void CompiledDirectStaticCall::set_stub_to_clean(static_stub_Relocation* static_
// Non-product mode code.
#ifndef PRODUCT
-void CompiledDirectStaticCall::verify() {
+void CompiledDirectCall::verify() {
ShouldNotReachHere(); // Only needed for COMPILER2.
}
diff --git a/src/hotspot/cpu/zero/icBuffer_zero.cpp b/src/hotspot/cpu/zero/icBuffer_zero.cpp
deleted file mode 100644
index adde916a4c4ad..0000000000000
--- a/src/hotspot/cpu/zero/icBuffer_zero.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2007 Red Hat, Inc.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "asm/assembler.inline.hpp"
-#include "code/icBuffer.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/resourceArea.hpp"
-#include "nativeInst_zero.hpp"
-#include "oops/oop.inline.hpp"
-
-int InlineCacheBuffer::ic_stub_code_size() {
- // NB set this once the functions below are implemented
- return 4;
-}
-
-void InlineCacheBuffer::assemble_ic_buffer_code(address code_begin,
- void* cached_oop,
- address entry_point) {
- // NB ic_stub_code_size() must return the size of the code we generate
- ShouldNotCallThis();
-}
-
-address InlineCacheBuffer::ic_buffer_entry_point(address code_begin) {
- // NB ic_stub_code_size() must return the size of the code we generate
- ShouldNotCallThis();
- return nullptr;
-}
-
-void* InlineCacheBuffer::ic_buffer_cached_value(address code_begin) {
- ShouldNotCallThis();
- return nullptr;
-}
diff --git a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
index 4244b5817db98..986cee685123b 100644
--- a/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
+++ b/src/hotspot/cpu/zero/sharedRuntime_zero.cpp
@@ -26,10 +26,8 @@
#include "precompiled.hpp"
#include "asm/assembler.inline.hpp"
#include "code/debugInfoRec.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
-#include "oops/compiledICHolder.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/vframeArray.hpp"
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index e701e0aef6082..0c1c0dbc6dcb1 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -29,7 +29,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "interpreter/interpreter.hpp"
@@ -1128,10 +1127,9 @@ bool os::dll_address_to_library_name(address addr, char* buf,
return true;
}
-void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
+static void* dll_load_library(const char *filename, char *ebuf, int ebuflen) {
log_info(os)("attempting shared library load of %s", filename);
-
if (ebuf && ebuflen > 0) {
ebuf[0] = '\0';
ebuf[ebuflen - 1] = '\0';
@@ -1179,6 +1177,26 @@ void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
}
return nullptr;
}
+// Load library named
+// If filename matches .so, and loading fails, repeat with .a.
+void *os::dll_load(const char *filename, char *ebuf, int ebuflen) {
+ void* result = nullptr;
+ char* const file_path = strdup(filename);
+ char* const pointer_to_dot = strrchr(file_path, '.');
+ const char old_extension[] = ".so";
+ const char new_extension[] = ".a";
+ STATIC_ASSERT(sizeof(old_extension) >= sizeof(new_extension));
+ // First try to load the existing file.
+ result = dll_load_library(filename, ebuf, ebuflen);
+ // If the load fails,we try to reload by changing the extension to .a for .so files only.
+ // Shared object in .so format dont have braces, hence they get removed for archives with members.
+ if (result == nullptr && pointer_to_dot != nullptr && strcmp(pointer_to_dot, old_extension) == 0) {
+ snprintf(pointer_to_dot, sizeof(old_extension), "%s", new_extension);
+ result = dll_load_library(file_path, ebuf, ebuflen);
+ }
+ FREE_C_HEAP_ARRAY(char, file_path);
+ return result;
+}
void os::print_dll_info(outputStream *st) {
st->print_cr("Dynamic libraries:");
@@ -2609,56 +2627,6 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags = MAP_PRIVATE;
-
- if (read_only) {
- prot = PROT_READ;
- flags = MAP_SHARED;
- } else {
- prot = PROT_READ | PROT_WRITE;
- flags = MAP_PRIVATE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- // Allow anonymous mappings if 'fd' is -1.
- if (fd == -1) {
- flags |= MAP_ANONYMOUS;
- }
-
- char* mapped_address = (char*)::mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
-}
-
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
// are used by JVM M&M and JVMTI to get user+sys or user CPU time
// of a thread.
diff --git a/src/hotspot/os/aix/os_perf_aix.cpp b/src/hotspot/os/aix/os_perf_aix.cpp
index e1719df48c331..b5ae1a6a725a5 100644
--- a/src/hotspot/os/aix/os_perf_aix.cpp
+++ b/src/hotspot/os/aix/os_perf_aix.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2022, IBM Corp.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, IBM Corp.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -87,6 +87,7 @@ static bool read_psinfo(const u_longlong_t& pid, psinfo_t& psinfo) {
}
len = fread(&psinfo, 1, sizeof(psinfo_t), fp);
+ fclose(fp);
return len == sizeof(psinfo_t);
}
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index 42a0b9c083239..0e67228de7b91 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -24,7 +24,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/disassembler.hpp"
@@ -1269,7 +1268,8 @@ void * os::dll_load(const char *filename, char *ebuf, int ebuflen) {
}
#endif // !__APPLE__
-int _print_dll_info_cb(const char * name, address base_address, address top_address, void * param) {
+static int _print_dll_info_cb(const char * name, address base_address,
+ address top_address, void * param) {
outputStream * out = (outputStream *) param;
out->print_cr(INTPTR_FORMAT " \t%s", (intptr_t)base_address, name);
return 0;
@@ -2345,53 +2345,6 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags;
-
- if (read_only) {
- prot = PROT_READ;
- flags = MAP_SHARED;
- } else {
- prot = PROT_READ | PROT_WRITE;
- flags = MAP_PRIVATE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
-}
-
// current_thread_cpu_time(bool) and thread_cpu_time(Thread*, bool)
// are used by JVM M&M and JVMTI to get user+sys or user CPU time
// of a thread.
diff --git a/src/hotspot/os/linux/hugepages.cpp b/src/hotspot/os/linux/hugepages.cpp
index 54e6c1adb7a97..b71593487cf8a 100644
--- a/src/hotspot/os/linux/hugepages.cpp
+++ b/src/hotspot/os/linux/hugepages.cpp
@@ -311,6 +311,19 @@ ExplicitHugePageSupport HugePages::_explicit_hugepage_support;
THPSupport HugePages::_thp_support;
ShmemTHPSupport HugePages::_shmem_thp_support;
+size_t HugePages::thp_pagesize_fallback() {
+ // Older kernels won't publish the THP page size. Fall back to default explicit huge page size,
+ // since that is likely to be the THP page size as well. Don't do it if the page size is considered
+ // too large to avoid large alignment waste. If explicit huge page size is unknown, use educated guess.
+ if (thp_pagesize() != 0) {
+ return thp_pagesize();
+ }
+ if (supports_explicit_hugepages()) {
+ return MIN2(default_explicit_hugepage_size(), 16 * M);
+ }
+ return 2 * M;
+}
+
void HugePages::initialize() {
_explicit_hugepage_support.scan_os();
_thp_support.scan_os();
diff --git a/src/hotspot/os/linux/hugepages.hpp b/src/hotspot/os/linux/hugepages.hpp
index 2e61fabc5a507..efd27c55fd60f 100644
--- a/src/hotspot/os/linux/hugepages.hpp
+++ b/src/hotspot/os/linux/hugepages.hpp
@@ -138,6 +138,7 @@ class HugePages : public AllStatic {
static bool supports_thp() { return thp_mode() == THPMode::madvise || thp_mode() == THPMode::always; }
static THPMode thp_mode() { return _thp_support.mode(); }
static size_t thp_pagesize() { return _thp_support.pagesize(); }
+ static size_t thp_pagesize_fallback();
static bool supports_shmem_thp() { return _shmem_thp_support.is_enabled(); }
static ShmemTHPMode shmem_thp_mode() { return _shmem_thp_support.mode(); }
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index f02ca95be5593..c51aeb0ae17c1 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/disassembler.hpp"
@@ -86,6 +85,8 @@
#endif
// put OS-includes here
+# include
+# include
# include
# include
# include
@@ -343,6 +344,29 @@ static void next_line(FILE *f) {
} while (c != '\n' && c != EOF);
}
+void os::Linux::kernel_version(long* major, long* minor) {
+ *major = -1;
+ *minor = -1;
+
+ struct utsname buffer;
+ int ret = uname(&buffer);
+ if (ret != 0) {
+ log_warning(os)("uname(2) failed to get kernel version: %s", os::errno_name(ret));
+ return;
+ }
+
+ char* walker = buffer.release;
+ long* set_v = major;
+ while (*minor == -1 && walker != nullptr) {
+ if (isdigit(walker[0])) {
+ *set_v = strtol(walker, &walker, 10);
+ set_v = minor;
+ } else {
+ ++walker;
+ }
+ }
+}
+
bool os::Linux::get_tick_information(CPUPerfTicks* pticks, int which_logical_cpu) {
FILE* fh;
uint64_t userTicks, niceTicks, systemTicks, idleTicks;
@@ -3936,8 +3960,12 @@ void os::Linux::large_page_init() {
// In THP mode:
// - os::large_page_size() is the *THP page size*
// - os::pagesizes() has two members, the THP page size and the system page size
- assert(HugePages::thp_pagesize() > 0, "Missing OS info");
_large_page_size = HugePages::thp_pagesize();
+ if (_large_page_size == 0) {
+ log_info(pagesize) ("Cannot determine THP page size (kernel < 4.10 ?)");
+ _large_page_size = HugePages::thp_pagesize_fallback();
+ log_info(pagesize) ("Assuming THP page size to be: " EXACTFMT " (heuristics)", EXACTFMTARGS(_large_page_size));
+ }
_page_sizes.add(_large_page_size);
_page_sizes.add(os::vm_page_size());
// +UseTransparentHugePages implies +UseLargePages
@@ -5064,51 +5092,6 @@ jlong os::seek_to_file_offset(int fd, jlong offset) {
return (jlong)::lseek(fd, (off_t)offset, SEEK_SET);
}
-// Map a block of memory.
-char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- int prot;
- int flags = MAP_PRIVATE;
-
- if (read_only) {
- prot = PROT_READ;
- } else {
- prot = PROT_READ | PROT_WRITE;
- }
-
- if (allow_exec) {
- prot |= PROT_EXEC;
- }
-
- if (addr != nullptr) {
- flags |= MAP_FIXED;
- }
-
- char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
- fd, file_offset);
- if (mapped_address == MAP_FAILED) {
- return nullptr;
- }
- return mapped_address;
-}
-
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // same as map_memory() on this OS
- return os::map_memory(fd, file_name, file_offset, addr, bytes, read_only,
- allow_exec);
-}
-
-
-// Unmap a block of memory.
-bool os::pd_unmap_memory(char* addr, size_t bytes) {
- return munmap(addr, bytes) == 0;
-}
-
static jlong slow_thread_cpu_time(Thread *thread, bool user_sys_cpu_time);
static jlong fast_cpu_time(Thread *thread) {
diff --git a/src/hotspot/os/linux/os_linux.hpp b/src/hotspot/os/linux/os_linux.hpp
index 4b2ccf8e370db..6b902e8280244 100644
--- a/src/hotspot/os/linux/os_linux.hpp
+++ b/src/hotspot/os/linux/os_linux.hpp
@@ -93,6 +93,8 @@ class os::Linux {
bool has_steal_ticks;
};
+ static void kernel_version(long* major, long* minor);
+
// which_logical_cpu=-1 returns accumulated ticks for all cpus.
static bool get_tick_information(CPUPerfTicks* pticks, int which_logical_cpu);
static bool _stack_is_executable;
diff --git a/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp b/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
index 446449a40e094..892d825b40cdc 100644
--- a/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
+++ b/src/hotspot/os/linux/systemMemoryBarrier_linux.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "logging/log.hpp"
-#include "runtime/os.hpp"
+#include "os_linux.hpp"
#include "utilities/debug.hpp"
#include "utilities/systemMemoryBarrier.hpp"
@@ -61,6 +61,18 @@ static long membarrier(int cmd, unsigned int flags, int cpu_id) {
}
bool LinuxSystemMemoryBarrier::initialize() {
+#if defined(RISCV)
+// RISCV port was introduced in kernel 4.4.
+// 4.4 also made membar private expedited mandatory.
+// But RISCV actually don't support it until 6.9.
+ long major, minor;
+ os::Linux::kernel_version(&major, &minor);
+ if (!(major > 6 || (major == 6 && minor >= 9))) {
+ log_info(os)("Linux kernel %ld.%ld does not support MEMBARRIER PRIVATE_EXPEDITED on RISC-V.",
+ major, minor);
+ return false;
+ }
+#endif
long ret = membarrier(MEMBARRIER_CMD_QUERY, 0, 0);
if (ret < 0) {
log_info(os)("MEMBARRIER_CMD_QUERY unsupported");
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 339cc475b34ae..39a6779b3fc04 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -2031,3 +2031,53 @@ void os::die() {
const char* os::file_separator() { return "/"; }
const char* os::line_separator() { return "\n"; }
const char* os::path_separator() { return ":"; }
+
+// Map file into memory; uses mmap().
+// Notes:
+// - if caller specifies addr, MAP_FIXED is used. That means existing
+// mappings will be replaced.
+// - The file descriptor must be valid (to create anonymous mappings, use
+// os::reserve_memory()).
+// Returns address to mapped memory, nullptr on error
+char* os::pd_map_memory(int fd, const char* unused,
+ size_t file_offset, char *addr, size_t bytes,
+ bool read_only, bool allow_exec) {
+
+ assert(fd != -1, "Specify a valid file descriptor");
+
+ int prot;
+ int flags = MAP_PRIVATE;
+
+ if (read_only) {
+ prot = PROT_READ;
+ } else {
+ prot = PROT_READ | PROT_WRITE;
+ }
+
+ if (allow_exec) {
+ prot |= PROT_EXEC;
+ }
+
+ if (addr != nullptr) {
+ flags |= MAP_FIXED;
+ }
+
+ char* mapped_address = (char*)mmap(addr, (size_t)bytes, prot, flags,
+ fd, file_offset);
+ if (mapped_address == MAP_FAILED) {
+ return nullptr;
+ }
+
+ // If we did specify an address, and the mapping succeeded, it should
+ // have returned that address since we specify MAP_FIXED
+ assert(addr == nullptr || addr == mapped_address,
+ "mmap+MAP_FIXED returned " PTR_FORMAT ", expected " PTR_FORMAT,
+ p2i(mapped_address), p2i(addr));
+
+ return mapped_address;
+}
+
+// Unmap a block of memory. Uses munmap.
+bool os::pd_unmap_memory(char* addr, size_t bytes) {
+ return munmap(addr, bytes) == 0;
+}
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index eaadb36731518..6a958f8903b8e 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -340,7 +340,7 @@ static const struct {
////////////////////////////////////////////////////////////////////////////////
// sun.misc.Signal and BREAK_SIGNAL support
-void jdk_misc_signal_init() {
+static void jdk_misc_signal_init() {
// Initialize signal structures
::memset((void*)pending_signals, 0, sizeof(pending_signals));
@@ -380,7 +380,7 @@ int os::signal_wait() {
////////////////////////////////////////////////////////////////////////////////
// signal chaining support
-struct sigaction* get_chained_signal_action(int sig) {
+static struct sigaction* get_chained_signal_action(int sig) {
struct sigaction *actp = nullptr;
if (libjsig_is_loaded) {
@@ -1245,7 +1245,7 @@ int os::get_signal_number(const char* signal_name) {
return -1;
}
-void set_signal_handler(int sig) {
+static void set_signal_handler(int sig) {
// Check for overwrite.
struct sigaction oldAct;
sigaction(sig, (struct sigaction*)nullptr, &oldAct);
@@ -1292,7 +1292,7 @@ void set_signal_handler(int sig) {
// install signal handlers for signals that HotSpot needs to
// handle in order to support Java-level exception handling.
-void install_signal_handlers() {
+static void install_signal_handlers() {
// signal-chaining
typedef void (*signal_setting_t)();
signal_setting_t begin_signal_setting = nullptr;
@@ -1723,7 +1723,7 @@ static void SR_handler(int sig, siginfo_t* siginfo, void* context) {
errno = old_errno;
}
-int SR_initialize() {
+static int SR_initialize() {
struct sigaction act;
char *s;
// Get signal number to use for suspend/resume
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 3613edfc7d9e6..2ddebc9bf6dee 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -27,7 +27,6 @@
// no precompiled headers
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
@@ -5168,22 +5167,6 @@ char* os::pd_map_memory(int fd, const char* file_name, size_t file_offset,
return base;
}
-
-// Remap a block of memory.
-char* os::pd_remap_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec) {
- // This OS does not allow existing memory maps to be remapped so we
- // would have to unmap the memory before we remap it.
-
- // Because there is a small window between unmapping memory and mapping
- // it in again with different protections, CDS archives are mapped RW
- // on windows, so this function isn't called.
- ShouldNotReachHere();
- return nullptr;
-}
-
-
// Unmap a block of memory.
// Returns true=success, otherwise false.
diff --git a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
index 5e0086521aad9..242042d4247aa 100644
--- a/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/os_aix_ppc.cpp
@@ -28,7 +28,6 @@
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S b/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
index 7b286820a9a8f..187cd20ddbdad 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
+++ b/src/hotspot/os_cpu/bsd_aarch64/copy_bsd_aarch64.S
@@ -28,6 +28,11 @@
.global CFUNC(_Copy_conjoint_words)
.global CFUNC(_Copy_disjoint_words)
+#ifdef __APPLE__
+ .private_extern CFUNC(_Copy_conjoint_words)
+ .private_extern CFUNC(_Copy_disjoint_words)
+#endif
+
s .req x0
d .req x1
count .req x2
diff --git a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
index fbd7c4eccd403..4750ed8805644 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/os_bsd_aarch64.cpp
@@ -29,7 +29,6 @@
#include "classfile/classLoader.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
index 34d7b8e34a739..b9b6df9b23aa0 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
+++ b/src/hotspot/os_cpu/bsd_aarch64/safefetch_bsd_aarch64.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -39,6 +39,15 @@
.global SYMBOL(_SafeFetch32_fault)
.global SYMBOL(_SafeFetch32_continuation)
+#ifdef __APPLE__
+ .private_extern SYMBOL(SafeFetchN_impl)
+ .private_extern SYMBOL(_SafeFetchN_fault)
+ .private_extern SYMBOL(_SafeFetchN_continuation)
+ .private_extern SYMBOL(SafeFetch32_impl)
+ .private_extern SYMBOL(_SafeFetch32_fault)
+ .private_extern SYMBOL(_SafeFetch32_continuation)
+#endif
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x0 : address
diff --git a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S b/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
index 02231040e15bd..5cad379df3f2b 100644
--- a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
+++ b/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
#endif
.globl SYMBOL(fixcw)
+ .globl SYMBOL(SpinPause)
# NOTE WELL! The _Copy functions are called directly
# from server-compiler-generated code via CallLeafNoFP,
@@ -50,6 +51,20 @@
.globl SYMBOL(_Atomic_cmpxchg_long)
.globl SYMBOL(_Atomic_move_long)
+#ifdef __APPLE__
+ .private_extern SYMBOL(fixcw)
+ .private_extern SYMBOL(SpinPause)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_bytes)
+ .private_extern SYMBOL(_Copy_conjoint_jshorts_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Copy_conjoint_jints_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jints)
+ .private_extern SYMBOL(_Copy_conjoint_jlongs_atomic)
+ .private_extern SYMBOL(_mmx_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Atomic_cmpxchg_long)
+ .private_extern SYMBOL(_Atomic_move_long)
+#endif
+
.text
# Support for void os::Solaris::init_thread_fpu_state() in os_solaris_i486.cpp
@@ -62,7 +77,6 @@ SYMBOL(fixcw):
popl %eax
ret
- .globl SYMBOL(SpinPause)
ELF_TYPE(SpinPause,@function)
.p2align 4,,15
SYMBOL(SpinPause):
diff --git a/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S b/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
index 95cea3bf2a3d3..5e2addc4e6f43 100644
--- a/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
+++ b/src/hotspot/os_cpu/bsd_x86/bsd_x86_64.S
@@ -1,5 +1,5 @@
-#
-# Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+#
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -31,22 +31,33 @@
#endif
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
+
+ .globl SYMBOL(SpinPause)
.globl SYMBOL(_Copy_arrayof_conjoint_bytes)
- .globl SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .globl SYMBOL(_Copy_arrayof_conjoint_jshorts)
.globl SYMBOL(_Copy_conjoint_jshorts_atomic)
.globl SYMBOL(_Copy_arrayof_conjoint_jints)
.globl SYMBOL(_Copy_conjoint_jints_atomic)
.globl SYMBOL(_Copy_arrayof_conjoint_jlongs)
.globl SYMBOL(_Copy_conjoint_jlongs_atomic)
- .text
+#ifdef __APPLE__
+ .private_extern SYMBOL(SpinPause)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_bytes)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jshorts)
+ .private_extern SYMBOL(_Copy_conjoint_jshorts_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jints)
+ .private_extern SYMBOL(_Copy_conjoint_jints_atomic)
+ .private_extern SYMBOL(_Copy_arrayof_conjoint_jlongs)
+ .private_extern SYMBOL(_Copy_conjoint_jlongs_atomic)
+#endif
+
+ .text
- .globl SYMBOL(SpinPause)
.p2align 4,,15
ELF_TYPE(SpinPause,@function)
SYMBOL(SpinPause):
@@ -63,7 +74,7 @@ SYMBOL(SpinPause):
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_bytes,@function)
SYMBOL(_Copy_arrayof_conjoint_bytes):
movq %rdx,%r8 # byte count
shrq $3,%rdx # qword count
@@ -71,7 +82,7 @@ SYMBOL(_Copy_arrayof_conjoint_bytes):
leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1
jbe acb_CopyRight
cmpq %rax,%rsi
- jbe acb_CopyLeft
+ jbe acb_CopyLeft
acb_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -165,8 +176,8 @@ acb_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function)
- ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jshorts,@function)
+ ELF_TYPE(_Copy_conjoint_jshorts_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jshorts):
SYMBOL(_Copy_conjoint_jshorts_atomic):
movq %rdx,%r8 # word count
@@ -175,7 +186,7 @@ SYMBOL(_Copy_conjoint_jshorts_atomic):
leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2
jbe acs_CopyRight
cmpq %rax,%rsi
- jbe acs_CopyLeft
+ jbe acs_CopyLeft
acs_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -255,8 +266,8 @@ acs_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jints,@function)
- ELF_TYPE(_Copy_conjoint_jints_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jints,@function)
+ ELF_TYPE(_Copy_conjoint_jints_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jints):
SYMBOL(_Copy_conjoint_jints_atomic):
movq %rdx,%r8 # dword count
@@ -265,7 +276,7 @@ SYMBOL(_Copy_conjoint_jints_atomic):
leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4
jbe aci_CopyRight
cmpq %rax,%rsi
- jbe aci_CopyLeft
+ jbe aci_CopyLeft
aci_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -334,15 +345,15 @@ aci_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function)
- ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function)
+ ELF_TYPE(_Copy_arrayof_conjoint_jlongs,@function)
+ ELF_TYPE(_Copy_conjoint_jlongs_atomic,@function)
SYMBOL(_Copy_arrayof_conjoint_jlongs):
SYMBOL(_Copy_conjoint_jlongs_atomic):
cmpq %rdi,%rsi
leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8
jbe acl_CopyRight
cmpq %rax,%rsi
- jbe acl_CopyLeft
+ jbe acl_CopyLeft
acl_CopyRight:
leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
negq %rdx
diff --git a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
index 37b92bc7ffd48..c73e83996ff57 100644
--- a/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
+++ b/src/hotspot/os_cpu/bsd_x86/os_bsd_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -351,7 +350,7 @@ frame os::get_sender_for_C_frame(frame* fr) {
return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
}
-intptr_t* _get_previous_fp() {
+static intptr_t* _get_previous_fp() {
#if defined(__clang__) || defined(__llvm__)
intptr_t **ebp;
__asm__("mov %%" SPELL_REG_FP ", %0":"=r"(ebp));
diff --git a/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
index 2a75f3dac94b3..1697f6f03b581 100644
--- a/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
+++ b/src/hotspot/os_cpu/bsd_x86/safefetch_bsd_x86_64.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -38,13 +38,22 @@
.globl SYMBOL(_SafeFetch32_continuation)
.globl SYMBOL(_SafeFetchN_continuation)
+#ifdef __APPLE__
+ .private_extern SYMBOL(SafeFetch32_impl)
+ .private_extern SYMBOL(SafeFetchN_impl)
+ .private_extern SYMBOL(_SafeFetch32_fault)
+ .private_extern SYMBOL(_SafeFetchN_fault)
+ .private_extern SYMBOL(_SafeFetch32_continuation)
+ .private_extern SYMBOL(_SafeFetchN_continuation)
+#endif
+
.text
# Support for int SafeFetch32(int* address, int defaultval);
#
# %rdi : address
# %esi : defaultval
- ELF_TYPE(SafeFetch32_impl,@function)
+ ELF_TYPE(SafeFetch32_impl,@function)
SYMBOL(SafeFetch32_impl:)
SYMBOL(_SafeFetch32_fault:)
movl (%rdi), %eax
diff --git a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
index 012f85ac0ff4a..0fc9484ce23ef 100644
--- a/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
+++ b/src/hotspot/os_cpu/bsd_zero/os_bsd_zero.cpp
@@ -27,7 +27,6 @@
#include "asm/assembler.inline.hpp"
#include "atomic_bsd_zero.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
index 4621e44ca3c47..e67206a9d497f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/atomic_linux_aarch64.S
@@ -24,6 +24,7 @@
.text
.globl aarch64_atomic_fetch_add_8_default_impl
+ .hidden aarch64_atomic_fetch_add_8_default_impl
.align 5
aarch64_atomic_fetch_add_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -40,6 +41,7 @@ aarch64_atomic_fetch_add_8_default_impl:
ret
.globl aarch64_atomic_fetch_add_4_default_impl
+ .hidden aarch64_atomic_fetch_add_4_default_impl
.align 5
aarch64_atomic_fetch_add_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -56,6 +58,7 @@ aarch64_atomic_fetch_add_4_default_impl:
ret
.global aarch64_atomic_fetch_add_8_relaxed_default_impl
+ .hidden aarch64_atomic_fetch_add_8_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_8_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -71,6 +74,7 @@ aarch64_atomic_fetch_add_8_relaxed_default_impl:
ret
.global aarch64_atomic_fetch_add_4_relaxed_default_impl
+ .hidden aarch64_atomic_fetch_add_4_relaxed_default_impl
.align 5
aarch64_atomic_fetch_add_4_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -86,6 +90,7 @@ aarch64_atomic_fetch_add_4_relaxed_default_impl:
ret
.globl aarch64_atomic_xchg_4_default_impl
+ .hidden aarch64_atomic_xchg_4_default_impl
.align 5
aarch64_atomic_xchg_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -101,6 +106,7 @@ aarch64_atomic_xchg_4_default_impl:
ret
.globl aarch64_atomic_xchg_8_default_impl
+ .hidden aarch64_atomic_xchg_8_default_impl
.align 5
aarch64_atomic_xchg_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -116,6 +122,7 @@ aarch64_atomic_xchg_8_default_impl:
ret
.globl aarch64_atomic_cmpxchg_1_default_impl
+ .hidden aarch64_atomic_cmpxchg_1_default_impl
.align 5
aarch64_atomic_cmpxchg_1_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -136,6 +143,7 @@ aarch64_atomic_cmpxchg_1_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_default_impl
.align 5
aarch64_atomic_cmpxchg_4_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -155,6 +163,7 @@ aarch64_atomic_cmpxchg_4_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_default_impl
.align 5
aarch64_atomic_cmpxchg_8_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -174,6 +183,7 @@ aarch64_atomic_cmpxchg_8_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_release_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_release_default_impl
.align 5
aarch64_atomic_cmpxchg_4_release_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -191,6 +201,7 @@ aarch64_atomic_cmpxchg_4_release_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_release_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_release_default_impl
.align 5
aarch64_atomic_cmpxchg_8_release_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -208,6 +219,7 @@ aarch64_atomic_cmpxchg_8_release_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_seq_cst_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_seq_cst_default_impl
.align 5
aarch64_atomic_cmpxchg_4_seq_cst_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -225,6 +237,7 @@ aarch64_atomic_cmpxchg_4_seq_cst_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_seq_cst_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_seq_cst_default_impl
.align 5
aarch64_atomic_cmpxchg_8_seq_cst_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -242,6 +255,7 @@ aarch64_atomic_cmpxchg_8_seq_cst_default_impl:
ret
.globl aarch64_atomic_cmpxchg_1_relaxed_default_impl
+.hidden aarch64_atomic_cmpxchg_1_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_1_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -260,6 +274,7 @@ aarch64_atomic_cmpxchg_1_relaxed_default_impl:
ret
.globl aarch64_atomic_cmpxchg_4_relaxed_default_impl
+ .hidden aarch64_atomic_cmpxchg_4_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_4_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
@@ -277,6 +292,7 @@ aarch64_atomic_cmpxchg_4_relaxed_default_impl:
ret
.globl aarch64_atomic_cmpxchg_8_relaxed_default_impl
+ .hidden aarch64_atomic_cmpxchg_8_relaxed_default_impl
.align 5
aarch64_atomic_cmpxchg_8_relaxed_default_impl:
#ifdef __ARM_FEATURE_ATOMICS
diff --git a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
index 4b8ed597c59fc..ade867ace016b 100644
--- a/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/copy_linux_aarch64.S
@@ -24,6 +24,9 @@
.global _Copy_conjoint_words
.global _Copy_disjoint_words
+ .hidden _Copy_conjoint_words
+ .hidden _Copy_disjoint_words
+
s .req x0
d .req x1
count .req x2
diff --git a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
index 4835eb9405a1b..3698896abb78a 100644
--- a/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
+++ b/src/hotspot/os_cpu/linux_aarch64/os_linux_aarch64.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "code/nativeInst.hpp"
#include "interpreter/interpreter.hpp"
diff --git a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
index fcb7e62e6d5e9..cfbd8f45f285f 100644
--- a/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/safefetch_linux_aarch64.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x0 : address
diff --git a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
index ac60d6aa94168..f9f5aab2a6bd8 100644
--- a/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
+++ b/src/hotspot/os_cpu/linux_aarch64/threadLS_linux_aarch64.S
@@ -25,22 +25,23 @@
// Clobber x1, flags.
// All other registers are preserved,
- .global _ZN10JavaThread25aarch64_get_thread_helperEv
- .type _ZN10JavaThread25aarch64_get_thread_helperEv, %function
+ .global _ZN10JavaThread25aarch64_get_thread_helperEv
+ .hidden _ZN10JavaThread25aarch64_get_thread_helperEv
+ .type _ZN10JavaThread25aarch64_get_thread_helperEv, %function
_ZN10JavaThread25aarch64_get_thread_helperEv:
- hint #0x19 // paciasp
- stp x29, x30, [sp, -16]!
- adrp x0, :tlsdesc:_ZN6Thread12_thr_currentE
- ldr x1, [x0, #:tlsdesc_lo12:_ZN6Thread12_thr_currentE]
- add x0, x0, :tlsdesc_lo12:_ZN6Thread12_thr_currentE
- .tlsdesccall _ZN6Thread12_thr_currentE
- blr x1
- mrs x1, tpidr_el0
- add x0, x1, x0
- ldr x0, [x0]
- ldp x29, x30, [sp], 16
- hint #0x1d // autiasp
- ret
+ hint #0x19 // paciasp
+ stp x29, x30, [sp, -16]!
+ adrp x0, :tlsdesc:_ZN6Thread12_thr_currentE
+ ldr x1, [x0, #:tlsdesc_lo12:_ZN6Thread12_thr_currentE]
+ add x0, x0, :tlsdesc_lo12:_ZN6Thread12_thr_currentE
+ .tlsdesccall _ZN6Thread12_thr_currentE
+ blr x1
+ mrs x1, tpidr_el0
+ add x0, x1, x0
+ ldr x0, [x0]
+ ldp x29, x30, [sp], 16
+ hint #0x1d // autiasp
+ ret
- .size _ZN10JavaThread25aarch64_get_thread_helperEv, .-_ZN10JavaThread25aarch64_get_thread_helperEv
+ .size _ZN10JavaThread25aarch64_get_thread_helperEv, .-_ZN10JavaThread25aarch64_get_thread_helperEv
diff --git a/src/hotspot/os_cpu/linux_arm/linux_arm_32.S b/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
index eb560d8f0c78b..ad88c58ce78ce 100644
--- a/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
+++ b/src/hotspot/os_cpu/linux_arm/linux_arm_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -23,38 +23,46 @@
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
- .globl _Copy_conjoint_bytes
- .type _Copy_conjoint_bytes, %function
+ .globl SpinPause
+ .hidden SpinPause
+ .type SpinPause, %function
.globl _Copy_arrayof_conjoint_bytes
- .type _Copy_arrayof_conjoint_bytes, %function
- .globl _Copy_disjoint_words
- .type _Copy_disjoint_words, %function
- .globl _Copy_conjoint_words
- .type _Copy_conjoint_words, %function
+ .hidden _Copy_arrayof_conjoint_bytes
+ .type _Copy_arrayof_conjoint_bytes, %function
+ .globl _Copy_disjoint_words
+ .hidden _Copy_disjoint_words
+ .type _Copy_disjoint_words, %function
+ .globl _Copy_conjoint_words
+ .hidden _Copy_conjoint_words
+ .type _Copy_conjoint_words, %function
.globl _Copy_conjoint_jshorts_atomic
- .type _Copy_conjoint_jshorts_atomic, %function
- .globl _Copy_arrayof_conjoint_jshorts
- .type _Copy_arrayof_conjoint_jshorts, %function
+ .hidden _Copy_conjoint_jshorts_atomic
+ .type _Copy_conjoint_jshorts_atomic, %function
+ .globl _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .type _Copy_arrayof_conjoint_jshorts, %function
.globl _Copy_conjoint_jints_atomic
- .type _Copy_conjoint_jints_atomic, %function
+ .hidden _Copy_conjoint_jints_atomic
+ .type _Copy_conjoint_jints_atomic, %function
.globl _Copy_arrayof_conjoint_jints
- .type _Copy_arrayof_conjoint_jints, %function
- .globl _Copy_conjoint_jlongs_atomic
- .type _Copy_conjoint_jlongs_atomic, %function
- .globl _Copy_arrayof_conjoint_jlongs
- .type _Copy_arrayof_conjoint_jlongs, %function
+ .hidden _Copy_arrayof_conjoint_jints
+ .type _Copy_arrayof_conjoint_jints, %function
+ .globl _Copy_conjoint_jlongs_atomic
+ .hidden _Copy_conjoint_jlongs_atomic
+ .type _Copy_conjoint_jlongs_atomic, %function
+ .globl _Copy_arrayof_conjoint_jlongs
+ .hidden _Copy_arrayof_conjoint_jlongs
+ .type _Copy_arrayof_conjoint_jlongs, %function
from .req r0
to .req r1
- .text
- .globl SpinPause
- .type SpinPause, %function
+ .text
SpinPause:
bx LR
@@ -70,7 +78,7 @@ _Copy_arrayof_conjoint_bytes:
# size_t count)
_Copy_disjoint_words:
stmdb sp!, {r3 - r9, ip}
-
+
cmp r2, #0
beq disjoint_words_finish
@@ -81,17 +89,17 @@ _Copy_disjoint_words:
.align 3
dw_f2b_loop_32:
subs r2, #32
- blt dw_f2b_loop_32_finish
+ blt dw_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt dw_f2b_loop_32
dw_f2b_loop_32_finish:
addlts r2, #32
beq disjoint_words_finish
cmp r2, #16
- blt disjoint_words_small
+ blt disjoint_words_small
ldmia from!, {r3 - r6}
subge r2, r2, #16
stmia to!, {r3 - r6}
@@ -116,8 +124,8 @@ disjoint_words_finish:
_Copy_conjoint_words:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_words_finish
+ cmp r2, #0
+ beq conjoint_words_finish
pld [from, #0]
cmp r2, #12
@@ -129,17 +137,17 @@ _Copy_conjoint_words:
.align 3
cw_f2b_loop_32:
subs r2, #32
- blt cw_f2b_loop_32_finish
+ blt cw_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt cw_f2b_loop_32
cw_f2b_loop_32_finish:
addlts r2, #32
beq conjoint_words_finish
cmp r2, #16
- blt conjoint_words_small
+ blt conjoint_words_small
ldmia from!, {r3 - r6}
subge r2, r2, #16
stmia to!, {r3 - r6}
@@ -154,7 +162,7 @@ conjoint_words_small:
strgt r9, [to], #4
b conjoint_words_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cw_b2f_copy:
add from, r2
pld [from, #-32]
@@ -162,17 +170,17 @@ cw_b2f_copy:
.align 3
cw_b2f_loop_32:
subs r2, #32
- blt cw_b2f_loop_32_finish
+ blt cw_b2f_loop_32_finish
ldmdb from!, {r3-r9,ip}
nop
- pld [from, #-32]
+ pld [from, #-32]
stmdb to!, {r3-r9,ip}
bgt cw_b2f_loop_32
cw_b2f_loop_32_finish:
addlts r2, #32
beq conjoint_words_finish
cmp r2, #16
- blt cw_b2f_copy_small
+ blt cw_b2f_copy_small
ldmdb from!, {r3 - r6}
subge r2, r2, #16
stmdb to!, {r3 - r6}
@@ -196,8 +204,8 @@ conjoint_words_finish:
_Copy_conjoint_jshorts_atomic:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_shorts_finish
+ cmp r2, #0
+ beq conjoint_shorts_finish
subs r3, to, from
cmphi r2, r3
@@ -210,11 +218,11 @@ _Copy_conjoint_jshorts_atomic:
ands r3, from, #3
bne cs_f2b_src_u
- # Aligned source address
+ # Aligned source address
.align 3
cs_f2b_loop_32:
subs r2, #32
- blt cs_f2b_loop_32_finish
+ blt cs_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
pld [from]
@@ -244,14 +252,14 @@ cs_f2b_4:
strgth r5, [to], #2
b conjoint_shorts_finish
- # Destination not aligned
+ # Destination not aligned
cs_f2b_dest_u:
ldrh r3, [from], #2
subs r2, #2
strh r3, [to], #2
beq conjoint_shorts_finish
- # Check to see if source is not aligned ether
+ # Check to see if source is not aligned ether
ands r3, from, #3
beq cs_f2b_loop_32
@@ -259,11 +267,11 @@ cs_f2b_src_u:
cmp r2, #16
blt cs_f2b_8_u
- # Load 2 first bytes to r7 and make src ptr word aligned
+ # Load 2 first bytes to r7 and make src ptr word aligned
bic from, #3
ldr r7, [from], #4
- # Destination aligned, source not
+ # Destination aligned, source not
mov r8, r2, lsr #4
.align 3
cs_f2b_16_u_loop:
@@ -306,7 +314,7 @@ cs_f2b_4_u:
strgth r5, [to], #2
b conjoint_shorts_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cs_b2f_copy:
add from, r2
pld [from, #-32]
@@ -319,7 +327,7 @@ cs_b2f_copy:
.align 3
cs_b2f_loop_32:
subs r2, #32
- blt cs_b2f_loop_32_finish
+ blt cs_b2f_loop_32_finish
ldmdb from!, {r3-r9,ip}
nop
pld [from, #-32]
@@ -359,16 +367,16 @@ cs_b2f_all_copy:
strgth r5, [to, #-2]!
b conjoint_shorts_finish
- # Destination not aligned
+ # Destination not aligned
cs_b2f_dest_u:
ldrh r3, [from, #-2]!
strh r3, [to, #-2]!
sub r2, #2
- # Check source alignment as well
+ # Check source alignment as well
ands r3, from, #3
beq cs_b2f_loop_32
- # Source not aligned
+ # Source not aligned
cs_b2f_src_u:
bic from, #3
.align 3
@@ -393,7 +401,7 @@ cs_b2f_16_loop_u:
cs_b2f_16_loop_u_finished:
addlts r2, #16
ldr r3, [from]
- cmp r2, #10
+ cmp r2, #10
blt cs_b2f_2_u_loop
ldmdb from!, {r4 - r5}
mov r6, r4, lsr #16
@@ -402,7 +410,7 @@ cs_b2f_16_loop_u_finished:
orr r7, r7, r3, lsl #16
stmdb to!, {r6-r7}
sub r2, #8
- .align 3
+ .align 3
cs_b2f_2_u_loop:
subs r2, #2
ldrh r3, [from], #-2
@@ -426,7 +434,7 @@ _Copy_arrayof_conjoint_jshorts:
_Copy_conjoint_jints_atomic:
_Copy_arrayof_conjoint_jints:
swi 0x9f0001
-
+
# Support for void Copy::conjoint_jlongs_atomic(jlong* from,
# jlong* to,
# size_t count)
@@ -434,8 +442,8 @@ _Copy_conjoint_jlongs_atomic:
_Copy_arrayof_conjoint_jlongs:
stmdb sp!, {r3 - r9, ip}
- cmp r2, #0
- beq conjoint_longs_finish
+ cmp r2, #0
+ beq conjoint_longs_finish
pld [from, #0]
cmp r2, #24
@@ -447,10 +455,10 @@ _Copy_arrayof_conjoint_jlongs:
.align 3
cl_f2b_loop_32:
subs r2, #32
- blt cl_f2b_loop_32_finish
+ blt cl_f2b_loop_32_finish
ldmia from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmia to!, {r3 - r9, ip}
bgt cl_f2b_loop_32
cl_f2b_loop_32_finish:
@@ -458,21 +466,21 @@ cl_f2b_loop_32_finish:
beq conjoint_longs_finish
conjoint_longs_small:
cmp r2, #16
- blt cl_f2b_copy_8
- bgt cl_f2b_copy_24
+ blt cl_f2b_copy_8
+ bgt cl_f2b_copy_24
ldmia from!, {r3 - r6}
stmia to!, {r3 - r6}
- b conjoint_longs_finish
+ b conjoint_longs_finish
cl_f2b_copy_8:
ldmia from!, {r3 - r4}
stmia to!, {r3 - r4}
b conjoint_longs_finish
cl_f2b_copy_24:
- ldmia from!, {r3 - r8}
+ ldmia from!, {r3 - r8}
stmia to!, {r3 - r8}
b conjoint_longs_finish
- # Src and dest overlap, copy in a descending order
+ # Src and dest overlap, copy in a descending order
cl_b2f_copy:
add from, r2
pld [from, #-32]
@@ -480,31 +488,29 @@ cl_b2f_copy:
.align 3
cl_b2f_loop_32:
subs r2, #32
- blt cl_b2f_loop_32_finish
+ blt cl_b2f_loop_32_finish
ldmdb from!, {r3 - r9, ip}
nop
- pld [from]
+ pld [from]
stmdb to!, {r3 - r9, ip}
bgt cl_b2f_loop_32
cl_b2f_loop_32_finish:
addlts r2, #32
beq conjoint_longs_finish
cmp r2, #16
- blt cl_b2f_copy_8
- bgt cl_b2f_copy_24
+ blt cl_b2f_copy_8
+ bgt cl_b2f_copy_24
ldmdb from!, {r3 - r6}
stmdb to!, {r3 - r6}
b conjoint_longs_finish
cl_b2f_copy_8:
- ldmdb from!, {r3 - r4}
+ ldmdb from!, {r3 - r4}
stmdb to!, {r3 - r4}
b conjoint_longs_finish
cl_b2f_copy_24:
- ldmdb from!, {r3 - r8}
+ ldmdb from!, {r3 - r8}
stmdb to!, {r3 - r8}
conjoint_longs_finish:
ldmia sp!, {r3 - r9, ip}
bx lr
-
-
diff --git a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
index 86e8ed25618c1..551270588438e 100644
--- a/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
+++ b/src/hotspot/os_cpu/linux_arm/os_linux_arm.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
index 5196b199f05f6..07e90fa3079f2 100644
--- a/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
+++ b/src/hotspot/os_cpu/linux_arm/safefetch_linux_arm.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,11 @@
.globl SafeFetch32_impl
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
.type SafeFetch32_impl, %function
# Support for int SafeFetch32(int* address, int defaultval);
diff --git a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
index b570e3b6d7f12..0b666f29c312b 100644
--- a/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
+++ b/src/hotspot/os_cpu/linux_ppc/os_linux_ppc.cpp
@@ -28,7 +28,6 @@
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
index c8d20cc1b4328..8c96edf01b4d0 100644
--- a/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
+++ b/src/hotspot/os_cpu/linux_ppc/safefetch_linux_ppc.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# r3 : address
diff --git a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
index 282467bc9e096..9f13e2bdd2cb7 100644
--- a/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/os_linux_riscv.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
@@ -39,6 +38,7 @@
#include "prims/jvm_misc.hpp"
#include "runtime/arguments.hpp"
#include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/java.hpp"
#include "runtime/javaCalls.hpp"
@@ -406,6 +406,14 @@ static inline void atomic_copy64(const volatile void *src, volatile void *dst) {
extern "C" {
int SpinPause() {
+ if (UseZihintpause) {
+ // PAUSE is encoded as a FENCE instruction with pred=W, succ=0, fm=0, rd=x0, and rs1=x0.
+ // To do: __asm__ volatile("pause " : : : );
+ // Since we're currently not passing '-march=..._zihintpause' to the compiler,
+ // it will not recognize the "pause" instruction, hence the hard-coded instruction.
+ __asm__ volatile(".word 0x0100000f " : : : );
+ return 1;
+ }
return 0;
}
diff --git a/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
index ecf0bac6f9e78..150df7567bdb8 100644
--- a/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
+++ b/src/hotspot/os_cpu/linux_riscv/safefetch_linux_riscv.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# x10 (a0) : address
diff --git a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
index 033ea14ead6a4..5aa65e705d9ed 100644
--- a/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
+++ b/src/hotspot/os_cpu/linux_s390/os_linux_s390.cpp
@@ -28,7 +28,6 @@
// no precompiled headers
#include "asm/assembler.inline.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/nativeInst.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/disassembler.hpp"
diff --git a/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
index 47fe82f5a278b..43d50c798e534 100644
--- a/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
+++ b/src/hotspot/os_cpu/linux_s390/safefetch_linux_s390.S
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2022 SAP SE. All rights reserved.
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,13 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetchN_continuation
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
# Support for int SafeFetch32(int* address, int defaultval);
#
# r2 : address
diff --git a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S b/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
index 344358172defd..e23cd2b9164ae 100644
--- a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
+++ b/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2004, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -21,28 +21,41 @@
# questions.
#
+ .globl SpinPause
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
.globl _Copy_arrayof_conjoint_bytes
.globl _Copy_conjoint_jshorts_atomic
- .globl _Copy_arrayof_conjoint_jshorts
+ .globl _Copy_arrayof_conjoint_jshorts
.globl _Copy_conjoint_jints_atomic
.globl _Copy_arrayof_conjoint_jints
- .globl _Copy_conjoint_jlongs_atomic
- .globl _mmx_Copy_arrayof_conjoint_jshorts
+ .globl _Copy_conjoint_jlongs_atomic
+ .globl _mmx_Copy_arrayof_conjoint_jshorts
.globl _Atomic_cmpxchg_long
.globl _Atomic_move_long
- .text
+ .hidden SpinPause
- .globl SpinPause
- .type SpinPause,@function
+ .hidden _Copy_arrayof_conjoint_bytes
+ .hidden _Copy_conjoint_jshorts_atomic
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_conjoint_jints_atomic
+ .hidden _Copy_arrayof_conjoint_jints
+ .hidden _Copy_conjoint_jlongs_atomic
+ .hidden _mmx_Copy_arrayof_conjoint_jshorts
+
+ .hidden _Atomic_cmpxchg_long
+ .hidden _Atomic_move_long
+
+ .text
+
+ .type SpinPause,@function
.p2align 4,,15
SpinPause:
rep
@@ -55,7 +68,7 @@ SpinPause:
# size_t count)
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_bytes,@function
+ .type _Copy_arrayof_conjoint_bytes,@function
_Copy_arrayof_conjoint_bytes:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -115,7 +128,7 @@ acb_CopyLeft:
jbe 2f # <= 32 dwords
rep; smovl
jmp 4f
- .space 8
+ .space 8
2: subl %esi,%edi
.p2align 4,,15
3: movl (%esi),%edx
@@ -131,7 +144,7 @@ acb_CopyLeft:
addl $3,%esi
6: movb (%esi),%dl
movb %dl,(%edi,%esi,1)
- subl $1,%esi
+ subl $1,%esi
subl $1,%ecx
jnz 6b
7: cld
@@ -143,7 +156,7 @@ acb_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _Copy_conjoint_jshorts_atomic,@function
+ .type _Copy_conjoint_jshorts_atomic,@function
_Copy_conjoint_jshorts_atomic:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -230,7 +243,7 @@ cs_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jshorts,@function
+ .type _Copy_arrayof_conjoint_jshorts,@function
_Copy_arrayof_conjoint_jshorts:
pushl %esi
movl 4+12(%esp),%ecx # count
@@ -307,8 +320,8 @@ acs_CopyLeft:
# Equivalent to
# arrayof_conjoint_jints
.p2align 4,,15
- .type _Copy_conjoint_jints_atomic,@function
- .type _Copy_arrayof_conjoint_jints,@function
+ .type _Copy_conjoint_jints_atomic,@function
+ .type _Copy_arrayof_conjoint_jints,@function
_Copy_conjoint_jints_atomic:
_Copy_arrayof_conjoint_jints:
pushl %esi
@@ -384,7 +397,7 @@ ci_CopyLeft:
# }
*/
.p2align 4,,15
- .type _Copy_conjoint_jlongs_atomic,@function
+ .type _Copy_conjoint_jlongs_atomic,@function
_Copy_conjoint_jlongs_atomic:
movl 4+8(%esp),%ecx # count
movl 4+0(%esp),%eax # from
@@ -413,7 +426,7 @@ cla_CopyLeft:
# void* to,
# size_t count)
.p2align 4,,15
- .type _mmx_Copy_arrayof_conjoint_jshorts,@function
+ .type _mmx_Copy_arrayof_conjoint_jshorts,@function
_mmx_Copy_arrayof_conjoint_jshorts:
pushl %esi
movl 4+12(%esp),%ecx
@@ -465,8 +478,8 @@ mmx_acs_CopyRight:
cmpl $16,%ecx
jge 4b
emms
- testl %ecx,%ecx
- ja 1b
+ testl %ecx,%ecx
+ ja 1b
5: andl $1,%eax
je 7f
6: movw (%esi),%dx
@@ -511,7 +524,7 @@ mmx_acs_CopyLeft:
# jlong exchange_value)
#
.p2align 4,,15
- .type _Atomic_cmpxchg_long,@function
+ .type _Atomic_cmpxchg_long,@function
_Atomic_cmpxchg_long:
# 8(%esp) : return PC
pushl %ebx # 4(%esp) : old %ebx
@@ -530,7 +543,7 @@ _Atomic_cmpxchg_long:
# Support for jlong Atomic::load and Atomic::store.
# void _Atomic_move_long(const volatile jlong* src, volatile jlong* dst)
.p2align 4,,15
- .type _Atomic_move_long,@function
+ .type _Atomic_move_long,@function
_Atomic_move_long:
movl 4(%esp), %eax # src
fildll (%eax)
diff --git a/src/hotspot/os_cpu/linux_x86/linux_x86_64.S b/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
index 89d98cb583786..65580a194afab 100644
--- a/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
+++ b/src/hotspot/os_cpu/linux_x86/linux_x86_64.S
@@ -1,5 +1,5 @@
-#
-# Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+#
+# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -21,24 +21,34 @@
# questions.
#
+ .globl SpinPause
# NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
+ # from server-compiler-generated code via CallLeafNoFP,
+ # which means that they *must* either not use floating
+ # point or use it in the same manner as does the server
+ # compiler.
+
.globl _Copy_arrayof_conjoint_bytes
- .globl _Copy_arrayof_conjoint_jshorts
+ .globl _Copy_arrayof_conjoint_jshorts
.globl _Copy_conjoint_jshorts_atomic
.globl _Copy_arrayof_conjoint_jints
.globl _Copy_conjoint_jints_atomic
.globl _Copy_arrayof_conjoint_jlongs
.globl _Copy_conjoint_jlongs_atomic
- .text
+ .hidden SpinPause
+
+ .hidden _Copy_arrayof_conjoint_bytes
+ .hidden _Copy_arrayof_conjoint_jshorts
+ .hidden _Copy_conjoint_jshorts_atomic
+ .hidden _Copy_arrayof_conjoint_jints
+ .hidden _Copy_conjoint_jints_atomic
+ .hidden _Copy_arrayof_conjoint_jlongs
+ .hidden _Copy_conjoint_jlongs_atomic
+
+ .text
- .globl SpinPause
.align 16
.type SpinPause,@function
SpinPause:
@@ -55,7 +65,7 @@ SpinPause:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_bytes,@function
+ .type _Copy_arrayof_conjoint_bytes,@function
_Copy_arrayof_conjoint_bytes:
movq %rdx,%r8 # byte count
shrq $3,%rdx # qword count
@@ -63,7 +73,7 @@ _Copy_arrayof_conjoint_bytes:
leaq -1(%rdi,%r8,1),%rax # from + bcount*1 - 1
jbe acb_CopyRight
cmpq %rax,%rsi
- jbe acb_CopyLeft
+ jbe acb_CopyLeft
acb_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -157,8 +167,8 @@ acb_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jshorts,@function
- .type _Copy_conjoint_jshorts_atomic,@function
+ .type _Copy_arrayof_conjoint_jshorts,@function
+ .type _Copy_conjoint_jshorts_atomic,@function
_Copy_arrayof_conjoint_jshorts:
_Copy_conjoint_jshorts_atomic:
movq %rdx,%r8 # word count
@@ -167,7 +177,7 @@ _Copy_conjoint_jshorts_atomic:
leaq -2(%rdi,%r8,2),%rax # from + wcount*2 - 2
jbe acs_CopyRight
cmpq %rax,%rsi
- jbe acs_CopyLeft
+ jbe acs_CopyLeft
acs_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -247,8 +257,8 @@ acs_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jints,@function
- .type _Copy_conjoint_jints_atomic,@function
+ .type _Copy_arrayof_conjoint_jints,@function
+ .type _Copy_conjoint_jints_atomic,@function
_Copy_arrayof_conjoint_jints:
_Copy_conjoint_jints_atomic:
movq %rdx,%r8 # dword count
@@ -257,7 +267,7 @@ _Copy_conjoint_jints_atomic:
leaq -4(%rdi,%r8,4),%rax # from + dcount*4 - 4
jbe aci_CopyRight
cmpq %rax,%rsi
- jbe aci_CopyLeft
+ jbe aci_CopyLeft
aci_CopyRight:
leaq -8(%rdi,%rdx,8),%rax # from + qcount*8 - 8
leaq -8(%rsi,%rdx,8),%rcx # to + qcount*8 - 8
@@ -326,15 +336,15 @@ aci_CopyLeft:
# rdx - count, treated as ssize_t
#
.p2align 4,,15
- .type _Copy_arrayof_conjoint_jlongs,@function
- .type _Copy_conjoint_jlongs_atomic,@function
+ .type _Copy_arrayof_conjoint_jlongs,@function
+ .type _Copy_conjoint_jlongs_atomic,@function
_Copy_arrayof_conjoint_jlongs:
_Copy_conjoint_jlongs_atomic:
cmpq %rdi,%rsi
leaq -8(%rdi,%rdx,8),%rax # from + count*8 - 8
jbe acl_CopyRight
cmpq %rax,%rsi
- jbe acl_CopyLeft
+ jbe acl_CopyLeft
acl_CopyRight:
leaq -8(%rsi,%rdx,8),%rcx # to + count*8 - 8
negq %rdx
diff --git a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
index b211330409d59..4dcaedf71da8c 100644
--- a/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
+++ b/src/hotspot/os_cpu/linux_x86/os_linux_x86.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
@@ -165,7 +164,7 @@ frame os::get_sender_for_C_frame(frame* fr) {
return frame(fr->sender_sp(), fr->link(), fr->sender_pc());
}
-intptr_t* _get_previous_fp() {
+static intptr_t* _get_previous_fp() {
#if defined(__clang__)
intptr_t **ebp;
__asm__ __volatile__ ("mov %%" SPELL_REG_FP ", %0":"=r"(ebp):);
diff --git a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
index 492b1207db6e2..54775cb7e8ede 100644
--- a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
+++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_32.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,11 @@
.globl _SafeFetch32_fault
.globl _SafeFetch32_continuation
- .text
+ .hidden SafeFetch32_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetch32_continuation
+
+ .text
# Support for int SafeFetch32(int* address, int defaultval);
#
diff --git a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
index 617851e8327d4..1937e71708897 100644
--- a/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
+++ b/src/hotspot/os_cpu/linux_x86/safefetch_linux_x86_64.S
@@ -1,6 +1,6 @@
#
# Copyright (c) 2022 SAP SE. All rights reserved.
-# Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
#
# This code is free software; you can redistribute it and/or modify it
@@ -28,7 +28,14 @@
.globl _SafeFetch32_continuation
.globl _SafeFetchN_continuation
- .text
+ .hidden SafeFetch32_impl
+ .hidden SafeFetchN_impl
+ .hidden _SafeFetch32_fault
+ .hidden _SafeFetchN_fault
+ .hidden _SafeFetch32_continuation
+ .hidden _SafeFetchN_continuation
+
+ .text
# Support for int SafeFetch32(int* address, int defaultval);
diff --git a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
index 1ce73588524c1..d593c46d15d91 100644
--- a/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
+++ b/src/hotspot/os_cpu/linux_zero/os_linux_zero.cpp
@@ -27,7 +27,6 @@
#include "asm/assembler.inline.hpp"
#include "atomic_linux_zero.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
index 46f718a9cd0f5..78e98609b6bdc 100644
--- a/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
+++ b/src/hotspot/os_cpu/windows_aarch64/os_windows_aarch64.cpp
@@ -27,7 +27,6 @@
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "code/nativeInst.hpp"
#include "interpreter/interpreter.hpp"
diff --git a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
index 4e18334315a37..7e0814c014bec 100644
--- a/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
+++ b/src/hotspot/os_cpu/windows_x86/os_windows_x86.cpp
@@ -25,7 +25,6 @@
// no precompiled headers
#include "asm/macroAssembler.hpp"
#include "classfile/vmSymbols.hpp"
-#include "code/icBuffer.hpp"
#include "code/vtableStubs.hpp"
#include "interpreter/interpreter.hpp"
#include "jvm.h"
diff --git a/src/hotspot/share/adlc/archDesc.cpp b/src/hotspot/share/adlc/archDesc.cpp
index d27bf0865608a..93fa7451dc0b9 100644
--- a/src/hotspot/share/adlc/archDesc.cpp
+++ b/src/hotspot/share/adlc/archDesc.cpp
@@ -1,5 +1,5 @@
//
-// Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
// archDesc.cpp - Internal format for architecture definition
+#include
#include "adlc.hpp"
static FILE *errfile = stderr;
@@ -684,6 +685,98 @@ bool ArchDesc::verify() {
return true;
}
+class MarkUsageFormClosure : public FormClosure {
+private:
+ ArchDesc* _ad;
+ std::unordered_set *_visited;
+
+public:
+ MarkUsageFormClosure(ArchDesc* ad, std::unordered_set *visit_map) {
+ _ad = ad;
+ _visited = visit_map;
+ }
+ virtual ~MarkUsageFormClosure() = default;
+
+ virtual void do_form(Form *form) {
+ if (_visited->find(form) == _visited->end()) {
+ _visited->insert(form);
+ form->forms_do(this);
+ }
+ }
+
+ virtual void do_form_by_name(const char* name) {
+ const Form* form = _ad->globalNames()[name];
+ if (form) {
+ do_form(const_cast(form));
+ return;
+ }
+ RegisterForm* regs = _ad->get_registers();
+ if (regs->getRegClass(name)) {
+ do_form(regs->getRegClass(name));
+ return;
+ }
+ }
+};
+
+// check unused operands
+bool ArchDesc::check_usage() {
+ std::unordered_set visited;
+ MarkUsageFormClosure callback(this, &visited);
+ _instructions.reset();
+ // iterate all instruction to mark used form
+ InstructForm* instr;
+ for ( ; (instr = (InstructForm*)_instructions.iter()) != nullptr; ) {
+ callback.do_form(instr);
+ }
+
+ // these forms are coded in OperandForm::is_user_name_for_sReg
+ // it may happen no instruction use these operands, like stackSlotP in aarch64,
+ // but we can not desclare they are useless.
+ callback.do_form_by_name("stackSlotI");
+ callback.do_form_by_name("stackSlotP");
+ callback.do_form_by_name("stackSlotD");
+ callback.do_form_by_name("stackSlotF");
+ callback.do_form_by_name("stackSlotL");
+
+ // sReg* are initial created by adlc in ArchDesc::initBaseOpTypes()
+ // In ARM, no definition or usage in adfile, but they are reported as unused
+ callback.do_form_by_name("sRegI");
+ callback.do_form_by_name("sRegP");
+ callback.do_form_by_name("sRegD");
+ callback.do_form_by_name("sRegF");
+ callback.do_form_by_name("sRegL");
+
+ // special generic vector operands only used in Matcher::pd_specialize_generic_vector_operand
+#if defined(AARCH64)
+ callback.do_form_by_name("vecA");
+ callback.do_form_by_name("vecD");
+ callback.do_form_by_name("vecX");
+#elif defined(AMD64)
+ callback.do_form_by_name("vecS");
+ callback.do_form_by_name("vecD");
+ callback.do_form_by_name("vecX");
+ callback.do_form_by_name("vecY");
+ callback.do_form_by_name("vecZ");
+ callback.do_form_by_name("legVecS");
+ callback.do_form_by_name("legVecD");
+ callback.do_form_by_name("legVecX");
+ callback.do_form_by_name("legVecY");
+ callback.do_form_by_name("legVecZ");
+#endif
+
+ int cnt = 0;
+ _operands.reset();
+ OperandForm* operand;
+ for ( ; (operand = (OperandForm*)_operands.iter()) != nullptr; ) {
+ if(visited.find(operand) == visited.end() && !operand->ideal_only()) {
+ fprintf(stderr, "\nWarning: unused operand (%s)", operand->_ident);
+ cnt++;
+ }
+ }
+ if (cnt) fprintf(stderr, "\n-------Warning: total %d unused operands\n", cnt);
+
+ return true;
+}
void ArchDesc::dump() {
_pre_header.dump();
diff --git a/src/hotspot/share/adlc/archDesc.hpp b/src/hotspot/share/adlc/archDesc.hpp
index 42369f6c2d13a..99e4947e110e1 100644
--- a/src/hotspot/share/adlc/archDesc.hpp
+++ b/src/hotspot/share/adlc/archDesc.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -226,6 +226,7 @@ class ArchDesc {
inline void getForm(EncodeForm **ptr) { *ptr = _encode; }
bool verify();
+ bool check_usage();
void dump();
// Helper utility that gets MatchList components from inside MatchRule
diff --git a/src/hotspot/share/adlc/forms.cpp b/src/hotspot/share/adlc/forms.cpp
index fd8e5c4fd50b1..068d745254e3d 100644
--- a/src/hotspot/share/adlc/forms.cpp
+++ b/src/hotspot/share/adlc/forms.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -362,6 +362,15 @@ void FormDict::dump() {
_form.print(dumpkey, dumpform);
}
+void FormDict::forms_do(FormClosure* f) {;
+ DictI iter(&_form);
+ for( ; iter.test(); ++iter ) {
+ Form* form = (Form*) iter._value;
+ assert(form != nullptr, "sanity");
+ f->do_form(form);
+ }
+}
+
//------------------------------SourceForm-------------------------------------
SourceForm::SourceForm(char* code) : _code(code) { }; // Constructor
SourceForm::~SourceForm() {
@@ -374,3 +383,11 @@ void SourceForm::dump() { // Debug printer
void SourceForm::output(FILE *fp) {
fprintf(fp,"\n//%s\n%s\n",classname(),(_code?_code:""));
}
+
+void FormClosure::do_form(Form* form) {
+ assert(false, "should not reach here");
+}
+
+void FormClosure::do_form_by_name(const char* name) {
+ assert(false, "should not reach here");
+}
diff --git a/src/hotspot/share/adlc/forms.hpp b/src/hotspot/share/adlc/forms.hpp
index c3dd85eb98bae..a82b9bbb3382d 100644
--- a/src/hotspot/share/adlc/forms.hpp
+++ b/src/hotspot/share/adlc/forms.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -58,6 +58,7 @@ class Flag;
class RewriteRule;
class ConstructRule;
class FormatRule;
+class FormClosure;
class Peephole;
class EncClass;
class Interface;
@@ -114,6 +115,8 @@ class FormDict {
const Form *operator [](const char *name) const; // Do a lookup
void dump();
+ // iterate child forms recursively
+ void forms_do(FormClosure *f);
};
// ***** Master Class for ADL Parser Forms *****
@@ -163,6 +166,9 @@ class Form {
// Write info to output files
virtual void output(FILE *fp) { fprintf(fp,"Form Output"); }
+ // iterate child forms recursively
+ virtual void forms_do (FormClosure* f) { return; }
+
public:
// ADLC types, match the last character on ideal operands and instructions
enum DataType {
@@ -255,6 +261,16 @@ class Form {
};
+class FormClosure {
+public:
+ FormClosure() = default;
+ virtual ~FormClosure() = default;
+
+ virtual void do_form(Form* form);
+ virtual void do_form_by_name(const char* name);
+};
+
+
//------------------------------FormList---------------------------------------
class FormList {
private:
diff --git a/src/hotspot/share/adlc/formsopt.cpp b/src/hotspot/share/adlc/formsopt.cpp
index 13d29ef3947cc..e1e4ed96c2ea0 100644
--- a/src/hotspot/share/adlc/formsopt.cpp
+++ b/src/hotspot/share/adlc/formsopt.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,6 +198,20 @@ void RegisterForm::output(FILE *fp) { // Write info to output files
fprintf(fp,"-------------------- end RegisterForm --------------------\n");
}
+void RegisterForm::forms_do(FormClosure *f) {
+ const char *name = nullptr;
+ if (_current_ac) f->do_form(_current_ac);
+ for(_rdefs.reset(); (name = _rdefs.iter()) != nullptr;) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+ for (_rclasses.reset(); (name = _rclasses.iter()) != nullptr;) {
+ f->do_form((RegClass*)_regClass[name]);
+ }
+ for (_aclasses.reset(); (name = _aclasses.iter()) != nullptr;) {
+ f->do_form((AllocClass*)_allocClass[name]);
+ }
+}
+
//------------------------------RegDef-----------------------------------------
// Constructor
RegDef::RegDef(char *regname, char *callconv, char *c_conv, char * idealtype, char * encode, char * concrete)
@@ -322,6 +336,13 @@ void RegClass::output(FILE *fp) { // Write info to output files
fprintf(fp,"--- done with entries for reg_class %s\n\n",_classid);
}
+void RegClass::forms_do(FormClosure *f) {
+ const char *name = nullptr;
+ for( _regDefs.reset(); (name = _regDefs.iter()) != nullptr; ) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+}
+
void RegClass::declare_register_masks(FILE* fp) {
const char* prefix = "";
const char* rc_name_to_upper = toUpper(_classid);
@@ -436,6 +457,14 @@ void AllocClass::output(FILE *fp) { // Write info to output files
fprintf(fp,"--- done with entries for alloc_class %s\n\n",_classid);
}
+void AllocClass::forms_do(FormClosure* f) {
+ const char *name;
+ for(_regDefs.reset(); (name = _regDefs.iter()) != nullptr;) {
+ f->do_form((RegDef*)_regDef[name]);
+ }
+ return;
+}
+
//==============================Frame Handling=================================
//------------------------------FrameForm--------------------------------------
FrameForm::FrameForm() {
@@ -706,6 +735,15 @@ void Peephole::output(FILE *fp) { // Write info to output files
if( _next ) _next->output(fp);
}
+void Peephole::forms_do(FormClosure *f) {
+ if (_predicate) f->do_form(_predicate);
+ if (_match) f->do_form(_match);
+ if (_procedure) f->do_form(_procedure);
+ if (_constraint) f->do_form(_constraint);
+ if (_replace) f->do_form(_replace);
+ return;
+}
+
//----------------------------PeepPredicate------------------------------------
PeepPredicate::PeepPredicate(const char* rule) : _rule(rule) {
}
diff --git a/src/hotspot/share/adlc/formsopt.hpp b/src/hotspot/share/adlc/formsopt.hpp
index 1a2b5dadd30ee..d183a46b87545 100644
--- a/src/hotspot/share/adlc/formsopt.hpp
+++ b/src/hotspot/share/adlc/formsopt.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -123,6 +123,7 @@ class RegisterForm : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------RegDef-----------------------------------------
@@ -199,6 +200,7 @@ class RegClass : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
virtual bool has_stack_version() {
return _stack_or_reg;
@@ -305,6 +307,11 @@ class ConditionalRegClass : public RegClass {
char* condition_code() {
return _condition_code;
}
+
+ virtual void forms_do(FormClosure* f) {
+ if (_rclasses[0]) f->do_form(_rclasses[0]);
+ if (_rclasses[1]) f->do_form(_rclasses[1]);
+ }
};
//------------------------------AllocClass-------------------------------------
@@ -325,6 +332,7 @@ class AllocClass : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
@@ -568,6 +576,7 @@ class Peephole : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
class PeepPredicate : public Form {
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index ecbf472c59ef4..be97547f8ce11 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1498,6 +1498,24 @@ void InstructForm::output(FILE *fp) {
if (_peephole) _peephole->output(fp);
}
+void InstructForm::forms_do(FormClosure *f) {
+ if (_cisc_spill_alternate) f->do_form(_cisc_spill_alternate);
+ if (_short_branch_form) f->do_form(_short_branch_form);
+ _localNames.forms_do(f);
+ if (_matrule) f->do_form(_matrule);
+ if (_opcode) f->do_form(_opcode);
+ if (_insencode) f->do_form(_insencode);
+ if (_constant) f->do_form(_constant);
+ if (_attribs) f->do_form(_attribs);
+ if (_predicate) f->do_form(_predicate);
+ _effects.forms_do(f);
+ if (_exprule) f->do_form(_exprule);
+ if (_rewrule) f->do_form(_rewrule);
+ if (_format) f->do_form(_format);
+ if (_peephole) f->do_form(_peephole);
+ assert(_components.count() == 0, "skip components");
+}
+
void MachNodeForm::dump() {
output(stderr);
}
@@ -1615,6 +1633,14 @@ void EncodeForm::output(FILE *fp) { // Write info to output files
}
fprintf(fp,"-------------------- end EncodeForm --------------------\n");
}
+
+void EncodeForm::forms_do(FormClosure* f) {
+ const char *name;
+ for (_eclasses.reset(); (name = _eclasses.iter()) != nullptr;) {
+ f->do_form((EncClass*)_encClass[name]);
+ }
+}
+
//------------------------------EncClass---------------------------------------
EncClass::EncClass(const char *name)
: _localNames(cmpstr,hashstr, Form::arena), _name(name) {
@@ -1705,6 +1731,15 @@ void EncClass::output(FILE *fp) {
}
+void EncClass::forms_do(FormClosure *f) {
+ _parameter_type.reset();
+ const char *type = _parameter_type.iter();
+ for ( ; type != nullptr ; type = _parameter_type.iter() ) {
+ f->do_form_by_name(type);
+ }
+ _localNames.forms_do(f);
+}
+
//------------------------------Opcode-----------------------------------------
Opcode::Opcode(char *primary, char *secondary, char *tertiary)
: _primary(primary), _secondary(secondary), _tertiary(tertiary) {
@@ -1835,6 +1870,15 @@ void InsEncode::output(FILE *fp) {
fprintf(fp,"\n");
}
+void InsEncode::forms_do(FormClosure *f) {
+ _encoding.reset();
+ NameAndList *encoding = (NameAndList*)_encoding.iter();
+ for( ; encoding != nullptr; encoding = (NameAndList*)_encoding.iter() ) {
+ // just check name, other operands will be checked as instruction parameters
+ f->do_form_by_name(encoding->name());
+ }
+}
+
//------------------------------Effect-----------------------------------------
static int effect_lookup(const char *name) {
if (!strcmp(name, "USE")) return Component::USE;
@@ -1968,6 +2012,19 @@ void ExpandRule::output(FILE *fp) { // Write info to output files
}
}
+void ExpandRule::forms_do(FormClosure *f) {
+ NameAndList *expand_instr = nullptr;
+ // Iterate over the instructions 'node' expands into
+ for(reset_instructions(); (expand_instr = iter_instructions()) != nullptr; ) {
+ f->do_form_by_name(expand_instr->name());
+ }
+ _newopers.reset();
+ const char* oper = _newopers.iter();
+ for(; oper != nullptr; oper = _newopers.iter()) {
+ f->do_form_by_name(oper);
+ }
+}
+
//------------------------------RewriteRule------------------------------------
RewriteRule::RewriteRule(char* params, char* block)
: _tempParams(params), _tempBlock(block) { }; // Constructor
@@ -1984,6 +2041,12 @@ void RewriteRule::output(FILE *fp) { // Write info to output files
(_tempBlock?_tempBlock:""));
}
+void RewriteRule::forms_do(FormClosure *f) {
+ if (_condition) f->do_form(_condition);
+ if (_instrs) f->do_form(_instrs);
+ if (_opers) f->do_form(_opers);
+}
+
//==============================MachNodes======================================
//------------------------------MachNodeForm-----------------------------------
@@ -2066,6 +2129,13 @@ void OpClassForm::output(FILE *fp) {
fprintf(fp,"\n");
}
+void OpClassForm::forms_do(FormClosure* f) {
+ const char *name;
+ for(_oplst.reset(); (name = _oplst.iter()) != nullptr;) {
+ f->do_form_by_name(name);
+ }
+}
+
//==============================Operands=======================================
//------------------------------OperandForm------------------------------------
@@ -2691,6 +2761,22 @@ void OperandForm::output(FILE *fp) {
if (_format) _format->dump();
}
+void OperandForm::forms_do(FormClosure* f) {
+ if (_matrule) f->do_form(_matrule);
+ if (_interface) f->do_form(_interface);
+ if (_attribs) f->do_form(_attribs);
+ if (_predicate) f->do_form(_predicate);
+ if (_constraint) f->do_form(_constraint);
+ if (_construct) f->do_form(_construct);
+ if (_format) f->do_form(_format);
+ _localNames.forms_do(f);
+ const char* opclass = nullptr;
+ for ( _classes.reset(); (opclass = _classes.iter()) != nullptr; ) {
+ f->do_form_by_name(opclass);
+ }
+ assert(_components.count() == 0, "skip _compnets");
+}
+
//------------------------------Constraint-------------------------------------
Constraint::Constraint(const char *func, const char *arg)
: _func(func), _arg(arg) {
@@ -2712,6 +2798,10 @@ void Constraint::output(FILE *fp) { // Write info to output files
fprintf(fp,"Constraint: %s ( %s )\n", _func, _arg);
}
+void Constraint::forms_do(FormClosure *f) {
+ f->do_form_by_name(_arg);
+}
+
//------------------------------Predicate--------------------------------------
Predicate::Predicate(char *pr)
: _pred(pr) {
@@ -3539,6 +3629,12 @@ void MatchNode::output(FILE *fp) {
}
}
+void MatchNode::forms_do(FormClosure *f) {
+ f->do_form_by_name(_name);
+ if (_lChild) f->do_form(_lChild);
+ if (_rChild) f->do_form(_rChild);
+}
+
int MatchNode::needs_ideal_memory_edge(FormDict &globals) const {
static const char *needs_ideal_memory_list[] = {
"StoreI","StoreL","StoreP","StoreN","StoreNKlass","StoreD","StoreF" ,
@@ -3608,6 +3704,7 @@ int InstructForm::needs_base_oop_edge(FormDict &globals) const {
}
+
//-------------------------cisc spilling methods-------------------------------
// helper routines and methods for detecting cisc-spilling instructions
//-------------------------cisc_spill_merge------------------------------------
@@ -4334,6 +4431,18 @@ void MatchRule::output(FILE *fp) {
fprintf(fp,"\n");
}
+void MatchRule::forms_do(FormClosure* f) {
+ // keep sync with MatchNode::forms_do
+ f->do_form_by_name(_name);
+ if (_lChild) f->do_form(_lChild);
+ if (_rChild) f->do_form(_rChild);
+
+ // handle next rule
+ if (_next) {
+ f->do_form(_next);
+ }
+}
+
//------------------------------Attribute--------------------------------------
Attribute::Attribute(char *id, char* val, int type)
: _ident(id), _val(val), _atype(type) {
diff --git a/src/hotspot/share/adlc/formssel.hpp b/src/hotspot/share/adlc/formssel.hpp
index eabd94c323a29..61d0fb40f18a8 100644
--- a/src/hotspot/share/adlc/formssel.hpp
+++ b/src/hotspot/share/adlc/formssel.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -310,6 +310,7 @@ class InstructForm : public Form {
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure *f);
};
//------------------------------EncodeForm-------------------------------------
@@ -333,6 +334,7 @@ class EncodeForm : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure *f);
};
//------------------------------EncClass---------------------------------------
@@ -377,6 +379,7 @@ class EncClass : public Form {
bool verify();
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------MachNode---------------------------------------
@@ -468,6 +471,7 @@ class InsEncode : public Form {
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure *f);
};
//------------------------------Effect-----------------------------------------
@@ -515,6 +519,7 @@ class ExpandRule : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure *f);
};
//---------------------------------Flag----------------------------------------
@@ -554,6 +559,7 @@ class RewriteRule : public Form {
~RewriteRule(); // Destructor
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
@@ -584,6 +590,7 @@ class OpClassForm : public Form {
virtual bool ideal_only() const;
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------OperandForm------------------------------------
@@ -711,6 +718,7 @@ class OperandForm : public OpClassForm {
virtual void dump(); // Debug printer
virtual void output(FILE *fp); // Write to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Constraint-------------------------------------
@@ -729,6 +737,7 @@ class Constraint : public Form {
void dump(); // Debug printer
void output(FILE *fp); // Write info to output files
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Predicate--------------------------------------
@@ -1014,6 +1023,7 @@ class MatchNode : public Form {
void dump();
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------MatchRule--------------------------------------
@@ -1075,6 +1085,7 @@ class MatchRule : public MatchNode {
void dump();
void output_short(FILE *fp);
void output(FILE *fp);
+ virtual void forms_do(FormClosure* f);
};
//------------------------------Attribute--------------------------------------
diff --git a/src/hotspot/share/adlc/main.cpp b/src/hotspot/share/adlc/main.cpp
index 6d921cf5e7bee..fd270b0944369 100644
--- a/src/hotspot/share/adlc/main.cpp
+++ b/src/hotspot/share/adlc/main.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -186,6 +186,9 @@ int main(int argc, char *argv[])
// Verify that the results of the parse are consistent
AD.verify();
+ // Check defined operands are used
+ AD.check_usage();
+
// Prepare to generate the result files:
AD.generateMatchLists();
AD.identify_unique_operands();
@@ -216,7 +219,6 @@ int main(int argc, char *argv[])
AD.addInclude(AD._CPP_file, "code/nativeInst.hpp");
AD.addInclude(AD._CPP_file, "code/vmreg.inline.hpp");
AD.addInclude(AD._CPP_file, "gc/shared/collectedHeap.inline.hpp");
- AD.addInclude(AD._CPP_file, "oops/compiledICHolder.hpp");
AD.addInclude(AD._CPP_file, "oops/compressedOops.hpp");
AD.addInclude(AD._CPP_file, "oops/markWord.hpp");
AD.addInclude(AD._CPP_file, "oops/method.hpp");
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 7a0a31abf597f..5b1e113f15d10 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "asm/codeBuffer.hpp"
+#include "code/compiledIC.hpp"
#include "code/oopRecorder.inline.hpp"
#include "compiler/disassembler.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/asm/codeBuffer.inline.hpp b/src/hotspot/share/asm/codeBuffer.inline.hpp
index 838447ad882dc..06ec9174b34bb 100644
--- a/src/hotspot/share/asm/codeBuffer.inline.hpp
+++ b/src/hotspot/share/asm/codeBuffer.inline.hpp
@@ -48,7 +48,7 @@ bool emit_shared_stubs_to_interp(CodeBuffer* cb, SharedStubToInterpRequests* sha
shared_stub_to_interp_requests->sort(by_shared_method);
MacroAssembler masm(cb);
for (int i = 0; i < shared_stub_to_interp_requests->length();) {
- address stub = __ start_a_stub(CompiledStaticCall::to_interp_stub_size());
+ address stub = __ start_a_stub(CompiledDirectCall::to_interp_stub_size());
if (stub == nullptr) {
return false;
}
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp
index 0dbf29300d11f..f82262d8e529e 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.cpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp
@@ -844,9 +844,18 @@ void Canonicalizer::do_LookupSwitch(LookupSwitch* x) {
if (x->tag()->type()->is_constant()) {
int v = x->tag()->type()->as_IntConstant()->value();
BlockBegin* sux = x->default_sux();
- for (int i = 0; i < x->length(); i++) {
- if (v == x->key_at(i)) {
- sux = x->sux_at(i);
+ int low = 0;
+ int high = x->length() - 1;
+ while (low <= high) {
+ int mid = low + ((high - low) >> 1);
+ int key = x->key_at(mid);
+ if (key == v) {
+ sux = x->sux_at(mid);
+ break;
+ } else if (key > v) {
+ high = mid - 1;
+ } else {
+ low = mid + 1;
}
}
set_canonical(new Goto(sux, x->state_before(), is_safepoint(x, sux)));
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index 6e5fb99242c8c..396c83c6ab976 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -523,7 +523,7 @@ inline bool BlockListBuilder::is_successor(BlockBegin* block, BlockBegin* sux) {
#ifndef PRODUCT
-int compare_depth_first(BlockBegin** a, BlockBegin** b) {
+static int compare_depth_first(BlockBegin** a, BlockBegin** b) {
return (*a)->depth_first_number() - (*b)->depth_first_number();
}
diff --git a/src/hotspot/share/c1/c1_IR.cpp b/src/hotspot/share/c1/c1_IR.cpp
index 628f14776154e..e375d16aafb18 100644
--- a/src/hotspot/share/c1/c1_IR.cpp
+++ b/src/hotspot/share/c1/c1_IR.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -306,22 +306,22 @@ void IR::eliminate_null_checks() {
}
}
-
-static int sort_pairs(BlockPair** a, BlockPair** b) {
- if ((*a)->from() == (*b)->from()) {
- return (*a)->to()->block_id() - (*b)->to()->block_id();
- } else {
- return (*a)->from()->block_id() - (*b)->from()->block_id();
- }
-}
-
-
+// The functionality of this class is to insert a new block between
+// the 'from' and 'to' block of a critical edge.
+// It first collects the block pairs, and then processes them.
+//
+// Some instructions may introduce more than one edge between two blocks.
+// By checking if the current 'to' block sets critical_edge_split_flag
+// (all new blocks set this flag) we can avoid repeated processing.
+// This is why BlockPair contains the index rather than the original 'to' block.
class CriticalEdgeFinder: public BlockClosure {
BlockPairList blocks;
- IR* _ir;
public:
- CriticalEdgeFinder(IR* ir): _ir(ir) {}
+ CriticalEdgeFinder(IR* ir) {
+ ir->iterate_preorder(this);
+ }
+
void block_do(BlockBegin* bb) {
BlockEnd* be = bb->end();
int nos = be->number_of_sux();
@@ -329,20 +329,22 @@ class CriticalEdgeFinder: public BlockClosure {
for (int i = 0; i < nos; i++) {
BlockBegin* sux = be->sux_at(i);
if (sux->number_of_preds() >= 2) {
- blocks.append(new BlockPair(bb, sux));
+ blocks.append(new BlockPair(bb, i));
}
}
}
}
void split_edges() {
- BlockPair* last_pair = nullptr;
- blocks.sort(sort_pairs);
for (int i = 0; i < blocks.length(); i++) {
BlockPair* pair = blocks.at(i);
- if (last_pair != nullptr && pair->is_same(last_pair)) continue;
BlockBegin* from = pair->from();
- BlockBegin* to = pair->to();
+ int index = pair->index();
+ BlockBegin* to = from->end()->sux_at(index);
+ if (to->is_set(BlockBegin::critical_edge_split_flag)) {
+ // inserted
+ continue;
+ }
BlockBegin* split = from->insert_block_between(to);
#ifndef PRODUCT
if ((PrintIR || PrintIR1) && Verbose) {
@@ -350,15 +352,12 @@ class CriticalEdgeFinder: public BlockClosure {
from->block_id(), to->block_id(), split->block_id());
}
#endif
- last_pair = pair;
}
}
};
void IR::split_critical_edges() {
CriticalEdgeFinder cef(this);
-
- iterate_preorder(&cef);
cef.split_edges();
}
diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp
index b2d31e3c1a83f..431bcea42cb5b 100644
--- a/src/hotspot/share/c1/c1_Instruction.cpp
+++ b/src/hotspot/share/c1/c1_Instruction.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -586,6 +586,8 @@ void BlockBegin::substitute_sux(BlockBegin* old_sux, BlockBegin* new_sux) {
// of the inserted block, without recomputing the values of the other blocks
// in the CFG. Therefore the value of "depth_first_number" in BlockBegin becomes meaningless.
BlockBegin* BlockBegin::insert_block_between(BlockBegin* sux) {
+ assert(!sux->is_set(critical_edge_split_flag), "sanity check");
+
int bci = sux->bci();
// critical edge splitting may introduce a goto after a if and array
// bound check elimination may insert a predicate between the if and
diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp
index 363f4f6e561f1..8f7fd698e7944 100644
--- a/src/hotspot/share/c1/c1_Instruction.hpp
+++ b/src/hotspot/share/c1/c1_Instruction.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -2423,15 +2423,11 @@ LEAF(MemBar, Instruction)
class BlockPair: public CompilationResourceObj {
private:
BlockBegin* _from;
- BlockBegin* _to;
+ int _index; // sux index of 'to' block
public:
- BlockPair(BlockBegin* from, BlockBegin* to): _from(from), _to(to) {}
+ BlockPair(BlockBegin* from, int index): _from(from), _index(index) {}
BlockBegin* from() const { return _from; }
- BlockBegin* to() const { return _to; }
- bool is_same(BlockBegin* from, BlockBegin* to) const { return _from == from && _to == to; }
- bool is_same(BlockPair* p) const { return _from == p->from() && _to == p->to(); }
- void set_to(BlockBegin* b) { _to = b; }
- void set_from(BlockBegin* b) { _from = b; }
+ int index() const { return _index; }
};
typedef GrowableArray BlockPairList;
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp
index a601696d8df7e..51fb851d00c0e 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.cpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp
@@ -606,13 +606,14 @@ void LIR_Assembler::emit_op0(LIR_Op0* op) {
Unimplemented();
break;
- case lir_std_entry:
+ case lir_std_entry: {
// init offsets
offsets()->set_value(CodeOffsets::OSR_Entry, _masm->offset());
- _masm->align(CodeEntryAlignment);
if (needs_icache(compilation()->method())) {
- check_icache();
+ int offset = check_icache();
+ offsets()->set_value(CodeOffsets::Entry, offset);
}
+ _masm->align(CodeEntryAlignment);
offsets()->set_value(CodeOffsets::Verified_Entry, _masm->offset());
_masm->verified_entry(compilation()->directive()->BreakAtExecuteOption);
if (needs_clinit_barrier_on_entry(compilation()->method())) {
@@ -621,6 +622,7 @@ void LIR_Assembler::emit_op0(LIR_Op0* op) {
build_frame();
offsets()->set_value(CodeOffsets::Frame_Complete, _masm->offset());
break;
+ }
case lir_osr_entry:
offsets()->set_value(CodeOffsets::OSR_Entry, _masm->offset());
diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp
index 9e9195a0d60d0..a4d955e52a004 100644
--- a/src/hotspot/share/c1/c1_LinearScan.cpp
+++ b/src/hotspot/share/c1/c1_LinearScan.cpp
@@ -1446,12 +1446,12 @@ int LinearScan::interval_cmp(Interval** a, Interval** b) {
}
}
-#ifndef PRODUCT
-int interval_cmp(Interval* const& l, Interval* const& r) {
+#ifdef ASSERT
+static int interval_cmp(Interval* const& l, Interval* const& r) {
return l->from() - r->from();
}
-bool find_interval(Interval* interval, IntervalArray* intervals) {
+static bool find_interval(Interval* interval, IntervalArray* intervals) {
bool found;
int idx = intervals->find_sorted(interval, found);
@@ -2303,11 +2303,11 @@ void assert_no_register_values(GrowableArray* values) {
}
}
-void assert_equal(Location l1, Location l2) {
+static void assert_equal(Location l1, Location l2) {
assert(l1.where() == l2.where() && l1.type() == l2.type() && l1.offset() == l2.offset(), "");
}
-void assert_equal(ScopeValue* v1, ScopeValue* v2) {
+static void assert_equal(ScopeValue* v1, ScopeValue* v2) {
if (v1->is_location()) {
assert(v2->is_location(), "");
assert_equal(((LocationValue*)v1)->location(), ((LocationValue*)v2)->location());
@@ -2328,12 +2328,12 @@ void assert_equal(ScopeValue* v1, ScopeValue* v2) {
}
}
-void assert_equal(MonitorValue* m1, MonitorValue* m2) {
+static void assert_equal(MonitorValue* m1, MonitorValue* m2) {
assert_equal(m1->owner(), m2->owner());
assert_equal(m1->basic_lock(), m2->basic_lock());
}
-void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
+static void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
assert(d1->scope() == d2->scope(), "not equal");
assert(d1->bci() == d2->bci(), "not equal");
@@ -2375,7 +2375,7 @@ void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
}
}
-void check_stack_depth(CodeEmitInfo* info, int stack_end) {
+static void check_stack_depth(CodeEmitInfo* info, int stack_end) {
if (info->stack()->bci() != SynchronizationEntryBCI && !info->scope()->method()->is_native()) {
Bytecodes::Code code = info->scope()->method()->java_code_at_bci(info->stack()->bci());
switch (code) {
diff --git a/src/hotspot/share/c1/c1_MacroAssembler.hpp b/src/hotspot/share/c1/c1_MacroAssembler.hpp
index 6a8304bd405fa..1e193ce086961 100644
--- a/src/hotspot/share/c1/c1_MacroAssembler.hpp
+++ b/src/hotspot/share/c1/c1_MacroAssembler.hpp
@@ -38,7 +38,6 @@ class C1_MacroAssembler: public MacroAssembler {
//----------------------------------------------------
void explicit_null_check(Register base);
- void inline_cache_check(Register receiver, Register iCache);
void build_frame(int frame_size_in_bytes, int bang_size_in_bytes);
void remove_frame(int frame_size_in_bytes);
diff --git a/src/hotspot/share/c1/c1_Optimizer.cpp b/src/hotspot/share/c1/c1_Optimizer.cpp
index 6cd282d02ad7e..dd428a5895bc4 100644
--- a/src/hotspot/share/c1/c1_Optimizer.cpp
+++ b/src/hotspot/share/c1/c1_Optimizer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -335,7 +335,7 @@ void Optimizer::eliminate_conditional_expressions() {
}
// This removes others' relation to block, but doesn't empty block's lists
-void disconnect_from_graph(BlockBegin* block) {
+static void disconnect_from_graph(BlockBegin* block) {
for (int p = 0; p < block->number_of_preds(); p++) {
BlockBegin* pred = block->pred_at(p);
int idx;
diff --git a/src/hotspot/share/cds/archiveHeapWriter.cpp b/src/hotspot/share/cds/archiveHeapWriter.cpp
index 3e7d42dd8f157..131b43307d5c6 100644
--- a/src/hotspot/share/cds/archiveHeapWriter.cpp
+++ b/src/hotspot/share/cds/archiveHeapWriter.cpp
@@ -44,7 +44,7 @@
#if INCLUDE_G1GC
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#endif
#if INCLUDE_CDS_JAVA_HEAP
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 5a0bf2e61b5d0..4c3b9fe018ee4 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -71,7 +71,7 @@
#include "utilities/ostream.hpp"
#if INCLUDE_G1GC
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#endif
# include
@@ -1664,9 +1664,9 @@ void FileMapInfo::close() {
/*
* Same as os::map_memory() but also pretouches if AlwaysPreTouch is enabled.
*/
-char* map_memory(int fd, const char* file_name, size_t file_offset,
- char *addr, size_t bytes, bool read_only,
- bool allow_exec, MEMFLAGS flags = mtNone) {
+static char* map_memory(int fd, const char* file_name, size_t file_offset,
+ char *addr, size_t bytes, bool read_only,
+ bool allow_exec, MEMFLAGS flags = mtNone) {
char* mem = os::map_memory(fd, file_name, file_offset, addr, bytes,
AlwaysPreTouch ? false : read_only,
allow_exec, flags);
@@ -1690,9 +1690,12 @@ bool FileMapInfo::remap_shared_readonly_as_readwrite() {
return false;
}
char *addr = r->mapped_base();
- char *base = os::remap_memory(_fd, _full_path, r->file_offset(),
- addr, size, false /* !read_only */,
- r->allow_exec());
+ // This path should not be reached for Windows; see JDK-8222379.
+ assert(WINDOWS_ONLY(false) NOT_WINDOWS(true), "Don't call on Windows");
+ // Replace old mapping with new one that is writable.
+ char *base = os::map_memory(_fd, _full_path, r->file_offset(),
+ addr, size, false /* !read_only */,
+ r->allow_exec());
close();
// These have to be errors because the shared region is now unmapped.
if (base == nullptr) {
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index fedbc5841b682..ae5d92fedff36 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -1327,6 +1327,9 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
release_reserved_spaces(total_space_rs, archive_space_rs, class_space_rs);
return nullptr;
}
+ // NMT: fix up the space tags
+ MemTracker::record_virtual_memory_type(archive_space_rs.base(), mtClassShared);
+ MemTracker::record_virtual_memory_type(class_space_rs.base(), mtClass);
} else {
if (use_archive_base_addr && base_address != nullptr) {
total_space_rs = ReservedSpace(total_range_size, archive_space_alignment,
@@ -1356,16 +1359,13 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
(size_t)archive_space_alignment);
class_space_rs = total_space_rs.last_part(ccs_begin_offset);
MemTracker::record_virtual_memory_split_reserved(total_space_rs.base(), total_space_rs.size(),
- ccs_begin_offset);
+ ccs_begin_offset, mtClassShared, mtClass);
}
assert(is_aligned(archive_space_rs.base(), archive_space_alignment), "Sanity");
assert(is_aligned(archive_space_rs.size(), archive_space_alignment), "Sanity");
assert(is_aligned(class_space_rs.base(), class_space_alignment), "Sanity");
assert(is_aligned(class_space_rs.size(), class_space_alignment), "Sanity");
- // NMT: fix up the space tags
- MemTracker::record_virtual_memory_type(archive_space_rs.base(), mtClassShared);
- MemTracker::record_virtual_memory_type(class_space_rs.base(), mtClass);
return archive_space_rs.base();
diff --git a/src/hotspot/share/classfile/altHashing.cpp b/src/hotspot/share/classfile/altHashing.cpp
index 158a8a232a7b4..1d43d6ebf1ed0 100644
--- a/src/hotspot/share/classfile/altHashing.cpp
+++ b/src/hotspot/share/classfile/altHashing.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -120,7 +120,7 @@ static void halfsiphash_init64(uint32_t v[4], uint64_t seed) {
v[1] ^= 0xee;
}
-uint32_t halfsiphash_finish32(uint32_t v[4], int rounds) {
+static uint32_t halfsiphash_finish32(uint32_t v[4], int rounds) {
v[2] ^= 0xff;
halfsiphash_rounds(v, rounds);
return (v[1] ^ v[3]);
diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp
index 43aa82b67f85f..40a7d178f7561 100644
--- a/src/hotspot/share/classfile/classLoader.cpp
+++ b/src/hotspot/share/classfile/classLoader.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -134,7 +134,8 @@ ClassPathEntry* ClassLoader::_last_module_path_entry = nullptr;
#endif
// helper routines
-bool string_starts_with(const char* str, const char* str_to_find) {
+#if INCLUDE_CDS
+static bool string_starts_with(const char* str, const char* str_to_find) {
size_t str_len = strlen(str);
size_t str_to_find_len = strlen(str_to_find);
if (str_to_find_len > str_len) {
@@ -142,6 +143,7 @@ bool string_starts_with(const char* str, const char* str_to_find) {
}
return (strncmp(str, str_to_find, str_to_find_len) == 0);
}
+#endif
static const char* get_jimage_version_string() {
static char version_string[10] = "";
@@ -1009,8 +1011,8 @@ const char* ClassLoader::file_name_for_class_name(const char* class_name,
return file_name;
}
-ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
- const GrowableArray* const module_list) {
+static ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
+ const GrowableArray* const module_list) {
int num_of_entries = module_list->length();
const Symbol* class_module_name = mod_entry->name();
@@ -1028,17 +1030,14 @@ ClassPathEntry* find_first_module_cpe(ModuleEntry* mod_entry,
}
-// Search either the patch-module or exploded build entries for class.
+// Search the module list for the class file stream based on the file name and java package
ClassFileStream* ClassLoader::search_module_entries(JavaThread* current,
const GrowableArray* const module_list,
- const char* const class_name,
+ PackageEntry* pkg_entry, // Java package entry derived from the class name
const char* const file_name) {
ClassFileStream* stream = nullptr;
- // Find the class' defining module in the boot loader's module entry table
- TempNewSymbol class_name_symbol = SymbolTable::new_symbol(class_name);
- TempNewSymbol pkg_name = package_from_class_name(class_name_symbol);
- PackageEntry* pkg_entry = get_package_entry(pkg_name, ClassLoaderData::the_null_class_loader_data());
+ // Find the defining module in the boot loader's module entry table
ModuleEntry* mod_entry = (pkg_entry != nullptr) ? pkg_entry->module() : nullptr;
// If the module system has not defined java.base yet, then
@@ -1083,7 +1082,7 @@ ClassFileStream* ClassLoader::search_module_entries(JavaThread* current,
}
// Called by the boot classloader to load classes
-InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TRAPS) {
+InstanceKlass* ClassLoader::load_class(Symbol* name, PackageEntry* pkg_entry, bool search_append_only, TRAPS) {
assert(name != nullptr, "invariant");
ResourceMark rm(THREAD);
@@ -1130,7 +1129,7 @@ InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TR
// is not supported with UseSharedSpaces, we can never come here during dynamic dumping.
assert(!CDSConfig::is_dumping_dynamic_archive(), "sanity");
if (!CDSConfig::is_dumping_static_archive()) {
- stream = search_module_entries(THREAD, _patch_mod_entries, class_name, file_name);
+ stream = search_module_entries(THREAD, _patch_mod_entries, pkg_entry, file_name);
}
}
@@ -1142,7 +1141,7 @@ InstanceKlass* ClassLoader::load_class(Symbol* name, bool search_append_only, TR
} else {
// Exploded build - attempt to locate class in its defining module's location.
assert(_exploded_entries != nullptr, "No exploded build entries present");
- stream = search_module_entries(THREAD, _exploded_entries, class_name, file_name);
+ stream = search_module_entries(THREAD, _exploded_entries, pkg_entry, file_name);
}
}
@@ -1355,7 +1354,7 @@ void ClassLoader::initialize(TRAPS) {
setup_bootstrap_search_path(THREAD);
}
-char* lookup_vm_resource(JImageFile *jimage, const char *jimage_version, const char *path) {
+static char* lookup_vm_resource(JImageFile *jimage, const char *jimage_version, const char *path) {
jlong size;
JImageLocationRef location = (*JImageFindResource)(jimage, "java.base", jimage_version, path, &size);
if (location == 0)
diff --git a/src/hotspot/share/classfile/classLoader.hpp b/src/hotspot/share/classfile/classLoader.hpp
index 10373dbcf9f91..c8ea47435dd93 100644
--- a/src/hotspot/share/classfile/classLoader.hpp
+++ b/src/hotspot/share/classfile/classLoader.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -292,14 +292,14 @@ class ClassLoader: AllStatic {
// Add a module's exploded directory to the boot loader's exploded module build list
static void add_to_exploded_build_list(JavaThread* current, Symbol* module_name);
- // Attempt load of individual class from either the patched or exploded modules build lists
+ // Search the module list for the class file stream based on the file name and java package
static ClassFileStream* search_module_entries(JavaThread* current,
const GrowableArray* const module_list,
- const char* const class_name,
+ PackageEntry* pkg_entry, // Java package entry derived from the class name
const char* const file_name);
// Load individual .class file
- static InstanceKlass* load_class(Symbol* class_name, bool search_append_only, TRAPS);
+ static InstanceKlass* load_class(Symbol* class_name, PackageEntry* pkg_entry, bool search_append_only, TRAPS);
// If the specified package has been loaded by the system, then returns
// the name of the directory or ZIP file that the package was loaded from.
diff --git a/src/hotspot/share/classfile/loaderConstraints.cpp b/src/hotspot/share/classfile/loaderConstraints.cpp
index e6021e00d3391..99d0c07ed42d4 100644
--- a/src/hotspot/share/classfile/loaderConstraints.cpp
+++ b/src/hotspot/share/classfile/loaderConstraints.cpp
@@ -296,8 +296,8 @@ void LoaderConstraintTable::purge_loader_constraints() {
_loader_constraint_table->unlink(&purge);
}
-void log_ldr_constraint_msg(Symbol* class_name, const char* reason,
- ClassLoaderData* loader1, ClassLoaderData* loader2) {
+static void log_ldr_constraint_msg(Symbol* class_name, const char* reason,
+ ClassLoaderData* loader1, ClassLoaderData* loader2) {
LogTarget(Info, class, loader, constraints) lt;
if (lt.is_enabled()) {
ResourceMark rm;
diff --git a/src/hotspot/share/classfile/moduleEntry.cpp b/src/hotspot/share/classfile/moduleEntry.cpp
index 24beecdcaf72f..9d2a76e9e51b3 100644
--- a/src/hotspot/share/classfile/moduleEntry.cpp
+++ b/src/hotspot/share/classfile/moduleEntry.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -481,7 +481,7 @@ void ModuleEntry::init_as_archived_entry() {
if (_location != nullptr) {
_location = ArchiveBuilder::get_buffered_symbol(_location);
}
- JFR_ONLY(set_trace_id(0));// re-init at runtime
+ JFR_ONLY(set_trace_id(0);) // re-init at runtime
ArchivePtrMarker::mark_pointer((address*)&_reads);
ArchivePtrMarker::mark_pointer((address*)&_version);
diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp
index bd4be93b86877..4664d9565d353 100644
--- a/src/hotspot/share/classfile/modules.cpp
+++ b/src/hotspot/share/classfile/modules.cpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -259,7 +259,7 @@ static void define_javabase_module(Handle module_handle, jstring version, jstrin
}
// Caller needs ResourceMark.
-void throw_dup_pkg_exception(const char* module_name, PackageEntry* package, TRAPS) {
+static void throw_dup_pkg_exception(const char* module_name, PackageEntry* package, TRAPS) {
const char* package_name = package->name()->as_C_string();
if (package->module()->is_named()) {
THROW_MSG(vmSymbols::java_lang_IllegalStateException(),
diff --git a/src/hotspot/share/classfile/packageEntry.cpp b/src/hotspot/share/classfile/packageEntry.cpp
index 1b315bc24be16..97cd041f3f103 100644
--- a/src/hotspot/share/classfile/packageEntry.cpp
+++ b/src/hotspot/share/classfile/packageEntry.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -252,7 +252,7 @@ void PackageEntry::init_as_archived_entry() {
_module = ModuleEntry::get_archived_entry(_module);
_qualified_exports = (GrowableArray*)archived_qualified_exports;
_defined_by_cds_in_class_path = 0;
- JFR_ONLY(set_trace_id(0)); // re-init at runtime
+ JFR_ONLY(set_trace_id(0);) // re-init at runtime
ArchivePtrMarker::mark_pointer((address*)&_name);
ArchivePtrMarker::mark_pointer((address*)&_module);
diff --git a/src/hotspot/share/classfile/placeholders.cpp b/src/hotspot/share/classfile/placeholders.cpp
index 1bb5f87870417..a6a86473ea794 100644
--- a/src/hotspot/share/classfile/placeholders.cpp
+++ b/src/hotspot/share/classfile/placeholders.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -198,8 +198,8 @@ void PlaceholderEntry::set_supername(Symbol* supername) {
// All threads examining the placeholder table must hold the
// SystemDictionary_lock, so we don't need special precautions
// on store ordering here.
-PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
- Symbol* supername){
+static PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
+ Symbol* supername){
assert_locked_or_safepoint(SystemDictionary_lock);
assert(class_name != nullptr, "adding nullptr obj");
@@ -213,7 +213,7 @@ PlaceholderEntry* add_entry(Symbol* class_name, ClassLoaderData* loader_data,
}
// Remove a placeholder object.
-void remove_entry(Symbol* class_name, ClassLoaderData* loader_data) {
+static void remove_entry(Symbol* class_name, ClassLoaderData* loader_data) {
assert_locked_or_safepoint(SystemDictionary_lock);
PlaceholderKey key(class_name, loader_data);
diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp
index 9e96340d82b9c..be2971288ef12 100644
--- a/src/hotspot/share/classfile/stringTable.cpp
+++ b/src/hotspot/share/classfile/stringTable.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -122,7 +122,7 @@ volatile bool _alt_hash = false;
static bool _rehashed = false;
static uint64_t _alt_hash_seed = 0;
-unsigned int hash_string(const jchar* s, int len, bool useAlt) {
+static unsigned int hash_string(const jchar* s, int len, bool useAlt) {
return useAlt ?
AltHashing::halfsiphash_32(_alt_hash_seed, s, len) :
java_lang_String::hash_code(s, len);
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 82c20a962ce14..30538926b4179 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -212,13 +212,13 @@ void SystemDictionary::set_platform_loader(ClassLoaderData *cld) {
// ----------------------------------------------------------------------------
// Parallel class loading check
-bool is_parallelCapable(Handle class_loader) {
+static bool is_parallelCapable(Handle class_loader) {
if (class_loader.is_null()) return true;
return java_lang_ClassLoader::parallelCapable(class_loader());
}
// ----------------------------------------------------------------------------
// ParallelDefineClass flag does not apply to bootclass loader
-bool is_parallelDefine(Handle class_loader) {
+static bool is_parallelDefine(Handle class_loader) {
if (class_loader.is_null()) return false;
if (AllowParallelDefineClass && java_lang_ClassLoader::parallelCapable(class_loader())) {
return true;
@@ -280,7 +280,7 @@ Symbol* SystemDictionary::class_name_symbol(const char* name, Symbol* exception,
#ifdef ASSERT
// Used to verify that class loading succeeded in adding k to the dictionary.
-void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) {
+static void verify_dictionary_entry(Symbol* class_name, InstanceKlass* k) {
MutexLocker mu(SystemDictionary_lock);
ClassLoaderData* loader_data = k->class_loader_data();
Dictionary* dictionary = loader_data->dictionary();
@@ -1276,7 +1276,7 @@ InstanceKlass* SystemDictionary::load_instance_class_impl(Symbol* class_name, Ha
if (k == nullptr) {
// Use VM class loader
PerfTraceTime vmtimer(ClassLoader::perf_sys_classload_time());
- k = ClassLoader::load_class(class_name, search_only_bootloader_append, CHECK_NULL);
+ k = ClassLoader::load_class(class_name, pkg_entry, search_only_bootloader_append, CHECK_NULL);
}
// find_or_define_instance_class may return a different InstanceKlass
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index 297483526846c..44d7da5c4a4f1 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1348,7 +1348,7 @@ void SystemDictionaryShared::update_shared_entry(InstanceKlass* k, int id) {
info->_id = id;
}
-const char* class_loader_name_for_shared(Klass* k) {
+static const char* class_loader_name_for_shared(Klass* k) {
assert(k != nullptr, "Sanity");
assert(k->is_shared(), "Must be");
assert(k->is_instance_klass(), "Must be");
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index b30a88e90cb48..d24e29c288d5c 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -25,7 +25,6 @@
#include "precompiled.hpp"
#include "code/codeBlob.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/relocInfo.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/disassembler.hpp"
@@ -649,11 +648,6 @@ void CodeBlob::dump_for_addr(address addr, outputStream* st, bool verbose) const
st->print_cr(INTPTR_FORMAT " is pointing to an (unnamed) stub routine", p2i(addr));
return;
}
- // the InlineCacheBuffer is using stubs generated into a buffer blob
- if (InlineCacheBuffer::contains(addr)) {
- st->print_cr(INTPTR_FORMAT " is pointing into InlineCacheBuffer", p2i(addr));
- return;
- }
VtableStub* v = VtableStubs::stub_containing(addr);
if (v != nullptr) {
st->print_cr(INTPTR_FORMAT " is at entry_point+%d in a vtable stub", p2i(addr), (int)(addr - v->entry_point()));
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index bf8c1d84e71a0..d56de671a1352 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -29,7 +29,6 @@
#include "code/compiledIC.hpp"
#include "code/dependencies.hpp"
#include "code/dependencyContext.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/pcDesc.hpp"
#include "compiler/compilationPolicy.hpp"
@@ -913,23 +912,6 @@ void CodeCache::verify_clean_inline_caches() {
#endif
}
-void CodeCache::verify_icholder_relocations() {
-#ifdef ASSERT
- // make sure that we aren't leaking icholders
- int count = 0;
- FOR_ALL_HEAPS(heap) {
- FOR_ALL_BLOBS(cb, *heap) {
- CompiledMethod *nm = cb->as_compiled_method_or_null();
- if (nm != nullptr) {
- count += nm->verify_icholder_relocations();
- }
- }
- }
- assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
- CompiledICHolder::live_count(), "must agree");
-#endif
-}
-
// Defer freeing of concurrently cleaned ExceptionCache entries until
// after a global handshake operation.
void CodeCache::release_exception_cache(ExceptionCache* entry) {
@@ -1748,6 +1730,10 @@ void CodeCache::print() {
void CodeCache::print_summary(outputStream* st, bool detailed) {
int full_count = 0;
+ julong total_used = 0;
+ julong total_max_used = 0;
+ julong total_free = 0;
+ julong total_size = 0;
FOR_ALL_HEAPS(heap_iterator) {
CodeHeap* heap = (*heap_iterator);
size_t total = (heap->high_boundary() - heap->low_boundary());
@@ -1756,10 +1742,17 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
} else {
st->print("CodeCache:");
}
+ size_t size = total/K;
+ size_t used = (total - heap->unallocated_capacity())/K;
+ size_t max_used = heap->max_allocated_capacity()/K;
+ size_t free = heap->unallocated_capacity()/K;
+ total_size += size;
+ total_used += used;
+ total_max_used += max_used;
+ total_free += free;
st->print_cr(" size=" SIZE_FORMAT "Kb used=" SIZE_FORMAT
"Kb max_used=" SIZE_FORMAT "Kb free=" SIZE_FORMAT "Kb",
- total/K, (total - heap->unallocated_capacity())/K,
- heap->max_allocated_capacity()/K, heap->unallocated_capacity()/K);
+ size, used, max_used, free);
if (detailed) {
st->print_cr(" bounds [" INTPTR_FORMAT ", " INTPTR_FORMAT ", " INTPTR_FORMAT "]",
@@ -1772,17 +1765,22 @@ void CodeCache::print_summary(outputStream* st, bool detailed) {
}
if (detailed) {
- st->print_cr(" total_blobs=" UINT32_FORMAT " nmethods=" UINT32_FORMAT
- " adapters=" UINT32_FORMAT,
- blob_count(), nmethod_count(), adapter_count());
- st->print_cr(" compilation: %s", CompileBroker::should_compile_new_jobs() ?
+ if (SegmentedCodeCache) {
+ st->print("CodeCache:");
+ st->print_cr(" size=" JULONG_FORMAT "Kb, used=" JULONG_FORMAT
+ "Kb, max_used=" JULONG_FORMAT "Kb, free=" JULONG_FORMAT "Kb",
+ total_size, total_used, total_max_used, total_free);
+ }
+ st->print_cr(" total_blobs=" UINT32_FORMAT ", nmethods=" UINT32_FORMAT
+ ", adapters=" UINT32_FORMAT ", full_count=" UINT32_FORMAT,
+ blob_count(), nmethod_count(), adapter_count(), full_count);
+ st->print_cr("Compilation: %s, stopped_count=%d, restarted_count=%d",
+ CompileBroker::should_compile_new_jobs() ?
"enabled" : Arguments::mode() == Arguments::_int ?
"disabled (interpreter mode)" :
- "disabled (not enough contiguous free space left)");
- st->print_cr(" stopped_count=%d, restarted_count=%d",
+ "disabled (not enough contiguous free space left)",
CompileBroker::get_total_compiler_stopped_count(),
CompileBroker::get_total_compiler_restarted_count());
- st->print_cr(" full_count=%d", full_count);
}
}
diff --git a/src/hotspot/share/code/codeCache.hpp b/src/hotspot/share/code/codeCache.hpp
index 103268c8ffcd1..d1c91727bf124 100644
--- a/src/hotspot/share/code/codeCache.hpp
+++ b/src/hotspot/share/code/codeCache.hpp
@@ -294,7 +294,6 @@ class CodeCache : AllStatic {
}
static void verify_clean_inline_caches();
- static void verify_icholder_relocations();
// Deoptimization
private:
diff --git a/src/hotspot/share/code/compiledIC.cpp b/src/hotspot/share/code/compiledIC.cpp
index c5063560ee570..250ef063a2a33 100644
--- a/src/hotspot/share/code/compiledIC.cpp
+++ b/src/hotspot/share/code/compiledIC.cpp
@@ -26,27 +26,19 @@
#include "code/codeBehaviours.hpp"
#include "code/codeCache.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/vtableStubs.hpp"
-#include "interpreter/interpreter.hpp"
-#include "interpreter/linkResolver.hpp"
-#include "memory/metadataFactory.hpp"
-#include "memory/oopFactory.hpp"
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
+#include "oops/compressedKlass.hpp"
#include "oops/klass.inline.hpp"
#include "oops/method.inline.hpp"
-#include "oops/oop.inline.hpp"
-#include "oops/symbol.hpp"
+#include "runtime/atomic.hpp"
#include "runtime/continuationEntry.hpp"
#include "runtime/handles.inline.hpp"
-#include "runtime/icache.hpp"
-#include "runtime/safepoint.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/sharedRuntime.hpp"
-#include "runtime/stubRoutines.hpp"
#include "sanitizers/leak.hpp"
-#include "utilities/events.hpp"
// Every time a compiled IC is changed or its type is being accessed,
@@ -75,191 +67,175 @@ bool CompiledICLocker::is_safe(address code) {
return CompiledICProtectionBehaviour::current()->is_safe(cm);
}
-//-----------------------------------------------------------------------------
-// Low-level access to an inline cache. Private, since they might not be
-// MT-safe to use.
+CompiledICData::CompiledICData()
+ : _speculated_method(),
+ _speculated_klass(),
+ _itable_defc_klass(),
+ _itable_refc_klass(),
+ _is_initialized() {}
-void* CompiledIC::cached_value() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert (!is_optimized(), "an optimized virtual call does not have a cached metadata");
-
- if (!is_in_transition_state()) {
- void* data = get_data();
- // If we let the metadata value here be initialized to zero...
- assert(data != nullptr || Universe::non_oop_word() == nullptr,
- "no raw nulls in CompiledIC metadatas, because of patching races");
- return (data == (void*)Universe::non_oop_word()) ? nullptr : data;
+// Inline cache callsite info is initialized once the first time it is resolved
+void CompiledICData::initialize(CallInfo* call_info, Klass* receiver_klass) {
+ _speculated_method = call_info->selected_method();
+ if (UseCompressedClassPointers) {
+ _speculated_klass = (uintptr_t)CompressedKlassPointers::encode_not_null(receiver_klass);
} else {
- return InlineCacheBuffer::cached_value_for((CompiledIC *)this);
+ _speculated_klass = (uintptr_t)receiver_klass;
}
+ if (call_info->call_kind() == CallInfo::itable_call) {
+ _itable_defc_klass = call_info->resolved_method()->method_holder();
+ _itable_refc_klass = call_info->resolved_klass();
+ }
+ _is_initialized = true;
}
+bool CompiledICData::is_speculated_klass_unloaded() const {
+ return is_initialized() && _speculated_klass == 0;
+}
-void CompiledIC::internal_set_ic_destination(address entry_point, bool is_icstub, void* cache, bool is_icholder) {
- assert(entry_point != nullptr, "must set legal entry point");
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert (!is_optimized() || cache == nullptr, "an optimized virtual call does not have a cached metadata");
- assert (cache == nullptr || cache != (Metadata*)badOopVal, "invalid metadata");
-
- assert(!is_icholder || is_icholder_entry(entry_point), "must be");
-
- // Don't use ic_destination for this test since that forwards
- // through ICBuffer instead of returning the actual current state of
- // the CompiledIC.
- if (is_icholder_entry(_call->destination())) {
- // When patching for the ICStub case the cached value isn't
- // overwritten until the ICStub copied into the CompiledIC during
- // the next safepoint. Make sure that the CompiledICHolder* is
- // marked for release at this point since it won't be identifiable
- // once the entry point is overwritten.
- InlineCacheBuffer::queue_for_release((CompiledICHolder*)get_data());
+void CompiledICData::clean_metadata() {
+ if (!is_initialized() || is_speculated_klass_unloaded()) {
+ return;
}
- if (TraceCompiledIC) {
- tty->print(" ");
- print_compiled_ic();
- tty->print(" changing destination to " INTPTR_FORMAT, p2i(entry_point));
- if (!is_optimized()) {
- tty->print(" changing cached %s to " INTPTR_FORMAT, is_icholder ? "icholder" : "metadata", p2i((address)cache));
- }
- if (is_icstub) {
- tty->print(" (icstub)");
- }
- tty->cr();
+ // GC cleaning doesn't need to change the state of the inline cache,
+ // only nuke stale speculated metadata if it gets unloaded. If the
+ // inline cache is monomorphic, the unverified entries will miss, and
+ // subsequent miss handlers will upgrade the callsite to megamorphic,
+ // which makes sense as it obviously is megamorphic then.
+ if (!speculated_klass()->is_loader_alive()) {
+ Atomic::store(&_speculated_klass, (uintptr_t)0);
+ Atomic::store(&_speculated_method, (Method*)nullptr);
}
-#ifdef ASSERT
- {
- CodeBlob* cb = CodeCache::find_blob(_call->instruction_address());
- assert(cb != nullptr && cb->is_compiled(), "must be compiled");
- }
-#endif
- _call->set_destination_mt_safe(entry_point);
+ assert(_speculated_method == nullptr || _speculated_method->method_holder()->is_loader_alive(),
+ "Speculated method is not unloaded despite class being unloaded");
+}
- if (is_optimized() || is_icstub) {
- // Optimized call sites don't have a cache value and ICStub call
- // sites only change the entry point. Changing the value in that
- // case could lead to MT safety issues.
- assert(cache == nullptr, "must be null");
+void CompiledICData::metadata_do(MetadataClosure* cl) {
+ if (!is_initialized()) {
return;
}
- if (cache == nullptr) cache = Universe::non_oop_word();
-
- set_data((intptr_t)cache);
-}
-
-
-void CompiledIC::set_ic_destination(ICStub* stub) {
- internal_set_ic_destination(stub->code_begin(), true, nullptr, false);
+ if (!is_speculated_klass_unloaded()) {
+ cl->do_metadata(_speculated_method);
+ cl->do_metadata(speculated_klass());
+ }
+ if (_itable_refc_klass != nullptr) {
+ cl->do_metadata(_itable_refc_klass);
+ }
+ if (_itable_defc_klass != nullptr) {
+ cl->do_metadata(_itable_defc_klass);
+ }
}
+Klass* CompiledICData::speculated_klass() const {
+ if (is_speculated_klass_unloaded()) {
+ return nullptr;
+ }
-
-address CompiledIC::ic_destination() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- if (!is_in_transition_state()) {
- return _call->destination();
+ if (UseCompressedClassPointers) {
+ return CompressedKlassPointers::decode_not_null((narrowKlass)_speculated_klass);
} else {
- return InlineCacheBuffer::ic_destination_for((CompiledIC *)this);
+ return (Klass*)_speculated_klass;
}
}
+//-----------------------------------------------------------------------------
+// High-level access to an inline cache. Guaranteed to be MT-safe.
-bool CompiledIC::is_in_transition_state() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- return InlineCacheBuffer::contains(_call->destination());;
+CompiledICData* CompiledIC::data() const {
+ return _data;
}
+CompiledICData* data_from_reloc_iter(RelocIterator* iter) {
+ assert(iter->type() == relocInfo::virtual_call_type, "wrong reloc. info");
+
+ virtual_call_Relocation* r = iter->virtual_call_reloc();
+ NativeMovConstReg* value = nativeMovConstReg_at(r->cached_value());
+
+ return (CompiledICData*)value->data();
+}
-bool CompiledIC::is_icholder_call() const {
+CompiledIC::CompiledIC(RelocIterator* iter)
+ : _method(iter->code()),
+ _data(data_from_reloc_iter(iter)),
+ _call(nativeCall_at(iter->addr()))
+{
+ assert(_method != nullptr, "must pass compiled method");
+ assert(_method->contains(iter->addr()), "must be in compiled method");
assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- return !_is_optimized && is_icholder_entry(ic_destination());
}
-// Returns native address of 'call' instruction in inline-cache. Used by
-// the InlineCacheBuffer when it needs to find the stub.
-address CompiledIC::stub_address() const {
- assert(is_in_transition_state(), "should only be called when we are in a transition state");
- return _call->destination();
+CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr) {
+ address call_site = nativeCall_before(return_addr)->instruction_address();
+ return CompiledIC_at(nm, call_site);
}
-// Clears the IC stub if the compiled IC is in transition state
-void CompiledIC::clear_ic_stub() {
- if (is_in_transition_state()) {
- ICStub* stub = ICStub::from_destination_address(stub_address());
- stub->clear();
- }
+CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site) {
+ RelocIterator iter(nm, call_site, call_site + 1);
+ iter.next();
+ return CompiledIC_at(&iter);
}
-//-----------------------------------------------------------------------------
-// High-level access to an inline cache. Guaranteed to be MT-safe.
+CompiledIC* CompiledIC_at(Relocation* call_reloc) {
+ address call_site = call_reloc->addr();
+ CompiledMethod* cm = CodeCache::find_blob(call_reloc->addr())->as_compiled_method();
+ return CompiledIC_at(cm, call_site);
+}
-void CompiledIC::initialize_from_iter(RelocIterator* iter) {
- assert(iter->addr() == _call->instruction_address(), "must find ic_call");
+CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
+ CompiledIC* c_ic = new CompiledIC(reloc_iter);
+ c_ic->verify();
+ return c_ic;
+}
- if (iter->type() == relocInfo::virtual_call_type) {
- virtual_call_Relocation* r = iter->virtual_call_reloc();
- _is_optimized = false;
- _value = _call->get_load_instruction(r);
- } else {
- assert(iter->type() == relocInfo::opt_virtual_call_type, "must be a virtual call");
- _is_optimized = true;
- _value = nullptr;
+void CompiledIC::ensure_initialized(CallInfo* call_info, Klass* receiver_klass) {
+ if (!_data->is_initialized()) {
+ _data->initialize(call_info, receiver_klass);
}
}
-CompiledIC::CompiledIC(CompiledMethod* cm, NativeCall* call)
- : _method(cm)
-{
- _call = _method->call_wrapper_at((address) call);
- address ic_call = _call->instruction_address();
-
- assert(ic_call != nullptr, "ic_call address must be set");
- assert(cm != nullptr, "must pass compiled method");
- assert(cm->contains(ic_call), "must be in compiled method");
-
- // Search for the ic_call at the given address.
- RelocIterator iter(cm, ic_call, ic_call+1);
- bool ret = iter.next();
- assert(ret == true, "relocInfo must exist at this address");
- assert(iter.addr() == ic_call, "must find ic_call");
-
- initialize_from_iter(&iter);
+void CompiledIC::set_to_clean() {
+ log_debug(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(_call->instruction_address()));
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_virtual_call_stub());
}
-CompiledIC::CompiledIC(RelocIterator* iter)
- : _method(iter->code())
-{
- _call = _method->call_wrapper_at(iter->addr());
- address ic_call = _call->instruction_address();
+void CompiledIC::set_to_monomorphic() {
+ assert(data()->is_initialized(), "must be initialized");
+ Method* method = data()->speculated_method();
+ CompiledMethod* code = method->code();
+ address entry;
+ bool to_compiled = code != nullptr && code->is_in_use() && !code->is_unloading();
+
+ if (to_compiled) {
+ entry = code->entry_point();
+ } else {
+ entry = method->get_c2i_unverified_entry();
+ }
- CompiledMethod* nm = iter->code();
- assert(ic_call != nullptr, "ic_call address must be set");
- assert(nm != nullptr, "must pass compiled method");
- assert(nm->contains(ic_call), "must be in compiled method");
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to %s: %s",
+ p2i(_call->instruction_address()),
+ to_compiled ? "compiled" : "interpreter",
+ method->print_value_string());
- initialize_from_iter(iter);
+ _call->set_destination_mt_safe(entry);
}
-// This function may fail for two reasons: either due to running out of vtable
-// stubs, or due to running out of IC stubs in an attempted transition to a
-// transitional state. The needs_ic_stub_refill value will be set if the failure
-// was due to running out of IC stubs, in which case the caller will refill IC
-// stubs and retry.
-bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecode,
- bool& needs_ic_stub_refill, TRAPS) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert(!is_optimized(), "cannot set an optimized virtual call to megamorphic");
- assert(is_call_to_compiled() || is_call_to_interpreted(), "going directly to megamorphic?");
+void CompiledIC::set_to_megamorphic(CallInfo* call_info) {
+ assert(data()->is_initialized(), "must be initialized");
address entry;
- if (call_info->call_kind() == CallInfo::itable_call) {
- assert(bytecode == Bytecodes::_invokeinterface, "");
+ if (call_info->call_kind() == CallInfo::direct_call) {
+ // C1 sometimes compiles a callsite before the target method is loaded, resulting in
+ // dynamically bound callsites that should really be statically bound. However, the
+ // target method might not have a vtable or itable. We just wait for better code to arrive
+ return;
+ } else if (call_info->call_kind() == CallInfo::itable_call) {
int itable_index = call_info->itable_index();
entry = VtableStubs::find_itable_stub(itable_index);
if (entry == nullptr) {
- return false;
+ return;
}
#ifdef ASSERT
int index = call_info->resolved_method()->itable_index();
@@ -267,401 +243,151 @@ bool CompiledIC::set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecod
InstanceKlass* k = call_info->resolved_method()->method_holder();
assert(k->verify_itable_index(itable_index), "sanity check");
#endif //ASSERT
- CompiledICHolder* holder = new CompiledICHolder(call_info->resolved_method()->method_holder(),
- call_info->resolved_klass(), false);
- holder->claim();
- if (!InlineCacheBuffer::create_transition_stub(this, holder, entry)) {
- delete holder;
- needs_ic_stub_refill = true;
- return false;
- }
- // LSan appears unable to follow malloc-based memory consistently when embedded as an immediate
- // in generated machine code. So we have to ignore it.
- LSAN_IGNORE_OBJECT(holder);
} else {
- assert(call_info->call_kind() == CallInfo::vtable_call, "either itable or vtable");
+ assert(call_info->call_kind() == CallInfo::vtable_call, "what else?");
// Can be different than selected_method->vtable_index(), due to package-private etc.
int vtable_index = call_info->vtable_index();
assert(call_info->resolved_klass()->verify_vtable_index(vtable_index), "sanity check");
entry = VtableStubs::find_vtable_stub(vtable_index);
if (entry == nullptr) {
- return false;
+ return;
}
- if (!InlineCacheBuffer::create_transition_stub(this, nullptr, entry)) {
- needs_ic_stub_refill = true;
- return false;
- }
- }
-
- {
- ResourceMark rm;
- assert(call_info->selected_method() != nullptr, "Unexpected null selected method");
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
- p2i(instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
}
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_megamorphic(), "sanity check");
- return true;
-}
-
+ log_trace(inlinecache)("IC@" INTPTR_FORMAT ": to megamorphic %s entry: " INTPTR_FORMAT,
+ p2i(_call->instruction_address()), call_info->selected_method()->print_value_string(), p2i(entry));
-// true if destination is megamorphic stub
-bool CompiledIC::is_megamorphic() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- assert(!is_optimized(), "an optimized call cannot be megamorphic");
-
- // Cannot rely on cached_value. It is either an interface or a method.
- return VtableStubs::entry_point(ic_destination()) != nullptr;
+ _call->set_destination_mt_safe(entry);
+ assert(is_megamorphic(), "sanity check");
}
-bool CompiledIC::is_call_to_compiled() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
-
- CodeBlob* cb = CodeCache::find_blob(ic_destination());
- bool is_monomorphic = (cb != nullptr && cb->is_compiled());
- // Check that the cached_value is a klass for non-optimized monomorphic calls
- // This assertion is invalid for compiler1: a call that does not look optimized (no static stub) can be used
- // for calling directly to vep without using the inline cache (i.e., cached_value == nullptr).
- // For JVMCI this occurs because CHA is only used to improve inlining so call sites which could be optimized
- // virtuals because there are no currently loaded subclasses of a type are left as virtual call sites.
-#ifdef ASSERT
- CodeBlob* caller = CodeCache::find_blob(instruction_address());
- bool is_c1_or_jvmci_method = caller->is_compiled_by_c1() || caller->is_compiled_by_jvmci();
- assert( is_c1_or_jvmci_method ||
- !is_monomorphic ||
- is_optimized() ||
- (cached_metadata() != nullptr && cached_metadata()->is_klass()), "sanity check");
-#endif // ASSERT
- return is_monomorphic;
-}
+void CompiledIC::update(CallInfo* call_info, Klass* receiver_klass) {
+ // If this is the first time we fix the inline cache, we ensure it's initialized
+ ensure_initialized(call_info, receiver_klass);
+ if (is_megamorphic()) {
+ // Terminal state for the inline cache
+ return;
+ }
-bool CompiledIC::is_call_to_interpreted() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- // Call to interpreter if destination is either calling to a stub (if it
- // is optimized), or calling to an I2C blob
- bool is_call_to_interpreted = false;
- if (!is_optimized()) {
- CodeBlob* cb = CodeCache::find_blob(ic_destination());
- is_call_to_interpreted = (cb != nullptr && cb->is_adapter_blob());
- assert(!is_call_to_interpreted || (is_icholder_call() && cached_icholder() != nullptr), "sanity check");
+ if (is_speculated_klass(receiver_klass)) {
+ // If the speculated class matches the receiver klass, we can speculate that will
+ // continue to be the case with a monomorphic inline cache
+ set_to_monomorphic();
} else {
- // Check if we are calling into our own codeblob (i.e., to a stub)
- address dest = ic_destination();
-#ifdef ASSERT
- {
- _call->verify_resolve_call(dest);
- }
-#endif /* ASSERT */
- is_call_to_interpreted = _call->is_call_to_interpreted(dest);
+ // If the dynamic type speculation fails, we try to transform to a megamorphic state
+ // for the inline cache using stubs to dispatch in tables
+ set_to_megamorphic(call_info);
}
- return is_call_to_interpreted;
}
-bool CompiledIC::set_to_clean(bool in_use) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- if (TraceInlineCacheClearing) {
- tty->print_cr("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
- print();
- }
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": set to clean", p2i(instruction_address()));
-
- address entry = _call->get_resolve_call_stub(is_optimized());
-
- bool safe_transition = _call->is_safe_for_patching() || !in_use || is_optimized() || SafepointSynchronize::is_at_safepoint();
+bool CompiledIC::is_clean() const {
+ return destination() == SharedRuntime::get_resolve_virtual_call_stub();
+}
- if (safe_transition) {
- // Kill any leftover stub we might have too
- clear_ic_stub();
- if (is_optimized()) {
- set_ic_destination(entry);
- } else {
- set_ic_destination_and_value(entry, (void*)nullptr);
- }
- } else {
- // Unsafe transition - create stub.
- if (!InlineCacheBuffer::create_transition_stub(this, nullptr, entry)) {
- return false;
- }
- }
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_clean(), "sanity check");
- return true;
+bool CompiledIC::is_monomorphic() const {
+ return !is_clean() && !is_megamorphic();
}
-bool CompiledIC::is_clean() const {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- bool is_clean = false;
- address dest = ic_destination();
- is_clean = dest == _call->get_resolve_call_stub(is_optimized());
- assert(!is_clean || is_optimized() || cached_value() == nullptr, "sanity check");
- return is_clean;
+bool CompiledIC::is_megamorphic() const {
+ return VtableStubs::entry_point(destination()) != nullptr;;
}
-bool CompiledIC::set_to_monomorphic(CompiledICInfo& info) {
- assert(CompiledICLocker::is_safe(_method), "mt unsafe call");
- // Updating a cache to the wrong entry can cause bugs that are very hard
- // to track down - if cache entry gets invalid - we just clean it. In
- // this way it is always the same code path that is responsible for
- // updating and resolving an inline cache
- //
- // The above is no longer true. SharedRuntime::fixup_callers_callsite will change optimized
- // callsites. In addition ic_miss code will update a site to monomorphic if it determines
- // that an monomorphic call to the interpreter can now be monomorphic to compiled code.
- //
- // In both of these cases the only thing being modified is the jump/call target and these
- // transitions are mt_safe
-
- Thread *thread = Thread::current();
- if (info.to_interpreter()) {
- // Call to interpreter
- if (info.is_optimized() && is_optimized()) {
- assert(is_clean(), "unsafe IC path");
- // the call analysis (callee structure) specifies that the call is optimized
- // (either because of CHA or the static target is final)
- // At code generation time, this call has been emitted as static call
- // Call via stub
- assert(info.cached_metadata() != nullptr && info.cached_metadata()->is_method(), "sanity check");
- methodHandle method (thread, (Method*)info.cached_metadata());
- _call->set_to_interpreted(method, info);
-
- {
- ResourceMark rm(thread);
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter: %s",
- p2i(instruction_address()),
- method->print_value_string());
- }
- } else {
- // Call via method-klass-holder
- CompiledICHolder* holder = info.claim_cached_icholder();
- if (!InlineCacheBuffer::create_transition_stub(this, holder, info.entry())) {
- delete holder;
- return false;
- }
- // LSan appears unable to follow malloc-based memory consistently when embedded as an
- // immediate in generated machine code. So we have to ignore it.
- LSAN_IGNORE_OBJECT(holder);
- {
- ResourceMark rm(thread);
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to interpreter via icholder ", p2i(instruction_address()));
- }
- }
- } else {
- // Call to compiled code
- bool static_bound = info.is_optimized() || (info.cached_metadata() == nullptr);
-#ifdef ASSERT
- CodeBlob* cb = CodeCache::find_blob(info.entry());
- assert (cb != nullptr && cb->is_compiled(), "must be compiled!");
-#endif /* ASSERT */
-
- // This is MT safe if we come from a clean-cache and go through a
- // non-verified entry point
- bool safe = SafepointSynchronize::is_at_safepoint() ||
- (!is_in_transition_state() && (info.is_optimized() || static_bound || is_clean()));
-
- if (!safe) {
- if (!InlineCacheBuffer::create_transition_stub(this, info.cached_metadata(), info.entry())) {
- return false;
- }
- } else {
- if (is_optimized()) {
- set_ic_destination(info.entry());
- } else {
- set_ic_destination_and_value(info.entry(), info.cached_metadata());
- }
- }
+bool CompiledIC::is_speculated_klass(Klass* receiver_klass) {
+ return data()->speculated_klass() == receiver_klass;
+}
- {
- ResourceMark rm(thread);
- assert(info.cached_metadata() == nullptr || info.cached_metadata()->is_klass(), "must be");
- log_trace(inlinecache)("IC@" INTPTR_FORMAT ": monomorphic to compiled (rcvr klass = %s) %s",
- p2i(instruction_address()),
- (info.cached_metadata() != nullptr) ? ((Klass*)info.cached_metadata())->print_value_string() : "nullptr",
- (safe) ? "" : " via stub");
- }
- }
- // We can't check this anymore. With lazy deopt we could have already
- // cleaned this IC entry before we even return. This is possible if
- // we ran out of space in the inline cache buffer trying to do the
- // set_next and we safepointed to free up space. This is a benign
- // race because the IC entry was complete when we safepointed so
- // cleaning it immediately is harmless.
- // assert(is_call_to_compiled() || is_call_to_interpreted(), "sanity check");
- return true;
-}
-
-
-// is_optimized: Compiler has generated an optimized call (i.e. fixed, no inline cache)
-// static_bound: The call can be static bound. If it isn't also optimized, the property
-// wasn't provable at time of compilation. An optimized call will have any necessary
-// null check, while a static_bound won't. A static_bound (but not optimized) must
-// therefore use the unverified entry point.
-void CompiledIC::compute_monomorphic_entry(const methodHandle& method,
- Klass* receiver_klass,
- bool is_optimized,
- bool static_bound,
- bool caller_is_nmethod,
- CompiledICInfo& info,
- TRAPS) {
- CompiledMethod* method_code = method->code();
-
- address entry = nullptr;
- if (method_code != nullptr && method_code->is_in_use() && !method_code->is_unloading()) {
- assert(method_code->is_compiled(), "must be compiled");
- // Call to compiled code
- //
- // Note: the following problem exists with Compiler1:
- // - at compile time we may or may not know if the destination is final
- // - if we know that the destination is final (is_optimized), we will emit
- // an optimized virtual call (no inline cache), and need a Method* to make
- // a call to the interpreter
- // - if we don't know if the destination is final, we emit a standard
- // virtual call, and use CompiledICHolder to call interpreted code
- // (no static call stub has been generated)
- // - In the case that we here notice the call is static bound we
- // convert the call into what looks to be an optimized virtual call,
- // but we must use the unverified entry point (since there will be no
- // null check on a call when the target isn't loaded).
- // This causes problems when verifying the IC because
- // it looks vanilla but is optimized. Code in is_call_to_interpreted
- // is aware of this and weakens its asserts.
- if (is_optimized) {
- entry = method_code->verified_entry_point();
- } else {
- entry = method_code->entry_point();
- }
- }
- if (entry != nullptr) {
- // Call to near compiled code.
- info.set_compiled_entry(entry, is_optimized ? nullptr : receiver_klass, is_optimized);
- } else {
- if (is_optimized) {
- // Use stub entry
- info.set_interpreter_entry(method()->get_c2i_entry(), method());
- } else {
- // Use icholder entry
- assert(method_code == nullptr || method_code->is_compiled(), "must be compiled");
- CompiledICHolder* holder = new CompiledICHolder(method(), receiver_klass);
- info.set_icholder_entry(method()->get_c2i_unverified_entry(), holder);
- }
- }
- assert(info.is_optimized() == is_optimized, "must agree");
+// GC support
+void CompiledIC::clean_metadata() {
+ data()->clean_metadata();
}
+void CompiledIC::metadata_do(MetadataClosure* cl) {
+ data()->metadata_do(cl);
+}
-bool CompiledIC::is_icholder_entry(address entry) {
- CodeBlob* cb = CodeCache::find_blob(entry);
- if (cb == nullptr) {
- return false;
- }
- if (cb->is_adapter_blob()) {
- return true;
- } else if (cb->is_vtable_blob()) {
- return VtableStubs::is_icholder_entry(entry);
- }
- return false;
+#ifndef PRODUCT
+void CompiledIC::print() {
+ tty->print("Inline cache at " INTPTR_FORMAT ", calling " INTPTR_FORMAT " cached_value " INTPTR_FORMAT,
+ p2i(instruction_address()), p2i(destination()), p2i(data()));
+ tty->cr();
}
-bool CompiledIC::is_icholder_call_site(virtual_call_Relocation* call_site, const CompiledMethod* cm) {
- // This call site might have become stale so inspect it carefully.
- address dest = cm->call_wrapper_at(call_site->addr())->destination();
- return is_icholder_entry(dest);
+void CompiledIC::verify() {
+ _call->verify();
}
+#endif
// ----------------------------------------------------------------------------
-bool CompiledStaticCall::set_to_clean(bool in_use) {
+void CompiledDirectCall::set_to_clean() {
// in_use is unused but needed to match template function in CompiledMethod
assert(CompiledICLocker::is_safe(instruction_address()), "mt unsafe call");
// Reset call site
- set_destination_mt_safe(resolve_call_stub());
+ RelocIterator iter((nmethod*)nullptr, instruction_address(), instruction_address() + 1);
+ while (iter.next()) {
+ switch(iter.type()) {
+ case relocInfo::static_call_type:
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_static_call_stub());
+ break;
+ case relocInfo::opt_virtual_call_type:
+ _call->set_destination_mt_safe(SharedRuntime::get_resolve_opt_virtual_call_stub());
+ break;
+ default:
+ ShouldNotReachHere();
+ }
+ }
+ assert(is_clean(), "should be clean after cleaning");
- // Do not reset stub here: It is too expensive to call find_stub.
- // Instead, rely on caller (nmethod::clear_inline_caches) to clear
- // both the call and its stub.
- return true;
+ log_debug(inlinecache)("DC@" INTPTR_FORMAT ": set to clean", p2i(_call->instruction_address()));
}
-bool CompiledStaticCall::is_clean() const {
- return destination() == resolve_call_stub();
-}
+void CompiledDirectCall::set(const methodHandle& callee_method) {
+ CompiledMethod* code = callee_method->code();
+ CompiledMethod* caller = CodeCache::find_compiled(instruction_address());
-bool CompiledStaticCall::is_call_to_compiled() const {
- return CodeCache::contains(destination());
-}
+ bool to_interp_cont_enter = caller->method()->is_continuation_enter_intrinsic() &&
+ ContinuationEntry::is_interpreted_call(instruction_address());
-bool CompiledDirectStaticCall::is_call_to_interpreted() const {
- // It is a call to interpreted, if it calls to a stub. Hence, the destination
- // must be in the stub part of the nmethod that contains the call
- CompiledMethod* cm = CodeCache::find_compiled(instruction_address());
- return cm->stub_contains(destination());
-}
+ bool to_compiled = !to_interp_cont_enter && code != nullptr && code->is_in_use() && !code->is_unloading();
-void CompiledStaticCall::set_to_compiled(address entry) {
- {
- ResourceMark rm;
- log_trace(inlinecache)("%s@" INTPTR_FORMAT ": set_to_compiled " INTPTR_FORMAT,
- name(),
- p2i(instruction_address()),
- p2i(entry));
+ if (to_compiled) {
+ _call->set_destination_mt_safe(code->verified_entry_point());
+ assert(is_call_to_compiled(), "should be compiled after set to compiled");
+ } else {
+ // Patch call site to C2I adapter if code is deoptimized or unloaded.
+ // We also need to patch the static call stub to set the rmethod register
+ // to the callee_method so the c2i adapter knows how to build the frame
+ set_to_interpreted(callee_method, callee_method->get_c2i_entry());
+ assert(is_call_to_interpreted(), "should be interpreted after set to interpreted");
}
- // Call to compiled code
- assert(CodeCache::contains(entry), "wrong entry point");
- set_destination_mt_safe(entry);
+
+ log_trace(inlinecache)("DC@" INTPTR_FORMAT ": set to %s: %s: " INTPTR_FORMAT,
+ p2i(_call->instruction_address()),
+ to_compiled ? "compiled" : "interpreter",
+ callee_method->print_value_string(),
+ p2i(_call->destination()));
}
-void CompiledStaticCall::set(const StaticCallInfo& info) {
- assert(CompiledICLocker::is_safe(instruction_address()), "mt unsafe call");
- // Updating a cache to the wrong entry can cause bugs that are very hard
- // to track down - if cache entry gets invalid - we just clean it. In
- // this way it is always the same code path that is responsible for
- // updating and resolving an inline cache
- assert(is_clean(), "do not update a call entry - use clean");
-
- if (info._to_interpreter) {
- // Call to interpreted code
- set_to_interpreted(info.callee(), info.entry());
- } else {
- set_to_compiled(info.entry());
- }
+bool CompiledDirectCall::is_clean() const {
+ return destination() == SharedRuntime::get_resolve_static_call_stub() ||
+ destination() == SharedRuntime::get_resolve_opt_virtual_call_stub();
}
-// Compute settings for a CompiledStaticCall. Since we might have to set
-// the stub when calling to the interpreter, we need to return arguments.
-void CompiledStaticCall::compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info) {
- CompiledMethod* m_code = m->code();
- info._callee = m;
- if (m_code != nullptr && m_code->is_in_use() && !m_code->is_unloading()) {
- info._to_interpreter = false;
- info._entry = m_code->verified_entry_point();
- } else {
- // Callee is interpreted code. In any case entering the interpreter
- // puts a converter-frame on the stack to save arguments.
- assert(!m->is_method_handle_intrinsic(), "Compiled code should never call interpreter MH intrinsics");
- info._to_interpreter = true;
- info._entry = m()->get_c2i_entry();
- }
+bool CompiledDirectCall::is_call_to_interpreted() const {
+ // It is a call to interpreted, if it calls to a stub. Hence, the destination
+ // must be in the stub part of the nmethod that contains the call
+ CompiledMethod* cm = CodeCache::find_compiled(instruction_address());
+ return cm->stub_contains(destination());
}
-void CompiledStaticCall::compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info) {
- if (ContinuationEntry::is_interpreted_call(instruction_address())) {
- info._to_interpreter = true;
- info._entry = m()->get_c2i_entry();
- }
+bool CompiledDirectCall::is_call_to_compiled() const {
+ CompiledMethod* caller = CodeCache::find_compiled(instruction_address());
+ CodeBlob* dest_cb = CodeCache::find_blob(destination());
+ return !caller->stub_contains(destination()) && dest_cb->is_compiled();
}
-address CompiledDirectStaticCall::find_stub_for(address instruction) {
+address CompiledDirectCall::find_stub_for(address instruction) {
// Find reloc. information containing this call-site
RelocIterator iter((nmethod*)nullptr, instruction);
while (iter.next()) {
@@ -673,8 +399,6 @@ address CompiledDirectStaticCall::find_stub_for(address instruction) {
// from the CompiledIC implementation
case relocInfo::opt_virtual_call_type:
return iter.opt_virtual_call_reloc()->static_stub();
- case relocInfo::poll_type:
- case relocInfo::poll_return_type: // A safepoint can't overlap a call.
default:
ShouldNotReachHere();
}
@@ -683,36 +407,13 @@ address CompiledDirectStaticCall::find_stub_for(address instruction) {
return nullptr;
}
-address CompiledDirectStaticCall::find_stub() {
- return CompiledDirectStaticCall::find_stub_for(instruction_address());
+address CompiledDirectCall::find_stub() {
+ return find_stub_for(instruction_address());
}
-address CompiledDirectStaticCall::resolve_call_stub() const {
- return SharedRuntime::get_resolve_static_call_stub();
-}
-
-//-----------------------------------------------------------------------------
-// Non-product mode code
#ifndef PRODUCT
-
-void CompiledIC::verify() {
- _call->verify();
- assert(is_clean() || is_call_to_compiled() || is_call_to_interpreted()
- || is_optimized() || is_megamorphic(), "sanity check");
-}
-
-void CompiledIC::print() {
- print_compiled_ic();
- tty->cr();
-}
-
-void CompiledIC::print_compiled_ic() {
- tty->print("Inline cache at " INTPTR_FORMAT ", calling %s " INTPTR_FORMAT " cached_value " INTPTR_FORMAT,
- p2i(instruction_address()), is_call_to_interpreted() ? "interpreted " : "", p2i(ic_destination()), p2i(is_optimized() ? nullptr : cached_value()));
-}
-
-void CompiledDirectStaticCall::print() {
- tty->print("static call at " INTPTR_FORMAT " -> ", p2i(instruction_address()));
+void CompiledDirectCall::print() {
+ tty->print("direct call at " INTPTR_FORMAT " to " INTPTR_FORMAT " -> ", p2i(instruction_address()), p2i(destination()));
if (is_clean()) {
tty->print("clean");
} else if (is_call_to_compiled()) {
@@ -723,9 +424,10 @@ void CompiledDirectStaticCall::print() {
tty->cr();
}
-void CompiledDirectStaticCall::verify_mt_safe(const methodHandle& callee, address entry,
- NativeMovConstReg* method_holder,
- NativeJump* jump) {
+void CompiledDirectCall::verify_mt_safe(const methodHandle& callee, address entry,
+ NativeMovConstReg* method_holder,
+ NativeJump* jump) {
+ _call->verify();
// A generated lambda form might be deleted from the Lambdaform
// cache in MethodTypeForm. If a jit compiled lambdaform method
// becomes not entrant and the cache access returns null, the new
@@ -743,4 +445,4 @@ void CompiledDirectStaticCall::verify_mt_safe(const methodHandle& callee, addres
|| old_method->is_old(), // may be race patching deoptimized nmethod due to redefinition.
"b) MT-unsafe modification of inline cache");
}
-#endif // !PRODUCT
+#endif
diff --git a/src/hotspot/share/code/compiledIC.hpp b/src/hotspot/share/code/compiledIC.hpp
index 17586fc57a05f..321bf280ed40a 100644
--- a/src/hotspot/share/code/compiledIC.hpp
+++ b/src/hotspot/share/code/compiledIC.hpp
@@ -27,42 +27,19 @@
#include "code/nativeInst.hpp"
#include "interpreter/linkResolver.hpp"
-#include "oops/compiledICHolder.hpp"
#include "runtime/safepointVerifiers.hpp"
//-----------------------------------------------------------------------------
// The CompiledIC represents a compiled inline cache.
//
-// In order to make patching of the inline cache MT-safe, we only allow the following
-// transitions (when not at a safepoint):
-//
-//
-// [1] --<-- Clean -->--- [1]
-// / (null) \
-// / \ /-<-\
-// / [2] \ / \
-// Interpreted ---------> Monomorphic | [3]
-// (CompiledICHolder*) (Klass*) |
-// \ / \ /
-// [4] \ / [4] \->-/
-// \->- Megamorphic -<-/
-// (CompiledICHolder*)
-//
-// The text in parentheses () refers to the value of the inline cache receiver (mov instruction)
-//
-// The numbers in square brackets refer to the kind of transition:
-// [1]: Initial fixup. Receiver it found from debug information
-// [2]: Compilation of a method
-// [3]: Recompilation of a method (note: only entry is changed. The Klass* must stay the same)
-// [4]: Inline cache miss. We go directly to megamorphic call.
-//
-// The class automatically inserts transition stubs (using the InlineCacheBuffer) when an MT-unsafe
-// transition is made to a stub.
+// It's safe to transition from any state to any state. Typically an inline cache starts
+// in the clean state, meaning it will resolve the call when called. Then it typically
+// transitions to monomorphic, assuming the first dynamic receiver will be the only one
+// observed. If that speculation fails, we transition to megamorphic.
//
class CompiledIC;
class CompiledICProtectionBehaviour;
class CompiledMethod;
-class ICStub;
class CompiledICLocker: public StackObj {
CompiledMethod* _method;
@@ -77,237 +54,105 @@ class CompiledICLocker: public StackObj {
static bool is_safe(address code);
};
-class CompiledICInfo : public StackObj {
- private:
- address _entry; // entry point for call
- void* _cached_value; // Value of cached_value (either in stub or inline cache)
- bool _is_icholder; // Is the cached value a CompiledICHolder*
- bool _is_optimized; // it is an optimized virtual call (i.e., can be statically bound)
- bool _to_interpreter; // Call it to interpreter
- bool _release_icholder;
+// A CompiledICData is a helper object for the inline cache implementation.
+// It comprises:
+// (1) The first receiver klass and its selected method
+// (2) Itable call metadata
+
+class CompiledICData : public CHeapObj {
+ friend class VMStructs;
+ friend class JVMCIVMStructs;
+
+ Method* volatile _speculated_method;
+ uintptr_t volatile _speculated_klass;
+ Klass* _itable_defc_klass;
+ Klass* _itable_refc_klass;
+ bool _is_initialized;
+
+ bool is_speculated_klass_unloaded() const;
+
public:
- address entry() const { return _entry; }
- Metadata* cached_metadata() const { assert(!_is_icholder, ""); return (Metadata*)_cached_value; }
- CompiledICHolder* claim_cached_icholder() {
- assert(_is_icholder, "");
- assert(_cached_value != nullptr, "must be non-null");
- _release_icholder = false;
- CompiledICHolder* icholder = (CompiledICHolder*)_cached_value;
- icholder->claim();
- return icholder;
- }
- bool is_optimized() const { return _is_optimized; }
- bool to_interpreter() const { return _to_interpreter; }
-
- void set_compiled_entry(address entry, Klass* klass, bool is_optimized) {
- _entry = entry;
- _cached_value = (void*)klass;
- _to_interpreter = false;
- _is_icholder = false;
- _is_optimized = is_optimized;
- _release_icholder = false;
- }
+ // Constructor
+ CompiledICData();
- void set_interpreter_entry(address entry, Method* method) {
- _entry = entry;
- _cached_value = (void*)method;
- _to_interpreter = true;
- _is_icholder = false;
- _is_optimized = true;
- _release_icholder = false;
- }
+ // accessors
+ Klass* speculated_klass() const;
+ Method* speculated_method() const { return _speculated_method; }
+ Klass* itable_defc_klass() const { return _itable_defc_klass; }
+ Klass* itable_refc_klass() const { return _itable_refc_klass; }
- void set_icholder_entry(address entry, CompiledICHolder* icholder) {
- _entry = entry;
- _cached_value = (void*)icholder;
- _to_interpreter = true;
- _is_icholder = true;
- _is_optimized = false;
- _release_icholder = true;
- }
+ static ByteSize speculated_method_offset() { return byte_offset_of(CompiledICData, _speculated_method); }
+ static ByteSize speculated_klass_offset() { return byte_offset_of(CompiledICData, _speculated_klass); }
- CompiledICInfo(): _entry(nullptr), _cached_value(nullptr), _is_icholder(false),
- _is_optimized(false), _to_interpreter(false), _release_icholder(false) {
- }
- ~CompiledICInfo() {
- // In rare cases the info is computed but not used, so release any
- // CompiledICHolder* that was created
- if (_release_icholder) {
- assert(_is_icholder, "must be");
- CompiledICHolder* icholder = (CompiledICHolder*)_cached_value;
- icholder->claim();
- delete icholder;
- }
- }
-};
+ static ByteSize itable_defc_klass_offset() { return byte_offset_of(CompiledICData, _itable_defc_klass); }
+ static ByteSize itable_refc_klass_offset() { return byte_offset_of(CompiledICData, _itable_refc_klass); }
-class NativeCallWrapper: public ResourceObj {
-public:
- virtual address destination() const = 0;
- virtual address instruction_address() const = 0;
- virtual address next_instruction_address() const = 0;
- virtual address return_address() const = 0;
- virtual address get_resolve_call_stub(bool is_optimized) const = 0;
- virtual void set_destination_mt_safe(address dest) = 0;
- virtual void set_to_interpreted(const methodHandle& method, CompiledICInfo& info) = 0;
- virtual void verify() const = 0;
- virtual void verify_resolve_call(address dest) const = 0;
-
- virtual bool is_call_to_interpreted(address dest) const = 0;
- virtual bool is_safe_for_patching() const = 0;
-
- virtual NativeInstruction* get_load_instruction(virtual_call_Relocation* r) const = 0;
-
- virtual void *get_data(NativeInstruction* instruction) const = 0;
- virtual void set_data(NativeInstruction* instruction, intptr_t data) = 0;
+ void initialize(CallInfo* call_info, Klass* receiver_klass);
+
+ bool is_initialized() const { return _is_initialized; }
+
+ // GC Support
+ void clean_metadata();
+ void metadata_do(MetadataClosure* cl);
};
class CompiledIC: public ResourceObj {
- friend class InlineCacheBuffer;
- friend class ICStub;
-
- private:
- NativeCallWrapper* _call;
- NativeInstruction* _value; // patchable value cell for this IC
- bool _is_optimized; // an optimized virtual call (i.e., no compiled IC)
+private:
CompiledMethod* _method;
+ CompiledICData* _data;
+ NativeCall* _call;
- CompiledIC(CompiledMethod* cm, NativeCall* ic_call);
CompiledIC(RelocIterator* iter);
- void initialize_from_iter(RelocIterator* iter);
+ // CompiledICData wrappers
+ void ensure_initialized(CallInfo* call_info, Klass* receiver_klass);
+ bool is_speculated_klass(Klass* receiver_klass);
- static bool is_icholder_entry(address entry);
+ // Inline cache states
+ void set_to_monomorphic();
+ void set_to_megamorphic(CallInfo* call_info);
- // low-level inline-cache manipulation. Cannot be accessed directly, since it might not be MT-safe
- // to change an inline-cache. These changes the underlying inline-cache directly. They *newer* make
- // changes to a transition stub.
- void internal_set_ic_destination(address entry_point, bool is_icstub, void* cache, bool is_icholder);
- void set_ic_destination(ICStub* stub);
- void set_ic_destination(address entry_point) {
- assert(_is_optimized, "use set_ic_destination_and_value instead");
- internal_set_ic_destination(entry_point, false, nullptr, false);
- }
- // This only for use by ICStubs where the type of the value isn't known
- void set_ic_destination_and_value(address entry_point, void* value) {
- internal_set_ic_destination(entry_point, false, value, is_icholder_entry(entry_point));
- }
- void set_ic_destination_and_value(address entry_point, Metadata* value) {
- internal_set_ic_destination(entry_point, false, value, false);
- }
- void set_ic_destination_and_value(address entry_point, CompiledICHolder* value) {
- internal_set_ic_destination(entry_point, false, value, true);
- }
-
- // Reads the location of the transition stub. This will fail with an assertion, if no transition stub is
- // associated with the inline cache.
- address stub_address() const;
- bool is_in_transition_state() const; // Use InlineCacheBuffer
-
- public:
+public:
// conversion (machine PC to CompiledIC*)
friend CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr);
friend CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site);
friend CompiledIC* CompiledIC_at(Relocation* call_site);
friend CompiledIC* CompiledIC_at(RelocIterator* reloc_iter);
- static bool is_icholder_call_site(virtual_call_Relocation* call_site, const CompiledMethod* cm);
-
- // Return the cached_metadata/destination associated with this inline cache. If the cache currently points
- // to a transition stub, it will read the values from the transition stub.
- void* cached_value() const;
- CompiledICHolder* cached_icholder() const {
- assert(is_icholder_call(), "must be");
- return (CompiledICHolder*) cached_value();
- }
- Metadata* cached_metadata() const {
- assert(!is_icholder_call(), "must be");
- return (Metadata*) cached_value();
- }
-
- void* get_data() const {
- return _call->get_data(_value);
- }
-
- void set_data(intptr_t data) {
- _call->set_data(_value, data);
- }
-
- address ic_destination() const;
-
- bool is_optimized() const { return _is_optimized; }
+ CompiledICData* data() const;
// State
- bool is_clean() const;
+ bool is_clean() const;
+ bool is_monomorphic() const;
bool is_megamorphic() const;
- bool is_call_to_compiled() const;
- bool is_call_to_interpreted() const;
-
- bool is_icholder_call() const;
- address end_of_call() const { return _call->return_address(); }
+ address end_of_call() const { return _call->return_address(); }
- // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledIC_ock
+ // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledICLocker
// so you are guaranteed that no patching takes place. The same goes for verify.
- //
- // Note: We do not provide any direct access to the stub code, to prevent parts of the code
- // to manipulate the inline cache in MT-unsafe ways.
- //
- // They all takes a TRAP argument, since they can cause a GC if the inline-cache buffer is full.
- //
- bool set_to_clean(bool in_use = true);
- bool set_to_monomorphic(CompiledICInfo& info);
- void clear_ic_stub();
-
- // Returns true if successful and false otherwise. The call can fail if memory
- // allocation in the code cache fails, or ic stub refill is required.
- bool set_to_megamorphic(CallInfo* call_info, Bytecodes::Code bytecode, bool& needs_ic_stub_refill, TRAPS);
-
- static void compute_monomorphic_entry(const methodHandle& method, Klass* receiver_klass,
- bool is_optimized, bool static_bound, bool caller_is_nmethod,
- CompiledICInfo& info, TRAPS);
+ void set_to_clean();
+ void update(CallInfo* call_info, Klass* receiver_klass);
+
+ // GC support
+ void clean_metadata();
+ void metadata_do(MetadataClosure* cl);
// Location
address instruction_address() const { return _call->instruction_address(); }
+ address destination() const { return _call->destination(); }
// Misc
void print() PRODUCT_RETURN;
- void print_compiled_ic() PRODUCT_RETURN;
void verify() PRODUCT_RETURN;
};
-inline CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr) {
- CompiledIC* c_ic = new CompiledIC(nm, nativeCall_before(return_addr));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site) {
- CompiledIC* c_ic = new CompiledIC(nm, nativeCall_at(call_site));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(Relocation* call_site) {
- assert(call_site->type() == relocInfo::virtual_call_type ||
- call_site->type() == relocInfo::opt_virtual_call_type, "wrong reloc. info");
- CompiledIC* c_ic = new CompiledIC(call_site->code(), nativeCall_at(call_site->addr()));
- c_ic->verify();
- return c_ic;
-}
-
-inline CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
- assert(reloc_iter->type() == relocInfo::virtual_call_type ||
- reloc_iter->type() == relocInfo::opt_virtual_call_type, "wrong reloc. info");
- CompiledIC* c_ic = new CompiledIC(reloc_iter);
- c_ic->verify();
- return c_ic;
-}
+CompiledIC* CompiledIC_before(CompiledMethod* nm, address return_addr);
+CompiledIC* CompiledIC_at(CompiledMethod* nm, address call_site);
+CompiledIC* CompiledIC_at(Relocation* call_site);
+CompiledIC* CompiledIC_at(RelocIterator* reloc_iter);
//-----------------------------------------------------------------------------
-// The CompiledStaticCall represents a call to a static method in the compiled
-//
-// Transition diagram of a static call site is somewhat simpler than for an inlined cache:
+// The CompiledDirectCall represents a call to a method in the compiled code
//
//
// -----<----- Clean ----->-----
@@ -321,63 +166,7 @@ inline CompiledIC* CompiledIC_at(RelocIterator* reloc_iter) {
//
//
-class StaticCallInfo {
- private:
- address _entry; // Entrypoint
- methodHandle _callee; // Callee (used when calling interpreter)
- bool _to_interpreter; // call to interpreted method (otherwise compiled)
-
- friend class CompiledStaticCall;
- friend class CompiledDirectStaticCall;
- friend class CompiledPltStaticCall;
- public:
- address entry() const { return _entry; }
- methodHandle callee() const { return _callee; }
-};
-
-class CompiledStaticCall : public ResourceObj {
- public:
- // Code
-
- // Returns null if CodeBuffer::expand fails
- static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = nullptr);
- static int to_interp_stub_size();
- static int to_trampoline_stub_size();
- static int reloc_to_interp_stub();
-
- // Compute entry point given a method
- static void compute_entry(const methodHandle& m, bool caller_is_nmethod, StaticCallInfo& info);
- void compute_entry_for_continuation_entry(const methodHandle& m, StaticCallInfo& info);
-
-public:
- // Clean static call (will force resolving on next use)
- virtual address destination() const = 0;
-
- // Clean static call (will force resolving on next use)
- bool set_to_clean(bool in_use = true);
-
- // Set state. The entry must be the same, as computed by compute_entry.
- // Computation and setting is split up, since the actions are separate during
- // a OptoRuntime::resolve_xxx.
- void set(const StaticCallInfo& info);
-
- // State
- bool is_clean() const;
- bool is_call_to_compiled() const;
- virtual bool is_call_to_interpreted() const = 0;
-
- virtual address instruction_address() const = 0;
- virtual address end_of_call() const = 0;
-protected:
- virtual address resolve_call_stub() const = 0;
- virtual void set_destination_mt_safe(address dest) = 0;
- virtual void set_to_interpreted(const methodHandle& callee, address entry) = 0;
- virtual const char* name() const = 0;
-
- void set_to_compiled(address entry);
-};
-
-class CompiledDirectStaticCall : public CompiledStaticCall {
+class CompiledDirectCall : public ResourceObj {
private:
friend class CompiledIC;
friend class DirectNativeCallWrapper;
@@ -392,22 +181,28 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
NativeCall* _call;
- CompiledDirectStaticCall(NativeCall* call) : _call(call) {}
+ CompiledDirectCall(NativeCall* call) : _call(call) {}
public:
- static inline CompiledDirectStaticCall* before(address return_addr) {
- CompiledDirectStaticCall* st = new CompiledDirectStaticCall(nativeCall_before(return_addr));
+ // Returns null if CodeBuffer::expand fails
+ static address emit_to_interp_stub(CodeBuffer &cbuf, address mark = nullptr);
+ static int to_interp_stub_size();
+ static int to_trampoline_stub_size();
+ static int reloc_to_interp_stub();
+
+ static inline CompiledDirectCall* before(address return_addr) {
+ CompiledDirectCall* st = new CompiledDirectCall(nativeCall_before(return_addr));
st->verify();
return st;
}
- static inline CompiledDirectStaticCall* at(address native_call) {
- CompiledDirectStaticCall* st = new CompiledDirectStaticCall(nativeCall_at(native_call));
+ static inline CompiledDirectCall* at(address native_call) {
+ CompiledDirectCall* st = new CompiledDirectCall(nativeCall_at(native_call));
st->verify();
return st;
}
- static inline CompiledDirectStaticCall* at(Relocation* call_site) {
+ static inline CompiledDirectCall* at(Relocation* call_site) {
return at(call_site->addr());
}
@@ -415,8 +210,15 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
address destination() const { return _call->destination(); }
address end_of_call() const { return _call->return_address(); }
+ // Clean static call (will force resolving on next use)
+ void set_to_clean();
+
+ void set(const methodHandle& callee_method);
+
// State
- virtual bool is_call_to_interpreted() const;
+ bool is_clean() const;
+ bool is_call_to_interpreted() const;
+ bool is_call_to_compiled() const;
// Stub support
static address find_stub_for(address instruction);
@@ -426,10 +228,6 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
// Misc.
void print() PRODUCT_RETURN;
void verify() PRODUCT_RETURN;
-
- protected:
- virtual address resolve_call_stub() const;
- virtual const char* name() const { return "CompiledDirectStaticCall"; }
};
#endif // SHARE_CODE_COMPILEDIC_HPP
diff --git a/src/hotspot/share/code/compiledMethod.cpp b/src/hotspot/share/code/compiledMethod.cpp
index a26d4a98aba9c..6553d6f79344f 100644
--- a/src/hotspot/share/code/compiledMethod.cpp
+++ b/src/hotspot/share/code/compiledMethod.cpp
@@ -28,7 +28,6 @@
#include "code/exceptionHandlerTable.hpp"
#include "code/scopeDesc.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/gcBehaviours.hpp"
@@ -36,7 +35,6 @@
#include "logging/log.hpp"
#include "logging/logTag.hpp"
#include "memory/resourceArea.hpp"
-#include "oops/compiledICHolder.inline.hpp"
#include "oops/klass.inline.hpp"
#include "oops/methodData.hpp"
#include "oops/method.inline.hpp"
@@ -335,28 +333,6 @@ address CompiledMethod::oops_reloc_begin() const {
return low_boundary;
}
-int CompiledMethod::verify_icholder_relocations() {
- ResourceMark rm;
- int count = 0;
-
- RelocIterator iter(this);
- while(iter.next()) {
- if (iter.type() == relocInfo::virtual_call_type) {
- if (CompiledIC::is_icholder_call_site(iter.virtual_call_reloc(), this)) {
- CompiledIC *ic = CompiledIC_at(&iter);
- if (TraceCompiledIC) {
- tty->print("noticed icholder " INTPTR_FORMAT " ", p2i(ic->cached_icholder()));
- ic->print();
- }
- assert(ic->cached_icholder() != nullptr, "must be non-nullptr");
- count++;
- }
- }
- }
-
- return count;
-}
-
// Method that knows how to preserve outgoing arguments at call. This method must be
// called with a frame corresponding to a Java invoke
void CompiledMethod::preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map, OopClosure* f) {
@@ -431,20 +407,6 @@ void CompiledMethod::clear_inline_caches() {
}
}
-// Clear IC callsites, releasing ICStubs of all compiled ICs
-// as well as any associated CompiledICHolders.
-void CompiledMethod::clear_ic_callsites() {
- assert(CompiledICLocker::is_safe(this), "mt unsafe call");
- ResourceMark rm;
- RelocIterator iter(this);
- while(iter.next()) {
- if (iter.type() == relocInfo::virtual_call_type) {
- CompiledIC* ic = CompiledIC_at(&iter);
- ic->set_to_clean(false);
- }
- }
-}
-
#ifdef ASSERT
// Check class_loader is alive for this bit of metadata.
class CheckClass : public MetadataClosure {
@@ -466,70 +428,22 @@ class CheckClass : public MetadataClosure {
#endif // ASSERT
-bool CompiledMethod::clean_ic_if_metadata_is_dead(CompiledIC *ic) {
- if (ic->is_clean()) {
- return true;
- }
- if (ic->is_icholder_call()) {
- // The only exception is compiledICHolder metadata which may
- // yet be marked below. (We check this further below).
- CompiledICHolder* cichk_metdata = ic->cached_icholder();
-
- if (cichk_metdata->is_loader_alive()) {
- return true;
- }
- } else {
- Metadata* ic_metdata = ic->cached_metadata();
- if (ic_metdata != nullptr) {
- if (ic_metdata->is_klass()) {
- if (((Klass*)ic_metdata)->is_loader_alive()) {
- return true;
- }
- } else if (ic_metdata->is_method()) {
- Method* method = (Method*)ic_metdata;
- assert(!method->is_old(), "old method should have been cleaned");
- if (method->method_holder()->is_loader_alive()) {
- return true;
- }
- } else {
- ShouldNotReachHere();
- }
- } else {
- // This inline cache is a megamorphic vtable call. Those ICs never hold
- // any Metadata and should therefore never be cleaned by this function.
- return true;
- }
- }
-
- return ic->set_to_clean();
+static void clean_ic_if_metadata_is_dead(CompiledIC *ic) {
+ ic->clean_metadata();
}
// Clean references to unloaded nmethods at addr from this one, which is not unloaded.
-template
-static bool clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address addr, CompiledMethod* from,
+template
+static void clean_if_nmethod_is_unloaded(CallsiteT* callsite, CompiledMethod* from,
bool clean_all) {
- CodeBlob *cb = CodeCache::find_blob(addr);
- CompiledMethod* nm = (cb != nullptr) ? cb->as_compiled_method_or_null() : nullptr;
- if (nm != nullptr) {
- // Clean inline caches pointing to bad nmethods
- if (clean_all || !nm->is_in_use() || nm->is_unloading() || (nm->method()->code() != nm)) {
- if (!ic->set_to_clean(!from->is_unloading())) {
- return false;
- }
- assert(ic->is_clean(), "nmethod " PTR_FORMAT "not clean %s", p2i(from), from->method()->name_and_sig_as_C_string());
- }
+ CodeBlob* cb = CodeCache::find_blob(callsite->destination());
+ if (!cb->is_compiled()) {
+ return;
+ }
+ CompiledMethod* cm = cb->as_compiled_method();
+ if (clean_all || !cm->is_in_use() || cm->is_unloading() || cm->method()->code() != cm) {
+ callsite->set_to_clean();
}
- return true;
-}
-
-static bool clean_if_nmethod_is_unloaded(CompiledIC *ic, CompiledMethod* from,
- bool clean_all) {
- return clean_if_nmethod_is_unloaded(ic, ic->ic_destination(), from, clean_all);
-}
-
-static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod* from,
- bool clean_all) {
- return clean_if_nmethod_is_unloaded(csc, csc->destination(), from, clean_all);
}
// Cleans caches in nmethods that point to either classes that are unloaded
@@ -539,7 +453,7 @@ static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, CompiledMethod
// nmethods are unloaded. Return postponed=true in the parallel case for
// inline caches found that point to nmethods that are not yet visited during
// the do_unloading walk.
-bool CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
+void CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
ResourceMark rm;
// Exception cache only needs to be called if unloading occurred
@@ -547,16 +461,13 @@ bool CompiledMethod::unload_nmethod_caches(bool unloading_occurred) {
clean_exception_cache();
}
- if (!cleanup_inline_caches_impl(unloading_occurred, false)) {
- return false;
- }
+ cleanup_inline_caches_impl(unloading_occurred, false);
#ifdef ASSERT
// Check that the metadata embedded in the nmethod is alive
CheckClass check_class;
metadata_do(&check_class);
#endif
- return true;
}
void CompiledMethod::run_nmethod_entry_barrier() {
@@ -578,8 +489,7 @@ void CompiledMethod::run_nmethod_entry_barrier() {
void CompiledMethod::cleanup_inline_caches_whitebox() {
assert_locked_or_safepoint(CodeCache_lock);
CompiledICLocker ic_locker(this);
- guarantee(cleanup_inline_caches_impl(false /* unloading_occurred */, true /* clean_all */),
- "Inline cache cleaning in a safepoint can't fail");
+ cleanup_inline_caches_impl(false /* unloading_occurred */, true /* clean_all */);
}
address* CompiledMethod::orig_pc_addr(const frame* fr) {
@@ -587,7 +497,7 @@ address* CompiledMethod::orig_pc_addr(const frame* fr) {
}
// Called to clean up after class unloading for live nmethods
-bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) {
+void CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) {
assert(CompiledICLocker::is_safe(this), "mt unsafe call");
ResourceMark rm;
@@ -602,26 +512,15 @@ bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool cl
if (unloading_occurred) {
// If class unloading occurred we first clear ICs where the cached metadata
// is referring to an unloaded klass or method.
- if (!clean_ic_if_metadata_is_dead(CompiledIC_at(&iter))) {
- return false;
- }
+ clean_ic_if_metadata_is_dead(CompiledIC_at(&iter));
}
- if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) {
- return false;
- }
+ clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all);
break;
case relocInfo::opt_virtual_call_type:
- if (!clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), this, clean_all)) {
- return false;
- }
- break;
-
case relocInfo::static_call_type:
- if (!clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), this, clean_all)) {
- return false;
- }
+ clean_if_nmethod_is_unloaded(CompiledDirectCall::at(iter.reloc()), this, clean_all);
break;
case relocInfo::static_stub_type: {
@@ -672,8 +571,6 @@ bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool cl
break;
}
}
-
- return true;
}
address CompiledMethod::continuation_for_implicit_exception(address pc, bool for_div0_check) {
diff --git a/src/hotspot/share/code/compiledMethod.hpp b/src/hotspot/share/code/compiledMethod.hpp
index afe0905266259..42d68bda55472 100644
--- a/src/hotspot/share/code/compiledMethod.hpp
+++ b/src/hotspot/share/code/compiledMethod.hpp
@@ -35,7 +35,7 @@ class ExceptionHandlerTable;
class ImplicitExceptionTable;
class AbstractCompiler;
class xmlStream;
-class CompiledStaticCall;
+class CompiledDirectCall;
class NativeCallWrapper;
class ScopeDesc;
class CompiledIC;
@@ -364,7 +364,7 @@ class CompiledMethod : public CodeBlob {
// Inline cache support for class unloading and nmethod unloading
private:
- bool cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all);
+ void cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all);
address continuation_for_implicit_exception(address pc, bool for_div0_check);
@@ -373,13 +373,10 @@ class CompiledMethod : public CodeBlob {
void cleanup_inline_caches_whitebox();
virtual void clear_inline_caches();
- void clear_ic_callsites();
// Execute nmethod barrier code, as if entering through nmethod call.
void run_nmethod_entry_barrier();
- // Verify and count cached icholder relocations.
- int verify_icholder_relocations();
void verify_oop_relocations();
bool has_evol_metadata();
@@ -389,14 +386,8 @@ class CompiledMethod : public CodeBlob {
// corresponds to the given method as well.
virtual bool is_dependent_on_method(Method* dependee) = 0;
- virtual NativeCallWrapper* call_wrapper_at(address call) const = 0;
- virtual NativeCallWrapper* call_wrapper_before(address return_pc) const = 0;
virtual address call_instruction_address(address pc) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_at(address addr) const = 0;
- virtual CompiledStaticCall* compiledStaticCall_before(address addr) const = 0;
-
Method* attached_method(address call_pc);
Method* attached_method_before_pc(address pc);
@@ -406,16 +397,13 @@ class CompiledMethod : public CodeBlob {
protected:
address oops_reloc_begin() const;
- private:
- bool static clean_ic_if_metadata_is_dead(CompiledIC *ic);
-
public:
// GC unloading support
// Cleans unloaded klasses and unloaded nmethods in inline caches
virtual bool is_unloading() = 0;
- bool unload_nmethod_caches(bool class_unloading_occurred);
+ void unload_nmethod_caches(bool class_unloading_occurred);
virtual void do_unloading(bool unloading_occurred) = 0;
private:
diff --git a/src/hotspot/share/code/icBuffer.cpp b/src/hotspot/share/code/icBuffer.cpp
deleted file mode 100644
index ec489eff9c882..0000000000000
--- a/src/hotspot/share/code/icBuffer.cpp
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "code/codeCache.hpp"
-#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
-#include "code/nmethod.hpp"
-#include "code/scopeDesc.hpp"
-#include "gc/shared/collectedHeap.inline.hpp"
-#include "interpreter/interpreter.hpp"
-#include "interpreter/linkResolver.hpp"
-#include "memory/resourceArea.hpp"
-#include "oops/method.hpp"
-#include "oops/oop.inline.hpp"
-#include "runtime/atomic.hpp"
-#include "runtime/handles.inline.hpp"
-#include "runtime/javaThread.hpp"
-#include "runtime/mutexLocker.hpp"
-#include "runtime/stubRoutines.hpp"
-#include "runtime/vmOperations.hpp"
-
-DEF_STUB_INTERFACE(ICStub);
-
-StubQueue* InlineCacheBuffer::_buffer = nullptr;
-
-CompiledICHolder* volatile InlineCacheBuffer::_pending_released = nullptr;
-volatile int InlineCacheBuffer::_pending_count = 0;
-
-#ifdef ASSERT
-ICRefillVerifier::ICRefillVerifier()
- : _refill_requested(false),
- _refill_remembered(false)
-{
- Thread* thread = Thread::current();
- assert(thread->missed_ic_stub_refill_verifier() == nullptr, "nesting not supported");
- thread->set_missed_ic_stub_refill_verifier(this);
-}
-
-ICRefillVerifier::~ICRefillVerifier() {
- assert(!_refill_requested || _refill_remembered,
- "Forgot to refill IC stubs after failed IC transition");
- Thread::current()->set_missed_ic_stub_refill_verifier(nullptr);
-}
-
-ICRefillVerifierMark::ICRefillVerifierMark(ICRefillVerifier* verifier) {
- Thread* thread = Thread::current();
- assert(thread->missed_ic_stub_refill_verifier() == nullptr, "nesting not supported");
- thread->set_missed_ic_stub_refill_verifier(verifier);
-}
-
-ICRefillVerifierMark::~ICRefillVerifierMark() {
- Thread::current()->set_missed_ic_stub_refill_verifier(nullptr);
-}
-
-static ICRefillVerifier* current_ic_refill_verifier() {
- Thread* current = Thread::current();
- ICRefillVerifier* verifier = current->missed_ic_stub_refill_verifier();
- assert(verifier != nullptr, "need a verifier for safety");
- return verifier;
-}
-#endif
-
-void ICStub::finalize() {
- if (!is_empty()) {
- ResourceMark rm;
- CompiledIC *ic = CompiledIC_at(CodeCache::find_compiled(ic_site()), ic_site());
- assert(CodeCache::find_compiled(ic->instruction_address()) != nullptr, "inline cache in non-compiled?");
-
- assert(this == ICStub::from_destination_address(ic->stub_address()), "wrong owner of ic buffer");
- ic->set_ic_destination_and_value(destination(), cached_value());
- }
-}
-
-
-address ICStub::destination() const {
- return InlineCacheBuffer::ic_buffer_entry_point(code_begin());
-}
-
-void* ICStub::cached_value() const {
- return InlineCacheBuffer::ic_buffer_cached_value(code_begin());
-}
-
-
-void ICStub::set_stub(CompiledIC *ic, void* cached_val, address dest_addr) {
- // We cannot store a pointer to the 'ic' object, since it is resource allocated. Instead we
- // store the location of the inline cache. Then we have enough information recreate the CompiledIC
- // object when we need to remove the stub.
- _ic_site = ic->instruction_address();
-
- // Assemble new stub
- InlineCacheBuffer::assemble_ic_buffer_code(code_begin(), cached_val, dest_addr);
- assert(destination() == dest_addr, "can recover destination");
- assert(cached_value() == cached_val, "can recover destination");
-}
-
-
-void ICStub::clear() {
- if (CompiledIC::is_icholder_entry(destination())) {
- InlineCacheBuffer::queue_for_release((CompiledICHolder*)cached_value());
- }
- _ic_site = nullptr;
-}
-
-
-#ifndef PRODUCT
-// anybody calling to this stub will trap
-
-void ICStub::verify() {
-}
-
-void ICStub::print() {
- tty->print_cr("ICStub: site: " INTPTR_FORMAT, p2i(_ic_site));
-}
-#endif
-
-//-----------------------------------------------------------------------------------------------
-// Implementation of InlineCacheBuffer
-
-
-void InlineCacheBuffer::initialize() {
- if (_buffer != nullptr) return; // already initialized
- _buffer = new StubQueue(new ICStubInterface, checked_cast(InlineCacheBufferSize), InlineCacheBuffer_lock, "InlineCacheBuffer");
- assert (_buffer != nullptr, "cannot allocate InlineCacheBuffer");
-}
-
-
-void InlineCacheBuffer::refill_ic_stubs() {
-#ifdef ASSERT
- ICRefillVerifier* verifier = current_ic_refill_verifier();
- verifier->request_remembered();
-#endif
- // we ran out of inline cache buffer space; must enter safepoint.
- // We do this by forcing a safepoint
- VM_ICBufferFull ibf;
- VMThread::execute(&ibf);
-}
-
-bool InlineCacheBuffer::needs_update_inline_caches() {
- // Stub removal
- if (buffer()->number_of_stubs() > 0) {
- return true;
- }
-
- // Release pending CompiledICHolder
- if (pending_icholder_count() > 0) {
- return true;
- }
-
- return false;
-}
-
-void InlineCacheBuffer::update_inline_caches() {
- if (buffer()->number_of_stubs() > 0) {
- if (TraceICBuffer) {
- tty->print_cr("[updating inline caches with %d stubs]", buffer()->number_of_stubs());
- }
- buffer()->remove_all();
- }
- release_pending_icholders();
-}
-
-
-bool InlineCacheBuffer::contains(address instruction_address) {
- return buffer()->contains(instruction_address);
-}
-
-
-bool InlineCacheBuffer::is_empty() {
- return buffer()->number_of_stubs() == 0;
-}
-
-
-void InlineCacheBuffer_init() {
- InlineCacheBuffer::initialize();
-}
-
-bool InlineCacheBuffer::create_transition_stub(CompiledIC *ic, void* cached_value, address entry) {
- assert(!SafepointSynchronize::is_at_safepoint(), "should not be called during a safepoint");
- assert(CompiledICLocker::is_safe(ic->instruction_address()), "mt unsafe call");
- if (TraceICBuffer) {
- tty->print_cr(" create transition stub for " INTPTR_FORMAT " destination " INTPTR_FORMAT " cached value " INTPTR_FORMAT,
- p2i(ic->instruction_address()), p2i(entry), p2i(cached_value));
- }
-
- // allocate and initialize new "out-of-line" inline-cache
- ICStub* ic_stub = (ICStub*) buffer()->request_committed(ic_stub_code_size());
- if (ic_stub == nullptr) {
-#ifdef ASSERT
- ICRefillVerifier* verifier = current_ic_refill_verifier();
- verifier->request_refill();
-#endif
- return false;
- }
-
-#ifdef ASSERT
- {
- ICStub* rev_stub = ICStub::from_destination_address(ic_stub->code_begin());
- assert(ic_stub == rev_stub,
- "ICStub mapping is reversible: stub=" PTR_FORMAT ", code=" PTR_FORMAT ", rev_stub=" PTR_FORMAT,
- p2i(ic_stub), p2i(ic_stub->code_begin()), p2i(rev_stub));
- }
-#endif
-
- // If an transition stub is already associate with the inline cache, then we remove the association.
- if (ic->is_in_transition_state()) {
- ICStub* old_stub = ICStub::from_destination_address(ic->stub_address());
- old_stub->clear();
- }
-
- ic_stub->set_stub(ic, cached_value, entry);
-
- // Update inline cache in nmethod to point to new "out-of-line" allocated inline cache
- ic->set_ic_destination(ic_stub);
- return true;
-}
-
-
-address InlineCacheBuffer::ic_destination_for(CompiledIC *ic) {
- ICStub* stub = ICStub::from_destination_address(ic->stub_address());
- return stub->destination();
-}
-
-
-void* InlineCacheBuffer::cached_value_for(CompiledIC *ic) {
- ICStub* stub = ICStub::from_destination_address(ic->stub_address());
- return stub->cached_value();
-}
-
-
-// Free CompiledICHolder*s that are no longer in use
-void InlineCacheBuffer::release_pending_icholders() {
- assert(SafepointSynchronize::is_at_safepoint(), "should only be called during a safepoint");
- CompiledICHolder* holder = Atomic::load(&_pending_released);
- _pending_released = nullptr;
- int count = 0;
- while (holder != nullptr) {
- CompiledICHolder* next = holder->next();
- delete holder;
- holder = next;
- count++;
- }
- assert(pending_icholder_count() == count, "wrong count");
- Atomic::store(&_pending_count, 0);
-}
-
-// Enqueue this icholder for release during the next safepoint. It's
-// not safe to free them until then since they might be visible to
-// another thread.
-void InlineCacheBuffer::queue_for_release(CompiledICHolder* icholder) {
- assert(icholder->next() == nullptr, "multiple enqueue?");
-
- CompiledICHolder* old = Atomic::load(&_pending_released);
- for (;;) {
- icholder->set_next(old);
- // The only reader runs at a safepoint serially so there is no need for a more strict atomic.
- CompiledICHolder* cur = Atomic::cmpxchg(&_pending_released, old, icholder, memory_order_relaxed);
- if (cur == old) {
- break;
- }
- old = cur;
- }
- Atomic::inc(&_pending_count, memory_order_relaxed);
-
- if (TraceICBuffer) {
- tty->print_cr("enqueueing icholder " INTPTR_FORMAT " to be freed", p2i(icholder));
- }
-}
-
-int InlineCacheBuffer::pending_icholder_count() {
- return Atomic::load(&_pending_count);
-}
diff --git a/src/hotspot/share/code/icBuffer.hpp b/src/hotspot/share/code/icBuffer.hpp
deleted file mode 100644
index f67080e6b5852..0000000000000
--- a/src/hotspot/share/code/icBuffer.hpp
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_CODE_ICBUFFER_HPP
-#define SHARE_CODE_ICBUFFER_HPP
-
-#include "asm/codeBuffer.hpp"
-#include "code/stubs.hpp"
-#include "interpreter/bytecodes.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/safepointVerifiers.hpp"
-#include "utilities/align.hpp"
-#include "utilities/debug.hpp"
-#include "utilities/macros.hpp"
-
-class CompiledIC;
-class CompiledICHolder;
-
-//
-// For CompiledIC's:
-//
-// In cases where we do not have MT-safe state transformation,
-// we go to a transition state, using ICStubs. At a safepoint,
-// the inline caches are transferred from the transitional code:
-//
-// instruction_address --> 01 set xxx_oop, Ginline_cache_klass
-// 23 jump_to Gtemp, yyyy
-// 4 nop
-
-class ICStub: public Stub {
- private:
- int _size; // total size of the stub incl. code
- address _ic_site; // points at call instruction of owning ic-buffer
- /* stub code follows here */
- protected:
- friend class ICStubInterface;
- // This will be called only by ICStubInterface
- void initialize(int size) { _size = size; _ic_site = nullptr; }
- void finalize(); // called when a method is removed
-
- // General info
- int size() const { return _size; }
-
- // To be cautious, we want to make sure that each ICStub is in a separate instruction
- // cache line. This would allow for piggybacking on instruction cache coherency on
- // some architectures to order the updates to ICStub and setting the destination to
- // the ICStub. Note that cache line size might be larger than CodeEntryAlignment
- // that is normal alignment for CodeBlobs.
- static int alignment() { return DEFAULT_CACHE_LINE_SIZE; }
-
- // Aligning the code section is normally done for performance reasons, which is not
- // required for ICStubs, as these stubs are transitional. Setting code alignment
- // to CodeEntryAlignment would waste a lot of memory in ICBuffer. Aligning to
- // word size should be enough. This also offsets the costs of aligning the entire
- // ICStub to cache line (see above), as smaller code alignment would allow ICStub
- // to fit a _single_ cache line.
- static int code_alignment() { return HeapWordSize; }
-
- public:
- // Creation
- void set_stub(CompiledIC *ic, void* cached_value, address dest_addr);
-
- // Code info
- address code_begin() const { return align_up((address)this + sizeof(ICStub), code_alignment()); }
- address code_end() const { return (address)this + size(); }
-
- // Call site info
- address ic_site() const { return _ic_site; }
- void clear();
- bool is_empty() const { return _ic_site == nullptr; }
-
- // stub info
- address destination() const; // destination of jump instruction
- void* cached_value() const; // cached_value for stub
-
- // Debugging
- void verify() PRODUCT_RETURN;
- void print() PRODUCT_RETURN;
-
- // Creation
- static inline ICStub* from_destination_address(address destination_address) {
- ICStub* stub = (ICStub*) align_down(destination_address - sizeof(ICStub), alignment());
-#ifdef ASSERT
- stub->verify();
-#endif
- return stub;
- }
-};
-
-#ifdef ASSERT
-// The ICRefillVerifier class is a stack allocated RAII object used to
-// detect if a failed IC transition that required IC stub refilling has
-// been accidentally missed. It is up to the caller to in that case
-// refill IC stubs.
-class ICRefillVerifier: StackObj {
- bool _refill_requested;
- bool _refill_remembered;
-
- public:
- ICRefillVerifier();
- ~ICRefillVerifier();
-
- void request_refill() { _refill_requested = true; }
- void request_remembered() { _refill_remembered = true; }
-};
-
-// The ICRefillVerifierMark is used to set the thread's current
-// ICRefillVerifier to a provided one. This is useful in particular
-// when transitioning IC stubs in parallel and refilling from the
-// master thread invoking the IC stub transitioning code.
-class ICRefillVerifierMark: StackObj {
- public:
- ICRefillVerifierMark(ICRefillVerifier* verifier);
- ~ICRefillVerifierMark();
-};
-#else
-class ICRefillVerifier: StackObj {
- public:
- ICRefillVerifier() {}
-};
-class ICRefillVerifierMark: StackObj {
- public:
- ICRefillVerifierMark(ICRefillVerifier* verifier) {}
-};
-#endif
-
-class InlineCacheBuffer: public AllStatic {
- private:
- // friends
- friend class ICStub;
-
- static int ic_stub_code_size();
-
- static StubQueue* _buffer;
-
- static CompiledICHolder* volatile _pending_released;
- static volatile int _pending_count;
-
- static StubQueue* buffer() { return _buffer; }
-
- // Machine-dependent implementation of ICBuffer
- static void assemble_ic_buffer_code(address code_begin, void* cached_value, address entry_point);
- static address ic_buffer_entry_point (address code_begin);
- static void* ic_buffer_cached_value (address code_begin);
-
- public:
-
- // Initialization; must be called before first usage
- static void initialize();
-
- // Access
- static bool contains(address instruction_address);
-
- // removes the ICStubs after backpatching
- static bool needs_update_inline_caches();
- static void update_inline_caches();
- static void refill_ic_stubs();
-
- // for debugging
- static bool is_empty();
-
- static void release_pending_icholders();
- static void queue_for_release(CompiledICHolder* icholder);
- static int pending_icholder_count();
-
- // New interface
- static bool create_transition_stub(CompiledIC *ic, void* cached_value, address entry);
- static address ic_destination_for(CompiledIC *ic);
- static void* cached_value_for(CompiledIC *ic);
-};
-
-#endif // SHARE_CODE_ICBUFFER_HPP
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index cc2dcf21deebf..2755df3251396 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -640,6 +640,7 @@ nmethod::nmethod(
ByteSize basic_lock_sp_offset,
OopMapSet* oop_maps )
: CompiledMethod(method, "native nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+ _compiled_ic_data(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(basic_lock_owner_sp_offset),
_native_basic_lock_sp_offset(basic_lock_sp_offset),
@@ -697,12 +698,12 @@ nmethod::nmethod(
clear_unloading_state();
+ finalize_relocations();
+
Universe::heap()->register_nmethod(this);
debug_only(Universe::heap()->verify_nmethod(this));
CodeCache::commit(this);
-
- finalize_relocations();
}
if (PrintNativeNMethods || PrintDebugInfo || PrintRelocations || PrintDependencies) {
@@ -784,6 +785,7 @@ nmethod::nmethod(
#endif
)
: CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, true),
+ _compiled_ic_data(nullptr),
_is_unlinked(false),
_native_receiver_sp_offset(in_ByteSize(-1)),
_native_basic_lock_sp_offset(in_ByteSize(-1)),
@@ -887,13 +889,13 @@ nmethod::nmethod(
}
#endif
+ finalize_relocations();
+
Universe::heap()->register_nmethod(this);
debug_only(Universe::heap()->verify_nmethod(this));
CodeCache::commit(this);
- finalize_relocations();
-
// Copy contents of ExceptionHandlerTable to nmethod
handler_table->copy_to(this);
nul_chk_table->copy_to(this);
@@ -1145,16 +1147,33 @@ static void install_post_call_nop_displacement(nmethod* nm, address pc) {
void nmethod::finalize_relocations() {
NoSafepointVerifier nsv;
+ GrowableArray virtual_call_data;
+
// Make sure that post call nops fill in nmethod offsets eagerly so
// we don't have to race with deoptimization
RelocIterator iter(this);
while (iter.next()) {
- if (iter.type() == relocInfo::post_call_nop_type) {
+ if (iter.type() == relocInfo::virtual_call_type) {
+ virtual_call_Relocation* r = iter.virtual_call_reloc();
+ NativeMovConstReg* value = nativeMovConstReg_at(r->cached_value());
+ virtual_call_data.append(value);
+ } else if (iter.type() == relocInfo::post_call_nop_type) {
post_call_nop_Relocation* const reloc = iter.post_call_nop_reloc();
address pc = reloc->addr();
install_post_call_nop_displacement(this, pc);
}
}
+
+ if (virtual_call_data.length() > 0) {
+ // We allocate a block of CompiledICData per nmethod so the GC can purge this faster.
+ _compiled_ic_data = new CompiledICData[virtual_call_data.length()];
+ CompiledICData* next_data = _compiled_ic_data;
+
+ for (NativeMovConstReg* value : virtual_call_data) {
+ value->set_data((intptr_t)next_data);
+ next_data++;
+ }
+ }
}
void nmethod::make_deoptimized() {
@@ -1180,8 +1199,7 @@ void nmethod::make_deoptimized() {
while (iter.next()) {
switch (iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledIC *ic = CompiledIC_at(&iter);
address pc = ic->end_of_call();
NativePostCallNop* nop = nativePostCallNop_at(pc);
@@ -1191,8 +1209,9 @@ void nmethod::make_deoptimized() {
assert(NativeDeoptInstruction::is_deopt_at(pc), "check");
break;
}
- case relocInfo::static_call_type: {
- CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
+ case relocInfo::static_call_type:
+ case relocInfo::opt_virtual_call_type: {
+ CompiledDirectCall *csc = CompiledDirectCall::at(iter.reloc());
address pc = csc->end_of_call();
NativePostCallNop* nop = nativePostCallNop_at(pc);
//tty->print_cr(" - static pc %p", pc);
@@ -1219,29 +1238,29 @@ void nmethod::verify_clean_inline_caches() {
RelocIterator iter(this, oops_reloc_begin());
while(iter.next()) {
switch(iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledIC *ic = CompiledIC_at(&iter);
- CodeBlob *cb = CodeCache::find_blob(ic->ic_destination());
+ CodeBlob *cb = CodeCache::find_blob(ic->destination());
assert(cb != nullptr, "destination not in CodeBlob?");
nmethod* nm = cb->as_nmethod_or_null();
- if( nm != nullptr ) {
+ if (nm != nullptr) {
// Verify that inline caches pointing to bad nmethods are clean
- if (!nm->is_in_use() || (nm->method()->code() != nm)) {
+ if (!nm->is_in_use() || nm->is_unloading()) {
assert(ic->is_clean(), "IC should be clean");
}
}
break;
}
- case relocInfo::static_call_type: {
- CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
- CodeBlob *cb = CodeCache::find_blob(csc->destination());
+ case relocInfo::static_call_type:
+ case relocInfo::opt_virtual_call_type: {
+ CompiledDirectCall *cdc = CompiledDirectCall::at(iter.reloc());
+ CodeBlob *cb = CodeCache::find_blob(cdc->destination());
assert(cb != nullptr, "destination not in CodeBlob?");
nmethod* nm = cb->as_nmethod_or_null();
- if( nm != nullptr ) {
+ if (nm != nullptr) {
// Verify that inline caches pointing to bad nmethods are clean
- if (!nm->is_in_use() || (nm->method()->code() != nm)) {
- assert(csc->is_clean(), "IC should be clean");
+ if (!nm->is_in_use() || nm->is_unloading() || nm->method()->code() != nm) {
+ assert(cdc->is_clean(), "IC should be clean");
}
}
break;
@@ -1405,9 +1424,7 @@ bool nmethod::make_not_entrant() {
// For concurrent GCs, there must be a handshake between unlink and flush
void nmethod::unlink() {
if (_is_unlinked) {
- // Already unlinked. It can be invoked twice because concurrent code cache
- // unloading might need to restart when inline cache cleaning fails due to
- // running out of ICStubs, which can only be refilled at safepoints
+ // Already unlinked.
return;
}
@@ -1418,7 +1435,6 @@ void nmethod::unlink() {
// the Method, because it is only concurrently unlinked by
// the entry barrier, which acquires the per nmethod lock.
unlink_from_method();
- clear_ic_callsites();
if (is_osr_method()) {
invalidate_osr_method();
@@ -1463,10 +1479,11 @@ void nmethod::purge(bool free_code_cache_data, bool unregister_nmethod) {
ec = next;
}
+ delete[] _compiled_ic_data;
+
if (unregister_nmethod) {
Universe::heap()->unregister_nmethod(this);
}
-
CodeCache::unregister_old_nmethod(this);
CodeBlob::purge(free_code_cache_data, unregister_nmethod);
@@ -1604,16 +1621,7 @@ void nmethod::metadata_do(MetadataClosure* f) {
// Check compiledIC holders associated with this nmethod
ResourceMark rm;
CompiledIC *ic = CompiledIC_at(&iter);
- if (ic->is_icholder_call()) {
- CompiledICHolder* cichk = ic->cached_icholder();
- f->do_metadata(cichk->holder_metadata());
- f->do_metadata(cichk->holder_klass());
- } else {
- Metadata* ic_oop = ic->cached_metadata();
- if (ic_oop != nullptr) {
- f->do_metadata(ic_oop);
- }
- }
+ ic->metadata_do(f);
}
}
}
@@ -1750,8 +1758,7 @@ void nmethod::do_unloading(bool unloading_occurred) {
if (is_unloading()) {
unlink();
} else {
- guarantee(unload_nmethod_caches(unloading_occurred),
- "Should not need transition stubs");
+ unload_nmethod_caches(unloading_occurred);
BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
if (bs_nm != nullptr) {
bs_nm->disarm(this);
@@ -2284,15 +2291,23 @@ void nmethod::verify() {
}
-void nmethod::verify_interrupt_point(address call_site) {
+void nmethod::verify_interrupt_point(address call_site, bool is_inline_cache) {
// Verify IC only when nmethod installation is finished.
if (!is_not_installed()) {
if (CompiledICLocker::is_safe(this)) {
- CompiledIC_at(this, call_site);
+ if (is_inline_cache) {
+ CompiledIC_at(this, call_site);
+ } else {
+ CompiledDirectCall::at(call_site);
+ }
} else {
CompiledICLocker ml_verify(this);
- CompiledIC_at(this, call_site);
+ if (is_inline_cache) {
+ CompiledIC_at(this, call_site);
+ } else {
+ CompiledDirectCall::at(call_site);
+ }
}
}
@@ -2316,15 +2331,15 @@ void nmethod::verify_scopes() {
address stub = nullptr;
switch (iter.type()) {
case relocInfo::virtual_call_type:
- verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), true /* is_inline_cache */);
break;
case relocInfo::opt_virtual_call_type:
stub = iter.opt_virtual_call_reloc()->static_stub();
- verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), false /* is_inline_cache */);
break;
case relocInfo::static_call_type:
stub = iter.static_call_reloc()->static_stub();
- //verify_interrupt_point(iter.addr());
+ verify_interrupt_point(iter.addr(), false /* is_inline_cache */);
break;
case relocInfo::runtime_call_type:
case relocInfo::runtime_call_w_cp_type: {
@@ -3239,75 +3254,6 @@ void nmethod::print_code_comment_on(outputStream* st, int column, address begin,
#endif
-class DirectNativeCallWrapper: public NativeCallWrapper {
-private:
- NativeCall* _call;
-
-public:
- DirectNativeCallWrapper(NativeCall* call) : _call(call) {}
-
- virtual address destination() const { return _call->destination(); }
- virtual address instruction_address() const { return _call->instruction_address(); }
- virtual address next_instruction_address() const { return _call->next_instruction_address(); }
- virtual address return_address() const { return _call->return_address(); }
-
- virtual address get_resolve_call_stub(bool is_optimized) const {
- if (is_optimized) {
- return SharedRuntime::get_resolve_opt_virtual_call_stub();
- }
- return SharedRuntime::get_resolve_virtual_call_stub();
- }
-
- virtual void set_destination_mt_safe(address dest) {
- _call->set_destination_mt_safe(dest);
- }
-
- virtual void set_to_interpreted(const methodHandle& method, CompiledICInfo& info) {
- CompiledDirectStaticCall* csc = CompiledDirectStaticCall::at(instruction_address());
- {
- csc->set_to_interpreted(method, info.entry());
- }
- }
-
- virtual void verify() const {
- // make sure code pattern is actually a call imm32 instruction
- _call->verify();
- _call->verify_alignment();
- }
-
- virtual void verify_resolve_call(address dest) const {
- CodeBlob* db = CodeCache::find_blob(dest);
- assert(db != nullptr && !db->is_adapter_blob(), "must use stub!");
- }
-
- virtual bool is_call_to_interpreted(address dest) const {
- CodeBlob* cb = CodeCache::find_blob(_call->instruction_address());
- return cb->contains(dest);
- }
-
- virtual bool is_safe_for_patching() const { return false; }
-
- virtual NativeInstruction* get_load_instruction(virtual_call_Relocation* r) const {
- return nativeMovConstReg_at(r->cached_value());
- }
-
- virtual void *get_data(NativeInstruction* instruction) const {
- return (void*)((NativeMovConstReg*) instruction)->data();
- }
-
- virtual void set_data(NativeInstruction* instruction, intptr_t data) {
- ((NativeMovConstReg*) instruction)->set_data(data);
- }
-};
-
-NativeCallWrapper* nmethod::call_wrapper_at(address call) const {
- return new DirectNativeCallWrapper((NativeCall*) call);
-}
-
-NativeCallWrapper* nmethod::call_wrapper_before(address return_pc) const {
- return new DirectNativeCallWrapper(nativeCall_before(return_pc));
-}
-
address nmethod::call_instruction_address(address pc) const {
if (NativeCall::is_call_before(pc)) {
NativeCall *ncall = nativeCall_before(pc);
@@ -3316,18 +3262,6 @@ address nmethod::call_instruction_address(address pc) const {
return nullptr;
}
-CompiledStaticCall* nmethod::compiledStaticCall_at(Relocation* call_site) const {
- return CompiledDirectStaticCall::at(call_site);
-}
-
-CompiledStaticCall* nmethod::compiledStaticCall_at(address call_site) const {
- return CompiledDirectStaticCall::at(call_site);
-}
-
-CompiledStaticCall* nmethod::compiledStaticCall_before(address return_addr) const {
- return CompiledDirectStaticCall::before(return_addr);
-}
-
#if defined(SUPPORT_DATA_STRUCTS)
void nmethod::print_value_on(outputStream* st) const {
st->print("nmethod");
@@ -3341,15 +3275,15 @@ void nmethod::print_calls(outputStream* st) {
RelocIterator iter(this);
while (iter.next()) {
switch (iter.type()) {
- case relocInfo::virtual_call_type:
- case relocInfo::opt_virtual_call_type: {
+ case relocInfo::virtual_call_type: {
CompiledICLocker ml_verify(this);
CompiledIC_at(&iter)->print();
break;
}
case relocInfo::static_call_type:
- st->print_cr("Static call at " INTPTR_FORMAT, p2i(iter.reloc()->addr()));
- CompiledDirectStaticCall::at(iter.reloc())->print();
+ case relocInfo::opt_virtual_call_type:
+ st->print_cr("Direct call at " INTPTR_FORMAT, p2i(iter.reloc()->addr()));
+ CompiledDirectCall::at(iter.reloc())->print();
break;
default:
break;
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index 13cd2f799a76b..2993db21305ee 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -27,6 +27,7 @@
#include "code/compiledMethod.hpp"
+class CompiledICData;
class CompileTask;
class DepChange;
class DirectiveSet;
@@ -196,6 +197,7 @@ class nmethod : public CompiledMethod {
address _verified_entry_point; // entry point without class check
address _osr_entry_point; // entry point for on stack replacement
+ CompiledICData* _compiled_ic_data;
bool _is_unlinked;
// Shared fields for all nmethod's
@@ -604,7 +606,7 @@ class nmethod : public CompiledMethod {
// verify operations
void verify();
void verify_scopes();
- void verify_interrupt_point(address interrupt_point);
+ void verify_interrupt_point(address interrupt_point, bool is_inline_cache);
// Disassemble this nmethod with additional debug information, e.g. information about blocks.
void decode2(outputStream* st) const;
@@ -699,14 +701,8 @@ class nmethod : public CompiledMethod {
virtual void metadata_do(MetadataClosure* f);
- NativeCallWrapper* call_wrapper_at(address call) const;
- NativeCallWrapper* call_wrapper_before(address return_pc) const;
address call_instruction_address(address pc) const;
- virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const;
- virtual CompiledStaticCall* compiledStaticCall_at(address addr) const;
- virtual CompiledStaticCall* compiledStaticCall_before(address addr) const;
-
virtual void make_deoptimized();
void finalize_relocations();
};
diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp
index bfb3db72d7265..ef90875767503 100644
--- a/src/hotspot/share/code/relocInfo.cpp
+++ b/src/hotspot/share/code/relocInfo.cpp
@@ -641,12 +641,10 @@ Method* virtual_call_Relocation::method_value() {
return (Method*)m;
}
-bool virtual_call_Relocation::clear_inline_cache() {
- // No stubs for ICs
- // Clean IC
+void virtual_call_Relocation::clear_inline_cache() {
ResourceMark rm;
CompiledIC* icache = CompiledIC_at(this);
- return icache->set_to_clean();
+ icache->set_to_clean();
}
@@ -669,18 +667,10 @@ Method* opt_virtual_call_Relocation::method_value() {
return (Method*)m;
}
-template
-static bool set_to_clean_no_ic_refill(CompiledICorStaticCall* ic) {
- guarantee(ic->set_to_clean(), "Should not need transition stubs");
- return true;
-}
-
-bool opt_virtual_call_Relocation::clear_inline_cache() {
- // No stubs for ICs
- // Clean IC
+void opt_virtual_call_Relocation::clear_inline_cache() {
ResourceMark rm;
- CompiledIC* icache = CompiledIC_at(this);
- return set_to_clean_no_ic_refill(icache);
+ CompiledDirectCall* callsite = CompiledDirectCall::at(this);
+ callsite->set_to_clean();
}
address opt_virtual_call_Relocation::static_stub() {
@@ -717,10 +707,10 @@ void static_call_Relocation::unpack_data() {
_method_index = unpack_1_int();
}
-bool static_call_Relocation::clear_inline_cache() {
- // Safe call site info
- CompiledStaticCall* handler = this->code()->compiledStaticCall_at(this);
- return set_to_clean_no_ic_refill(handler);
+void static_call_Relocation::clear_inline_cache() {
+ ResourceMark rm;
+ CompiledDirectCall* callsite = CompiledDirectCall::at(this);
+ callsite->set_to_clean();
}
@@ -759,11 +749,10 @@ address trampoline_stub_Relocation::get_trampoline_for(address call, nmethod* co
return nullptr;
}
-bool static_stub_Relocation::clear_inline_cache() {
+void static_stub_Relocation::clear_inline_cache() {
// Call stub is only used when calling the interpreted code.
// It does not really need to be cleared, except that we want to clean out the methodoop.
- CompiledDirectStaticCall::set_stub_to_clean(this);
- return true;
+ CompiledDirectCall::set_stub_to_clean(this);
}
diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp
index 77e24708bb143..5f67f94bdadc5 100644
--- a/src/hotspot/share/code/relocInfo.hpp
+++ b/src/hotspot/share/code/relocInfo.hpp
@@ -862,7 +862,7 @@ class Relocation {
// all relocations are able to reassert their values
virtual void set_value(address x);
- virtual bool clear_inline_cache() { return true; }
+ virtual void clear_inline_cache() {}
// This method assumes that all virtual/static (inline) caches are cleared (since for static_call_type and
// ic_call_type is not always position dependent (depending on the state of the cache)). However, this is
@@ -1141,7 +1141,7 @@ class virtual_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
};
@@ -1170,7 +1170,7 @@ class opt_virtual_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
// find the matching static_stub
address static_stub();
@@ -1202,7 +1202,7 @@ class static_call_Relocation : public CallRelocation {
void pack_data_to(CodeSection* dest) override;
void unpack_data() override;
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
// find the matching static_stub
address static_stub();
@@ -1227,7 +1227,7 @@ class static_stub_Relocation : public Relocation {
static_stub_Relocation() : Relocation(relocInfo::static_stub_type) { }
public:
- bool clear_inline_cache() override;
+ void clear_inline_cache() override;
address static_call() { return _static_call; }
diff --git a/src/hotspot/share/code/vtableStubs.cpp b/src/hotspot/share/code/vtableStubs.cpp
index eed3dc8e7876a..5a54426d6a420 100644
--- a/src/hotspot/share/code/vtableStubs.cpp
+++ b/src/hotspot/share/code/vtableStubs.cpp
@@ -283,13 +283,6 @@ VtableStub* VtableStubs::entry_point(address pc) {
return (s == stub) ? s : nullptr;
}
-bool VtableStubs::is_icholder_entry(address pc) {
- assert(contains(pc), "must contain all vtable blobs");
- VtableStub* stub = (VtableStub*)(pc - VtableStub::entry_offset());
- // itable stubs use CompiledICHolder.
- return stub->is_itable_stub();
-}
-
bool VtableStubs::contains(address pc) {
// simple solution for now - we may want to use
// a faster way if this function is called often
diff --git a/src/hotspot/share/code/vtableStubs.hpp b/src/hotspot/share/code/vtableStubs.hpp
index 7076e50f3e3d8..3993e1e72d5cf 100644
--- a/src/hotspot/share/code/vtableStubs.hpp
+++ b/src/hotspot/share/code/vtableStubs.hpp
@@ -107,7 +107,6 @@ class VtableStubs : AllStatic {
static address find_itable_stub(int itable_index) { return find_stub(false, itable_index); }
static VtableStub* entry_point(address pc); // vtable stub entry point for a pc
- static bool is_icholder_entry(address pc); // is the blob containing pc (which must be a vtable blob) an icholder?
static bool contains(address pc); // is pc within any stub?
static VtableStub* stub_containing(address pc); // stub containing pc or nullptr
static void initialize();
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.cpp b/src/hotspot/share/compiler/compilationFailureInfo.cpp
index e3f3353589e54..fb94102ef1654 100644
--- a/src/hotspot/share/compiler/compilationFailureInfo.cpp
+++ b/src/hotspot/share/compiler/compilationFailureInfo.cpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -42,11 +42,16 @@
#include "utilities/ostream.hpp"
#include "utilities/nativeCallStack.hpp"
+int CompilationFailureInfo::current_compile_id_or_0() {
+ ciEnv* env = ciEnv::current();
+ return (env != nullptr) ? env->compile_id() : 0;
+}
+
CompilationFailureInfo::CompilationFailureInfo(const char* failure_reason) :
_stack(2),
_failure_reason(os::strdup(failure_reason)),
_elapsed_seconds(os::elapsedTime()),
- _compile_id(ciEnv::current()->task()->compile_id())
+ _compile_id(current_compile_id_or_0())
{}
CompilationFailureInfo::~CompilationFailureInfo() {
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.hpp b/src/hotspot/share/compiler/compilationFailureInfo.hpp
index 3de62eb69da5a..470865a2f6608 100644
--- a/src/hotspot/share/compiler/compilationFailureInfo.hpp
+++ b/src/hotspot/share/compiler/compilationFailureInfo.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,7 @@ class CompilationFailureInfo : public CHeapObj {
char* const _failure_reason;
const double _elapsed_seconds;
const int _compile_id;
+ static int current_compile_id_or_0();
public:
CompilationFailureInfo(const char* failure_reason);
~CompilationFailureInfo();
diff --git a/src/hotspot/share/compiler/compileBroker.cpp b/src/hotspot/share/compiler/compileBroker.cpp
index 1ade28278aedd..3a25112891484 100644
--- a/src/hotspot/share/compiler/compileBroker.cpp
+++ b/src/hotspot/share/compiler/compileBroker.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -582,7 +582,7 @@ CompilerCounters::CompilerCounters() {
// c2 uses explicit CompilerPhaseType idToPhase mapping in opto/phasetype.hpp,
// so if c2 is used, it should be always registered first.
// This function is called during vm initialization.
-void register_jfr_phasetype_serializer(CompilerType compiler_type) {
+static void register_jfr_phasetype_serializer(CompilerType compiler_type) {
ResourceMark rm;
static bool first_registration = true;
if (compiler_type == compiler_jvmci) {
diff --git a/src/hotspot/share/compiler/compilerOracle.cpp b/src/hotspot/share/compiler/compilerOracle.cpp
index 089fc5a4d995e..a8eee10fac539 100644
--- a/src/hotspot/share/compiler/compilerOracle.cpp
+++ b/src/hotspot/share/compiler/compilerOracle.cpp
@@ -48,7 +48,7 @@ static const char* optiontype_names[] = {
#undef enum_of_types
};
-const char* optiontype2name(enum OptionType type) {
+static const char* optiontype2name(enum OptionType type) {
return optiontype_names[static_cast(type)];
}
@@ -58,7 +58,7 @@ static enum OptionType option_types[] = {
#undef enum_of_options
};
-enum OptionType option2type(enum CompileCommand option) {
+static enum OptionType option2type(enum CompileCommand option) {
return option_types[static_cast(option)];
}
@@ -68,7 +68,7 @@ static const char* option_names[] = {
#undef enum_of_options
};
-const char* option2name(enum CompileCommand option) {
+static const char* option2name(enum CompileCommand option) {
return option_names[static_cast(option)];
}
@@ -108,7 +108,7 @@ static bool print_final_memstat_report = false;
// A filter for quick lookup if an option is set
static bool option_filter[static_cast(CompileCommand::Unknown) + 1] = { 0 };
-void command_set_in_filter(enum CompileCommand option) {
+static void command_set_in_filter(enum CompileCommand option) {
assert(option != CompileCommand::Unknown, "sanity");
assert(option2type(option) != OptionType::Unknown, "sanity");
@@ -120,7 +120,7 @@ void command_set_in_filter(enum CompileCommand option) {
option_filter[static_cast(option)] = true;
}
-bool has_command(enum CompileCommand option) {
+static bool has_command(enum CompileCommand option) {
return option_filter[static_cast(option)];
}
@@ -547,7 +547,7 @@ enum OptionType CompilerOracle::parse_option_type(const char* type_str) {
return OptionType::Unknown;
}
-void print_tip() { // CMH Update info
+static void print_tip() { // CMH Update info
tty->cr();
tty->print_cr("Usage: '-XX:CompileCommand=,' - to set boolean option to true");
tty->print_cr("Usage: '-XX:CompileCommand=,,'");
@@ -555,13 +555,13 @@ void print_tip() { // CMH Update info
tty->cr();
}
-void print_option(enum CompileCommand option, const char* name, enum OptionType type) {
+static void print_option(enum CompileCommand option, const char* name, enum OptionType type) {
if (type != OptionType::Unknown) {
tty->print_cr(" %s (%s)", name, optiontype2name(type));
}
}
-void print_commands() {
+static void print_commands() {
tty->cr();
tty->print_cr("All available options:");
#define enum_of_options(option, name, ctype) print_option(CompileCommand::option, name, OptionType::ctype);
@@ -621,7 +621,7 @@ static void usage() {
tty->cr();
};
-int skip_whitespace(char* &line) {
+static int skip_whitespace(char* &line) {
// Skip any leading spaces
int whitespace_read = 0;
sscanf(line, "%*[ \t]%n", &whitespace_read);
@@ -629,7 +629,7 @@ int skip_whitespace(char* &line) {
return whitespace_read;
}
-void skip_comma(char* &line) {
+static void skip_comma(char* &line) {
// Skip any leading spaces
if (*line == ',') {
line++;
diff --git a/src/hotspot/share/compiler/methodMatcher.cpp b/src/hotspot/share/compiler/methodMatcher.cpp
index 3e2ee288c80ed..de2e4e82bba3f 100644
--- a/src/hotspot/share/compiler/methodMatcher.cpp
+++ b/src/hotspot/share/compiler/methodMatcher.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -246,7 +246,7 @@ static MethodMatcher::Mode check_mode(char name[], const char*& error_msg) {
}
// Skip any leading spaces
-void skip_leading_spaces(char*& line, int* total_bytes_read ) {
+static void skip_leading_spaces(char*& line, int* total_bytes_read ) {
int bytes_read = 0;
sscanf(line, "%*[ \t]%n", &bytes_read);
if (bytes_read > 0) {
diff --git a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
index ef2f23633b9ad..53d4aa221f214 100644
--- a/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
+++ b/src/hotspot/share/gc/g1/c1/g1BarrierSetC1.cpp
@@ -28,8 +28,8 @@
#include "gc/g1/c1/g1BarrierSetC1.hpp"
#include "gc/g1/g1BarrierSet.hpp"
#include "gc/g1/g1BarrierSetAssembler.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "utilities/macros.hpp"
#ifdef ASSERT
diff --git a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
index 4d45c227f8c07..06be2fd7606e5 100644
--- a/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
+++ b/src/hotspot/share/gc/g1/c2/g1BarrierSetC2.cpp
@@ -29,7 +29,7 @@
#include "gc/g1/g1BarrierSetRuntime.hpp"
#include "gc/g1/g1CardTable.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "opto/arraycopynode.hpp"
#include "opto/compile.hpp"
#include "opto/escape.hpp"
diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.hpp
index 600d155125936..b39a717d3b708 100644
--- a/src/hotspot/share/gc/g1/g1AllocRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1AllocRegion.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_GC_G1_G1ALLOCREGION_HPP
#define SHARE_GC_G1_G1ALLOCREGION_HPP
-#include "gc/g1/heapRegion.hpp"
#include "gc/g1/g1EvacStats.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
#include "gc/g1/g1NUMA.hpp"
diff --git a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
index 33c402b00871e..00ef254e84c37 100644
--- a/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1AllocRegion.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#define assert_alloc_region(p, message) \
do { \
diff --git a/src/hotspot/share/gc/g1/g1Allocator.cpp b/src/hotspot/share/gc/g1/g1Allocator.cpp
index ffd2b65f555e7..3964ad8c8af3b 100644
--- a/src/hotspot/share/gc/g1/g1Allocator.cpp
+++ b/src/hotspot/share/gc/g1/g1Allocator.cpp
@@ -28,11 +28,11 @@
#include "gc/g1/g1EvacInfo.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
-#include "gc/g1/heapRegionType.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "runtime/mutexLocker.hpp"
#include "utilities/align.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Arguments.cpp b/src/hotspot/share/gc/g1/g1Arguments.cpp
index 5f022bfdc3ced..d0d9cd3129709 100644
--- a/src/hotspot/share/gc/g1/g1Arguments.cpp
+++ b/src/hotspot/share/gc/g1/g1Arguments.cpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/cardTable.hpp"
#include "gc/shared/gcArguments.hpp"
#include "gc/shared/workerPolicy.hpp"
diff --git a/src/hotspot/share/gc/g1/g1BarrierSet.cpp b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
index c2bc7bf5425b0..448f2f5fccf04 100644
--- a/src/hotspot/share/gc/g1/g1BarrierSet.cpp
+++ b/src/hotspot/share/gc/g1/g1BarrierSet.cpp
@@ -27,10 +27,10 @@
#include "gc/g1/g1BarrierSetAssembler.hpp"
#include "gc/g1/g1CardTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "logging/log.hpp"
#include "oops/access.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
index 71107387564a2..af47ce92dc511 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "logging/log.hpp"
#include "nmt/memTracker.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
index a29050c04b34f..80697d38f7d4b 100644
--- a/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1BlockOffsetTable.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/memset_with_concurrent_readers.hpp"
#include "runtime/atomic.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CardSet.cpp b/src/hotspot/share/gc/g1/g1CardSet.cpp
index e6b2d5029e5f6..101822dbc4428 100644
--- a/src/hotspot/share/gc/g1/g1CardSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CardSet.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcLogPrecious.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "memory/allocation.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
index d48fce6cb9ed1..0585b68e33644 100644
--- a/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1CardTable.inline.hpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
inline uint G1CardTable::region_idx_for(CardValue* p) {
size_t const card_idx = pointer_delta(p, _byte_map, sizeof(CardValue));
diff --git a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
index 77211af436444..5e99b91e76a50 100644
--- a/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeBlobClosure.cpp
@@ -27,8 +27,8 @@
#include "gc/g1/g1CodeBlobClosure.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
index a1eb7b8543af2..135c1bca2e305 100644
--- a/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CodeRootSet.cpp
@@ -27,7 +27,7 @@
#include "code/codeCache.hpp"
#include "code/nmethod.hpp"
#include "gc/g1/g1CodeRootSet.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/atomic.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index 802049d9b2f05..ec3d5fa4ecc44 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -27,7 +27,6 @@
#include "classfile/metadataOnStackMark.hpp"
#include "classfile/systemDictionary.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1Arguments.hpp"
@@ -47,6 +46,9 @@
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1GCPauseType.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1HeapSizingPolicy.hpp"
#include "gc/g1/g1HeapTransition.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
@@ -72,9 +74,6 @@
#include "gc/g1/g1VMOperations.hpp"
#include "gc/g1/g1YoungCollector.hpp"
#include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/gcBehaviours.hpp"
@@ -2198,8 +2197,6 @@ void G1CollectedHeap::trace_heap(GCWhen::Type when, const GCTracer* gc_tracer) {
}
void G1CollectedHeap::gc_prologue(bool full) {
- assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
-
// Update common counters.
increment_total_collections(full /* full gc */);
if (full || collector_state()->in_concurrent_start_gc()) {
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
index 189a09fe04241..b4157f16493b5 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp
@@ -36,6 +36,8 @@
#include "gc/g1/g1EvacStats.hpp"
#include "gc/g1/g1GCPauseType.hpp"
#include "gc/g1/g1HeapRegionAttr.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HeapTransition.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1HRPrinter.hpp"
@@ -45,8 +47,6 @@
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
#include "gc/g1/g1YoungGCAllocationFailureInjector.hpp"
-#include "gc/g1/heapRegionManager.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/gcHeapSummary.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
index 537b6863fb94a..513b0a4505ba7 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.inline.hpp
@@ -31,13 +31,13 @@
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1EvacFailureRegions.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionPinCache.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/markBitMap.inline.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "oops/stackChunkOop.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSet.cpp b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
index 30787adbe543b..3ccd8b69644b4 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSet.cpp
@@ -28,11 +28,11 @@
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectorState.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1ParScanThreadState.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "logging/logStream.hpp"
#include "runtime/orderAccess.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
index 9f903bc924f1e..b4649414b10e6 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetCandidates.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/growableArray.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
index f19488a49a8ae..ca7f2b373985b 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetChooser.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSetCandidates.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/shared/space.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/quickSort.hpp"
diff --git a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
index 5b005205551b3..62f9b2a5cef02 100644
--- a/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
+++ b/src/hotspot/share/gc/g1/g1CollectionSetChooser.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
#define SHARE_GC_G1_G1COLLECTIONSETCHOOSER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/allStatic.hpp"
#include "runtime/globals.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
index cb7f90b08477c..5d81f49546c32 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.cpp
@@ -34,16 +34,16 @@
#include "gc/g1/g1ConcurrentMarkThread.inline.hpp"
#include "gc/g1/g1ConcurrentRebuildAndScrub.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#include "gc/g1/g1Trace.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "gc/shared/gcId.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
@@ -152,7 +152,8 @@ G1CMMarkStack::TaskQueueEntryChunk* G1CMMarkStack::ChunkAllocator::allocate_new_
MutexLocker x(MarkStackChunkList_lock, Mutex::_no_safepoint_check_flag);
if (Atomic::load_acquire(&_buckets[bucket]) == nullptr) {
- if (!expand()) {
+ size_t desired_capacity = bucket_size(bucket) * 2;
+ if (!try_expand_to(desired_capacity)) {
return nullptr;
}
}
@@ -196,23 +197,28 @@ bool G1CMMarkStack::ChunkAllocator::initialize(size_t initial_capacity, size_t m
return true;
}
-bool G1CMMarkStack::ChunkAllocator::expand() {
+bool G1CMMarkStack::ChunkAllocator::try_expand_to(size_t desired_capacity) {
if (_capacity == _max_capacity) {
log_debug(gc)("Can not expand overflow mark stack further, already at maximum capacity of " SIZE_FORMAT " chunks.", _capacity);
return false;
}
+
size_t old_capacity = _capacity;
- // Double capacity if possible.
- size_t new_capacity = MIN2(old_capacity * 2, _max_capacity);
+ desired_capacity = MIN2(desired_capacity, _max_capacity);
- if (reserve(new_capacity)) {
+ if (reserve(desired_capacity)) {
log_debug(gc)("Expanded the mark stack capacity from " SIZE_FORMAT " to " SIZE_FORMAT " chunks",
- old_capacity, new_capacity);
+ old_capacity, desired_capacity);
return true;
}
return false;
}
+bool G1CMMarkStack::ChunkAllocator::try_expand() {
+ size_t new_capacity = _capacity * 2;
+ return try_expand_to(new_capacity);
+}
+
G1CMMarkStack::ChunkAllocator::~ChunkAllocator() {
if (_buckets == nullptr) {
return;
@@ -234,6 +240,9 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
size_t highest_bucket = get_bucket(new_capacity - 1);
size_t i = get_bucket(_capacity);
+ // Allocate all buckets associated with indexes between the current capacity (_capacity)
+ // and the new capacity (new_capacity). This step ensures that there are no gaps in the
+ // array and that the capacity accurately reflects the reserved memory.
for (; i <= highest_bucket; i++) {
if (Atomic::load_acquire(&_buckets[i]) != nullptr) {
continue; // Skip over already allocated buckets.
@@ -261,7 +270,7 @@ bool G1CMMarkStack::ChunkAllocator::reserve(size_t new_capacity) {
}
void G1CMMarkStack::expand() {
- _chunk_allocator.expand();
+ _chunk_allocator.try_expand();
}
void G1CMMarkStack::add_chunk_to_list(TaskQueueEntryChunk* volatile* list, TaskQueueEntryChunk* elem) {
@@ -480,12 +489,10 @@ G1ConcurrentMark::G1ConcurrentMark(G1CollectedHeap* g1h,
// _verbose_level set below
- _init_times(),
_remark_times(),
_remark_mark_times(),
_remark_weak_ref_times(),
_cleanup_times(),
- _total_cleanup_time(0.0),
_accum_task_vtime(nullptr),
@@ -1549,9 +1556,7 @@ void G1ConcurrentMark::cleanup() {
verify_during_pause(G1HeapVerifier::G1VerifyCleanup, VerifyLocation::CleanupAfter);
// Local statistics
- double recent_cleanup_time = (os::elapsedTime() - start);
- _total_cleanup_time += recent_cleanup_time;
- _cleanup_times.add(recent_cleanup_time);
+ _cleanup_times.add((os::elapsedTime() - start) * 1000.0);
{
GCTraceTime(Debug, gc, phases) debug("Finalize Concurrent Mark Cleanup", _gc_timer_cm);
@@ -2117,7 +2122,6 @@ void G1ConcurrentMark::print_summary_info() {
}
log.trace(" Concurrent marking:");
- print_ms_time_info(" ", "init marks", _init_times);
print_ms_time_info(" ", "remarks", _remark_times);
{
print_ms_time_info(" ", "final marks", _remark_mark_times);
@@ -2126,9 +2130,9 @@ void G1ConcurrentMark::print_summary_info() {
}
print_ms_time_info(" ", "cleanups", _cleanup_times);
log.trace(" Finalize live data total time = %8.2f s (avg = %8.2f ms).",
- _total_cleanup_time, (_cleanup_times.num() > 0 ? _total_cleanup_time * 1000.0 / (double)_cleanup_times.num() : 0.0));
+ _cleanup_times.sum() / 1000.0, _cleanup_times.avg());
log.trace(" Total stop_world time = %8.2f s.",
- (_init_times.sum() + _remark_times.sum() + _cleanup_times.sum())/1000.0);
+ (_remark_times.sum() + _cleanup_times.sum())/1000.0);
log.trace(" Total concurrent time = %8.2f s (%8.2f s marking).",
cm_thread()->vtime_accum(), cm_thread()->vtime_mark_accum());
}
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
index a4315ec5f6c48..3d9693bed100b 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.hpp
@@ -27,9 +27,9 @@
#include "gc/g1/g1ConcurrentMarkBitMap.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1RegionMarkStatsCache.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "gc/shared/gcCause.hpp"
#include "gc/shared/taskTerminator.hpp"
#include "gc/shared/taskqueue.hpp"
@@ -145,7 +145,7 @@ class G1CMMarkStack {
// within the bucket. Additionally, each new bucket added to the growable array doubles the capacity of
// the growable array.
//
- // Illustration of the Growable Array data structure.
+ // Illustration of the growable array data structure.
//
// +----+ +----+----+
// | |------->| | |
@@ -174,7 +174,7 @@ class G1CMMarkStack {
size_t bucket_size(size_t bucket) {
return (bucket == 0) ?
_min_capacity :
- _min_capacity * ( 1ULL << (bucket -1));
+ _min_capacity * ( 1ULL << (bucket - 1));
}
static unsigned int find_highest_bit(uintptr_t mask) {
@@ -225,7 +225,9 @@ class G1CMMarkStack {
size_t capacity() const { return _capacity; }
- bool expand();
+ // Expand the mark stack doubling its size.
+ bool try_expand();
+ bool try_expand_to(size_t desired_capacity);
TaskQueueEntryChunk* allocate_new_chunk();
};
@@ -435,12 +437,10 @@ class G1ConcurrentMark : public CHeapObj {
G1OldTracer* _gc_tracer_cm;
// Timing statistics. All of them are in ms
- NumberSeq _init_times;
NumberSeq _remark_times;
NumberSeq _remark_mark_times;
NumberSeq _remark_weak_ref_times;
NumberSeq _cleanup_times;
- double _total_cleanup_time;
double* _accum_task_vtime; // Accumulated task vtime
@@ -561,7 +561,7 @@ class G1ConcurrentMark : public CHeapObj {
// Sets the internal top_at_region_start for the given region to current top of the region.
inline void update_top_at_rebuild_start(HeapRegion* r);
// TARS for the given region during remembered set rebuilding.
- inline HeapWord* top_at_rebuild_start(uint region) const;
+ inline HeapWord* top_at_rebuild_start(HeapRegion* r) const;
// Clear statistics gathered during the concurrent cycle for the given region after
// it has been reclaimed.
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
index 3601f4529a336..4a05b86ae5a7c 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMark.inline.hpp
@@ -30,12 +30,12 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RegionMarkStatsCache.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "utilities/bitMap.inline.hpp"
@@ -184,9 +184,8 @@ inline size_t G1CMTask::scan_objArray(objArrayOop obj, MemRegion mr) {
return mr.word_size();
}
-inline HeapWord* G1ConcurrentMark::top_at_rebuild_start(uint region) const {
- assert(region < _g1h->max_reserved_regions(), "Tried to access TARS for region %u out of bounds", region);
- return _top_at_rebuild_starts[region];
+inline HeapWord* G1ConcurrentMark::top_at_rebuild_start(HeapRegion* r) const {
+ return _top_at_rebuild_starts[r->hrm_index()];
}
inline void G1ConcurrentMark::update_top_at_rebuild_start(HeapRegion* r) {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
index e2ace6c8c0a97..438b58c2c02c7 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkBitMap.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/virtualspace.hpp"
G1CMBitMap::G1CMBitMap() : MarkBitMap(), _listener() {
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
index 80acada04726f..84823f8bf830d 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentMarkObjArrayProcessor.cpp
@@ -26,7 +26,7 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkObjArrayProcessor.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "memory/memRegion.hpp"
#include "utilities/globalDefinitions.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
index 8cf7601d65a25..74e242a0f3fa6 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRebuildAndScrub.cpp
@@ -28,9 +28,9 @@
#include "gc/g1/g1ConcurrentMark.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/workerThread.hpp"
#include "logging/log.hpp"
@@ -105,7 +105,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
// - been allocated after rebuild start, or
// - been reclaimed by a collection.
bool should_rebuild_or_scrub(HeapRegion* hr) const {
- return _cm->top_at_rebuild_start(hr->hrm_index()) != nullptr;
+ return _cm->top_at_rebuild_start(hr) != nullptr;
}
// Helper used by both humongous objects and when chunking an object larger than the
@@ -229,7 +229,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
assert(should_rebuild_or_scrub(hr), "must be");
log_trace(gc, marking)("Scrub and rebuild region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT " TAMS: " PTR_FORMAT,
- HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr->hrm_index())), p2i(hr->top_at_mark_start()));
+ HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)), p2i(hr->top_at_mark_start()));
if (scan_and_scrub_to_pb(hr, hr->bottom(), pb)) {
log_trace(gc, marking)("Scan and scrub aborted for region: %u", hr->hrm_index());
@@ -246,7 +246,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
hr->note_end_of_scrubbing();
// Rebuild from TAMS (= parsable_bottom) to TARS.
- if (scan_from_pb_to_tars(hr, pb, _cm->top_at_rebuild_start(hr->hrm_index()))) {
+ if (scan_from_pb_to_tars(hr, pb, _cm->top_at_rebuild_start(hr))) {
log_trace(gc, marking)("Rebuild aborted for region: %u (%s)", hr->hrm_index(), hr->get_short_type_str());
return true;
}
@@ -272,7 +272,7 @@ class G1RebuildRSAndScrubTask : public WorkerTask {
"Humongous object not live");
log_trace(gc, marking)("Rebuild for humongous region: " HR_FORMAT " pb: " PTR_FORMAT " TARS: " PTR_FORMAT,
- HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr->hrm_index())));
+ HR_FORMAT_PARAMS(hr), p2i(pb), p2i(_cm->top_at_rebuild_start(hr)));
// Scan the humongous object in chunks from bottom to top to rebuild remembered sets.
HeapWord* humongous_end = hr->humongous_start_region()->bottom() + humongous->size();
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
index b2ac4b25e9065..9ae9ad793c424 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefine.cpp
@@ -28,9 +28,9 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1Policy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/gc_globals.hpp"
#include "logging/log.hpp"
#include "memory/allocation.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp b/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
index 8767f8e1da99b..33943eb5113ab 100644
--- a/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
+++ b/src/hotspot/share/gc/g1/g1ConcurrentRefineThreadsNeeded.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1Analytics.hpp"
#include "gc/g1/g1ConcurrentRefineThreadsNeeded.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1Policy.hpp"
#include "utilities/globalDefinitions.hpp"
#include
diff --git a/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp b/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
index 93ff09134bfa6..67059b6ec1c47 100644
--- a/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
+++ b/src/hotspot/share/gc/g1/g1DirtyCardQueue.cpp
@@ -30,10 +30,10 @@
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
#include "gc/g1/g1FreeIdSet.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RedirtyCardsQueue.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/bufferNodeList.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
diff --git a/src/hotspot/share/gc/g1/g1EdenRegions.hpp b/src/hotspot/share/gc/g1/g1EdenRegions.hpp
index 23b97785ad358..c76099c284a19 100644
--- a/src/hotspot/share/gc/g1/g1EdenRegions.hpp
+++ b/src/hotspot/share/gc/g1/g1EdenRegions.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1EDENREGIONS_HPP
#include "gc/g1/g1RegionsOnNodes.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "runtime/globals.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
index a308fe7a3934a..845868c3e2415 100644
--- a/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
+++ b/src/hotspot/share/gc/g1/g1EvacFailureRegions.cpp
@@ -27,7 +27,7 @@
#include "gc/g1/g1BatchedTask.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/atomic.hpp"
#include "utilities/bitMap.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp
index 430d6f327886b..81a54a9c31f32 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp
@@ -312,6 +312,13 @@ void G1FullCollector::phase1_mark_live_objects() {
reference_processor()->set_active_mt_degree(old_active_mt_degree);
}
+ {
+ GCTraceTime(Debug, gc, phases) debug("Phase 1: Flush Mark Stats Cache", scope()->timer());
+ for (uint i = 0; i < workers(); i++) {
+ marker(i)->flush_mark_stats_cache();
+ }
+ }
+
// Weak oops cleanup.
{
GCTraceTime(Debug, gc, phases) debug("Phase 1: Weak Processing", scope()->timer());
diff --git a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
index 9150d2f2ccdfe..7aa04449bcf7b 100644
--- a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullCollector.hpp"
#include "gc/g1/g1FullGCHeapRegionAttr.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "oops/oopsHierarchy.hpp"
#include "runtime/atomic.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
index 76aebeb7a8334..3eca766b2fcf7 100644
--- a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp
@@ -32,7 +32,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCMarker.hpp"
#include "gc/g1/g1FullGCOopClosures.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
index c9b190acd0659..26ac183d10d75 100644
--- a/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp
@@ -27,8 +27,8 @@
#include "gc/g1/g1FullGCOopClosures.hpp"
#include "gc/g1/g1FullGCTask.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegionManager.hpp"
#include "gc/shared/weakProcessor.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
index 2ed6ccd4735fb..5e3a9687f9808 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCCompactTask.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "logging/log.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
index a7e2ea38c1775..7f7f144397baf 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp
@@ -28,7 +28,7 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/shared/referenceProcessor.hpp"
class G1CollectedHeap;
diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
index a1db2aa87ba2e..7b19aa431beb1 100644
--- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp
@@ -25,7 +25,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
index 209e89ae4abb1..e27439143d900 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarkTask.cpp
@@ -56,7 +56,6 @@ void G1FullGCMarkTask::work(uint worker_id) {
// Mark stack is populated, now process and drain it.
marker->complete_marking(collector()->oop_queue_set(), collector()->array_queue_set(), &_terminator);
- marker->flush_mark_stats_cache();
// This is the point where the entire marking should have completed.
assert(marker->oop_stack()->is_empty(), "Marking should have completed");
diff --git a/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp b/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
index 5e45b65c65faf..cf87bcfa7c882 100644
--- a/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCMarkTask.hpp
@@ -28,8 +28,8 @@
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
#include "gc/g1/g1FullGCTask.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegionManager.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
index aa194d16203b7..f10f884b24253 100644
--- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp
@@ -28,10 +28,10 @@
#include "gc/g1/g1FullGCOopClosures.hpp"
#include "gc/g1/g1Allocator.inline.hpp"
-#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1ConcurrentMarkBitMap.inline.hpp"
+#include "gc/g1/g1FullCollector.inline.hpp"
#include "gc/g1/g1FullGCMarker.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "memory/universe.hpp"
#include "oops/access.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
index 7cef60293974c..e116367133d91 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp
@@ -30,7 +30,7 @@
#include "gc/g1/g1FullGCMarker.hpp"
#include "gc/g1/g1FullGCOopClosures.inline.hpp"
#include "gc/g1/g1FullGCPrepareTask.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
index 7f09f0553e9f7..9618986dcdd0c 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1FULLGCPREPARETASK_HPP
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
class G1CollectedHeap;
diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
index 2a2add76f48c6..5cff8e5f412a1 100644
--- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp
@@ -31,7 +31,7 @@
#include "gc/g1/g1FullCollector.hpp"
#include "gc/g1/g1FullGCCompactionPoint.hpp"
#include "gc/g1/g1FullGCScope.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
void G1DetermineCompactionQueueClosure::free_empty_humongous_region(HeapRegion* hr) {
_g1h->free_humongous_region(hr, nullptr);
diff --git a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
index 25745ff0abcc4..f424906d56377 100644
--- a/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
+++ b/src/hotspot/share/gc/g1/g1FullGCResetMetadataTask.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1FULLGCRESETMETADATATASK_HPP
#define SHARE_GC_G1_G1FULLGCRESETMETADATATASK_HPP
#include "gc/g1/g1FullGCTask.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
class G1FullGCResetMetadataTask : public G1FullGCTask {
G1FullCollector* _collector;
diff --git a/src/hotspot/share/gc/g1/g1HRPrinter.cpp b/src/hotspot/share/gc/g1/g1HRPrinter.cpp
index af83284f0618d..7ec4f2a0d96e3 100644
--- a/src/hotspot/share/gc/g1/g1HRPrinter.cpp
+++ b/src/hotspot/share/gc/g1/g1HRPrinter.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1HRPrinter.hpp"
-#include "gc/g1/heapRegionSet.hpp"
void G1HRPrinter::cleanup(FreeRegionList* cleanup_list) {
if (is_active()) {
diff --git a/src/hotspot/share/gc/g1/g1HRPrinter.hpp b/src/hotspot/share/gc/g1/g1HRPrinter.hpp
index 876be95f0746c..9d6d151084fbd 100644
--- a/src/hotspot/share/gc/g1/g1HRPrinter.hpp
+++ b/src/hotspot/share/gc/g1/g1HRPrinter.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1HRPRINTER_HPP
#define SHARE_GC_G1_G1HRPRINTER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "logging/log.hpp"
#define SKIP_RETIRED_FULL_REGIONS 1
diff --git a/src/hotspot/share/gc/g1/heapRegion.cpp b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegion.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.cpp
index 69040028d98c5..cb230ee598890 100644
--- a/src/hotspot/share/gc/g1/heapRegion.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.cpp
@@ -29,14 +29,14 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1CollectionSet.hpp"
#include "gc/g1/g1CollectionSetCandidates.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
#include "memory/iterator.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegion.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegion.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.hpp
index 85f4a9e5f9ffb..b9187ddfddc2f 100644
--- a/src/hotspot/share/gc/g1/heapRegion.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.hpp
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGION_HPP
-#define SHARE_GC_G1_HEAPREGION_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGION_HPP
+#define SHARE_GC_G1_G1HEAPREGION_HPP
#include "gc/g1/g1BlockOffsetTable.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "gc/g1/g1SurvRateGroup.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
-#include "gc/g1/heapRegionType.hpp"
#include "gc/shared/ageTable.hpp"
#include "gc/shared/spaceDecorator.hpp"
#include "gc/shared/verifyOption.hpp"
@@ -622,4 +622,4 @@ class HeapRegionIndexClosure : public StackObj {
bool is_complete() { return _is_complete; }
};
-#endif // SHARE_GC_G1_HEAPREGION_HPP
+#endif // SHARE_GC_G1_G1HEAPREGION_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegion.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
index 608cc2025a2a8..146f5aff86680 100644
--- a/src/hotspot/share/gc/g1/heapRegion.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegion.inline.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGION_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGION_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "classfile/vmClasses.hpp"
#include "gc/g1/g1BlockOffsetTable.inline.hpp"
@@ -559,4 +559,4 @@ inline void HeapRegion::add_pinned_object_count(size_t value) {
Atomic::add(&_pinned_object_count, value, memory_order_relaxed);
}
-#endif // SHARE_GC_G1_HEAPREGION_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGION_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp b/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
index e5a10858ed126..c3c7b94e287b7 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionAttr.hpp
@@ -26,7 +26,7 @@
#define SHARE_GC_G1_G1HEAPREGIONATTR_HPP
#include "gc/g1/g1BiasedArray.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
// Per-region attributes often used during garbage collection to avoid costly
// lookups for that information all over the place.
diff --git a/src/hotspot/share/gc/g1/heapRegionBounds.hpp b/src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
similarity index 94%
rename from src/hotspot/share/gc/g1/heapRegionBounds.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
index 5a6db9fac3b14..3cde2f77d4a5b 100644
--- a/src/hotspot/share/gc/g1/heapRegionBounds.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionBounds.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
-#define SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
+#define SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
#include "memory/allStatic.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -55,4 +55,4 @@ class HeapRegionBounds : public AllStatic {
static inline size_t target_number();
};
-#endif // SHARE_GC_G1_HEAPREGIONBOUNDS_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONBOUNDS_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
similarity index 87%
rename from src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
index 73a44bcf1794f..473057144d30c 100644
--- a/src/hotspot/share/gc/g1/heapRegionBounds.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionBounds.inline.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
-#include "gc/g1/heapRegionBounds.hpp"
+#include "gc/g1/g1HeapRegionBounds.hpp"
size_t HeapRegionBounds::min_size() {
return MIN_REGION_SIZE;
@@ -43,4 +43,4 @@ size_t HeapRegionBounds::target_number() {
return TARGET_REGION_NUMBER;
}
-#endif // SHARE_GC_G1_HEAPREGIONBOUNDS_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONBOUNDS_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp b/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
index 19433e39c3235..5dce9b2aa909e 100644
--- a/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionEventSender.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1HeapRegionEventSender.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
#include "jfr/jfrEvents.hpp"
#include "runtime/vmThread.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.cpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegionManager.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
index 455aacc0931a1..0fb8e7499d68f 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.cpp
@@ -27,10 +27,10 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1CommittedRegionMap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1NUMAStats.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "logging/logStream.hpp"
#include "memory/allocation.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionManager.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
index 0a84010e73c55..8ecb5d6d66f64 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.hpp
@@ -22,13 +22,13 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONMANAGER_HPP
-#define SHARE_GC_G1_HEAPREGIONMANAGER_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
+#define SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
#include "gc/g1/g1BiasedArray.hpp"
#include "gc/g1/g1CommittedRegionMap.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1RegionToSpaceMapper.hpp"
-#include "gc/g1/heapRegionSet.hpp"
#include "memory/allocation.hpp"
#include "services/memoryUsage.hpp"
@@ -326,4 +326,4 @@ class HeapRegionClaimer : public StackObj {
// Claim the given region, returns true if successfully claimed.
bool claim_region(uint region_index);
};
-#endif // SHARE_GC_G1_HEAPREGIONMANAGER_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONMANAGER_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionManager.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
similarity index 91%
rename from src/hotspot/share/gc/g1/heapRegionManager.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
index 1bcba292ffa2b..9768ee358baae 100644
--- a/src/hotspot/share/gc/g1/heapRegionManager.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionManager.inline.hpp
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "gc/g1/g1CommittedRegionMap.inline.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
inline bool HeapRegionManager::is_available(uint region) const {
return _committed_map.active(region);
@@ -83,4 +83,4 @@ inline HeapRegion* HeapRegionManager::allocate_free_regions_starting_at(uint fir
return start;
}
-#endif // SHARE_GC_G1_HEAPREGIONMANAGER_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONMANAGER_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
index 19b383a08a39c..88a3e09bd8e9e 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.cpp
@@ -29,8 +29,8 @@
#include "gc/g1/g1CardSetContainers.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentRefine.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "memory/allocation.hpp"
#include "memory/padded.inline.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
index 2e5ae7b1c3076..7e76965835b6b 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONREMSET_HPP
-#define SHARE_GC_G1_HEAPREGIONREMSET_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
+#define SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
#include "gc/g1/g1CardSet.hpp"
#include "gc/g1/g1CardSetMemory.hpp"
@@ -185,4 +185,4 @@ class HeapRegionRemSet : public CHeapObj {
#endif
};
-#endif // SHARE_GC_G1_HEAPREGIONREMSET_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONREMSET_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
similarity index 95%
rename from src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
index ca8f675e6dfd7..78153cc0c3023 100644
--- a/src/hotspot/share/gc/g1/heapRegionRemSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionRemSet.inline.hpp
@@ -22,14 +22,14 @@
*
*/
-#ifndef SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
-#define SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
+#ifndef SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
+#define SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
-#include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1CardSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "runtime/atomic.hpp"
#include "utilities/bitMap.inline.hpp"
@@ -144,4 +144,4 @@ void HeapRegionRemSet::print_info(outputStream* st, OopOrNarrowOopStar from) {
_card_set.print_info(st, to_card(from));
}
-#endif // SHARE_VM_GC_G1_HEAPREGIONREMSET_INLINE_HPP
+#endif // SHARE_VM_GC_G1_G1HEAPREGIONREMSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.cpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
similarity index 99%
rename from src/hotspot/share/gc/g1/heapRegionSet.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
index 7243677ede8e2..832770ddb4c42 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
+#include "gc/g1/g1HeapRegionSet.inline.hpp"
#include "gc/g1/g1NUMA.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
-#include "gc/g1/heapRegionSet.inline.hpp"
uint FreeRegionList::_unrealistically_long_length = 0;
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.hpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionSet.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
index 5db2fbb696aa7..bfc60992c3d40 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONSET_HPP
-#define SHARE_GC_G1_HEAPREGIONSET_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONSET_HPP
+#define SHARE_GC_G1_G1HEAPREGIONSET_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "utilities/macros.hpp"
#define assert_heap_region_set(p, message) \
@@ -271,4 +271,4 @@ class FreeRegionListIterator : public StackObj {
}
};
-#endif // SHARE_GC_G1_HEAPREGIONSET_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONSET_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp b/src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
index 0d7c7ed315d13..fbd19466d9f9a 100644
--- a/src/hotspot/share/gc/g1/heapRegionSet.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionSet.inline.hpp
@@ -22,10 +22,10 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
-#define SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
-#include "gc/g1/heapRegionSet.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/g1/g1NUMA.hpp"
@@ -271,4 +271,4 @@ inline uint FreeRegionList::length(uint node_index) const {
}
}
-#endif // SHARE_GC_G1_HEAPREGIONSET_INLINE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionTracer.cpp b/src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionTracer.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
index e039ae219a11e..c66ae0cb06941 100644
--- a/src/hotspot/share/gc/g1/heapRegionTracer.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionTracer.cpp
@@ -23,7 +23,7 @@
*/
#include "precompiled.hpp"
-#include "gc/g1/heapRegionTracer.hpp"
+#include "gc/g1/g1HeapRegionTracer.hpp"
#include "jfr/jfrEvents.hpp"
void HeapRegionTracer::send_region_type_change(uint index,
diff --git a/src/hotspot/share/gc/g1/heapRegionTracer.hpp b/src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
similarity index 92%
rename from src/hotspot/share/gc/g1/heapRegionTracer.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
index 647744d0d5e23..137bc6189aed0 100644
--- a/src/hotspot/share/gc/g1/heapRegionTracer.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionTracer.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONTRACER_HPP
-#define SHARE_GC_G1_HEAPREGIONTRACER_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
+#define SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "memory/allStatic.hpp"
@@ -38,4 +38,4 @@ class HeapRegionTracer : AllStatic {
size_t used);
};
-#endif // SHARE_GC_G1_HEAPREGIONTRACER_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONTRACER_HPP
diff --git a/src/hotspot/share/gc/g1/heapRegionType.cpp b/src/hotspot/share/gc/g1/g1HeapRegionType.cpp
similarity index 98%
rename from src/hotspot/share/gc/g1/heapRegionType.cpp
rename to src/hotspot/share/gc/g1/g1HeapRegionType.cpp
index bb5c4e9392c79..e55759b3bf38d 100644
--- a/src/hotspot/share/gc/g1/heapRegionType.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionType.cpp
@@ -24,7 +24,7 @@
#include "precompiled.hpp"
#include "gc/g1/g1HeapRegionTraceType.hpp"
-#include "gc/g1/heapRegionType.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
const HeapRegionType HeapRegionType::Eden = HeapRegionType(EdenTag);
const HeapRegionType HeapRegionType::Survivor = HeapRegionType(SurvTag);
diff --git a/src/hotspot/share/gc/g1/heapRegionType.hpp b/src/hotspot/share/gc/g1/g1HeapRegionType.hpp
similarity index 97%
rename from src/hotspot/share/gc/g1/heapRegionType.hpp
rename to src/hotspot/share/gc/g1/g1HeapRegionType.hpp
index 85a89f77a9405..b62699cc3fb9e 100644
--- a/src/hotspot/share/gc/g1/heapRegionType.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapRegionType.hpp
@@ -22,8 +22,8 @@
*
*/
-#ifndef SHARE_GC_G1_HEAPREGIONTYPE_HPP
-#define SHARE_GC_G1_HEAPREGIONTYPE_HPP
+#ifndef SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
+#define SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
#include "gc/g1/g1HeapRegionTraceType.hpp"
#include "utilities/globalDefinitions.hpp"
@@ -167,4 +167,4 @@ friend class VMStructs;
static const HeapRegionType Humongous;
};
-#endif // SHARE_GC_G1_HEAPREGIONTYPE_HPP
+#endif // SHARE_GC_G1_G1HEAPREGIONTYPE_HPP
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
index 6cca6e8b05610..b7484727e1fb1 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.cpp
@@ -27,12 +27,12 @@
#include "gc/g1/g1Allocator.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMarkThread.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1HeapVerifier.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
diff --git a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
index f10787c528e36..221ae9576489e 100644
--- a/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
+++ b/src/hotspot/share/gc/g1/g1HeapVerifier.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1HEAPVERIFIER_HPP
#define SHARE_GC_G1_G1HEAPVERIFIER_HPP
-#include "gc/g1/heapRegionSet.hpp"
+#include "gc/g1/g1HeapRegionSet.hpp"
#include "gc/shared/verifyOption.hpp"
#include "memory/allocation.hpp"
#include "utilities/macros.hpp"
diff --git a/src/hotspot/share/gc/g1/g1MemoryPool.cpp b/src/hotspot/share/gc/g1/g1MemoryPool.cpp
index 428d2db212e2e..f924e5a7e6517 100644
--- a/src/hotspot/share/gc/g1/g1MemoryPool.cpp
+++ b/src/hotspot/share/gc/g1/g1MemoryPool.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1MemoryPool.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/gc_globals.hpp"
G1MemoryPoolSuper::G1MemoryPoolSuper(G1CollectedHeap* g1h,
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
index 4ba4469bd964f..337acfe24058d 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.cpp
@@ -26,10 +26,10 @@
#include "ci/ciUtilities.hpp"
#include "gc/g1/g1CardSetMemory.inline.hpp"
#include "gc/g1/g1CollectedHeap.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1MonotonicArenaFreeMemoryTask.hpp"
#include "gc/g1/g1MonotonicArenaFreePool.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
diff --git a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
index 88611c63b5c10..d55c844037c40 100644
--- a/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
+++ b/src/hotspot/share/gc/g1/g1MonotonicArenaFreeMemoryTask.hpp
@@ -26,9 +26,9 @@
#define SHARE_GC_G1_G1MONOTONICARENAFREEMEMORYTASK_HPP
#include "gc/g1/g1CardSetMemory.hpp"
+#include "gc/g1/g1HeapRegionRemSet.hpp"
#include "gc/g1/g1MonotonicArenaFreePool.hpp"
#include "gc/g1/g1ServiceThread.hpp"
-#include "gc/g1/heapRegionRemSet.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1NUMA.hpp b/src/hotspot/share/gc/g1/g1NUMA.hpp
index acf5a114aebd4..cbefab411dff6 100644
--- a/src/hotspot/share/gc/g1/g1NUMA.hpp
+++ b/src/hotspot/share/gc/g1/g1NUMA.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_VM_GC_G1_NUMA_HPP
#define SHARE_VM_GC_G1_NUMA_HPP
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1NUMAStats.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
#include "runtime/os.hpp"
diff --git a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
index 13890c026b2d6..eb86b81d2abbd 100644
--- a/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
+++ b/src/hotspot/share/gc/g1/g1OldGenAllocationTracker.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_VM_GC_G1_G1OLDGENALLOCATIONTRACKER_HPP
#define SHARE_VM_GC_G1_G1OLDGENALLOCATIONTRACKER_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
class G1AdaptiveIHOPControl;
diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
index ea7f690d325fc..9eebfbd6a8682 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1CollectedHeap.inline.hpp"
#include "gc/g1/g1ConcurrentMark.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
#include "memory/iterator.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Policy.cpp b/src/hotspot/share/gc/g1/g1Policy.cpp
index 9e22acc73ca33..10227dfe2add8 100644
--- a/src/hotspot/share/gc/g1/g1Policy.cpp
+++ b/src/hotspot/share/gc/g1/g1Policy.cpp
@@ -34,13 +34,13 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineStats.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1IHOPControl.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
#include "gc/g1/g1YoungGenSizer.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/concurrentGCBreakpoints.hpp"
#include "gc/shared/gcPolicyCounters.hpp"
#include "logging/log.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RegionPinCache.hpp b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp
index 6df1c1793c285..1c04936930a8f 100644
--- a/src/hotspot/share/gc/g1/g1RegionPinCache.hpp
+++ b/src/hotspot/share/gc/g1/g1RegionPinCache.hpp
@@ -25,7 +25,7 @@
#ifndef SHARE_GC_G1_G1REGIONPINCACHE_HPP
#define SHARE_GC_G1_G1REGIONPINCACHE_HPP
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "memory/allocation.hpp"
#include "utilities/globalDefinitions.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
index 5ec3c759369ce..88a4b9611dc2a 100644
--- a/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
+++ b/src/hotspot/share/gc/g1/g1RegionsOnNodes.cpp
@@ -23,9 +23,9 @@
*/
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1NUMA.hpp"
#include "gc/g1/g1RegionsOnNodes.hpp"
-#include "gc/g1/heapRegion.hpp"
G1RegionsOnNodes::G1RegionsOnNodes() : _count_per_node(nullptr), _numa(G1NUMA::numa()) {
_count_per_node = NEW_C_HEAP_ARRAY(uint, _numa->num_active_nodes(), mtGC);
diff --git a/src/hotspot/share/gc/g1/g1RemSet.cpp b/src/hotspot/share/gc/g1/g1RemSet.cpp
index ac445850085cd..2112795221459 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.cpp
@@ -35,14 +35,14 @@
#include "gc/g1/g1FromCardCache.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionManager.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RootClosures.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1_globals.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionManager.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/bufferNodeList.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
@@ -1174,24 +1174,27 @@ class G1MergeHeapRootsTask : public WorkerTask {
}
HeapRegion* r = g1h->region_at(region_index);
- if (r->rem_set()->is_empty()) {
- return false;
- }
+
+ assert(r->rem_set()->is_complete(), "humongous candidates must have complete remset");
guarantee(r->rem_set()->occupancy_less_or_equal_than(G1EagerReclaimRemSetThreshold),
"Found a not-small remembered set here. This is inconsistent with previous assumptions.");
- _cl.merge_card_set_for_region(r);
+ if (!r->rem_set()->is_empty()) {
+ _cl.merge_card_set_for_region(r);
- // We should only clear the card based remembered set here as we will not
- // implicitly rebuild anything else during eager reclaim. Note that at the moment
- // (and probably never) we do not enter this path if there are other kind of
- // remembered sets for this region.
- // We want to continue collecting remembered set entries for humongous regions
- // that were not reclaimed.
- r->rem_set()->clear(true /* only_cardset */, true /* keep_tracked */);
+ // We should only clear the card based remembered set here as we will not
+ // implicitly rebuild anything else during eager reclaim. Note that at the moment
+ // (and probably never) we do not enter this path if there are other kind of
+ // remembered sets for this region.
+ // We want to continue collecting remembered set entries for humongous regions
+ // that were not reclaimed.
+ r->rem_set()->clear(true /* only_cardset */, true /* keep_tracked */);
+ }
- assert(r->rem_set()->is_empty() && r->rem_set()->is_complete(), "must be for eager reclaim candidates");
+ // Postcondition
+ assert(r->rem_set()->is_empty(), "must be empty after flushing");
+ assert(r->rem_set()->is_complete(), "should still be after flushing");
return false;
}
diff --git a/src/hotspot/share/gc/g1/g1RemSet.hpp b/src/hotspot/share/gc/g1/g1RemSet.hpp
index 053fc39aa8365..f38e5e7f0f484 100644
--- a/src/hotspot/share/gc/g1/g1RemSet.hpp
+++ b/src/hotspot/share/gc/g1/g1RemSet.hpp
@@ -26,10 +26,10 @@
#define SHARE_GC_G1_G1REMSET_HPP
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/g1RemSetSummary.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "memory/allocation.hpp"
#include "memory/iterator.hpp"
#include "utilities/ticks.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
index 4156e85fbfa00..0cd1823a8ee57 100644
--- a/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetSummary.cpp
@@ -29,10 +29,10 @@
#include "gc/g1/g1ConcurrentRefine.hpp"
#include "gc/g1/g1ConcurrentRefineThread.hpp"
#include "gc/g1/g1DirtyCardQueue.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1RemSetSummary.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "memory/allocation.inline.hpp"
#include "memory/iterator.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
index c227b625b8de6..8738e2bc886c2 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectionSetChooser.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1RemSetTrackingPolicy.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "runtime/safepoint.hpp"
bool G1RemSetTrackingPolicy::needs_scan_for_rebuild(HeapRegion* r) const {
diff --git a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
index 2c2734d4abf4a..94ff2734fbf8e 100644
--- a/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
+++ b/src/hotspot/share/gc/g1/g1RemSetTrackingPolicy.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_GC_G1_G1REMSETTRACKINGPOLICY_HPP
#define SHARE_GC_G1_G1REMSETTRACKINGPOLICY_HPP
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionType.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionType.hpp"
#include "memory/allocation.hpp"
// The remembered set tracking policy determines for a given region the state of
diff --git a/src/hotspot/share/gc/g1/g1RootProcessor.cpp b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
index 3a9e30ec4034d..51a77eccfdf38 100644
--- a/src/hotspot/share/gc/g1/g1RootProcessor.cpp
+++ b/src/hotspot/share/gc/g1/g1RootProcessor.cpp
@@ -32,11 +32,11 @@
#include "gc/g1/g1CollectorState.hpp"
#include "gc/g1/g1GCParPhaseTimesTracker.hpp"
#include "gc/g1/g1GCPhaseTimes.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
#include "gc/g1/g1ParScanThreadState.inline.hpp"
#include "gc/g1/g1Policy.hpp"
#include "gc/g1/g1RootClosures.hpp"
#include "gc/g1/g1RootProcessor.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
#include "gc/shared/oopStorage.inline.hpp"
#include "gc/shared/oopStorageSet.hpp"
#include "gc/shared/oopStorageSetParState.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp b/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
index f1cf554ca462a..b7790d154d4cf 100644
--- a/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
+++ b/src/hotspot/share/gc/g1/g1SATBMarkQueueSet.cpp
@@ -25,9 +25,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1BarrierSet.inline.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SATBMarkQueueSet.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "gc/shared/satbMarkQueue.hpp"
#include "oops/oop.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
index 2b09dfc668b59..48773a6634e96 100644
--- a/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
+++ b/src/hotspot/share/gc/g1/g1SurvRateGroup.cpp
@@ -24,9 +24,9 @@
#include "precompiled.hpp"
#include "gc/g1/g1CollectedHeap.inline.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1Predictions.hpp"
#include "gc/g1/g1SurvRateGroup.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "logging/log.hpp"
#include "memory/allocation.hpp"
diff --git a/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp b/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
index dc821b135148f..fa86f02075deb 100644
--- a/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
+++ b/src/hotspot/share/gc/g1/g1SurvivorRegions.cpp
@@ -23,8 +23,8 @@
*/
#include "precompiled.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1SurvivorRegions.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/debug.hpp"
diff --git a/src/hotspot/share/gc/g1/g1Trace.cpp b/src/hotspot/share/gc/g1/g1Trace.cpp
index 453728f7e3dcd..598a238e93b92 100644
--- a/src/hotspot/share/gc/g1/g1Trace.cpp
+++ b/src/hotspot/share/gc/g1/g1Trace.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,7 +70,7 @@ static void register_jfr_type_constants() {
#endif
void G1NewTracer::initialize() {
- JFR_ONLY(register_jfr_type_constants());
+ JFR_ONLY(register_jfr_type_constants();)
}
void G1NewTracer::report_young_gc_pause(G1GCPauseType pause) {
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
index 377f5f550d17e..653402b82751e 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
@@ -34,12 +34,12 @@
#include "gc/g1/g1EvacFailureRegions.inline.hpp"
#include "gc/g1/g1EvacInfo.hpp"
#include "gc/g1/g1EvacStats.inline.hpp"
+#include "gc/g1/g1HeapRegion.inline.hpp"
+#include "gc/g1/g1HeapRegionRemSet.inline.hpp"
#include "gc/g1/g1OopClosures.inline.hpp"
#include "gc/g1/g1ParScanThreadState.hpp"
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
-#include "gc/g1/heapRegion.inline.hpp"
-#include "gc/g1/heapRegionRemSet.inline.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/preservedMarks.inline.hpp"
#include "jfr/jfrEvents.hpp"
diff --git a/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp b/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
index 13bbb22252cb9..30b9c6cb7a1a4 100644
--- a/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGenSizer.cpp
@@ -24,8 +24,8 @@
#include "precompiled.hpp"
#include "gc/g1/g1Arguments.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1YoungGenSizer.hpp"
-#include "gc/g1/heapRegion.hpp"
#include "logging/log.hpp"
#include "runtime/globals_extension.hpp"
diff --git a/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp b/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
index b04c02a3fc978..46c850cb643a4 100644
--- a/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
+++ b/src/hotspot/share/gc/g1/jvmFlagConstraintsG1.cpp
@@ -23,7 +23,7 @@
*/
#include "precompiled.hpp"
-#include "gc/g1/heapRegionBounds.inline.hpp"
+#include "gc/g1/g1HeapRegionBounds.inline.hpp"
#include "gc/g1/jvmFlagConstraintsG1.hpp"
#include "gc/shared/bufferNode.hpp"
#include "gc/shared/ptrQueue.hpp"
diff --git a/src/hotspot/share/gc/g1/vmStructs_g1.hpp b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
index 9ce61531989ee..62d91fe310099 100644
--- a/src/hotspot/share/gc/g1/vmStructs_g1.hpp
+++ b/src/hotspot/share/gc/g1/vmStructs_g1.hpp
@@ -26,8 +26,8 @@
#define SHARE_GC_G1_VMSTRUCTS_G1_HPP
#include "gc/g1/g1CollectedHeap.hpp"
-#include "gc/g1/heapRegion.hpp"
-#include "gc/g1/heapRegionManager.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
+#include "gc/g1/g1HeapRegionManager.hpp"
#include "utilities/macros.hpp"
#define VM_STRUCTS_G1GC(nonstatic_field, \
diff --git a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
index 0b8ff00e2cbcd..48dce98b55b71 100644
--- a/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
+++ b/src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
@@ -119,12 +119,6 @@ class ParallelScavengeHeap : public CollectedHeap {
_old_pool(nullptr),
_workers("GC Thread", ParallelGCThreads) { }
- // For use by VM operations
- enum CollectionType {
- Scavenge,
- MarkSweep
- };
-
Name kind() const override {
return CollectedHeap::Parallel;
}
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.cpp b/src/hotspot/share/gc/parallel/psCompactionManager.cpp
index e3b35db4bff52..fe9bcbff703b9 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.cpp
@@ -61,6 +61,7 @@ ParCompactionManager::ParCompactionManager() {
reset_bitmap_query_cache();
_deferred_obj_array = new (mtGC) GrowableArray(10, mtGC);
+ _marking_stats_cache = nullptr;
}
void ParCompactionManager::initialize(ParMarkBitMap* mbm) {
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.hpp b/src/hotspot/share/gc/parallel/psCompactionManager.hpp
index 458d33af74a87..b33ad06ee3ee1 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.hpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.hpp
@@ -111,7 +111,35 @@ class ParCompactionManager : public CHeapObj {
static RegionTaskQueueSet* region_task_queues() { return _region_task_queues; }
OopTaskQueue* oop_stack() { return &_oop_stack; }
- public:
+ // To collect per-region live-words in a worker local cache in order to
+ // reduce threads contention.
+ class MarkingStatsCache : public CHeapObj {
+ constexpr static size_t num_entries = 1024;
+ static_assert(is_power_of_2(num_entries), "inv");
+ static_assert(num_entries > 0, "inv");
+
+ constexpr static size_t entry_mask = num_entries - 1;
+
+ struct CacheEntry {
+ size_t region_id;
+ size_t live_words;
+ };
+
+ CacheEntry entries[num_entries] = {};
+
+ inline void push(size_t region_id, size_t live_words);
+
+ public:
+ inline void push(oop obj, size_t live_words);
+
+ inline void evict(size_t index);
+
+ inline void evict_all();
+ };
+
+ MarkingStatsCache* _marking_stats_cache;
+
+public:
static const size_t InvalidShadow = ~0;
static size_t pop_shadow_region_mt_safe(PSParallelCompact::RegionData* region_ptr);
static void push_shadow_region_mt_safe(size_t shadow_region);
@@ -198,6 +226,10 @@ class ParCompactionManager : public CHeapObj {
virtual void do_void();
};
+ inline void create_marking_stats_cache();
+
+ inline void flush_and_destroy_marking_stats_cache();
+
// Called after marking.
static void verify_all_marking_stack_empty() NOT_DEBUG_RETURN;
diff --git a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
index c0eb546022892..ae2e449e0b6ab 100644
--- a/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psCompactionManager.inline.hpp
@@ -107,6 +107,8 @@ inline void ParCompactionManager::mark_and_push(T* p) {
assert(ParallelScavengeHeap::heap()->is_in(obj), "should be in heap");
if (mark_bitmap()->is_unmarked(obj) && PSParallelCompact::mark_obj(obj)) {
+ assert(_marking_stats_cache != nullptr, "inv");
+ _marking_stats_cache->push(obj, obj->size());
push(obj);
if (StringDedup::is_enabled() &&
@@ -174,4 +176,73 @@ inline void ParCompactionManager::follow_contents(oop obj) {
}
}
+inline void ParCompactionManager::MarkingStatsCache::push(size_t region_id, size_t live_words) {
+ size_t index = (region_id & entry_mask);
+ if (entries[index].region_id == region_id) {
+ // Hit
+ entries[index].live_words += live_words;
+ return;
+ }
+ // Miss
+ if (entries[index].live_words != 0) {
+ evict(index);
+ }
+ entries[index].region_id = region_id;
+ entries[index].live_words = live_words;
+}
+
+inline void ParCompactionManager::MarkingStatsCache::push(oop obj, size_t live_words) {
+ ParallelCompactData& data = PSParallelCompact::summary_data();
+ const size_t region_size = ParallelCompactData::RegionSize;
+
+ HeapWord* addr = cast_from_oop(obj);
+ const size_t start_region_id = data.addr_to_region_idx(addr);
+ const size_t end_region_id = data.addr_to_region_idx(addr + live_words - 1);
+ if (start_region_id == end_region_id) {
+ // Completely inside this region
+ push(start_region_id, live_words);
+ return;
+ }
+
+ // First region
+ push(start_region_id, region_size - data.region_offset(addr));
+
+ // Middle regions; bypass cache
+ for (size_t i = start_region_id + 1; i < end_region_id; ++i) {
+ data.region(i)->set_partial_obj_size(region_size);
+ data.region(i)->set_partial_obj_addr(addr);
+ }
+
+ // Last region; bypass cache
+ const size_t end_offset = data.region_offset(addr + live_words - 1);
+ data.region(end_region_id)->set_partial_obj_size(end_offset + 1);
+ data.region(end_region_id)->set_partial_obj_addr(addr);
+}
+
+inline void ParCompactionManager::MarkingStatsCache::evict(size_t index) {
+ ParallelCompactData& data = PSParallelCompact::summary_data();
+ // flush to global data
+ data.region(entries[index].region_id)->add_live_obj(entries[index].live_words);
+}
+
+inline void ParCompactionManager::MarkingStatsCache::evict_all() {
+ for (size_t i = 0; i < num_entries; ++i) {
+ if (entries[i].live_words != 0) {
+ evict(i);
+ entries[i].live_words = 0;
+ }
+ }
+}
+
+inline void ParCompactionManager::create_marking_stats_cache() {
+ assert(_marking_stats_cache == nullptr, "precondition");
+ _marking_stats_cache = new MarkingStatsCache();
+}
+
+inline void ParCompactionManager::flush_and_destroy_marking_stats_cache() {
+ _marking_stats_cache->evict_all();
+ delete _marking_stats_cache;
+ _marking_stats_cache = nullptr;
+}
+
#endif // SHARE_GC_PARALLEL_PSCOMPACTIONMANAGER_INLINE_HPP
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
index e45b60851124f..9df0d7fb4d764 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -246,7 +246,7 @@ void PSParallelCompact::print_region_ranges() {
}
}
-void
+static void
print_generic_summary_region(size_t i, const ParallelCompactData::RegionData* c)
{
#define REGION_IDX_FORMAT SIZE_FORMAT_W(7)
@@ -312,7 +312,7 @@ print_generic_summary_data(ParallelCompactData& summary_data,
}
}
-void
+static void
print_initial_summary_data(ParallelCompactData& summary_data,
const MutableSpace* space) {
if (space->top() == space->bottom()) {
@@ -393,7 +393,7 @@ print_initial_summary_data(ParallelCompactData& summary_data,
max_reclaimed_ratio_region, max_dead_to_right, max_live_to_right, max_reclaimed_ratio);
}
-void
+static void
print_initial_summary_data(ParallelCompactData& summary_data,
SpaceInfo* space_info) {
if (!log_develop_is_enabled(Trace, gc, compaction)) {
@@ -1969,6 +1969,7 @@ class MarkFromRootsTask : public WorkerTask {
virtual void work(uint worker_id) {
ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(worker_id);
+ cm->create_marking_stats_cache();
PCMarkAndPushClosure mark_and_push_closure(cm);
{
@@ -2017,6 +2018,13 @@ class ParallelCompactRefProcProxyTask : public RefProcProxyTask {
}
};
+static void flush_marking_stats_cache(const uint num_workers) {
+ for (uint i = 0; i < num_workers; ++i) {
+ ParCompactionManager* cm = ParCompactionManager::gc_thread_compaction_manager(i);
+ cm->flush_and_destroy_marking_stats_cache();
+ }
+}
+
void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
// Recursively traverse all live objects and mark them
GCTraceTime(Info, gc, phases) tm("Marking Phase", &_gc_timer);
@@ -2046,6 +2054,12 @@ void PSParallelCompact::marking_phase(ParallelOldTracer *gc_tracer) {
pt.print_all_references();
}
+ {
+ GCTraceTime(Debug, gc, phases) tm("Flush Marking Stats", &_gc_timer);
+
+ flush_marking_stats_cache(active_gc_threads);
+ }
+
// This is the point where the entire marking should have completed.
ParCompactionManager::verify_all_marking_stack_empty();
diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
index ff6052d5070a7..bec95c0e13c13 100644
--- a/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psParallelCompact.inline.hpp
@@ -100,7 +100,6 @@ inline void PSParallelCompact::check_new_location(HeapWord* old_addr, HeapWord*
inline bool PSParallelCompact::mark_obj(oop obj) {
const size_t obj_size = obj->size();
if (mark_bitmap()->mark_obj(obj, obj_size)) {
- _summary_data.add_obj(obj, obj_size);
ContinuationGCSupport::transform_stack_chunk(obj);
return true;
} else {
diff --git a/src/hotspot/share/gc/serial/cardTableRS.cpp b/src/hotspot/share/gc/serial/cardTableRS.cpp
index 5d431b3eb1a05..87c0eba819298 100644
--- a/src/hotspot/share/gc/serial/cardTableRS.cpp
+++ b/src/hotspot/share/gc/serial/cardTableRS.cpp
@@ -32,27 +32,20 @@
#include "utilities/align.hpp"
void CardTableRS::scan_old_to_young_refs(TenuredSpace* sp) {
- verify_used_region_at_save_marks(sp);
-
+ const MemRegion ur = sp->used_region();
const MemRegion urasm = sp->used_region_at_save_marks();
- if (!urasm.is_empty()) {
- OldGenScanClosure cl(SerialHeap::heap()->young_gen());
- non_clean_card_iterate(sp, urasm, &cl);
- }
-}
-
-#ifdef ASSERT
-void CardTableRS::verify_used_region_at_save_marks(Space* sp) const {
- MemRegion ur = sp->used_region();
- MemRegion urasm = sp->used_region_at_save_marks();
assert(ur.contains(urasm),
"Did you forget to call save_marks()? "
"[" PTR_FORMAT ", " PTR_FORMAT ") is not contained in "
"[" PTR_FORMAT ", " PTR_FORMAT ")",
p2i(urasm.start()), p2i(urasm.end()), p2i(ur.start()), p2i(ur.end()));
+
+ if (!urasm.is_empty()) {
+ OldGenScanClosure cl(SerialHeap::heap()->young_gen());
+ non_clean_card_iterate(sp, urasm, &cl);
+ }
}
-#endif
void CardTableRS::maintain_old_to_young_invariant(TenuredGeneration* old_gen,
bool is_young_gen_empty) {
diff --git a/src/hotspot/share/gc/serial/cardTableRS.hpp b/src/hotspot/share/gc/serial/cardTableRS.hpp
index 545185fb54048..7ad6f8aef0b13 100644
--- a/src/hotspot/share/gc/serial/cardTableRS.hpp
+++ b/src/hotspot/share/gc/serial/cardTableRS.hpp
@@ -30,7 +30,6 @@
#include "oops/oop.hpp"
class OldGenScanClosure;
-class Space;
class TenuredGeneration;
class TenuredSpace;
@@ -63,8 +62,6 @@ class CardTableRS : public CardTable {
void scan_old_to_young_refs(TenuredSpace* sp);
- virtual void verify_used_region_at_save_marks(Space* sp) const NOT_DEBUG_RETURN;
-
void inline_write_ref_field_gc(void* field) {
CardValue* byte = byte_for(field);
*byte = dirty_card_val();
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index b85041c80a512..3f1cc7c6e4f93 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -335,9 +335,9 @@ void DefNewGeneration::compute_space_boundaries(uintx minimum_eden_size,
char *to_end = to_start + survivor_size;
assert(to_end == _virtual_space.high(), "just checking");
- assert(Space::is_aligned(eden_start), "checking alignment");
- assert(Space::is_aligned(from_start), "checking alignment");
- assert(Space::is_aligned(to_start), "checking alignment");
+ assert(is_aligned(eden_start, SpaceAlignment), "checking alignment");
+ assert(is_aligned(from_start, SpaceAlignment), "checking alignment");
+ assert(is_aligned(to_start, SpaceAlignment), "checking alignment");
MemRegion edenMR((HeapWord*)eden_start, (HeapWord*)from_start);
MemRegion fromMR((HeapWord*)from_start, (HeapWord*)to_start);
@@ -862,6 +862,9 @@ oop DefNewGeneration::copy_to_survivor_space(oop old) {
handle_promotion_failure(old);
return old;
}
+
+ ContinuationGCSupport::transform_stack_chunk(obj);
+
new_obj_is_tenured = true;
} else {
// Prefetch beyond obj
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.hpp b/src/hotspot/share/gc/serial/defNewGeneration.hpp
index 5bfce3c2eef03..ddc177f6ccc1e 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.hpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.hpp
@@ -201,9 +201,6 @@ class DefNewGeneration: public Generation {
// Return true if the expansion was successful.
bool expand(size_t bytes);
- // DefNewGeneration cannot currently expand except at
- // a GC.
- virtual bool is_maximal_no_gc() const { return true; }
// Iteration
void object_iterate(ObjectClosure* blk);
diff --git a/src/hotspot/share/gc/serial/genMarkSweep.cpp b/src/hotspot/share/gc/serial/genMarkSweep.cpp
index 41bd49edaf605..d0594a3a50037 100644
--- a/src/hotspot/share/gc/serial/genMarkSweep.cpp
+++ b/src/hotspot/share/gc/serial/genMarkSweep.cpp
@@ -30,7 +30,6 @@
#include "classfile/systemDictionary.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
#include "gc/serial/cardTableRS.hpp"
#include "gc/serial/defNewGeneration.hpp"
diff --git a/src/hotspot/share/gc/serial/generation.cpp b/src/hotspot/share/gc/serial/generation.cpp
index 0d1f6590b7545..c1c195aaf38bd 100644
--- a/src/hotspot/share/gc/serial/generation.cpp
+++ b/src/hotspot/share/gc/serial/generation.cpp
@@ -27,7 +27,6 @@
#include "gc/serial/generation.hpp"
#include "gc/serial/serialHeap.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
-#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/gcLocker.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
@@ -92,33 +91,3 @@ size_t Generation::max_contiguous_available() const {
}
return MAX2(avail, old_avail);
}
-
-// Ignores "ref" and calls allocate().
-oop Generation::promote(oop obj, size_t obj_size) {
- assert(obj_size == obj->size(), "bad obj_size passed in");
-
-#ifndef PRODUCT
- if (SerialHeap::heap()->promotion_should_fail()) {
- return nullptr;
- }
-#endif // #ifndef PRODUCT
-
- // Allocate new object.
- HeapWord* result = allocate(obj_size, false);
- if (result == nullptr) {
- // Promotion of obj into gen failed. Try to expand and allocate.
- result = expand_and_allocate(obj_size, false);
- if (result == nullptr) {
- return nullptr;
- }
- }
-
- // Copy to new location.
- Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size);
- oop new_obj = cast_to_oop(result);
-
- // Transform object if it is a stack chunk.
- ContinuationGCSupport::transform_stack_chunk(new_obj);
-
- return new_obj;
-}
diff --git a/src/hotspot/share/gc/serial/generation.hpp b/src/hotspot/share/gc/serial/generation.hpp
index c64b994d31766..e4545b56f44e5 100644
--- a/src/hotspot/share/gc/serial/generation.hpp
+++ b/src/hotspot/share/gc/serial/generation.hpp
@@ -105,12 +105,6 @@ class Generation: public CHeapObj {
// The largest number of contiguous free bytes in this or any higher generation.
virtual size_t max_contiguous_available() const;
- // Returns true if this generation cannot be expanded further
- // without a GC. Override as appropriate.
- virtual bool is_maximal_no_gc() const {
- return _virtual_space.uncommitted_size() == 0;
- }
-
MemRegion reserved() const { return _reserved; }
/* Returns "TRUE" iff "p" points into the reserved area of the generation. */
@@ -142,15 +136,6 @@ class Generation: public CHeapObj {
// Thread-local allocation buffers
virtual bool supports_tlab_allocation() const { return false; }
- // "obj" is the address of an object in a younger generation. Allocate space
- // for "obj" in the current (or some higher) generation, and copy "obj" into
- // the newly allocated space, if possible, returning the result (or null if
- // the allocation failed).
- //
- // The "obj_size" argument is just obj->size(), passed along so the caller can
- // avoid repeating the virtual call to retrieve it.
- virtual oop promote(oop obj, size_t obj_size);
-
// Returns "true" iff collect() should subsequently be called on this
// this generation. See comment below.
// This is a generic implementation which can be overridden.
diff --git a/src/hotspot/share/gc/serial/serialHeap.cpp b/src/hotspot/share/gc/serial/serialHeap.cpp
index 6d5ce55a1c853..2fd5dc1889aea 100644
--- a/src/hotspot/share/gc/serial/serialHeap.cpp
+++ b/src/hotspot/share/gc/serial/serialHeap.cpp
@@ -28,7 +28,6 @@
#include "classfile/symbolTable.hpp"
#include "classfile/vmSymbols.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "compiler/oopMap.hpp"
#include "gc/serial/cardTableRS.hpp"
#include "gc/serial/defNewGeneration.inline.hpp"
@@ -192,7 +191,7 @@ jint SerialHeap::initialize() {
ReservedSpace young_rs = heap_rs.first_part(MaxNewSize);
ReservedSpace old_rs = heap_rs.last_part(MaxNewSize);
- _rem_set = create_rem_set(heap_rs.region());
+ _rem_set = new CardTableRS(heap_rs.region());
_rem_set->initialize(young_rs.base(), old_rs.base());
CardTableBarrierSet *bs = new CardTableBarrierSet(_rem_set);
@@ -207,11 +206,6 @@ jint SerialHeap::initialize() {
return JNI_OK;
}
-
-CardTableRS* SerialHeap::create_rem_set(const MemRegion& reserved_region) {
- return new CardTableRS(reserved_region);
-}
-
ReservedHeapSpace SerialHeap::allocate(size_t alignment) {
// Now figure out the total size.
const size_t pageSize = UseLargePages ? os::large_page_size() : os::vm_page_size();
@@ -522,7 +516,7 @@ void SerialHeap::do_collection(bool full,
print_heap_before_gc();
- if (run_verification && VerifyGCLevel <= 0 && VerifyBeforeGC) {
+ if (run_verification && VerifyBeforeGC) {
prepare_for_verify();
prepared_for_verification = true;
}
@@ -534,7 +528,7 @@ void SerialHeap::do_collection(bool full,
full,
size,
is_tlab,
- run_verification && VerifyGCLevel <= 0,
+ run_verification,
do_clear_all_soft_refs);
if (size > 0 && (!is_tlab || _young_gen->supports_tlab_allocation()) &&
@@ -572,8 +566,7 @@ void SerialHeap::do_collection(bool full,
print_heap_before_gc();
- if (!prepared_for_verification && run_verification &&
- VerifyGCLevel <= 1 && VerifyBeforeGC) {
+ if (!prepared_for_verification && run_verification && VerifyBeforeGC) {
prepare_for_verify();
}
@@ -599,7 +592,7 @@ void SerialHeap::do_collection(bool full,
full,
size,
is_tlab,
- run_verification && VerifyGCLevel <= 1,
+ run_verification,
do_clear_all_soft_refs);
CodeCache::on_gc_marking_cycle_finish();
@@ -952,19 +945,9 @@ void SerialHeap::prepare_for_verify() {
ensure_parsability(false); // no need to retire TLABs
}
-void SerialHeap::generation_iterate(GenClosure* cl,
- bool old_to_young) {
- if (old_to_young) {
- cl->do_generation(_old_gen);
- cl->do_generation(_young_gen);
- } else {
- cl->do_generation(_young_gen);
- cl->do_generation(_old_gen);
- }
-}
-
bool SerialHeap::is_maximal_no_gc() const {
- return _young_gen->is_maximal_no_gc() && _old_gen->is_maximal_no_gc();
+ // We don't expand young-gen except at a GC.
+ return _old_gen->is_maximal_no_gc();
}
void SerialHeap::save_marks() {
@@ -1039,8 +1022,6 @@ void SerialHeap::print_heap_change(const PreGenGCValues& pre_gc_values) const {
}
void SerialHeap::gc_prologue(bool full) {
- assert(InlineCacheBuffer::is_empty(), "should have cleaned up ICBuffer");
-
// Fill TLAB's and such
ensure_parsability(true); // retire TLABs
@@ -1061,18 +1042,10 @@ void SerialHeap::gc_epilogue(bool full) {
};
#ifndef PRODUCT
-class GenGCSaveTopsBeforeGCClosure: public SerialHeap::GenClosure {
- private:
- public:
- void do_generation(Generation* gen) {
- gen->record_spaces_top();
- }
-};
-
void SerialHeap::record_gen_tops_before_GC() {
if (ZapUnusedHeapArea) {
- GenGCSaveTopsBeforeGCClosure blk;
- generation_iterate(&blk, false); // not old-to-young.
+ _young_gen->record_spaces_top();
+ _old_gen->record_spaces_top();
}
}
#endif // not PRODUCT
diff --git a/src/hotspot/share/gc/serial/serialHeap.hpp b/src/hotspot/share/gc/serial/serialHeap.hpp
index 2b04d04eaea6d..f1f2e5a1cc1a7 100644
--- a/src/hotspot/share/gc/serial/serialHeap.hpp
+++ b/src/hotspot/share/gc/serial/serialHeap.hpp
@@ -141,7 +141,6 @@ class SerialHeap : public CollectedHeap {
public:
// Returns JNI_OK on success
jint initialize() override;
- virtual CardTableRS* create_rem_set(const MemRegion& reserved_region);
// Does operations required after initialization has been done.
void post_initialize() override;
@@ -249,10 +248,6 @@ class SerialHeap : public CollectedHeap {
virtual void do_generation(Generation* gen) = 0;
};
- // Apply "cl.do_generation" to all generations in the heap
- // If "old_to_young" determines the order.
- void generation_iterate(GenClosure* cl, bool old_to_young);
-
// Return "true" if all generations have reached the
// maximal committed limit that they can reach, without a garbage
// collection.
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.cpp b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
index 954b063336d9c..fe37d9975498a 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.cpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.cpp
@@ -413,6 +413,31 @@ bool TenuredGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes)
return res;
}
+oop TenuredGeneration::promote(oop obj, size_t obj_size) {
+ assert(obj_size == obj->size(), "bad obj_size passed in");
+
+#ifndef PRODUCT
+ if (SerialHeap::heap()->promotion_should_fail()) {
+ return nullptr;
+ }
+#endif // #ifndef PRODUCT
+
+ // Allocate new object.
+ HeapWord* result = allocate(obj_size, false);
+ if (result == nullptr) {
+ // Promotion of obj into gen failed. Try to expand and allocate.
+ result = expand_and_allocate(obj_size, false);
+ if (result == nullptr) {
+ return nullptr;
+ }
+ }
+
+ // Copy to new location.
+ Copy::aligned_disjoint_words(cast_from_oop(obj), result, obj_size);
+ oop new_obj = cast_to_oop(result);
+ return new_obj;
+}
+
void TenuredGeneration::collect(bool full,
bool clear_all_soft_refs,
size_t size,
diff --git a/src/hotspot/share/gc/serial/tenuredGeneration.hpp b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
index 4e506343113a7..1b7347161e73e 100644
--- a/src/hotspot/share/gc/serial/tenuredGeneration.hpp
+++ b/src/hotspot/share/gc/serial/tenuredGeneration.hpp
@@ -98,6 +98,12 @@ class TenuredGeneration: public Generation {
MemRegion prev_used_region() const { return _prev_used_region; }
void save_used_region() { _prev_used_region = used_region(); }
+ // Returns true if this generation cannot be expanded further
+ // without a GC.
+ bool is_maximal_no_gc() const {
+ return _virtual_space.uncommitted_size() == 0;
+ }
+
HeapWord* block_start(const void* p) const;
void scan_old_to_young_refs();
@@ -160,6 +166,14 @@ class TenuredGeneration: public Generation {
// might be attempted in the worst case.
bool promotion_attempt_is_safe(size_t max_promoted_in_bytes) const;
+ // "obj" is the address of an object in young-gen. Allocate space for "obj"
+ // in the old-gen, and copy "obj" into the newly allocated space, if
+ // possible, returning the result (or null if the allocation failed).
+ //
+ // The "obj_size" argument is just obj->size(), passed along so the caller can
+ // avoid repeating the virtual call to retrieve it.
+ oop promote(oop obj, size_t obj_size);
+
virtual void verify();
virtual void print_on(outputStream* st) const;
};
diff --git a/src/hotspot/share/gc/serial/vmStructs_serial.hpp b/src/hotspot/share/gc/serial/vmStructs_serial.hpp
index f4941401314ec..9f646424100db 100644
--- a/src/hotspot/share/gc/serial/vmStructs_serial.hpp
+++ b/src/hotspot/share/gc/serial/vmStructs_serial.hpp
@@ -32,6 +32,13 @@
#define VM_STRUCTS_SERIALGC(nonstatic_field, \
volatile_nonstatic_field, \
static_field) \
+ nonstatic_field(Generation, _reserved, MemRegion) \
+ nonstatic_field(Generation, _virtual_space, VirtualSpace) \
+ nonstatic_field(Generation, _stat_record, Generation::StatRecord) \
+ \
+ nonstatic_field(Generation::StatRecord, invocations, int) \
+ nonstatic_field(Generation::StatRecord, accumulated_time, elapsedTimer) \
+ \
nonstatic_field(TenuredGeneration, _rs, CardTableRS*) \
nonstatic_field(TenuredGeneration, _bts, SerialBlockOffsetSharedArray*) \
nonstatic_field(TenuredGeneration, _shrink_factor, size_t) \
@@ -61,6 +68,8 @@
#define VM_TYPES_SERIALGC(declare_type, \
declare_toplevel_type, \
declare_integer_type) \
+ declare_toplevel_type(Generation) \
+ declare_toplevel_type(Generation::StatRecord) \
declare_type(SerialHeap, CollectedHeap) \
declare_type(TenuredGeneration, Generation) \
declare_type(TenuredSpace, ContiguousSpace) \
@@ -73,6 +82,8 @@
declare_toplevel_type(SerialBlockOffsetTable)
#define VM_INT_CONSTANTS_SERIALGC(declare_constant, \
- declare_constant_with_value)
+ declare_constant_with_value) \
+ declare_constant(Generation::LogOfGenGrain) \
+ declare_constant(Generation::GenGrain)
#endif // SHARE_GC_SERIAL_VMSTRUCTS_SERIAL_HPP
diff --git a/src/hotspot/share/gc/shared/collectedHeap.cpp b/src/hotspot/share/gc/shared/collectedHeap.cpp
index dbcef1ba575a9..02d20b92b5ce7 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.cpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -402,6 +402,13 @@ void CollectedHeap::set_gc_cause(GCCause::Cause v) {
_gc_cause = v;
}
+// Returns the header size in words aligned to the requirements of the
+// array object type.
+static int int_array_header_size() {
+ size_t typesize_in_bytes = arrayOopDesc::header_size_in_bytes();
+ return (int)align_up(typesize_in_bytes, HeapWordSize)/HeapWordSize;
+}
+
size_t CollectedHeap::max_tlab_size() const {
// TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE].
// This restriction could be removed by enabling filling with multiple arrays.
@@ -411,14 +418,14 @@ size_t CollectedHeap::max_tlab_size() const {
// We actually lose a little by dividing first,
// but that just makes the TLAB somewhat smaller than the biggest array,
// which is fine, since we'll be able to fill that.
- size_t max_int_size = typeArrayOopDesc::header_size(T_INT) +
+ size_t max_int_size = int_array_header_size() +
sizeof(jint) *
((juint) max_jint / (size_t) HeapWordSize);
return align_down(max_int_size, MinObjAlignment);
}
size_t CollectedHeap::filler_array_hdr_size() {
- return align_object_offset(arrayOopDesc::header_size(T_INT)); // align to Long
+ return align_object_offset(int_array_header_size()); // align to Long
}
size_t CollectedHeap::filler_array_min_size() {
diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp
index b503d38c5051e..1562a4b43ad04 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.hpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.hpp
@@ -46,9 +46,6 @@
// class defines the functions that a heap must implement, and contains
// infrastructure common to all heaps.
-class WorkerTask;
-class AdaptiveSizePolicy;
-class BarrierSet;
class GCHeapLog;
class GCHeapSummary;
class GCTimer;
diff --git a/src/hotspot/share/gc/shared/gcId.cpp b/src/hotspot/share/gc/shared/gcId.cpp
index 646de8339104c..8d123386180cb 100644
--- a/src/hotspot/share/gc/shared/gcId.cpp
+++ b/src/hotspot/share/gc/shared/gcId.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -43,7 +43,7 @@ void GCId::set_printer(GCIdPrinter* printer) {
_printer = printer;
}
-NamedThread* currentNamedthread() {
+static NamedThread* currentNamedthread() {
assert(Thread::current()->is_Named_thread(), "This thread must be NamedThread");
return (NamedThread*)Thread::current();
}
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index 652153922d10e..25e581cae6784 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -213,13 +213,13 @@
\
/* where does the range max value of (max_jint - 1) come from? */ \
product(size_t, MarkStackSizeMax, NOT_LP64(4*M) LP64_ONLY(512*M), \
- "Maximum size of marking stack") \
- range(1, (INT_MAX - 1)) \
+ "Maximum size of marking stack in bytes.") \
+ range(1, (INT_MAX - 1)) \
\
product(size_t, MarkStackSize, NOT_LP64(64*K) LP64_ONLY(4*M), \
- "Size of marking stack") \
+ "Size of marking stack in bytes.") \
constraint(MarkStackSizeConstraintFunc,AfterErgo) \
- range(1, (INT_MAX - 1)) \
+ range(1, (INT_MAX - 1)) \
\
product(bool, ParallelRefProcEnabled, false, \
"Enable parallel reference processing whenever possible") \
@@ -345,10 +345,6 @@
product(bool, UseAdaptiveSizePolicyWithSystemGC, false, \
"Include statistics from System.gc() for adaptive size policy") \
\
- develop(intx, PSAdaptiveSizePolicyResizeVirtualSpaceAlot, -1, \
- "Resize the virtual spaces of the young or old generations") \
- range(-1, 1) \
- \
product(uint, AdaptiveSizeThroughPutPolicy, 0, \
"Policy for changing generation size for throughput goals") \
range(0, 1) \
@@ -617,10 +613,6 @@
"GC invoke count where +VerifyBefore/AfterGC kicks in") \
range(0, max_uintx) \
\
- product(int, VerifyGCLevel, 0, DIAGNOSTIC, \
- "Generation level at which to start +VerifyBefore/AfterGC") \
- range(0, 1) \
- \
product(uint, MaxTenuringThreshold, 15, \
"Maximum value for tenuring threshold") \
range(0, markWord::max_age + 1) \
diff --git a/src/hotspot/share/gc/shared/memAllocator.cpp b/src/hotspot/share/gc/shared/memAllocator.cpp
index e4b91413537bd..dc94b83c13998 100644
--- a/src/hotspot/share/gc/shared/memAllocator.cpp
+++ b/src/hotspot/share/gc/shared/memAllocator.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,7 @@
#include "services/lowMemoryDetector.hpp"
#include "utilities/align.hpp"
#include "utilities/copy.hpp"
+#include "utilities/globalDefinitions.hpp"
class MemAllocator::Allocation: StackObj {
friend class MemAllocator;
@@ -408,11 +409,30 @@ oop ObjArrayAllocator::initialize(HeapWord* mem) const {
assert(_length >= 0, "length should be non-negative");
if (_do_zero) {
mem_clear(mem);
+ mem_zap_end_padding(mem);
}
arrayOopDesc::set_length(mem, _length);
return finish(mem);
}
+#ifndef PRODUCT
+void ObjArrayAllocator::mem_zap_end_padding(HeapWord* mem) const {
+ const size_t length_in_bytes = static_cast(_length) << ArrayKlass::cast(_klass)->log2_element_size();
+ const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
+ const size_t base_offset_in_bytes = arrayOopDesc::base_offset_in_bytes(element_type);
+ const size_t size_in_bytes = _word_size * BytesPerWord;
+
+ const address obj_end = reinterpret_cast(mem) + size_in_bytes;
+ const address base = reinterpret_cast(mem) + base_offset_in_bytes;
+ const address elements_end = base + length_in_bytes;
+ assert(elements_end <= obj_end, "payload must fit in object");
+ if (elements_end < obj_end) {
+ const size_t padding_in_bytes = obj_end - elements_end;
+ Copy::fill_to_bytes(elements_end, padding_in_bytes, heapPaddingByteVal);
+ }
+}
+#endif
+
oop ClassAllocator::initialize(HeapWord* mem) const {
// Set oop_size field before setting the _klass field because a
// non-null _klass field indicates that the object is parsable by
diff --git a/src/hotspot/share/gc/shared/memAllocator.hpp b/src/hotspot/share/gc/shared/memAllocator.hpp
index 93ed10dce0555..bd4ae04babfcd 100644
--- a/src/hotspot/share/gc/shared/memAllocator.hpp
+++ b/src/hotspot/share/gc/shared/memAllocator.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -98,6 +98,8 @@ class ObjArrayAllocator: public MemAllocator {
const int _length;
const bool _do_zero;
+ void mem_zap_end_padding(HeapWord* mem) const PRODUCT_RETURN;
+
public:
ObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero,
Thread* thread = Thread::current())
diff --git a/src/hotspot/share/gc/shared/parallelCleaning.cpp b/src/hotspot/share/gc/shared/parallelCleaning.cpp
index 396f7e6c31e1f..ef82c8b676ef1 100644
--- a/src/hotspot/share/gc/shared/parallelCleaning.cpp
+++ b/src/hotspot/share/gc/shared/parallelCleaning.cpp
@@ -47,7 +47,6 @@ CodeCacheUnloadingTask::CodeCacheUnloadingTask(uint num_workers, bool unloading_
CodeCacheUnloadingTask::~CodeCacheUnloadingTask() {
CodeCache::verify_clean_inline_caches();
- CodeCache::verify_icholder_relocations();
}
void CodeCacheUnloadingTask::claim_nmethods(CompiledMethod** claimed_nmethods, int *num_claimed_nmethods) {
diff --git a/src/hotspot/share/gc/shared/space.cpp b/src/hotspot/share/gc/shared/space.cpp
index 069b672b54426..7d141710d47e1 100644
--- a/src/hotspot/share/gc/shared/space.cpp
+++ b/src/hotspot/share/gc/shared/space.cpp
@@ -40,7 +40,9 @@
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
-ContiguousSpace::ContiguousSpace(): Space(),
+ContiguousSpace::ContiguousSpace():
+ _bottom(nullptr),
+ _end(nullptr),
_next_compaction_space(nullptr),
_top(nullptr) {
_mangler = new GenSpaceMangler(this);
@@ -75,9 +77,6 @@ void ContiguousSpace::clear(bool mangle_space) {
#ifndef PRODUCT
-void ContiguousSpace::set_top_for_allocations(HeapWord* v) {
- mangler()->set_top_for_allocations(v);
-}
void ContiguousSpace::set_top_for_allocations() {
mangler()->set_top_for_allocations(top());
}
@@ -100,36 +99,14 @@ void ContiguousSpace::mangle_unused_area_complete() {
}
#endif // NOT_PRODUCT
-
-void Space::print_short() const { print_short_on(tty); }
-
-void Space::print_short_on(outputStream* st) const {
- st->print(" space " SIZE_FORMAT "K, %3d%% used", capacity() / K,
- (int) ((double) used() * 100 / capacity()));
-}
-
-void Space::print() const { print_on(tty); }
-
-void Space::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(end()));
-}
+void ContiguousSpace::print() const { print_on(tty); }
void ContiguousSpace::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(top()), p2i(end()));
+ st->print_cr(" space %zuK, %3d%% used [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
+ capacity() / K, (int) ((double) used() * 100 / capacity()),
+ p2i(bottom()), p2i(top()), p2i(end()));
}
-#if INCLUDE_SERIALGC
-void TenuredSpace::print_on(outputStream* st) const {
- print_short_on(st);
- st->print_cr(" [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(bottom()), p2i(top()), p2i(end()));
-}
-#endif
-
void ContiguousSpace::verify() const {
HeapWord* p = bottom();
HeapWord* t = top();
@@ -168,26 +145,6 @@ HeapWord* ContiguousSpace::block_start_const(const void* p) const {
}
}
-size_t ContiguousSpace::block_size(const HeapWord* p) const {
- assert(MemRegion(bottom(), end()).contains(p),
- "p (" PTR_FORMAT ") not in space [" PTR_FORMAT ", " PTR_FORMAT ")",
- p2i(p), p2i(bottom()), p2i(end()));
- HeapWord* current_top = top();
- assert(p <= current_top,
- "p > current top - p: " PTR_FORMAT ", current top: " PTR_FORMAT,
- p2i(p), p2i(current_top));
- assert(p == current_top || oopDesc::is_oop(cast_to_oop(p)),
- "p (" PTR_FORMAT ") is not a block start - "
- "current_top: " PTR_FORMAT ", is_oop: %s",
- p2i(p), p2i(current_top), BOOL_TO_STR(oopDesc::is_oop(cast_to_oop(p))));
- if (p < current_top) {
- return cast_to_oop(p)->size();
- } else {
- assert(p == current_top, "just checking");
- return pointer_delta(end(), (HeapWord*) p);
- }
-}
-
// This version requires locking.
inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
assert(Heap_lock->owned_by_self() ||
@@ -197,7 +154,7 @@ inline HeapWord* ContiguousSpace::allocate_impl(size_t size) {
if (pointer_delta(end(), obj) >= size) {
HeapWord* new_top = obj + size;
set_top(new_top);
- assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
+ assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
return obj;
} else {
return nullptr;
@@ -215,7 +172,7 @@ inline HeapWord* ContiguousSpace::par_allocate_impl(size_t size) {
// the old top value: the exchange succeeded
// otherwise: the new value of the top is returned.
if (result == obj) {
- assert(is_aligned(obj) && is_aligned(new_top), "checking alignment");
+ assert(is_object_aligned(obj) && is_object_aligned(new_top), "checking alignment");
return obj;
}
} else {
diff --git a/src/hotspot/share/gc/shared/space.hpp b/src/hotspot/share/gc/shared/space.hpp
index 8e6f4f88a6bad..6eb8fe9818c6b 100644
--- a/src/hotspot/share/gc/shared/space.hpp
+++ b/src/hotspot/share/gc/shared/space.hpp
@@ -45,47 +45,53 @@
// for iterating over objects and free blocks, etc.
// Forward decls.
-class Space;
class ContiguousSpace;
class Generation;
class ContiguousSpace;
class CardTableRS;
class DirtyCardToOopClosure;
+class GenSpaceMangler;
-// A Space describes a heap area. Class Space is an abstract
-// base class.
-//
-// Space supports allocation, size computation and GC support is provided.
+// A space in which the free area is contiguous. It therefore supports
+// faster allocation, and compaction.
//
// Invariant: bottom() and end() are on page_size boundaries and
// bottom() <= top() <= end()
// top() is inclusive and end() is exclusive.
-
-class Space: public CHeapObj {
+class ContiguousSpace: public CHeapObj {
friend class VMStructs;
- protected:
+
+private:
HeapWord* _bottom;
HeapWord* _end;
// Used in support of save_marks()
HeapWord* _saved_mark_word;
- Space():
- _bottom(nullptr), _end(nullptr) { }
+ ContiguousSpace* _next_compaction_space;
+
+ HeapWord* _top;
+ // A helper for mangling the unused area of the space in debug builds.
+ GenSpaceMangler* _mangler;
+
+ GenSpaceMangler* mangler() { return _mangler; }
+
+ // Allocation helpers (return null if full).
+ inline HeapWord* allocate_impl(size_t word_size);
+ inline HeapWord* par_allocate_impl(size_t word_size);
+
+public:
+ ContiguousSpace();
+ ~ContiguousSpace();
- public:
// Accessors
HeapWord* bottom() const { return _bottom; }
HeapWord* end() const { return _end; }
- virtual void set_bottom(HeapWord* value) { _bottom = value; }
- virtual void set_end(HeapWord* value) { _end = value; }
+ void set_bottom(HeapWord* value) { _bottom = value; }
+ void set_end(HeapWord* value) { _end = value; }
HeapWord* saved_mark_word() const { return _saved_mark_word; }
- // Returns a subregion of the space containing only the allocated objects in
- // the space.
- virtual MemRegion used_region() const = 0;
-
// Returns a region that is guaranteed to contain (at least) all objects
// allocated at the time of the last call to "save_marks". If the space
// initializes its DirtyCardToOopClosure's specifying the "contig" option
@@ -98,13 +104,6 @@ class Space: public CHeapObj {
return MemRegion(bottom(), saved_mark_word());
}
- // For detecting GC bugs. Should only be called at GC boundaries, since
- // some unused space may be used as scratch space during GC's.
- // We also call this when expanding a space to satisfy an allocation
- // request. See bug #4668531
- virtual void mangle_unused_area() = 0;
- virtual void mangle_unused_area_complete() = 0;
-
// Testers
bool is_empty() const { return used() == 0; }
@@ -122,64 +121,13 @@ class Space: public CHeapObj {
// given address.
bool is_in_reserved(const void* p) const { return _bottom <= p && p < _end; }
- // Test whether p is double-aligned
- static bool is_aligned(void* p) {
- return ::is_aligned(p, sizeof(double));
- }
-
// Size computations. Sizes are in bytes.
- size_t capacity() const { return byte_size(bottom(), end()); }
- virtual size_t used() const = 0;
- virtual size_t free() const = 0;
-
- // If "p" is in the space, returns the address of the start of the
- // "block" that contains "p". We say "block" instead of "object" since
- // some heaps may not pack objects densely; a chunk may either be an
- // object or a non-object. If "p" is not in the space, return null.
- virtual HeapWord* block_start_const(const void* p) const = 0;
-
- // Requires "addr" to be the start of a chunk, and returns its size.
- // "addr + size" is required to be the start of a new chunk, or the end
- // of the active area of the heap.
- virtual size_t block_size(const HeapWord* addr) const = 0;
-
- // Allocation (return null if full). Assumes the caller has established
- // mutually exclusive access to the space.
- virtual HeapWord* allocate(size_t word_size) = 0;
-
- // Allocation (return null if full). Enforces mutual exclusion internally.
- virtual HeapWord* par_allocate(size_t word_size) = 0;
+ size_t capacity() const { return byte_size(bottom(), end()); }
+ size_t used() const { return byte_size(bottom(), top()); }
+ size_t free() const { return byte_size(top(), end()); }
void print() const;
- virtual void print_on(outputStream* st) const;
- void print_short() const;
- void print_short_on(outputStream* st) const;
-};
-
-class GenSpaceMangler;
-
-// A space in which the free area is contiguous. It therefore supports
-// faster allocation, and compaction.
-class ContiguousSpace: public Space {
- friend class VMStructs;
-
-private:
- ContiguousSpace* _next_compaction_space;
-
-protected:
- HeapWord* _top;
- // A helper for mangling the unused area of the space in debug builds.
- GenSpaceMangler* _mangler;
-
- GenSpaceMangler* mangler() { return _mangler; }
-
- // Allocation helpers (return null if full).
- inline HeapWord* allocate_impl(size_t word_size);
- inline HeapWord* par_allocate_impl(size_t word_size);
-
- public:
- ContiguousSpace();
- ~ContiguousSpace();
+ void print_on(outputStream* st) const;
// Initialization.
// "initialize" should be called once on a space, before it is used for
@@ -216,19 +164,18 @@ class ContiguousSpace: public Space {
bool saved_mark_at_top() const { return saved_mark_word() == top(); }
- // In debug mode mangle (write it with a particular bit
- // pattern) the unused part of a space.
-
- // Used to save the address in a space for later use during mangling.
- void set_top_for_allocations(HeapWord* v) PRODUCT_RETURN;
// Used to save the space's current top for later use during mangling.
void set_top_for_allocations() PRODUCT_RETURN;
+ // For detecting GC bugs. Should only be called at GC boundaries, since
+ // some unused space may be used as scratch space during GC's.
+ // We also call this when expanding a space to satisfy an allocation
+ // request. See bug #4668531
// Mangle regions in the space from the current top up to the
// previously mangled part of the space.
- void mangle_unused_area() override PRODUCT_RETURN;
+ void mangle_unused_area() PRODUCT_RETURN;
// Mangle [top, end)
- void mangle_unused_area_complete() override PRODUCT_RETURN;
+ void mangle_unused_area_complete() PRODUCT_RETURN;
// Do some sparse checking on the area that should have been mangled.
void check_mangled_unused_area(HeapWord* limit) PRODUCT_RETURN;
@@ -236,17 +183,13 @@ class ContiguousSpace: public Space {
// This code may be null depending on the macro DEBUG_MANGLING.
void check_mangled_unused_area_complete() PRODUCT_RETURN;
- // Size computations: sizes in bytes.
- size_t used() const override { return byte_size(bottom(), top()); }
- size_t free() const override { return byte_size(top(), end()); }
+ MemRegion used_region() const { return MemRegion(bottom(), top()); }
- // In a contiguous space we have a more obvious bound on what parts
- // contain objects.
- MemRegion used_region() const override { return MemRegion(bottom(), top()); }
-
- // Allocation (return null if full)
- HeapWord* allocate(size_t word_size) override;
- HeapWord* par_allocate(size_t word_size) override;
+ // Allocation (return null if full). Assumes the caller has established
+ // mutually exclusive access to the space.
+ virtual HeapWord* allocate(size_t word_size);
+ // Allocation (return null if full). Enforces mutual exclusion internally.
+ virtual HeapWord* par_allocate(size_t word_size);
// Iteration
void object_iterate(ObjectClosure* blk);
@@ -261,15 +204,15 @@ class ContiguousSpace: public Space {
template
void oop_since_save_marks_iterate(OopClosureType* blk);
- // Very inefficient implementation.
- HeapWord* block_start_const(const void* p) const override;
- size_t block_size(const HeapWord* p) const override;
+ // If "p" is in the space, returns the address of the start of the
+ // "block" that contains "p". We say "block" instead of "object" since
+ // some heaps may not pack objects densely; a chunk may either be an
+ // object or a non-object. If "p" is not in the space, return null.
+ virtual HeapWord* block_start_const(const void* p) const;
// Addresses for inlined allocation
HeapWord** top_addr() { return &_top; }
- void print_on(outputStream* st) const override;
-
// Debugging
void verify() const;
};
@@ -298,8 +241,6 @@ class TenuredSpace: public ContiguousSpace {
inline HeapWord* par_allocate(size_t word_size) override;
inline void update_for_block(HeapWord* start, HeapWord* end);
-
- void print_on(outputStream* st) const override;
};
#endif //INCLUDE_SERIALGC
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index 69bb8c848e812..ae9843c31eb6f 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -98,21 +98,13 @@
nonstatic_field(CollectedHeap, _is_gc_active, bool) \
nonstatic_field(CollectedHeap, _total_collections, unsigned int) \
\
+ nonstatic_field(ContiguousSpace, _bottom, HeapWord*) \
+ nonstatic_field(ContiguousSpace, _end, HeapWord*) \
nonstatic_field(ContiguousSpace, _top, HeapWord*) \
nonstatic_field(ContiguousSpace, _saved_mark_word, HeapWord*) \
\
- nonstatic_field(Generation, _reserved, MemRegion) \
- nonstatic_field(Generation, _virtual_space, VirtualSpace) \
- nonstatic_field(Generation, _stat_record, Generation::StatRecord) \
- \
- nonstatic_field(Generation::StatRecord, invocations, int) \
- nonstatic_field(Generation::StatRecord, accumulated_time, elapsedTimer) \
- \
nonstatic_field(MemRegion, _start, HeapWord*) \
- nonstatic_field(MemRegion, _word_size, size_t) \
- \
- nonstatic_field(Space, _bottom, HeapWord*) \
- nonstatic_field(Space, _end, HeapWord*)
+ nonstatic_field(MemRegion, _word_size, size_t)
#define VM_TYPES_GC(declare_type, \
declare_toplevel_type, \
@@ -142,9 +134,7 @@
/******************************************/ \
\
declare_toplevel_type(CollectedHeap) \
- declare_toplevel_type(Generation) \
- declare_toplevel_type(Space) \
- declare_type(ContiguousSpace, Space) \
+ declare_toplevel_type(ContiguousSpace) \
declare_toplevel_type(BarrierSet) \
declare_type(ModRefBarrierSet, BarrierSet) \
declare_type(CardTableBarrierSet, ModRefBarrierSet) \
@@ -155,7 +145,6 @@
\
declare_toplevel_type(AgeTable) \
declare_toplevel_type(CardTable::CardValue) \
- declare_toplevel_type(Generation::StatRecord) \
declare_toplevel_type(HeapWord) \
declare_toplevel_type(MemRegion) \
declare_toplevel_type(ThreadLocalAllocBuffer) \
@@ -173,7 +162,6 @@
declare_toplevel_type(HeapWord*) \
declare_toplevel_type(HeapWord* volatile) \
declare_toplevel_type(MemRegion*) \
- declare_toplevel_type(Space*) \
declare_toplevel_type(ThreadLocalAllocBuffer*) \
\
declare_toplevel_type(BarrierSet::FakeRtti)
@@ -212,9 +200,6 @@
declare_constant(CollectedHeap::Serial) \
declare_constant(CollectedHeap::Parallel) \
declare_constant(CollectedHeap::G1) \
- \
- declare_constant(Generation::LogOfGenGrain) \
- declare_constant(Generation::GenGrain) \
#define VM_LONG_CONSTANTS_GC(declare_constant) \
ZGC_ONLY(VM_LONG_CONSTANTS_Z_SHARED(declare_constant))
diff --git a/src/hotspot/share/gc/shared/weakProcessor.cpp b/src/hotspot/share/gc/shared/weakProcessor.cpp
index 02ef9bbe88da8..8adf3fab7d5c1 100644
--- a/src/hotspot/share/gc/shared/weakProcessor.cpp
+++ b/src/hotspot/share/gc/shared/weakProcessor.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,7 @@
#include "prims/jvmtiTagMap.hpp"
#endif // INCLUDE_JVMTI
-void notify_jvmti_tagmaps() {
+static void notify_jvmti_tagmaps() {
#if INCLUDE_JVMTI
// Notify JVMTI tagmaps that a STW weak reference processing might be
// clearing entries, so the tagmaps need cleaning. Doing this here allows
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
index c9c566ad611b6..14fb038a6c6a8 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp
@@ -1734,11 +1734,26 @@ bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoo
return true;
}
+bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) {
+ for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) {
+ Node* n = region->fast_out(i);
+ if (n->is_LoadStore()) {
+ // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support
+ // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes
+ // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get
+ // expanded around a LoadStore node.
+ return false;
+ }
+ }
+ return true;
+}
+
+
void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) {
assert(is_heap_stable_test(n), "no other tests");
if (identical_backtoback_ifs(n, phase)) {
Node* n_ctrl = n->in(0);
- if (phase->can_split_if(n_ctrl)) {
+ if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) {
IfNode* dom_if = phase->idom(n_ctrl)->as_If();
if (is_heap_stable_test(n)) {
Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1);
diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
index 032f338aa8895..7a6ed74f56393 100644
--- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
+++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp
@@ -65,6 +65,7 @@ class ShenandoahBarrierC2Support : public AllStatic {
static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase);
static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase);
static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase);
+ static bool merge_point_safe(Node* region);
static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase);
static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase);
static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase);
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
index c5b6d787b95a8..c6352b2749c78 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCodeRoots.cpp
@@ -24,7 +24,6 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
#include "gc/shenandoah/shenandoahClosures.inline.hpp"
@@ -128,26 +127,17 @@ void ShenandoahCodeRoots::disarm_nmethods() {
class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
private:
bool _unloading_occurred;
- volatile bool _failed;
ShenandoahHeap* const _heap;
BarrierSetNMethod* const _bs;
- void set_failed() {
- Atomic::store(&_failed, true);
- }
-
public:
ShenandoahNMethodUnlinkClosure(bool unloading_occurred) :
_unloading_occurred(unloading_occurred),
- _failed(false),
_heap(ShenandoahHeap::heap()),
_bs(ShenandoahBarrierSet::barrier_set()->barrier_set_nmethod()) {}
virtual void do_nmethod(nmethod* nm) {
assert(_heap->is_concurrent_weak_root_in_progress(), "Only this phase");
- if (failed()) {
- return;
- }
ShenandoahNMethod* nm_data = ShenandoahNMethod::gc_data(nm);
assert(!nm_data->is_unregistered(), "Should not see unregistered entry");
@@ -170,27 +160,19 @@ class ShenandoahNMethodUnlinkClosure : public NMethodClosure {
}
// Clear compiled ICs and exception caches
- if (!nm->unload_nmethod_caches(_unloading_occurred)) {
- set_failed();
- }
- }
-
- bool failed() const {
- return Atomic::load(&_failed);
+ nm->unload_nmethod_caches(_unloading_occurred);
}
};
class ShenandoahUnlinkTask : public WorkerTask {
private:
ShenandoahNMethodUnlinkClosure _cl;
- ICRefillVerifier* _verifier;
ShenandoahConcurrentNMethodIterator _iterator;
public:
- ShenandoahUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
+ ShenandoahUnlinkTask(bool unloading_occurred) :
WorkerTask("Shenandoah Unlink NMethods"),
_cl(unloading_occurred),
- _verifier(verifier),
_iterator(ShenandoahCodeRoots::table()) {
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
_iterator.nmethods_do_begin();
@@ -202,35 +184,15 @@ class ShenandoahUnlinkTask : public WorkerTask {
}
virtual void work(uint worker_id) {
- ICRefillVerifierMark mark(_verifier);
_iterator.nmethods_do(&_cl);
}
-
- bool success() const {
- return !_cl.failed();
- }
};
void ShenandoahCodeRoots::unlink(WorkerThreads* workers, bool unloading_occurred) {
assert(ShenandoahHeap::heap()->unload_classes(), "Only when running concurrent class unloading");
- for (;;) {
- ICRefillVerifier verifier;
-
- {
- ShenandoahUnlinkTask task(unloading_occurred, &verifier);
- workers->run_task(&task);
- if (task.success()) {
- return;
- }
- }
-
- // Cleaning failed because we ran out of transitional IC stubs,
- // so we have to refill and try again. Refilling requires taking
- // a safepoint, so we temporarily leave the suspendible thread set.
- SuspendibleThreadSetLeaver sts;
- InlineCacheBuffer::refill_ic_stubs();
- }
+ ShenandoahUnlinkTask task(unloading_occurred);
+ workers->run_task(&task);
}
void ShenandoahCodeRoots::purge() {
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
index 12fde72b7b491..d14bcc39f4591 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp
@@ -38,31 +38,17 @@ ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() :
_consecutive_degenerated_gcs(0),
_alloc_failure_degenerated(0),
_alloc_failure_degenerated_upgrade_to_full(0),
- _alloc_failure_full(0),
- _explicit_concurrent(0),
- _explicit_full(0),
- _implicit_concurrent(0),
- _implicit_full(0) {
+ _alloc_failure_full(0) {
- Copy::zero_to_bytes(_degen_points, sizeof(size_t) * ShenandoahGC::_DEGENERATED_LIMIT);
+ Copy::zero_to_bytes(_degen_point_counts, sizeof(size_t) * ShenandoahGC::_DEGENERATED_LIMIT);
+ Copy::zero_to_bytes(_collection_cause_counts, sizeof(size_t) * GCCause::_last_gc_cause);
_tracer = new ShenandoahTracer();
}
-void ShenandoahCollectorPolicy::record_explicit_to_concurrent() {
- _explicit_concurrent++;
-}
-
-void ShenandoahCollectorPolicy::record_explicit_to_full() {
- _explicit_full++;
-}
-
-void ShenandoahCollectorPolicy::record_implicit_to_concurrent() {
- _implicit_concurrent++;
-}
-
-void ShenandoahCollectorPolicy::record_implicit_to_full() {
- _implicit_full++;
+void ShenandoahCollectorPolicy::record_collection_cause(GCCause::Cause cause) {
+ assert(cause < GCCause::_last_gc_cause, "Invalid GCCause");
+ _collection_cause_counts[cause]++;
}
void ShenandoahCollectorPolicy::record_alloc_failure_to_full() {
@@ -72,7 +58,7 @@ void ShenandoahCollectorPolicy::record_alloc_failure_to_full() {
void ShenandoahCollectorPolicy::record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point) {
assert(point < ShenandoahGC::_DEGENERATED_LIMIT, "sanity");
_alloc_failure_degenerated++;
- _degen_points[point]++;
+ _degen_point_counts[point]++;
}
void ShenandoahCollectorPolicy::record_degenerated_upgrade_to_full() {
@@ -109,6 +95,44 @@ bool ShenandoahCollectorPolicy::is_at_shutdown() {
return _in_shutdown.is_set();
}
+bool is_explicit_gc(GCCause::Cause cause) {
+ return GCCause::is_user_requested_gc(cause)
+ || GCCause::is_serviceability_requested_gc(cause);
+}
+
+bool is_implicit_gc(GCCause::Cause cause) {
+ return cause != GCCause::_allocation_failure
+ && cause != GCCause::_shenandoah_concurrent_gc
+ && !is_explicit_gc(cause);
+}
+
+#ifdef ASSERT
+bool is_valid_request(GCCause::Cause cause) {
+ return is_explicit_gc(cause)
+ || cause == GCCause::_metadata_GC_clear_soft_refs
+ || cause == GCCause::_codecache_GC_aggressive
+ || cause == GCCause::_codecache_GC_threshold
+ || cause == GCCause::_full_gc_alot
+ || cause == GCCause::_wb_young_gc
+ || cause == GCCause::_wb_full_gc
+ || cause == GCCause::_wb_breakpoint
+ || cause == GCCause::_scavenge_alot;
+}
+#endif
+
+bool ShenandoahCollectorPolicy::should_run_full_gc(GCCause::Cause cause) {
+ return is_explicit_gc(cause) ? !ExplicitGCInvokesConcurrent : !ShenandoahImplicitGCInvokesConcurrent;
+}
+
+bool ShenandoahCollectorPolicy::should_handle_requested_gc(GCCause::Cause cause) {
+ assert(is_valid_request(cause), "only requested GCs here: %s", GCCause::to_string(cause));
+
+ if (DisableExplicitGC) {
+ return !is_explicit_gc(cause);
+ }
+ return true;
+}
+
void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
out->print_cr("Under allocation pressure, concurrent cycles may cancel, and either continue cycle");
out->print_cr("under stop-the-world pause or result in stop-the-world Full GC. Increase heap size,");
@@ -119,10 +143,32 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
size_t completed_gcs = _success_full_gcs + _success_degenerated_gcs + _success_concurrent_gcs;
out->print_cr(SIZE_FORMAT_W(5) " Completed GCs", completed_gcs);
- out->print_cr(SIZE_FORMAT_W(5) " Successful Concurrent GCs (%.2f%%)", _success_concurrent_gcs, percent_of(_success_concurrent_gcs, completed_gcs));
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", _explicit_concurrent, percent_of(_explicit_concurrent, _success_concurrent_gcs));
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", _implicit_concurrent, percent_of(_implicit_concurrent, _success_concurrent_gcs));
- out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_concurrent_gcs, percent_of(_abbreviated_concurrent_gcs, _success_concurrent_gcs));
+
+ size_t explicit_requests = 0;
+ size_t implicit_requests = 0;
+ for (int c = 0; c < GCCause::_last_gc_cause; c++) {
+ size_t cause_count = _collection_cause_counts[c];
+ if (cause_count > 0) {
+ auto cause = (GCCause::Cause) c;
+ if (is_explicit_gc(cause)) {
+ explicit_requests += cause_count;
+ } else if (is_implicit_gc(cause)) {
+ implicit_requests += cause_count;
+ }
+ const char* desc = GCCause::to_string(cause);
+ out->print_cr(" " SIZE_FORMAT_W(5) " caused by %s (%.2f%%)", cause_count, desc, percent_of(cause_count, completed_gcs));
+ }
+ }
+
+ out->cr();
+ out->print_cr(SIZE_FORMAT_W(5) " Successful Concurrent GCs (%.2f%%)", _success_concurrent_gcs, percent_of(_success_concurrent_gcs, completed_gcs));
+ if (ExplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", explicit_requests, percent_of(explicit_requests, _success_concurrent_gcs));
+ }
+ if (ShenandoahImplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", implicit_requests, percent_of(implicit_requests, _success_concurrent_gcs));
+ }
+ out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_concurrent_gcs, percent_of(_abbreviated_concurrent_gcs, _success_concurrent_gcs));
out->cr();
size_t degenerated_gcs = _alloc_failure_degenerated_upgrade_to_full + _success_degenerated_gcs;
@@ -131,16 +177,20 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const {
out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure (%.2f%%)", _alloc_failure_degenerated, percent_of(_alloc_failure_degenerated, degenerated_gcs));
out->print_cr(" " SIZE_FORMAT_W(5) " abbreviated (%.2f%%)", _abbreviated_degenerated_gcs, percent_of(_abbreviated_degenerated_gcs, degenerated_gcs));
for (int c = 0; c < ShenandoahGC::_DEGENERATED_LIMIT; c++) {
- if (_degen_points[c] > 0) {
+ if (_degen_point_counts[c] > 0) {
const char* desc = ShenandoahGC::degen_point_to_string((ShenandoahGC::ShenandoahDegenPoint)c);
- out->print_cr(" " SIZE_FORMAT_W(5) " happened at %s", _degen_points[c], desc);
+ out->print_cr(" " SIZE_FORMAT_W(5) " happened at %s", _degen_point_counts[c], desc);
}
}
out->cr();
- out->print_cr(SIZE_FORMAT_W(5) " Full GCs (%.2f%%)", _success_full_gcs, percent_of(_success_full_gcs, completed_gcs));
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", _explicit_full, percent_of(_explicit_full, _success_full_gcs));
- out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", _implicit_full, percent_of(_implicit_full, _success_full_gcs));
+ out->print_cr(SIZE_FORMAT_W(5) " Full GCs (%.2f%%)", _success_full_gcs, percent_of(_success_full_gcs, completed_gcs));
+ if (!ExplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly (%.2f%%)", explicit_requests, percent_of(explicit_requests, _success_concurrent_gcs));
+ }
+ if (!ShenandoahImplicitGCInvokesConcurrent) {
+ out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly (%.2f%%)", implicit_requests, percent_of(implicit_requests, _success_concurrent_gcs));
+ }
out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure (%.2f%%)", _alloc_failure_full, percent_of(_alloc_failure_full, _success_full_gcs));
out->print_cr(" " SIZE_FORMAT_W(5) " upgraded from Degenerated GC (%.2f%%)", _alloc_failure_degenerated_upgrade_to_full, percent_of(_alloc_failure_degenerated_upgrade_to_full, _success_full_gcs));
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
index 8d894c9144d4d..638acce14561f 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp
@@ -48,11 +48,8 @@ class ShenandoahCollectorPolicy : public CHeapObj {
size_t _alloc_failure_degenerated;
size_t _alloc_failure_degenerated_upgrade_to_full;
size_t _alloc_failure_full;
- size_t _explicit_concurrent;
- size_t _explicit_full;
- size_t _implicit_concurrent;
- size_t _implicit_full;
- size_t _degen_points[ShenandoahGC::_DEGENERATED_LIMIT];
+ size_t _collection_cause_counts[GCCause::_last_gc_cause];
+ size_t _degen_point_counts[ShenandoahGC::_DEGENERATED_LIMIT];
ShenandoahSharedFlag _in_shutdown;
ShenandoahTracer* _tracer;
@@ -72,10 +69,7 @@ class ShenandoahCollectorPolicy : public CHeapObj {
void record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point);
void record_alloc_failure_to_full();
void record_degenerated_upgrade_to_full();
- void record_explicit_to_concurrent();
- void record_explicit_to_full();
- void record_implicit_to_concurrent();
- void record_implicit_to_full();
+ void record_collection_cause(GCCause::Cause cause);
void record_shutdown();
bool is_at_shutdown();
@@ -94,6 +88,9 @@ class ShenandoahCollectorPolicy : public CHeapObj {
inline size_t consecutive_degenerated_gc_count() const {
return _consecutive_degenerated_gcs;
}
+
+ static bool should_run_full_gc(GCCause::Cause cause);
+ static bool should_handle_requested_gc(GCCause::Cause cause);
};
#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTORPOLICY_HPP
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
index 86887b161cf69..bdf1e5b912857 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp
@@ -55,8 +55,8 @@ ShenandoahControlThread::ShenandoahControlThread() :
void ShenandoahControlThread::run_service() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
- GCMode default_mode = concurrent_normal;
- GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
+ const GCMode default_mode = concurrent_normal;
+ const GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
int sleep = ShenandoahControlIntervalMin;
double last_shrink_time = os::elapsedTime();
@@ -66,23 +66,21 @@ void ShenandoahControlThread::run_service() {
// Having a period 10x lower than the delay would mean we hit the
// shrinking with lag of less than 1/10-th of true delay.
// ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
- double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
+ const double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
- ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
- ShenandoahHeuristics* heuristics = heap->heuristics();
+ ShenandoahCollectorPolicy* const policy = heap->shenandoah_policy();
+ ShenandoahHeuristics* const heuristics = heap->heuristics();
while (!in_graceful_shutdown() && !should_terminate()) {
// Figure out if we have pending requests.
- bool alloc_failure_pending = _alloc_failure_gc.is_set();
- bool is_gc_requested = _gc_requested.is_set();
- GCCause::Cause requested_gc_cause = _requested_gc_cause;
- bool explicit_gc_requested = is_gc_requested && is_explicit_gc(requested_gc_cause);
- bool implicit_gc_requested = is_gc_requested && !is_explicit_gc(requested_gc_cause);
+ const bool alloc_failure_pending = _alloc_failure_gc.is_set();
+ const bool is_gc_requested = _gc_requested.is_set();
+ const GCCause::Cause requested_gc_cause = _requested_gc_cause;
// This control loop iteration has seen this much allocation.
- size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed);
+ const size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed);
// Check if we have seen a new target for soft max heap size.
- bool soft_max_changed = heap->check_soft_max_changed();
+ const bool soft_max_changed = heap->check_soft_max_changed();
// Choose which GC mode to run in. The block below should select a single mode.
GCMode mode = none;
@@ -109,36 +107,17 @@ void ShenandoahControlThread::run_service() {
mode = stw_full;
}
- } else if (explicit_gc_requested) {
+ } else if (is_gc_requested) {
cause = requested_gc_cause;
- log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause));
-
+ log_info(gc)("Trigger: GC request (%s)", GCCause::to_string(cause));
heuristics->record_requested_gc();
- if (ExplicitGCInvokesConcurrent) {
- policy->record_explicit_to_concurrent();
- mode = default_mode;
- // Unload and clean up everything
- heap->set_unload_classes(heuristics->can_unload_classes());
- } else {
- policy->record_explicit_to_full();
+ if (ShenandoahCollectorPolicy::should_run_full_gc(cause)) {
mode = stw_full;
- }
- } else if (implicit_gc_requested) {
- cause = requested_gc_cause;
- log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause));
-
- heuristics->record_requested_gc();
-
- if (ShenandoahImplicitGCInvokesConcurrent) {
- policy->record_implicit_to_concurrent();
+ } else {
mode = default_mode;
-
// Unload and clean up everything
heap->set_unload_classes(heuristics->can_unload_classes());
- } else {
- policy->record_implicit_to_full();
- mode = stw_full;
}
} else {
// Potential normal cycle: ask heuristics if it wants to act
@@ -153,11 +132,11 @@ void ShenandoahControlThread::run_service() {
// Blow all soft references on this cycle, if handling allocation failure,
// either implicit or explicit GC request, or we are requested to do so unconditionally.
- if (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs) {
+ if (alloc_failure_pending || is_gc_requested || ShenandoahAlwaysClearSoftRefs) {
heap->soft_ref_policy()->set_should_clear_all_soft_refs(true);
}
- bool gc_requested = (mode != none);
+ const bool gc_requested = (mode != none);
assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set");
if (gc_requested) {
@@ -193,7 +172,7 @@ void ShenandoahControlThread::run_service() {
}
// If this was the requested GC cycle, notify waiters about it
- if (explicit_gc_requested || implicit_gc_requested) {
+ if (is_gc_requested) {
notify_gc_waiters();
}
@@ -266,14 +245,14 @@ void ShenandoahControlThread::run_service() {
}
}
- double current = os::elapsedTime();
+ const double current = os::elapsedTime();
- if (ShenandoahUncommit && (explicit_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
+ if (ShenandoahUncommit && (is_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
// Explicit GC tries to uncommit everything down to min capacity.
// Soft max change tries to uncommit everything down to target capacity.
// Periodic uncommit tries to uncommit suitable regions down to min capacity.
- double shrink_before = (explicit_gc_requested || soft_max_changed) ?
+ double shrink_before = (is_gc_requested || soft_max_changed) ?
current :
current - (ShenandoahUncommitDelay / 1000.0);
@@ -395,29 +374,8 @@ void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause
gc.collect(cause);
}
-bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const {
- return GCCause::is_user_requested_gc(cause) ||
- GCCause::is_serviceability_requested_gc(cause);
-}
-
void ShenandoahControlThread::request_gc(GCCause::Cause cause) {
- assert(GCCause::is_user_requested_gc(cause) ||
- GCCause::is_serviceability_requested_gc(cause) ||
- cause == GCCause::_metadata_GC_clear_soft_refs ||
- cause == GCCause::_codecache_GC_aggressive ||
- cause == GCCause::_codecache_GC_threshold ||
- cause == GCCause::_full_gc_alot ||
- cause == GCCause::_wb_young_gc ||
- cause == GCCause::_wb_full_gc ||
- cause == GCCause::_wb_breakpoint ||
- cause == GCCause::_scavenge_alot,
- "only requested GCs here: %s", GCCause::to_string(cause));
-
- if (is_explicit_gc(cause)) {
- if (!DisableExplicitGC) {
- handle_requested_gc(cause);
- }
- } else {
+ if (ShenandoahCollectorPolicy::should_handle_requested_gc(cause)) {
handle_requested_gc(cause);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
index 227a3ba7061a9..9da25b1a73ced 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp
@@ -84,8 +84,6 @@ class ShenandoahControlThread: public ConcurrentGCThread {
// Blocks until GC is over.
void handle_requested_gc(GCCause::Cause cause);
- bool is_explicit_gc(GCCause::Cause cause) const;
-
public:
// Constructor
ShenandoahControlThread();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
index 3552669abb661..d66bf052da4f1 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp
@@ -235,12 +235,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) {
phase3_update_references();
phase4_compact_objects(worker_slices);
- }
- {
- // Epilogue
- _preserved_marks->restore(heap->workers());
- _preserved_marks->reclaim();
+ phase5_epilog();
}
// Resize metaspace
@@ -280,6 +276,8 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure {
_ctx->capture_top_at_mark_start(r);
r->clear_live_data();
}
+
+ bool is_thread_safe() { return true; }
};
void ShenandoahFullGC::phase1_mark_heap() {
@@ -289,7 +287,7 @@ void ShenandoahFullGC::phase1_mark_heap() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
ShenandoahPrepareForMarkClosure cl;
- heap->heap_region_iterate(&cl);
+ heap->parallel_heap_region_iterate(&cl);
heap->set_unload_classes(heap->heuristics()->can_unload_classes());
@@ -328,7 +326,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
_from_region = from_region;
}
- void finish_region() {
+ void finish() {
assert(_to_region != nullptr, "should not happen");
_to_region->set_new_top(_compact_point);
}
@@ -348,7 +346,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure {
size_t obj_size = p->size();
if (_compact_point + obj_size > _to_region->end()) {
- finish_region();
+ finish();
// Object doesn't fit. Pick next empty region and start compacting there.
ShenandoahHeapRegion* new_to_region;
@@ -400,47 +398,61 @@ class ShenandoahPrepareForCompactionTask : public WorkerTask {
return r->is_stw_move_allowed() && !r->is_humongous();
}
- void work(uint worker_id) {
- ShenandoahParallelWorkerSession worker_session(worker_id);
- ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
- ShenandoahHeapRegionSetIterator it(slice);
- ShenandoahHeapRegion* from_region = it.next();
- // No work?
- if (from_region == nullptr) {
- return;
- }
-
- // Sliding compaction. Walk all regions in the slice, and compact them.
- // Remember empty regions and reuse them as needed.
- ResourceMark rm;
+ void work(uint worker_id) override;
+private:
+ template
+ void prepare_for_compaction(ClosureType& cl,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegionSetIterator& it,
+ ShenandoahHeapRegion* from_region);
+};
- GrowableArray empty_regions((int)_heap->num_regions());
+void ShenandoahPrepareForCompactionTask::work(uint worker_id) {
+ ShenandoahParallelWorkerSession worker_session(worker_id);
+ ShenandoahHeapRegionSet* slice = _worker_slices[worker_id];
+ ShenandoahHeapRegionSetIterator it(slice);
+ ShenandoahHeapRegion* from_region = it.next();
+ // No work?
+ if (from_region == nullptr) {
+ return;
+ }
- ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ // Sliding compaction. Walk all regions in the slice, and compact them.
+ // Remember empty regions and reuse them as needed.
+ ResourceMark rm;
- while (from_region != nullptr) {
- assert(is_candidate_region(from_region), "Sanity");
+ GrowableArray empty_regions((int)_heap->num_regions());
- cl.set_from_region(from_region);
- if (from_region->has_live()) {
- _heap->marked_object_iterate(from_region, &cl);
- }
+ ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region);
+ prepare_for_compaction(cl, empty_regions, it, from_region);
+}
- // Compacted the region to somewhere else? From-region is empty then.
- if (!cl.is_compact_same_region()) {
- empty_regions.append(from_region);
- }
- from_region = it.next();
+template
+void ShenandoahPrepareForCompactionTask::prepare_for_compaction(ClosureType& cl,
+ GrowableArray& empty_regions,
+ ShenandoahHeapRegionSetIterator& it,
+ ShenandoahHeapRegion* from_region) {
+ while (from_region != nullptr) {
+ assert(is_candidate_region(from_region), "Sanity");
+ cl.set_from_region(from_region);
+ if (from_region->has_live()) {
+ _heap->marked_object_iterate(from_region, &cl);
}
- cl.finish_region();
- // Mark all remaining regions as empty
- for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
- ShenandoahHeapRegion* r = empty_regions.at(pos);
- r->set_new_top(r->bottom());
+ // Compacted the region to somewhere else? From-region is empty then.
+ if (!cl.is_compact_same_region()) {
+ empty_regions.append(from_region);
}
+ from_region = it.next();
}
-};
+ cl.finish();
+
+ // Mark all remaining regions as empty
+ for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) {
+ ShenandoahHeapRegion* r = empty_regions.at(pos);
+ r->set_new_top(r->bottom());
+ }
+}
void ShenandoahFullGC::calculate_target_humongous_objects() {
ShenandoahHeap* heap = ShenandoahHeap::heap();
@@ -948,7 +960,7 @@ void ShenandoahFullGC::compact_humongous_objects() {
//
// This code is serial, because doing the in-slice parallel sliding is tricky. In most cases,
// humongous regions are already compacted, and do not require further moves, which alleviates
- // sliding costs. We may consider doing this in parallel in future.
+ // sliding costs. We may consider doing this in parallel in the future.
ShenandoahHeap* heap = ShenandoahHeap::heap();
@@ -1054,6 +1066,11 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong);
compact_humongous_objects();
}
+}
+
+void ShenandoahFullGC::phase5_epilog() {
+ GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer);
+ ShenandoahHeap* heap = ShenandoahHeap::heap();
// Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer
// and must ensure the bitmap is in sync.
@@ -1066,14 +1083,15 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s
// Bring regions in proper states after the collection, and set heap properties.
{
ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild);
-
ShenandoahPostCompactClosure post_compact;
heap->heap_region_iterate(&post_compact);
heap->set_used(post_compact.get_live());
heap->collection_set()->clear();
heap->free_set()->rebuild();
+ heap->clear_cancelled_gc();
}
- heap->clear_cancelled_gc();
+ _preserved_marks->restore(heap->workers());
+ _preserved_marks->reclaim();
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
index 1c1653e59ec56..6687116b21f78 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp
@@ -81,6 +81,7 @@ class ShenandoahFullGC : public ShenandoahGC {
void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices);
void phase3_update_references();
void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices);
+ void phase5_epilog();
void distribute_slices(ShenandoahHeapRegionSet** worker_slices);
void calculate_target_humongous_objects();
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
index de55dde8aca96..f0a2e53a12b1e 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -660,7 +660,7 @@ void ShenandoahHeap::post_initialize() {
_heuristics->initialize();
- JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers());
+ JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
}
size_t ShenandoahHeap::used() const {
@@ -1003,7 +1003,11 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
}
HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
- ShenandoahHeapLocker locker(lock());
+ // If we are dealing with mutator allocation, then we may need to block for safepoint.
+ // We cannot block for safepoint for GC allocations, because there is a high chance
+ // we are already running at safepoint or from stack watermark machinery, and we cannot
+ // block again.
+ ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
return _free_set->allocate(req, in_new_region);
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
index 3a9b612860b9f..39588d68bc83b 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.cpp
@@ -28,9 +28,47 @@
#include "gc/shenandoah/shenandoahLock.hpp"
#include "runtime/atomic.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/os.inline.hpp"
+// These are inline variants of Thread::SpinAcquire with optional blocking in VM.
+
+class ShenandoahNoBlockOp : public StackObj {
+public:
+ ShenandoahNoBlockOp(JavaThread* java_thread) {
+ assert(java_thread == nullptr, "Should not pass anything");
+ }
+};
+
+void ShenandoahLock::contended_lock(bool allow_block_for_safepoint) {
+ Thread* thread = Thread::current();
+ if (allow_block_for_safepoint && thread->is_Java_thread()) {
+ contended_lock_internal(JavaThread::cast(thread));
+ } else {
+ contended_lock_internal(nullptr);
+ }
+}
+
+template
+void ShenandoahLock::contended_lock_internal(JavaThread* java_thread) {
+ int ctr = 0;
+ int yields = 0;
+ while (Atomic::cmpxchg(&_state, unlocked, locked) != unlocked) {
+ if ((++ctr & 0xFFF) == 0) {
+ BlockOp block(java_thread);
+ if (yields > 5) {
+ os::naked_short_sleep(1);
+ } else {
+ os::naked_yield();
+ yields++;
+ }
+ } else {
+ SpinPause();
+ }
+ }
+}
+
ShenandoahSimpleLock::ShenandoahSimpleLock() {
assert(os::mutex_init_done(), "Too early!");
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
index 6237637619548..4412680bc8ce6 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahLock.hpp
@@ -35,34 +35,39 @@ class ShenandoahLock {
enum LockState { unlocked = 0, locked = 1 };
shenandoah_padding(0);
- volatile int _state;
+ volatile LockState _state;
shenandoah_padding(1);
volatile Thread* _owner;
shenandoah_padding(2);
+ template
+ void contended_lock_internal(JavaThread* java_thread);
+
public:
ShenandoahLock() : _state(unlocked), _owner(nullptr) {};
- void lock() {
-#ifdef ASSERT
- assert(_owner != Thread::current(), "reentrant locking attempt, would deadlock");
-#endif
- Thread::SpinAcquire(&_state, "Shenandoah Heap Lock");
-#ifdef ASSERT
- assert(_state == locked, "must be locked");
- assert(_owner == nullptr, "must not be owned");
- _owner = Thread::current();
-#endif
+ void lock(bool allow_block_for_safepoint) {
+ assert(Atomic::load(&_owner) != Thread::current(), "reentrant locking attempt, would deadlock");
+
+ // Try to lock fast, or dive into contended lock handling.
+ if (Atomic::cmpxchg(&_state, unlocked, locked) != unlocked) {
+ contended_lock(allow_block_for_safepoint);
+ }
+
+ assert(Atomic::load(&_state) == locked, "must be locked");
+ assert(Atomic::load(&_owner) == nullptr, "must not be owned");
+ DEBUG_ONLY(Atomic::store(&_owner, Thread::current());)
}
void unlock() {
-#ifdef ASSERT
- assert (_owner == Thread::current(), "sanity");
- _owner = nullptr;
-#endif
- Thread::SpinRelease(&_state);
+ assert(Atomic::load(&_owner) == Thread::current(), "sanity");
+ DEBUG_ONLY(Atomic::store(&_owner, (Thread*)nullptr);)
+ OrderAccess::fence();
+ Atomic::store(&_state, unlocked);
}
+ void contended_lock(bool allow_block_for_safepoint);
+
bool owned_by_self() {
#ifdef ASSERT
return _state == locked && _owner == Thread::current();
@@ -77,9 +82,9 @@ class ShenandoahLocker : public StackObj {
private:
ShenandoahLock* const _lock;
public:
- ShenandoahLocker(ShenandoahLock* lock) : _lock(lock) {
+ ShenandoahLocker(ShenandoahLock* lock, bool allow_block_for_safepoint = false) : _lock(lock) {
if (_lock != nullptr) {
- _lock->lock();
+ _lock->lock(allow_block_for_safepoint);
}
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
index bb13e9b8e224c..1017210e23e92 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp
@@ -106,7 +106,7 @@ class ShenandoahCompiledICProtectionBehaviour : public CompiledICProtectionBehav
}
virtual bool is_safe(CompiledMethod* method) {
- if (SafepointSynchronize::is_at_safepoint()) {
+ if (SafepointSynchronize::is_at_safepoint() || method->is_unloading()) {
return true;
}
diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp
index e4a8e38cba762..64074a9672cca 100644
--- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp
+++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp
@@ -44,6 +44,7 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause) :
_tracer(_heap->tracer()) {
assert(!ShenandoahGCPhase::is_current_phase_valid(), "No current GC phase");
+ _heap->shenandoah_policy()->record_collection_cause(cause);
_heap->set_gc_cause(cause);
_timer->register_gc_start();
_tracer->report_gc_start(cause, _timer->gc_start());
diff --git a/src/hotspot/share/gc/x/xDirector.cpp b/src/hotspot/share/gc/x/xDirector.cpp
index 05ef362374192..e1c69bd05b7de 100644
--- a/src/hotspot/share/gc/x/xDirector.cpp
+++ b/src/hotspot/share/gc/x/xDirector.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -168,7 +168,7 @@ static double select_gc_workers(double serial_gc_time, double parallelizable_gc_
return gc_workers;
}
-XDriverRequest rule_allocation_rate_dynamic() {
+static XDriverRequest rule_allocation_rate_dynamic() {
if (!XStatCycle::is_time_trustable()) {
// Rule disabled
return GCCause::_no_gc;
diff --git a/src/hotspot/share/gc/x/xNMethod.cpp b/src/hotspot/share/gc/x/xNMethod.cpp
index 613e190850225..5a368a8483b65 100644
--- a/src/hotspot/share/gc/x/xNMethod.cpp
+++ b/src/hotspot/share/gc/x/xNMethod.cpp
@@ -24,7 +24,6 @@
#include "precompiled.hpp"
#include "code/relocInfo.hpp"
#include "code/nmethod.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
@@ -305,9 +304,7 @@ class XNMethodUnlinkClosure : public NMethodClosure {
}
// Clear compiled ICs and exception caches
- if (!nm->unload_nmethod_caches(_unloading_occurred)) {
- set_failed();
- }
+ nm->unload_nmethod_caches(_unloading_occurred);
}
bool failed() const {
@@ -318,13 +315,11 @@ class XNMethodUnlinkClosure : public NMethodClosure {
class XNMethodUnlinkTask : public XTask {
private:
XNMethodUnlinkClosure _cl;
- ICRefillVerifier* _verifier;
public:
- XNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
+ XNMethodUnlinkTask(bool unloading_occurred) :
XTask("XNMethodUnlinkTask"),
- _cl(unloading_occurred),
- _verifier(verifier) {
+ _cl(unloading_occurred) {
XNMethodTable::nmethods_do_begin();
}
@@ -333,33 +328,13 @@ class XNMethodUnlinkTask : public XTask {
}
virtual void work() {
- ICRefillVerifierMark mark(_verifier);
XNMethodTable::nmethods_do(&_cl);
}
-
- bool success() const {
- return !_cl.failed();
- }
};
void XNMethod::unlink(XWorkers* workers, bool unloading_occurred) {
- for (;;) {
- ICRefillVerifier verifier;
-
- {
- XNMethodUnlinkTask task(unloading_occurred, &verifier);
- workers->run(&task);
- if (task.success()) {
- return;
- }
- }
-
- // Cleaning failed because we ran out of transitional IC stubs,
- // so we have to refill and try again. Refilling requires taking
- // a safepoint, so we temporarily leave the suspendible thread set.
- SuspendibleThreadSetLeaver sts;
- InlineCacheBuffer::refill_ic_stubs();
- }
+ XNMethodUnlinkTask task(unloading_occurred);
+ workers->run(&task);
}
void XNMethod::purge() {
diff --git a/src/hotspot/share/gc/x/xNMethodTable.cpp b/src/hotspot/share/gc/x/xNMethodTable.cpp
index f866c5816846c..52fcba755a70b 100644
--- a/src/hotspot/share/gc/x/xNMethodTable.cpp
+++ b/src/hotspot/share/gc/x/xNMethodTable.cpp
@@ -24,7 +24,6 @@
#include "precompiled.hpp"
#include "code/relocInfo.hpp"
#include "code/nmethod.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/x/xGlobals.hpp"
diff --git a/src/hotspot/share/gc/x/xObjArrayAllocator.cpp b/src/hotspot/share/gc/x/xObjArrayAllocator.cpp
index 9408e027cbd8c..0950b886a9b7b 100644
--- a/src/hotspot/share/gc/x/xObjArrayAllocator.cpp
+++ b/src/hotspot/share/gc/x/xObjArrayAllocator.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -50,7 +50,17 @@ oop XObjArrayAllocator::initialize(HeapWord* mem) const {
// time and time-to-safepoint
const size_t segment_max = XUtils::bytes_to_words(64 * K);
const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
- const size_t header = arrayOopDesc::header_size(element_type);
+
+ // Clear leading 32 bits, if necessary.
+ int base_offset = arrayOopDesc::base_offset_in_bytes(element_type);
+ if (!is_aligned(base_offset, HeapWordSize)) {
+ assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned");
+ *reinterpret_cast(reinterpret_cast(mem) + base_offset) = 0;
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned");
+
+ const size_t header = heap_word_size(base_offset);
const size_t payload_size = _word_size - header;
if (payload_size <= segment_max) {
diff --git a/src/hotspot/share/gc/x/xTracer.cpp b/src/hotspot/share/gc/x/xTracer.cpp
index 68fbdc28356ca..3a0bd2b00e3bf 100644
--- a/src/hotspot/share/gc/x/xTracer.cpp
+++ b/src/hotspot/share/gc/x/xTracer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -93,7 +93,7 @@ XTracer::XTracer() :
void XTracer::initialize() {
assert(_tracer == nullptr, "Already initialized");
_tracer = new XTracer();
- JFR_ONLY(register_jfr_type_serializers());
+ JFR_ONLY(register_jfr_type_serializers();)
}
void XTracer::send_stat_counter(const XStatCounter& counter, uint64_t increment, uint64_t value) {
diff --git a/src/hotspot/share/gc/x/xUnload.cpp b/src/hotspot/share/gc/x/xUnload.cpp
index 27d429d3635df..230dbf613a445 100644
--- a/src/hotspot/share/gc/x/xUnload.cpp
+++ b/src/hotspot/share/gc/x/xUnload.cpp
@@ -101,7 +101,7 @@ class XCompiledICProtectionBehaviour : public CompiledICProtectionBehaviour {
}
virtual bool is_safe(CompiledMethod* method) {
- if (SafepointSynchronize::is_at_safepoint()) {
+ if (SafepointSynchronize::is_at_safepoint() || method->is_unloading()) {
return true;
}
diff --git a/src/hotspot/share/gc/z/zBarrierSet.cpp b/src/hotspot/share/gc/z/zBarrierSet.cpp
index a82cc67754b5f..48228a3e1abc3 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.cpp
@@ -152,11 +152,12 @@ void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
deoptimize_allocation(thread);
}
-void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj, size_t size) {
+void ZBarrierSet::clone_obj_array(objArrayOop src_obj, objArrayOop dst_obj) {
volatile zpointer* src = (volatile zpointer*)src_obj->base();
volatile zpointer* dst = (volatile zpointer*)dst_obj->base();
+ const int length = src_obj->length();
- for (const zpointer* const end = cast_from_oop(src_obj) + size; src < end; src++, dst++) {
+ for (const volatile zpointer* const end = src + length; src < end; src++, dst++) {
zaddress elem = ZBarrier::load_barrier_on_oop_field(src);
// We avoid healing here because the store below colors the pointer store good,
// hence avoiding the cost of a CAS.
diff --git a/src/hotspot/share/gc/z/zBarrierSet.hpp b/src/hotspot/share/gc/z/zBarrierSet.hpp
index adba5ee15c14a..bf233df683afb 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.hpp
@@ -39,7 +39,7 @@ class ZBarrierSet : public BarrierSet {
static ZBarrierSetAssembler* assembler();
static bool barrier_needed(DecoratorSet decorators, BasicType type);
- static void clone_obj_array(objArrayOop src, objArrayOop dst, size_t size);
+ static void clone_obj_array(objArrayOop src, objArrayOop dst);
virtual void on_thread_create(Thread* thread);
virtual void on_thread_destroy(Thread* thread);
diff --git a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
index 997e5bae58dea..d53b69345dd98 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
@@ -439,7 +439,7 @@ inline void ZBarrierSet::AccessBarrier::clone_in_heap(o
// for cloning arrays transform the clone to an optimized allocation
// and arraycopy sequence, so the performance of this runtime call
// does not matter for object arrays.
- clone_obj_array(objArrayOop(src), objArrayOop(dst), size);
+ clone_obj_array(objArrayOop(src), objArrayOop(dst));
return;
}
diff --git a/src/hotspot/share/gc/z/zDirector.cpp b/src/hotspot/share/gc/z/zDirector.cpp
index 417ac33a6f86d..47e24063ead75 100644
--- a/src/hotspot/share/gc/z/zDirector.cpp
+++ b/src/hotspot/share/gc/z/zDirector.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -143,11 +143,11 @@ static double select_young_gc_workers(const ZDirectorStats& stats, double serial
return gc_workers;
}
-ZDriverRequest rule_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
- double serial_gc_time_passed,
- double parallel_gc_time_passed,
- bool conservative_alloc_rate,
- size_t capacity) {
+static ZDriverRequest rule_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+ double serial_gc_time_passed,
+ double parallel_gc_time_passed,
+ bool conservative_alloc_rate,
+ size_t capacity) {
if (!stats._old_stats._cycle._is_time_trustable) {
// Rule disabled
return ZDriverRequest(GCCause::_no_gc, ZYoungGCThreads, 0);
@@ -214,9 +214,9 @@ ZDriverRequest rule_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
return ZDriverRequest(GCCause::_z_allocation_rate, actual_gc_workers, 0);
}
-ZDriverRequest rule_soft_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
- double serial_gc_time_passed,
- double parallel_gc_time_passed) {
+static ZDriverRequest rule_soft_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+ double serial_gc_time_passed,
+ double parallel_gc_time_passed) {
return rule_minor_allocation_rate_dynamic(stats,
0.0 /* serial_gc_time_passed */,
0.0 /* parallel_gc_time_passed */,
@@ -224,9 +224,9 @@ ZDriverRequest rule_soft_minor_allocation_rate_dynamic(const ZDirectorStats& sta
stats._heap._soft_max_heap_size /* capacity */);
}
-ZDriverRequest rule_semi_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
- double serial_gc_time_passed,
- double parallel_gc_time_passed) {
+static ZDriverRequest rule_semi_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+ double serial_gc_time_passed,
+ double parallel_gc_time_passed) {
return rule_minor_allocation_rate_dynamic(stats,
0.0 /* serial_gc_time_passed */,
0.0 /* parallel_gc_time_passed */,
@@ -234,9 +234,9 @@ ZDriverRequest rule_semi_hard_minor_allocation_rate_dynamic(const ZDirectorStats
ZHeap::heap()->max_capacity() /* capacity */);
}
-ZDriverRequest rule_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
- double serial_gc_time_passed,
- double parallel_gc_time_passed) {
+static ZDriverRequest rule_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+ double serial_gc_time_passed,
+ double parallel_gc_time_passed) {
return rule_minor_allocation_rate_dynamic(stats,
0.0 /* serial_gc_time_passed */,
0.0 /* parallel_gc_time_passed */,
diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp
index 71d514face15a..a7d23124b683f 100644
--- a/src/hotspot/share/gc/z/zNMethod.cpp
+++ b/src/hotspot/share/gc/z/zNMethod.cpp
@@ -25,7 +25,6 @@
#include "code/codeCache.hpp"
#include "code/relocInfo.hpp"
#include "code/nmethod.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/shared/classUnloadingContext.hpp"
@@ -334,23 +333,13 @@ oop ZNMethod::load_oop(oop* p, DecoratorSet decorators) {
class ZNMethodUnlinkClosure : public NMethodClosure {
private:
- bool _unloading_occurred;
- volatile bool _failed;
-
- void set_failed() {
- Atomic::store(&_failed, true);
- }
+ bool _unloading_occurred;
public:
ZNMethodUnlinkClosure(bool unloading_occurred)
- : _unloading_occurred(unloading_occurred),
- _failed(false) {}
+ : _unloading_occurred(unloading_occurred) {}
virtual void do_nmethod(nmethod* nm) {
- if (failed()) {
- return;
- }
-
if (nm->is_unloading()) {
// Unlink from the ZNMethodTable
ZNMethod::unregister_nmethod(nm);
@@ -386,26 +375,18 @@ class ZNMethodUnlinkClosure : public NMethodClosure {
}
// Clear compiled ICs and exception caches
- if (!nm->unload_nmethod_caches(_unloading_occurred)) {
- set_failed();
- }
- }
-
- bool failed() const {
- return Atomic::load(&_failed);
+ nm->unload_nmethod_caches(_unloading_occurred);
}
};
class ZNMethodUnlinkTask : public ZTask {
private:
ZNMethodUnlinkClosure _cl;
- ICRefillVerifier* _verifier;
public:
- ZNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier)
+ ZNMethodUnlinkTask(bool unloading_occurred)
: ZTask("ZNMethodUnlinkTask"),
- _cl(unloading_occurred),
- _verifier(verifier) {
+ _cl(unloading_occurred) {
ZNMethodTable::nmethods_do_begin(false /* secondary */);
}
@@ -414,33 +395,13 @@ class ZNMethodUnlinkTask : public ZTask {
}
virtual void work() {
- ICRefillVerifierMark mark(_verifier);
ZNMethodTable::nmethods_do(false /* secondary */, &_cl);
}
-
- bool success() const {
- return !_cl.failed();
- }
};
void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) {
- for (;;) {
- ICRefillVerifier verifier;
-
- {
- ZNMethodUnlinkTask task(unloading_occurred, &verifier);
- workers->run(&task);
- if (task.success()) {
- return;
- }
- }
-
- // Cleaning failed because we ran out of transitional IC stubs,
- // so we have to refill and try again. Refilling requires taking
- // a safepoint, so we temporarily leave the suspendible thread set.
- SuspendibleThreadSetLeaver sts_leaver;
- InlineCacheBuffer::refill_ic_stubs();
- }
+ ZNMethodUnlinkTask task(unloading_occurred);
+ workers->run(&task);
}
void ZNMethod::purge() {
diff --git a/src/hotspot/share/gc/z/zNMethodTable.cpp b/src/hotspot/share/gc/z/zNMethodTable.cpp
index f75af7af616c7..a4b56292c522b 100644
--- a/src/hotspot/share/gc/z/zNMethodTable.cpp
+++ b/src/hotspot/share/gc/z/zNMethodTable.cpp
@@ -24,7 +24,6 @@
#include "precompiled.hpp"
#include "code/relocInfo.hpp"
#include "code/nmethod.hpp"
-#include "code/icBuffer.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/barrierSetNMethod.hpp"
#include "gc/z/zHash.inline.hpp"
diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
index c65f0d613d068..b5aaed3a27411 100644
--- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
+++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -50,7 +50,17 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
// time and time-to-safepoint
const size_t segment_max = ZUtils::bytes_to_words(64 * K);
const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
- const size_t header = arrayOopDesc::header_size(element_type);
+
+ // Clear leading 32 bits, if necessary.
+ int base_offset = arrayOopDesc::base_offset_in_bytes(element_type);
+ if (!is_aligned(base_offset, HeapWordSize)) {
+ assert(is_aligned(base_offset, BytesPerInt), "array base must be 32 bit aligned");
+ *reinterpret_cast(reinterpret_cast(mem) + base_offset) = 0;
+ base_offset += BytesPerInt;
+ }
+ assert(is_aligned(base_offset, HeapWordSize), "remaining array base must be 64 bit aligned");
+
+ const size_t header = heap_word_size(base_offset);
const size_t payload_size = _word_size - header;
if (payload_size <= segment_max) {
@@ -132,6 +142,8 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
assert(result, "Array initialization should always succeed the second time");
}
+ mem_zap_end_padding(mem);
+
ZThreadLocalData::clear_invisible_root(_thread);
// Signal to the ZIterator that this is no longer an invisible root
diff --git a/src/hotspot/share/gc/z/zTracer.cpp b/src/hotspot/share/gc/z/zTracer.cpp
index bc0af4680d414..7c7fff5624ccc 100644
--- a/src/hotspot/share/gc/z/zTracer.cpp
+++ b/src/hotspot/share/gc/z/zTracer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -119,7 +119,7 @@ void ZOldTracer::report_end(const Ticks& timestamp) {
}
void ZTracer::initialize() {
- JFR_ONLY(register_jfr_type_serializers());
+ JFR_ONLY(register_jfr_type_serializers();)
}
void ZTracer::send_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value) {
diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp
index 76c275df35599..3ab4cd5b19f22 100644
--- a/src/hotspot/share/gc/z/zUnload.cpp
+++ b/src/hotspot/share/gc/z/zUnload.cpp
@@ -104,7 +104,7 @@ class ZCompiledICProtectionBehaviour : public CompiledICProtectionBehaviour {
}
virtual bool is_safe(CompiledMethod* method) {
- if (SafepointSynchronize::is_at_safepoint()) {
+ if (SafepointSynchronize::is_at_safepoint() || method->is_unloading()) {
return true;
}
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
index db534b6df3c43..f087e92f52705 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethod.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -36,6 +36,7 @@
#include "jfr/recorder/repository/jfrEmergencyDump.hpp"
#include "jfr/recorder/service/jfrEventThrottler.hpp"
#include "jfr/recorder/service/jfrOptionSet.hpp"
+#include "jfr/recorder/service/jfrRecorderService.hpp"
#include "jfr/recorder/stacktrace/jfrStackFilter.hpp"
#include "jfr/recorder/stacktrace/jfrStackFilterRegistry.hpp"
#include "jfr/recorder/stacktrace/jfrStackTraceRepository.hpp"
@@ -347,9 +348,10 @@ JVM_ENTRY_NO_ENV(void, jfr_set_force_instrumentation(JNIEnv* env, jclass jvm, jb
JfrEventClassTransformer::set_force_instrumentation(force_instrumentation == JNI_TRUE);
JVM_END
-JVM_ENTRY_NO_ENV(void, jfr_emit_old_object_samples(JNIEnv* env, jclass jvm, jlong cutoff_ticks, jboolean emit_all, jboolean skip_bfs))
- LeakProfiler::emit_events(cutoff_ticks, emit_all == JNI_TRUE, skip_bfs == JNI_TRUE);
-JVM_END
+NO_TRANSITION(void, jfr_emit_old_object_samples(JNIEnv* env, jclass jvm, jlong cutoff_ticks, jboolean emit_all, jboolean skip_bfs))
+ JfrRecorderService service;
+ service.emit_leakprofiler_events(cutoff_ticks, emit_all == JNI_TRUE, skip_bfs == JNI_TRUE);
+NO_TRANSITION_END
JVM_ENTRY_NO_ENV(void, jfr_exclude_thread(JNIEnv* env, jclass jvm, jobject t))
JfrJavaSupport::exclude(thread, t);
@@ -401,6 +403,15 @@ JVM_ENTRY_NO_ENV(jlong, jfr_host_total_memory(JNIEnv* env, jclass jvm))
#endif
JVM_END
+JVM_ENTRY_NO_ENV(jlong, jfr_host_total_swap_memory(JNIEnv* env, jclass jvm))
+#ifdef LINUX
+ // We want the host swap memory, not the container value.
+ return os::Linux::host_swap();
+#else
+ return os::total_swap_space();
+#endif
+JVM_END
+
JVM_ENTRY_NO_ENV(void, jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes))
EventDataLoss::commit(bytes, min_jlong);
JVM_END
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
index c75a4c9721913..72351d3e1c1db 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethod.hpp
@@ -157,6 +157,8 @@ jboolean JNICALL jfr_is_containerized(JNIEnv* env, jclass jvm);
jlong JNICALL jfr_host_total_memory(JNIEnv* env, jclass jvm);
+jlong JNICALL jfr_host_total_swap_memory(JNIEnv* env, jclass jvm);
+
void JNICALL jfr_emit_data_loss(JNIEnv* env, jclass jvm, jlong bytes);
jlong JNICALL jfr_register_stack_filter(JNIEnv* env, jobject classes, jobject methods);
diff --git a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
index 338f63fbc4f37..7ef831f6282f0 100644
--- a/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
+++ b/src/hotspot/share/jfr/jni/jfrJniMethodRegistration.cpp
@@ -97,6 +97,7 @@ JfrJniMethodRegistration::JfrJniMethodRegistration(JNIEnv* env) {
(char*)"isInstrumented", (char*)"(Ljava/lang/Class;)Z", (void*) jfr_is_class_instrumented,
(char*)"isContainerized", (char*)"()Z", (void*) jfr_is_containerized,
(char*)"hostTotalMemory", (char*)"()J", (void*) jfr_host_total_memory,
+ (char*)"hostTotalSwapMemory", (char*)"()J", (void*) jfr_host_total_swap_memory,
(char*)"emitDataLoss", (char*)"(J)V", (void*)jfr_emit_data_loss,
(char*)"registerStackFilter", (char*)"([Ljava/lang/String;[Ljava/lang/String;)J", (void*)jfr_register_stack_filter,
(char*)"unregisterStackFilter", (char*)"(J)V", (void*)jfr_unregister_stack_filter
diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp
index 9f6679c93ebfc..d13b31e89a037 100644
--- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp
+++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleCheckpoint.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -205,6 +205,15 @@ static bool stack_trace_precondition(const ObjectSample* sample) {
return sample->has_stack_trace_id() && !sample->is_dead();
}
+static void add_to_leakp_set(const ObjectSample* sample) {
+ assert(sample != nullptr, "invariant");
+ oop object = sample->object();
+ if (object == nullptr) {
+ return;
+ }
+ JfrTraceId::load_leakp(object->klass());
+}
+
class StackTraceBlobInstaller {
private:
BlobCache _cache;
@@ -214,11 +223,9 @@ class StackTraceBlobInstaller {
StackTraceBlobInstaller() : _cache(JfrOptionSet::old_object_queue_size()) {
prepare_for_resolution();
}
- ~StackTraceBlobInstaller() {
- JfrStackTraceRepository::clear_leak_profiler();
- }
void sample_do(ObjectSample* sample) {
if (stack_trace_precondition(sample)) {
+ add_to_leakp_set(sample);
install(sample);
}
}
@@ -270,11 +277,14 @@ void ObjectSampleCheckpoint::on_rotation(const ObjectSampler* sampler) {
assert(LeakProfiler::is_running(), "invariant");
JavaThread* const thread = JavaThread::current();
DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread);)
- // can safepoint here
- ThreadInVMfromNative transition(thread);
- MutexLocker lock(ClassLoaderDataGraph_lock);
- // the lock is needed to ensure the unload lists do not grow in the middle of inspection.
- install_stack_traces(sampler);
+ {
+ // can safepoint here
+ ThreadInVMfromNative transition(thread);
+ MutexLocker lock(ClassLoaderDataGraph_lock);
+ // the lock is needed to ensure the unload lists do not grow in the middle of inspection.
+ install_stack_traces(sampler);
+ }
+ JfrStackTraceRepository::clear_leak_profiler();
}
static bool is_klass_unloaded(traceid klass_id) {
diff --git a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleWriter.cpp b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleWriter.cpp
index 2c2d7fe7b3da6..87851c71cdb75 100644
--- a/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleWriter.cpp
+++ b/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleWriter.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -199,7 +199,7 @@ static ArrayInfo* array_infos = nullptr;
static FieldTable* field_infos = nullptr;
static RootDescriptionInfo* root_infos = nullptr;
-int __write_sample_info__(JfrCheckpointWriter* writer, const void* si) {
+static int __write_sample_info__(JfrCheckpointWriter* writer, const void* si) {
assert(writer != nullptr, "invariant");
assert(si != nullptr, "invariant");
const OldObjectSampleInfo* const oosi = (const OldObjectSampleInfo*)si;
@@ -224,7 +224,7 @@ static void write_sample_infos(JfrCheckpointWriter& writer) {
}
}
-int __write_reference_info__(JfrCheckpointWriter* writer, const void* ri) {
+static int __write_reference_info__(JfrCheckpointWriter* writer, const void* ri) {
assert(writer != nullptr, "invariant");
assert(ri != nullptr, "invariant");
const ReferenceInfo* const ref_info = (const ReferenceInfo*)ri;
@@ -246,7 +246,7 @@ static void write_reference_infos(JfrCheckpointWriter& writer) {
}
}
-int __write_array_info__(JfrCheckpointWriter* writer, const void* ai) {
+static int __write_array_info__(JfrCheckpointWriter* writer, const void* ai) {
assert(writer != nullptr, "invariant");
assert(ai != nullptr, "invariant");
const ObjectSampleArrayInfo* const osai = (const ObjectSampleArrayInfo*)ai;
@@ -283,7 +283,7 @@ static void write_array_infos(JfrCheckpointWriter& writer) {
}
}
-int __write_field_info__(JfrCheckpointWriter* writer, const void* fi) {
+static int __write_field_info__(JfrCheckpointWriter* writer, const void* fi) {
assert(writer != nullptr, "invariant");
assert(fi != nullptr, "invariant");
const FieldTable::FieldInfoEntry* field_info_entry = (const FieldTable::FieldInfoEntry*)fi;
@@ -340,7 +340,7 @@ static const char* description(const ObjectSampleRootDescriptionInfo* osdi) {
return description.description();
}
-int __write_root_description_info__(JfrCheckpointWriter* writer, const void* di) {
+static int __write_root_description_info__(JfrCheckpointWriter* writer, const void* di) {
assert(writer != nullptr, "invariant");
assert(di != nullptr, "invariant");
const ObjectSampleRootDescriptionInfo* const osdi = (const ObjectSampleRootDescriptionInfo*)di;
@@ -367,11 +367,11 @@ typedef JfrTypeWriterImplHost RootDescriptionWriter;
-int _edge_reference_compare_(uintptr_t lhs, uintptr_t rhs) {
+static int _edge_reference_compare_(uintptr_t lhs, uintptr_t rhs) {
return lhs > rhs ? 1 : (lhs < rhs) ? -1 : 0;
}
-int _root_desc_compare_(const ObjectSampleRootDescriptionInfo*const & lhs, const ObjectSampleRootDescriptionInfo* const& rhs) {
+static int _root_desc_compare_(const ObjectSampleRootDescriptionInfo*const & lhs, const ObjectSampleRootDescriptionInfo* const& rhs) {
const uintptr_t lhs_ref = lhs->_data._root_edge->reference().addr();
const uintptr_t rhs_ref = rhs->_data._root_edge->reference().addr();
return _edge_reference_compare_(lhs_ref, rhs_ref);
diff --git a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp
index a732191af3d13..21d3338f51528 100644
--- a/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp
+++ b/src/hotspot/share/jfr/leakprofiler/sampling/objectSample.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "jfr/leakprofiler/sampling/objectSample.hpp"
#include "jfr/leakprofiler/sampling/objectSampler.hpp"
+#include "jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp"
#include "oops/weakHandle.inline.hpp"
#include "runtime/handles.inline.hpp"
@@ -36,7 +37,7 @@ void ObjectSample::reset() {
}
oop ObjectSample::object() const {
- return _object.resolve();
+ return is_dead() ? nullptr :_object.resolve();
}
bool ObjectSample::is_dead() const {
@@ -48,6 +49,7 @@ const oop* ObjectSample::object_addr() const {
}
void ObjectSample::set_object(oop object) {
+ assert(object != nullptr, "invariant");
assert(_object.is_empty(), "should be empty");
Handle h(Thread::current(), object);
_object = WeakHandle(ObjectSampler::oop_storage(), h);
diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml
index 34925674f8aaa..78cd95840b22d 100644
--- a/src/hotspot/share/jfr/metadata/metadata.xml
+++ b/src/hotspot/share/jfr/metadata/metadata.xml
@@ -1138,7 +1138,7 @@
+
diff --git a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
index 0ab787a7f3a7d..5cef25c54cb51 100644
--- a/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
+++ b/src/hotspot/share/jfr/periodic/sampling/jfrThreadSampler.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -668,7 +668,7 @@ JfrThreadSampling::~JfrThreadSampling() {
}
#ifdef ASSERT
-void assert_periods(const JfrThreadSampler* sampler, int64_t java_period_millis, int64_t native_period_millis) {
+static void assert_periods(const JfrThreadSampler* sampler, int64_t java_period_millis, int64_t native_period_millis) {
assert(sampler != nullptr, "invariant");
assert(sampler->get_java_period() == java_period_millis, "invariant");
assert(sampler->get_native_period() == native_period_millis, "invariant");
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
index 3a86f204e619d..3ef7abd8a4336 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSet.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -126,7 +126,6 @@ static traceid artifact_tag(const T* ptr, bool leakp) {
SET_LEAKP(ptr);
}
assert(IS_LEAKP(ptr), "invariant");
- return artifact_id(ptr);
}
if (not_used(ptr)) {
SET_TRANSIENT(ptr);
@@ -153,7 +152,7 @@ static inline bool should_do_cld_klass(const Klass* cld_klass, bool leakp) {
static inline bool should_enqueue(const Klass* cld_klass) {
assert(cld_klass != nullptr, "invariant");
- if (previous_epoch()) {
+ if (unloading() || previous_epoch()) {
return false;
}
CldPtr cld = get_cld(cld_klass);
@@ -253,7 +252,11 @@ class ModuleFieldSelector {
static TypePtr select(KlassPtr klass) {
assert(klass != nullptr, "invariant");
PkgPtr pkg = klass->package();
- return pkg != nullptr ? pkg->module() : nullptr;
+ if (pkg == nullptr) {
+ return nullptr;
+ }
+ assert(current_epoch() ? IS_SERIALIZED(pkg) : true, "invariant");
+ return pkg->module();
}
};
@@ -272,7 +275,11 @@ class ModuleCldFieldSelector {
static TypePtr select(KlassPtr klass) {
assert(klass != nullptr, "invariant");
ModPtr mod = ModuleFieldSelector::select(klass);
- return mod != nullptr ? mod->loader_data() : nullptr;
+ if (mod == nullptr) {
+ return nullptr;
+ }
+ assert(current_epoch() ? IS_SERIALIZED(mod) : true, "invariant");
+ return mod->loader_data();
}
};
@@ -283,18 +290,7 @@ class SerializePredicate {
SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
bool operator()(T const& value) {
assert(value != nullptr, "invariant");
- return _class_unload ? _artifacts->should_do_unloading_artifact(value) : IS_NOT_SERIALIZED(value);
- }
-};
-
-template <>
-class SerializePredicate {
- bool _class_unload;
-public:
- SerializePredicate(bool class_unload) : _class_unload(class_unload) {}
- bool operator()(const Klass* klass) {
- assert(klass != nullptr, "invariant");
- return _class_unload ? true : IS_NOT_SERIALIZED(klass);
+ return _class_unload ? true : IS_NOT_SERIALIZED(value);
}
};
@@ -352,13 +348,19 @@ static void do_write_klass(JfrCheckpointWriter* writer, CldPtr cld, KlassPtr kla
writer->write(package_id(klass, leakp));
writer->write(klass->modifier_flags());
writer->write(klass->is_hidden());
- if (!leakp) {
- set_serialized(klass);
+ if (leakp) {
+ assert(IS_LEAKP(klass), "invariant");
+ CLEAR_LEAKP(klass);
+ assert(IS_NOT_LEAKP(klass), "invariant");
+ return;
}
+ assert(used(klass), "invariant");
+ assert(unloading() ? true : IS_NOT_SERIALIZED(klass), "invariant");
+ set_serialized(klass);
}
static inline bool should_write_cld_klass(KlassPtr klass, bool leakp) {
- return klass != nullptr && (leakp || IS_NOT_SERIALIZED(klass));
+ return klass != nullptr && (leakp ? IS_LEAKP(klass) : unloading() ? true : IS_NOT_SERIALIZED(klass));
}
static void write_klass(JfrCheckpointWriter* writer, KlassPtr klass, bool leakp, int& elements) {
@@ -373,10 +375,10 @@ static void write_klass(JfrCheckpointWriter* writer, KlassPtr klass, bool leakp,
write_klass(writer, cld_klass, leakp, elements);
}
}
- KlassPtr mod_klass = get_module_cld_klass(klass, leakp);
- if (should_write_cld_klass(mod_klass, leakp)) {
+ KlassPtr mod_cld_klass = get_module_cld_klass(klass, leakp);
+ if (should_write_cld_klass(mod_cld_klass, leakp)) {
// Write the klass for the module cld.
- write_klass(writer, mod_klass, leakp, elements);
+ write_klass(writer, mod_cld_klass, leakp, elements);
}
}
@@ -398,7 +400,6 @@ int write__klass(JfrCheckpointWriter* writer, const void* k) {
int write__klass__leakp(JfrCheckpointWriter* writer, const void* k) {
assert(k != nullptr, "invariant");
KlassPtr klass = static_cast(k);
- CLEAR_LEAKP(klass);
int elements = 0;
write_klass(writer, klass, true, elements);
return elements;
@@ -978,15 +979,12 @@ class MethodIteratorHost {
MethodUsedPredicate _method_used_predicate;
MethodFlagPredicate _method_flag_predicate;
public:
- MethodIteratorHost(JfrCheckpointWriter* writer,
- bool current_epoch = false,
- bool class_unload = false,
- bool skip_header = false) :
- _method_cb(writer, class_unload, skip_header),
- _klass_cb(writer, class_unload, skip_header),
- _klass_used_predicate(current_epoch),
- _method_used_predicate(current_epoch),
- _method_flag_predicate(current_epoch) {}
+ MethodIteratorHost(JfrCheckpointWriter* writer) :
+ _method_cb(writer, unloading(), false),
+ _klass_cb(writer, unloading(), false),
+ _klass_used_predicate(current_epoch()),
+ _method_used_predicate(current_epoch()),
+ _method_flag_predicate(current_epoch()) {}
bool operator()(KlassPtr klass) {
if (_method_used_predicate(klass)) {
@@ -1037,14 +1035,13 @@ typedef LeakPredicate LeakMethodPredicate;
typedef JfrPredicatedTypeWriterImplHost LeakMethodWriterImplTarget;
typedef JfrTypeWriterHost LeakMethodWriterImpl;
typedef MethodIteratorHost LeakMethodWriter;
-typedef MethodIteratorHost LeakMethodWriter;
typedef CompositeFunctor CompositeMethodWriter;
static void write_methods_with_leakp(MethodWriter& mw) {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- LeakMethodWriter lpmw(_leakp_writer, current_epoch(), unloading());
+ LeakMethodWriter lpmw(_leakp_writer);
CompositeMethodWriter cmw(&lpmw, &mw);
_artifacts->iterate_klasses(cmw);
_artifacts->tally(mw);
@@ -1052,7 +1049,7 @@ static void write_methods_with_leakp(MethodWriter& mw) {
static void write_methods() {
assert(_writer != nullptr, "invariant");
- MethodWriter mw(_writer, current_epoch(), unloading());
+ MethodWriter mw(_writer);
if (_leakp_writer == nullptr) {
_artifacts->iterate_klasses(mw);
_artifacts->tally(mw);
@@ -1065,7 +1062,7 @@ static void write_methods_on_clear() {
assert(_writer != nullptr, "invariant");
assert(_leakp_writer != nullptr, "invariant");
assert(previous_epoch(), "invariant");
- MethodWriter mw(_writer, current_epoch(), unloading());
+ MethodWriter mw(_writer);
write_methods_with_leakp(mw);
}
@@ -1092,14 +1089,14 @@ static int write_symbol(JfrCheckpointWriter* writer, SymbolEntryPtr entry, bool
return 1;
}
-int write__symbol(JfrCheckpointWriter* writer, const void* e) {
+static int write__symbol(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
SymbolEntryPtr entry = static_cast(e);
set_serialized(entry);
return write_symbol(writer, entry, false);
}
-int write__symbol__leakp(JfrCheckpointWriter* writer, const void* e) {
+static int write__symbol__leakp(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
SymbolEntryPtr entry = static_cast(e);
return write_symbol(writer, entry, true);
@@ -1113,14 +1110,14 @@ static int write_string(JfrCheckpointWriter* writer, StringEntryPtr entry, bool
return 1;
}
-int write__string(JfrCheckpointWriter* writer, const void* e) {
+static int write__string(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
StringEntryPtr entry = static_cast(e);
set_serialized(entry);
return write_string(writer, entry, false);
}
-int write__string__leakp(JfrCheckpointWriter* writer, const void* e) {
+static int write__string__leakp(JfrCheckpointWriter* writer, const void* e) {
assert(e != nullptr, "invariant");
StringEntryPtr entry = static_cast(e);
return write_string(writer, entry, true);
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
index 883821f853c0a..3db940156a800 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -54,10 +54,6 @@ void JfrArtifactSet::initialize(bool class_unload) {
_klass_list = new GrowableArray(initial_klass_list_size);
_klass_loader_set = new GrowableArray(initial_klass_loader_set_size);
_klass_loader_leakp_set = new GrowableArray(initial_klass_loader_set_size);
-
- if (class_unload) {
- _unloading_set = new GrowableArray(initial_klass_list_size);
- }
}
void JfrArtifactSet::clear() {
@@ -117,18 +113,9 @@ bool JfrArtifactSet::should_do_cld_klass(const Klass* k, bool leakp) {
return not_in_set(leakp ? _klass_loader_leakp_set : _klass_loader_set, k);
}
-bool JfrArtifactSet::should_do_unloading_artifact(const void* ptr) {
- assert(ptr != nullptr, "invariant");
- assert(_class_unload, "invariant");
- assert(_unloading_set != nullptr, "invariant");
- // The incoming pointers are of all kinds of different types.
- // However, we are only interested in set membership.
- // Treat them uniformly as const Klass* for simplicity and code reuse.
- return not_in_set(_unloading_set, static_cast(ptr));
-}
-
void JfrArtifactSet::register_klass(const Klass* k) {
assert(k != nullptr, "invariant");
+ assert(IS_SERIALIZED(k), "invariant");
assert(_klass_list != nullptr, "invariant");
_klass_list->append(k);
}
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
index 24424fdef3aa9..237745b13d93e 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/jfrTypeSetUtils.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -80,6 +80,7 @@ class KlassToFieldEnvelope {
public:
KlassToFieldEnvelope(Letter* letter) : _letter(letter) {}
bool operator()(const Klass* klass) {
+ assert(IS_SERIALIZED(klass), "invariant");
typename FieldSelector::TypePtr t = FieldSelector::select(klass);
return t != nullptr ? (*_letter)(t) : true;
}
@@ -130,7 +131,7 @@ class SymbolPredicate {
class KlassUsedPredicate {
bool _current_epoch;
-public:
+ public:
KlassUsedPredicate(bool current_epoch) : _current_epoch(current_epoch) {}
bool operator()(const Klass* klass) {
return _current_epoch ? USED_THIS_EPOCH(klass) : USED_PREVIOUS_EPOCH(klass);
@@ -201,7 +202,6 @@ class JfrArtifactSet : public JfrCHeapObj {
GrowableArray* _klass_list;
GrowableArray* _klass_loader_set;
GrowableArray* _klass_loader_leakp_set;
- GrowableArray* _unloading_set;
size_t _total_count;
bool _class_unload;
@@ -229,23 +229,8 @@ class JfrArtifactSet : public JfrCHeapObj {
size_t total_count() const;
void register_klass(const Klass* k);
bool should_do_cld_klass(const Klass* k, bool leakp);
- bool should_do_unloading_artifact(const void* ptr);
void increment_checkpoint_id();
- template
- void iterate_klasses(Functor& functor) const {
- for (int i = 0; i < _klass_list->length(); ++i) {
- if (!functor(_klass_list->at(i))) {
- return;
- }
- }
- for (int i = 0; i < _klass_loader_set->length(); ++i) {
- if (!functor(_klass_loader_set->at(i))) {
- return;
- }
- }
- }
-
template
void iterate_symbols(T& functor) {
_symbol_table->iterate_symbols(functor);
@@ -261,6 +246,24 @@ class JfrArtifactSet : public JfrCHeapObj {
_total_count += writer.count();
}
+ template
+ void iterate_klasses(Functor& functor) const {
+ if (iterate(functor, _klass_list)) {
+ iterate(functor, _klass_loader_set);
+ }
+ }
+
+ private:
+ template
+ bool iterate(Functor& functor, GrowableArray* list) const {
+ assert(list != nullptr, "invariant");
+ for (int i = 0; i < list->length(); ++i) {
+ if (!functor(list->at(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
};
class KlassArtifactRegistrator {
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp
index 36b9d491d30b6..d9211e8cb8c23 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2011, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -92,6 +92,7 @@ class JfrTraceId : public AllStatic {
static traceid load(const ModuleEntry* module);
static traceid load(const PackageEntry* package);
static traceid load(const ClassLoaderData* cld);
+ static traceid load_leakp(const Klass* klass); // leak profiler
static traceid load_leakp(const Klass* klass, const Method* method); // leak profiler
static traceid load_leakp_previous_epoch(const Klass* klass, const Method* method); // leak profiler
static traceid load_no_enqueue(const Method* method);
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp
index 4f1ce813ea6f6..aa99a8383eb56 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -68,6 +68,10 @@ inline traceid JfrTraceId::load(const ClassLoaderData* cld) {
return JfrTraceIdLoadBarrier::load(cld);
}
+inline traceid JfrTraceId::load_leakp(const Klass* klass) {
+ return JfrTraceIdLoadBarrier::load_leakp(klass);
+}
+
inline traceid JfrTraceId::load_leakp(const Klass* klass, const Method* method) {
return JfrTraceIdLoadBarrier::load_leakp(klass, method);
}
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp
index 0795c98940e88..ff984d05c19fa 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -87,6 +87,7 @@ class JfrTraceIdLoadBarrier : AllStatic {
static traceid load(const Method* method);
static traceid load(const ModuleEntry* module);
static traceid load(const PackageEntry* package);
+ static traceid load_leakp(const Klass* klass); // leak profiler
static traceid load_leakp(const Klass* klass, const Method* method); // leak profiler
static traceid load_leakp_previuos_epoch(const Klass* klass, const Method* method); // leak profiler
static void do_klasses(void f(Klass*), bool previous_epoch = false);
diff --git a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
index 13853e14a1334..ed30233527400 100644
--- a/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
+++ b/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdLoadBarrier.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -82,7 +82,7 @@ inline traceid JfrTraceIdLoadBarrier::load(const Klass* klass) {
if (should_tag(klass)) {
load_barrier(klass);
}
- assert(USED_THIS_EPOCH(klass), "invariant");
+ assert(METHOD_AND_CLASS_USED_THIS_EPOCH(klass), "invariant");
return TRACE_ID(klass);
}
@@ -148,6 +148,13 @@ inline traceid JfrTraceIdLoadBarrier::load(const PackageEntry* package) {
return set_used_and_get(package);
}
+inline traceid JfrTraceIdLoadBarrier::load_leakp(const Klass* klass) {
+ assert(klass != nullptr, "invariant");
+ load(klass); // Ensure tagged and enqueued.
+ SET_LEAKP(klass);
+ return TRACE_ID(klass);
+}
+
inline traceid JfrTraceIdLoadBarrier::load_leakp(const Klass* klass, const Method* method) {
assert(klass != nullptr, "invariant");
assert(METHOD_AND_CLASS_USED_THIS_EPOCH(klass), "invariant");
diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
index fe674bc961064..4f87916248345 100644
--- a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
+++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -573,9 +573,7 @@ void JfrRecorderService::pre_safepoint_write() {
ObjectSampleCheckpoint::on_rotation(ObjectSampler::acquire());
}
write_storage(_storage, _chunkwriter);
- if (_stack_trace_repository.is_modified()) {
- write_stacktrace(_stack_trace_repository, _chunkwriter, false);
- }
+ write_stacktrace(_stack_trace_repository, _chunkwriter, true);
}
void JfrRecorderService::invoke_safepoint_write() {
@@ -691,3 +689,16 @@ void JfrRecorderService::evaluate_chunk_size_for_rotation() {
DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current()));
JfrChunkRotation::evaluate(_chunkwriter);
}
+
+void JfrRecorderService::emit_leakprofiler_events(int64_t cutoff_ticks, bool emit_all, bool skip_bfs) {
+ DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current()));
+ // Take the rotation lock to exclude flush() during event emits. This is because event emit
+ // also creates a number checkpoint events. Those checkpoint events require a future typeset checkpoint
+ // event for completeness, i.e. to be generated before being flushed to a segment.
+ // The upcoming flush() or rotation() after event emit completes this typeset checkpoint
+ // and serializes all event emit checkpoint events to the same segment.
+ JfrRotationLock lock;
+ // Take the rotation lock before the transition.
+ ThreadInVMfromNative transition(JavaThread::current());
+ LeakProfiler::emit_events(cutoff_ticks, emit_all, skip_bfs);
+}
diff --git a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.hpp b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.hpp
index 4ba775d8970bd..89c0437dd13bf 100644
--- a/src/hotspot/share/jfr/recorder/service/jfrRecorderService.hpp
+++ b/src/hotspot/share/jfr/recorder/service/jfrRecorderService.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -70,6 +70,7 @@ class JfrRecorderService : public StackObj {
void flushpoint();
void process_full_buffers();
void evaluate_chunk_size_for_rotation();
+ void emit_leakprofiler_events(int64_t cutoff_ticks, bool emit_all, bool skip_bfs);
static bool is_recording();
};
diff --git a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp
index 3cc93545097ff..a1d9f6efb19cb 100644
--- a/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp
+++ b/src/hotspot/share/jfr/recorder/stacktrace/jfrStackTraceRepository.cpp
@@ -98,11 +98,10 @@ bool JfrStackTraceRepository::is_modified() const {
}
size_t JfrStackTraceRepository::write(JfrChunkWriter& sw, bool clear) {
+ MutexLocker lock(JfrStacktrace_lock, Mutex::_no_safepoint_check_flag);
if (_entries == 0) {
return 0;
}
- MutexLocker lock(JfrStacktrace_lock, Mutex::_no_safepoint_check_flag);
- assert(_entries > 0, "invariant");
int count = 0;
for (u4 i = 0; i < TABLE_SIZE; ++i) {
JfrStackTrace* stacktrace = _table[i];
diff --git a/src/hotspot/share/jfr/support/jfrKlassUnloading.cpp b/src/hotspot/share/jfr/support/jfrKlassUnloading.cpp
index 46d9cea90e90c..e0f26a800ba52 100644
--- a/src/hotspot/share/jfr/support/jfrKlassUnloading.cpp
+++ b/src/hotspot/share/jfr/support/jfrKlassUnloading.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -101,13 +101,12 @@ void JfrKlassUnloading::clear() {
}
}
-static bool add_to_unloaded_klass_set(traceid klass_id, bool current_epoch) {
+static void add_to_unloaded_klass_set(traceid klass_id) {
assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
- GrowableArray* const unload_set = current_epoch ? get_unload_set() : get_unload_set_previous_epoch();
+ GrowableArray* const unload_set = get_unload_set();
assert(unload_set != nullptr, "invariant");
assert(unload_set->find(klass_id) == -1, "invariant");
unload_set->append(klass_id);
- return true;
}
#if INCLUDE_MANAGEMENT
@@ -129,7 +128,8 @@ bool JfrKlassUnloading::on_unload(const Klass* k) {
if (IS_JDK_JFR_EVENT_SUBKLASS(k)) {
++event_klass_unloaded_count;
}
- return USED_ANY_EPOCH(k) && add_to_unloaded_klass_set(JfrTraceId::load_raw(k), USED_THIS_EPOCH(k));
+ add_to_unloaded_klass_set(JfrTraceId::load_raw(k));
+ return USED_THIS_EPOCH(k);
}
bool JfrKlassUnloading::is_unloaded(traceid klass_id, bool previous_epoch /* false */) {
diff --git a/src/hotspot/share/jfr/support/jfrTraceIdExtension.hpp b/src/hotspot/share/jfr/support/jfrTraceIdExtension.hpp
index d319e307884f6..353e5c3f07c2b 100644
--- a/src/hotspot/share/jfr/support/jfrTraceIdExtension.hpp
+++ b/src/hotspot/share/jfr/support/jfrTraceIdExtension.hpp
@@ -1,5 +1,5 @@
/*
-* Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 2012, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -95,6 +95,10 @@ class JfrTraceFlag {
} \
uint8_t* trace_meta_addr() const { \
return _trace_flags.meta_addr(); \
+ } \
+ void copy_trace_flags(uint8_t src_flags) const { \
+ uint8_t flags = *_trace_flags.flags_addr(); \
+ _trace_flags.set_flags(flags | src_flags); \
}
#endif // SHARE_JFR_SUPPORT_JFRTRACEIDEXTENSION_HPP
diff --git a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
index 51281b0f5564e..a5a20a1310ef9 100644
--- a/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
+++ b/src/hotspot/share/jvmci/jvmciCodeInstaller.cpp
@@ -896,8 +896,8 @@ int CodeInstaller::estimate_stubs_size(HotSpotCompiledCodeStream* stream, JVMCI_
// Estimate the number of static call stubs that might be emitted.
u2 static_call_stubs = stream->read_u2("numStaticCallStubs");
u2 trampoline_stubs = stream->read_u2("numTrampolineStubs");
- int size = static_call_stubs * CompiledStaticCall::to_interp_stub_size();
- size += trampoline_stubs * CompiledStaticCall::to_trampoline_stub_size();
+ int size = static_call_stubs * CompiledDirectCall::to_interp_stub_size();
+ size += trampoline_stubs * CompiledDirectCall::to_trampoline_stub_size();
return size;
}
@@ -1243,7 +1243,7 @@ void CodeInstaller::site_Call(CodeBuffer& buffer, u1 tag, jint pc_offset, HotSpo
CodeInstaller::pd_relocate_JavaMethod(buffer, method, pc_offset, JVMCI_CHECK);
if (_next_call_type == INVOKESTATIC || _next_call_type == INVOKESPECIAL) {
// Need a static call stub for transitions from compiled to interpreted.
- if (CompiledStaticCall::emit_to_interp_stub(buffer, _instructions->start() + pc_offset) == nullptr) {
+ if (CompiledDirectCall::emit_to_interp_stub(buffer, _instructions->start() + pc_offset) == nullptr) {
JVMCI_ERROR("could not emit to_interp stub - code cache is full");
}
}
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
index 0999363688f50..23e726fb18048 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVM.cpp
@@ -182,7 +182,7 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
// Entry to native method implementation that transitions
// current thread to '_thread_in_vm'.
#define C2V_VMENTRY(result_type, name, signature) \
- JNIEXPORT result_type JNICALL c2v_ ## name signature { \
+ result_type JNICALL c2v_ ## name signature { \
JavaThread* thread = JavaThread::current_or_null(); \
if (thread == nullptr) { \
env->ThrowNew(JNIJVMCI::InternalError::clazz(), \
@@ -193,7 +193,7 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
JVMCITraceMark jtm("CompilerToVM::" #name);
#define C2V_VMENTRY_(result_type, name, signature, result) \
- JNIEXPORT result_type JNICALL c2v_ ## name signature { \
+ result_type JNICALL c2v_ ## name signature { \
JavaThread* thread = JavaThread::current_or_null(); \
if (thread == nullptr) { \
env->ThrowNew(JNIJVMCI::InternalError::clazz(), \
@@ -209,7 +209,7 @@ Handle JavaArgumentUnboxer::next_arg(BasicType expectedType) {
// Entry to native method implementation that does not transition
// current thread to '_thread_in_vm'.
#define C2V_VMENTRY_PREFIX(result_type, name, signature) \
- JNIEXPORT result_type JNICALL c2v_ ## name signature { \
+ result_type JNICALL c2v_ ## name signature { \
JavaThread* thread = JavaThread::current_or_null();
#define C2V_END }
@@ -1380,7 +1380,7 @@ C2V_END
/*
* Used by matches() to convert a ResolvedJavaMethod[] to an array of Method*.
*/
-GrowableArray* init_resolved_methods(jobjectArray methods, JVMCIEnv* JVMCIENV) {
+static GrowableArray* init_resolved_methods(jobjectArray methods, JVMCIEnv* JVMCIENV) {
objArrayOop methods_oop = (objArrayOop) JNIHandles::resolve(methods);
GrowableArray* resolved_methods = new GrowableArray(methods_oop->length());
for (int i = 0; i < methods_oop->length(); i++) {
@@ -1399,7 +1399,7 @@ GrowableArray* init_resolved_methods(jobjectArray methods, JVMCIEnv* JV
* The ResolvedJavaMethod[] array is converted to a Method* array that is then cached in the resolved_methods_ref in/out parameter.
* In case of a match, the matching ResolvedJavaMethod is returned in matched_jvmci_method_ref.
*/
-bool matches(jobjectArray methods, Method* method, GrowableArray** resolved_methods_ref, Handle* matched_jvmci_method_ref, Thread* THREAD, JVMCIEnv* JVMCIENV) {
+static bool matches(jobjectArray methods, Method* method, GrowableArray** resolved_methods_ref, Handle* matched_jvmci_method_ref, Thread* THREAD, JVMCIEnv* JVMCIENV) {
GrowableArray* resolved_methods = *resolved_methods_ref;
if (resolved_methods == nullptr) {
resolved_methods = init_resolved_methods(methods, JVMCIENV);
@@ -1420,7 +1420,7 @@ bool matches(jobjectArray methods, Method* method, GrowableArray** reso
/*
* Resolves an interface call to a concrete method handle.
*/
-methodHandle resolve_interface_call(Klass* spec_klass, Symbol* name, Symbol* signature, JavaCallArguments* args, TRAPS) {
+static methodHandle resolve_interface_call(Klass* spec_klass, Symbol* name, Symbol* signature, JavaCallArguments* args, TRAPS) {
CallInfo callinfo;
Handle receiver = args->receiver();
Klass* recvrKlass = receiver.is_null() ? (Klass*)nullptr : receiver->klass();
@@ -1435,7 +1435,7 @@ methodHandle resolve_interface_call(Klass* spec_klass, Symbol* name, Symbol* sig
/*
* Used by c2v_iterateFrames to make a new vframeStream at the given compiled frame id (stack pointer) and vframe id.
*/
-void resync_vframestream_to_compiled_frame(vframeStream& vfst, intptr_t* stack_pointer, int vframe_id, JavaThread* thread, TRAPS) {
+static void resync_vframestream_to_compiled_frame(vframeStream& vfst, intptr_t* stack_pointer, int vframe_id, JavaThread* thread, TRAPS) {
vfst = vframeStream(thread);
while (vfst.frame_id() != stack_pointer && !vfst.at_end()) {
vfst.next();
@@ -1458,7 +1458,7 @@ void resync_vframestream_to_compiled_frame(vframeStream& vfst, intptr_t* stack_p
/*
* Used by c2v_iterateFrames. Returns an array of any unallocated scope objects or null if none.
*/
-GrowableArray* get_unallocated_objects_or_null(GrowableArray* scope_objects) {
+static GrowableArray* get_unallocated_objects_or_null(GrowableArray* scope_objects) {
GrowableArray* unallocated = nullptr;
for (int i = 0; i < scope_objects->length(); i++) {
ObjectValue* sv = (ObjectValue*) scope_objects->at(i);
@@ -2442,7 +2442,7 @@ C2V_END
C2V_VMENTRY_0(jint, arrayBaseOffset, (JNIEnv* env, jobject, jchar type_char))
BasicType type = JVMCIENV->typeCharToBasicType(type_char, JVMCI_CHECK_0);
- return arrayOopDesc::header_size(type) * HeapWordSize;
+ return arrayOopDesc::base_offset_in_bytes(type);
C2V_END
C2V_VMENTRY_0(jint, arrayIndexScale, (JNIEnv* env, jobject, jchar type_char))
diff --git a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
index 2ad442db93483..242933c18af94 100644
--- a/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
+++ b/src/hotspot/share/jvmci/vmStructs_jvmci.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
#include "precompiled.hpp"
#include "code/codeCache.hpp"
+#include "code/compiledIC.hpp"
#include "compiler/compileBroker.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "jvmci/jvmciCodeInstaller.hpp"
@@ -42,7 +43,7 @@
#include "runtime/vm_version.hpp"
#if INCLUDE_G1GC
#include "gc/g1/g1CardTable.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/g1/g1ThreadLocalData.hpp"
#endif
@@ -144,6 +145,11 @@
\
nonstatic_field(CompileTask, _num_inlined_bytecodes, int) \
\
+ volatile_nonstatic_field(CompiledICData, _speculated_method, Method*) \
+ volatile_nonstatic_field(CompiledICData, _speculated_klass, uintptr_t) \
+ nonstatic_field(CompiledICData, _itable_defc_klass, Klass*) \
+ nonstatic_field(CompiledICData, _itable_refc_klass, Klass*) \
+ \
nonstatic_field(ConstantPool, _tags, Array*) \
nonstatic_field(ConstantPool, _pool_holder, InstanceKlass*) \
nonstatic_field(ConstantPool, _length, int) \
@@ -430,6 +436,7 @@
declare_toplevel_type(oopDesc) \
declare_type(arrayOopDesc, oopDesc) \
\
+ declare_toplevel_type(CompiledICData) \
declare_toplevel_type(MetaspaceObj) \
declare_type(Metadata, MetaspaceObj) \
declare_type(Klass, Metadata) \
@@ -1016,14 +1023,6 @@ int JVMCIVMStructs::localHotSpotVMAddresses_count() {
return (sizeof(localHotSpotVMAddresses) / sizeof(VMAddressEntry)) - 1;
}
-extern "C" {
-JNIEXPORT VMStructEntry* jvmciHotSpotVMStructs = JVMCIVMStructs::localHotSpotVMStructs;
-JNIEXPORT VMTypeEntry* jvmciHotSpotVMTypes = JVMCIVMStructs::localHotSpotVMTypes;
-JNIEXPORT VMIntConstantEntry* jvmciHotSpotVMIntConstants = JVMCIVMStructs::localHotSpotVMIntConstants;
-JNIEXPORT VMLongConstantEntry* jvmciHotSpotVMLongConstants = JVMCIVMStructs::localHotSpotVMLongConstants;
-JNIEXPORT VMAddressEntry* jvmciHotSpotVMAddresses = JVMCIVMStructs::localHotSpotVMAddresses;
-}
-
#ifdef ASSERT
// This is used both to check the types of referenced fields and
// to ensure that all of the field types are present.
diff --git a/src/hotspot/share/memory/memRegion.hpp b/src/hotspot/share/memory/memRegion.hpp
index 3d5cf5f06e3ee..5d3d635c650cd 100644
--- a/src/hotspot/share/memory/memRegion.hpp
+++ b/src/hotspot/share/memory/memRegion.hpp
@@ -54,10 +54,6 @@ class MemRegion {
_start(start), _word_size(pointer_delta(end, start)) {
assert(end >= start, "incorrect constructor arguments");
}
- MemRegion(MetaWord* start, MetaWord* end) :
- _start((HeapWord*)start), _word_size(pointer_delta(end, start)) {
- assert(end >= start, "incorrect constructor arguments");
- }
MemRegion intersection(const MemRegion mr2) const;
// regions must overlap or be adjacent
diff --git a/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp
index dc5f7a6cdf110..84f9c9ecc4385 100644
--- a/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp
+++ b/src/hotspot/share/memory/metaspace/virtualSpaceNode.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2018, 2021 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -56,12 +56,12 @@ namespace metaspace {
#define LOGFMT_ARGS p2i(this), p2i(_base)
#ifdef ASSERT
-void check_pointer_is_aligned_to_commit_granule(const MetaWord* p) {
+static void check_pointer_is_aligned_to_commit_granule(const MetaWord* p) {
assert(is_aligned(p, Settings::commit_granule_bytes()),
"Pointer not aligned to commit granule size: " PTR_FORMAT ".",
p2i(p));
}
-void check_word_size_is_aligned_to_commit_granule(size_t word_size) {
+static void check_word_size_is_aligned_to_commit_granule(size_t word_size) {
assert(is_aligned(word_size, Settings::commit_granule_words()),
"Not aligned to commit granule size: " SIZE_FORMAT ".", word_size);
}
diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp
index dfeb6b11f6f21..2a3d532725ffb 100644
--- a/src/hotspot/share/memory/universe.cpp
+++ b/src/hotspot/share/memory/universe.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -298,7 +298,7 @@ void Universe::check_alignment(uintx size, uintx alignment, const char* name) {
}
}
-void initialize_basic_type_klass(Klass* k, TRAPS) {
+static void initialize_basic_type_klass(Klass* k, TRAPS) {
Klass* ok = vmClasses::Object_klass();
#if INCLUDE_CDS
if (UseSharedSpaces) {
@@ -924,11 +924,11 @@ void universe_oopstorage_init() {
Universe::oopstorage_init();
}
-void initialize_known_method(LatestMethodCache* method_cache,
- InstanceKlass* ik,
- const char* method,
- Symbol* signature,
- bool is_static, TRAPS)
+static void initialize_known_method(LatestMethodCache* method_cache,
+ InstanceKlass* ik,
+ const char* method,
+ Symbol* signature,
+ bool is_static, TRAPS)
{
TempNewSymbol name = SymbolTable::new_symbol(method);
Method* m = nullptr;
diff --git a/src/hotspot/share/nmt/memTracker.hpp b/src/hotspot/share/nmt/memTracker.hpp
index a863c45a65854..c7fbda4713acc 100644
--- a/src/hotspot/share/nmt/memTracker.hpp
+++ b/src/hotspot/share/nmt/memTracker.hpp
@@ -175,12 +175,12 @@ class MemTracker : AllStatic {
//
// The two new memory regions will be both registered under stack and
// memory flags of the original region.
- static inline void record_virtual_memory_split_reserved(void* addr, size_t size, size_t split) {
+ static inline void record_virtual_memory_split_reserved(void* addr, size_t size, size_t split, MEMFLAGS flag, MEMFLAGS split_flag) {
assert_post_init();
if (!enabled()) return;
if (addr != nullptr) {
ThreadCritical tc;
- VirtualMemoryTracker::split_reserved_region((address)addr, size, split);
+ VirtualMemoryTracker::split_reserved_region((address)addr, size, split, flag, split_flag);
}
}
diff --git a/src/hotspot/share/nmt/virtualMemoryTracker.cpp b/src/hotspot/share/nmt/virtualMemoryTracker.cpp
index 471c9eafd0efe..06f9469f25e14 100644
--- a/src/hotspot/share/nmt/virtualMemoryTracker.cpp
+++ b/src/hotspot/share/nmt/virtualMemoryTracker.cpp
@@ -560,7 +560,7 @@ bool VirtualMemoryTracker::remove_released_region(address addr, size_t size) {
// Given an existing memory mapping registered with NMT, split the mapping in
// two. The newly created two mappings will be registered under the call
// stack and the memory flags of the original section.
-bool VirtualMemoryTracker::split_reserved_region(address addr, size_t size, size_t split) {
+bool VirtualMemoryTracker::split_reserved_region(address addr, size_t size, size_t split, MEMFLAGS flag, MEMFLAGS split_flag) {
ReservedMemoryRegion rgn(addr, size);
ReservedMemoryRegion* reserved_rgn = _reserved_regions->find(rgn);
@@ -576,8 +576,8 @@ bool VirtualMemoryTracker::split_reserved_region(address addr, size_t size, size
log_debug(nmt)("Split region \'%s\' (" INTPTR_FORMAT ", " SIZE_FORMAT ") with size " SIZE_FORMAT,
name, p2i(rgn.base()), rgn.size(), split);
// Now, create two new regions.
- add_reserved_region(addr, split, original_stack, original_flags);
- add_reserved_region(addr + split, size - split, original_stack, original_flags);
+ add_reserved_region(addr, split, original_stack, flag);
+ add_reserved_region(addr + split, size - split, original_stack, split_flag);
return true;
}
diff --git a/src/hotspot/share/nmt/virtualMemoryTracker.hpp b/src/hotspot/share/nmt/virtualMemoryTracker.hpp
index b4dd891f1c21f..db49f108c84b9 100644
--- a/src/hotspot/share/nmt/virtualMemoryTracker.hpp
+++ b/src/hotspot/share/nmt/virtualMemoryTracker.hpp
@@ -390,7 +390,7 @@ class VirtualMemoryTracker : AllStatic {
// Given an existing memory mapping registered with NMT, split the mapping in
// two. The newly created two mappings will be registered under the call
// stack and the memory flags of the original section.
- static bool split_reserved_region(address addr, size_t size, size_t split);
+ static bool split_reserved_region(address addr, size_t size, size_t split, MEMFLAGS flag, MEMFLAGS split_flag);
// Walk virtual memory data structure for creating baseline, etc.
static bool walk_virtual_memory(VirtualMemoryWalker* walker);
diff --git a/src/hotspot/share/oops/accessBackend.cpp b/src/hotspot/share/oops/accessBackend.cpp
index 853f42f6f7e4c..be191316f25d2 100644
--- a/src/hotspot/share/oops/accessBackend.cpp
+++ b/src/hotspot/share/oops/accessBackend.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,12 @@
#include "utilities/debug.hpp"
#include "utilities/vmError.hpp"
+#if defined(TARGET_COMPILER_gcc)
+#define HIDDEN __attribute__ ((visibility ("hidden")))
+#else
+#define HIDDEN
+#endif
+
namespace AccessInternal {
// These forward copying calls to Copy without exposing the Copy type in headers unnecessarily
@@ -58,102 +64,102 @@ namespace AccessInternal {
reinterpret_cast(dst), length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jboolean* src, jboolean* dst, size_t length) {
Copy::conjoint_jbytes(reinterpret_cast(src), reinterpret_cast(dst), length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jbyte* src, jbyte* dst, size_t length) {
Copy::conjoint_jbytes(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jchar* src, jchar* dst, size_t length) {
Copy::conjoint_jshorts_atomic(reinterpret_cast(src), reinterpret_cast(dst), length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jshort* src, jshort* dst, size_t length) {
Copy::conjoint_jshorts_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jint* src, jint* dst, size_t length) {
Copy::conjoint_jints_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jfloat* src, jfloat* dst, size_t length) {
Copy::conjoint_jints_atomic(reinterpret_cast(src), reinterpret_cast(dst), length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jlong* src, jlong* dst, size_t length) {
Copy::conjoint_jlongs_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(jdouble* src, jdouble* dst, size_t length) {
Copy::conjoint_jlongs_atomic(reinterpret_cast(src), reinterpret_cast(dst), length);
}
- template<>
+ template<> HIDDEN
void arraycopy_arrayof_conjoint(jbyte* src, jbyte* dst, size_t length) {
Copy::arrayof_conjoint_jbytes(reinterpret_cast(src),
reinterpret_cast(dst),
length);
}
- template<>
+ template<> HIDDEN
void arraycopy_arrayof_conjoint(jshort* src, jshort* dst, size_t length) {
Copy::arrayof_conjoint_jshorts(reinterpret_cast(src),
reinterpret_cast(dst),
length);
}
- template<>
+ template<> HIDDEN
void arraycopy_arrayof_conjoint(jint* src, jint* dst, size_t length) {
Copy::arrayof_conjoint_jints(reinterpret_cast(src),
reinterpret_cast(dst),
length);
}
- template<>
+ template<> HIDDEN
void arraycopy_arrayof_conjoint(jlong* src, jlong* dst, size_t length) {
Copy::arrayof_conjoint_jlongs(reinterpret_cast(src),
reinterpret_cast(dst),
length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint(void* src, void* dst, size_t length) {
Copy::conjoint_jbytes(reinterpret_cast(src),
reinterpret_cast(dst),
length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint_atomic(jbyte* src, jbyte* dst, size_t length) {
Copy::conjoint_jbytes_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint_atomic(jshort* src, jshort* dst, size_t length) {
Copy::conjoint_jshorts_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint_atomic(jint* src, jint* dst, size_t length) {
Copy::conjoint_jints_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint_atomic(jlong* src, jlong* dst, size_t length) {
Copy::conjoint_jlongs_atomic(src, dst, length);
}
- template<>
+ template<> HIDDEN
void arraycopy_conjoint_atomic(void* src, void* dst, size_t length) {
Copy::conjoint_memory_atomic(src, dst, length);
}
diff --git a/src/hotspot/share/oops/arrayOop.hpp b/src/hotspot/share/oops/arrayOop.hpp
index a557268fe7772..0aa26500bd8d2 100644
--- a/src/hotspot/share/oops/arrayOop.hpp
+++ b/src/hotspot/share/oops/arrayOop.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@
#include "oops/oop.hpp"
#include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
// arrayOopDesc is the abstract baseclass for all arrays. It doesn't
// declare pure virtual to enforce this because that would allocate a vtbl
@@ -45,13 +46,29 @@ class arrayOopDesc : public oopDesc {
// Interpreter/Compiler offsets
+private:
+ // Returns the address of the length "field". See length_offset_in_bytes().
+ static int* length_addr_impl(void* obj_ptr) {
+ char* ptr = static_cast(obj_ptr);
+ return reinterpret_cast(ptr + length_offset_in_bytes());
+ }
+
+ // Given a type, return true if elements of that type must be aligned to 64-bit.
+ static bool element_type_should_be_aligned(BasicType type) {
+#ifdef _LP64
+ if (type == T_OBJECT || type == T_ARRAY) {
+ return !UseCompressedOops;
+ }
+#endif
+ return type == T_DOUBLE || type == T_LONG;
+ }
+
+ public:
// Header size computation.
// The header is considered the oop part of this type plus the length.
- // Returns the aligned header_size_in_bytes. This is not equivalent to
- // sizeof(arrayOopDesc) which should not appear in the code.
+ // This is not equivalent to sizeof(arrayOopDesc) which should not appear in the code.
static int header_size_in_bytes() {
- size_t hs = align_up(length_offset_in_bytes() + sizeof(int),
- HeapWordSize);
+ size_t hs = length_offset_in_bytes() + sizeof(int);
#ifdef ASSERT
// make sure it isn't called before UseCompressedOops is initialized.
static size_t arrayoopdesc_hs = 0;
@@ -61,20 +78,6 @@ class arrayOopDesc : public oopDesc {
return (int)hs;
}
- // Returns the address of the length "field". See length_offset_in_bytes().
- static int* length_addr_impl(void* obj_ptr) {
- char* ptr = static_cast(obj_ptr);
- return reinterpret_cast(ptr + length_offset_in_bytes());
- }
-
- // Check whether an element of a typeArrayOop with the given type must be
- // aligned 0 mod 8. The typeArrayOop itself must be aligned at least this
- // strongly.
- static bool element_type_should_be_aligned(BasicType type) {
- return type == T_DOUBLE || type == T_LONG;
- }
-
- public:
// The _length field is not declared in C++. It is allocated after the
// declared nonstatic fields in arrayOopDesc if not compressed, otherwise
// it occupies the second half of the _klass field in oopDesc.
@@ -85,7 +88,8 @@ class arrayOopDesc : public oopDesc {
// Returns the offset of the first element.
static int base_offset_in_bytes(BasicType type) {
- return header_size(type) * HeapWordSize;
+ size_t hs = header_size_in_bytes();
+ return (int)(element_type_should_be_aligned(type) ? align_up(hs, BytesPerLong) : hs);
}
// Returns the address of the first element. The elements in the array will not
@@ -122,18 +126,7 @@ class arrayOopDesc : public oopDesc {
*length_addr_impl(mem) = length;
}
- // Should only be called with constants as argument
- // (will not constant fold otherwise)
- // Returns the header size in words aligned to the requirements of the
- // array object type.
- static int header_size(BasicType type) {
- size_t typesize_in_bytes = header_size_in_bytes();
- return (int)(element_type_should_be_aligned(type)
- ? align_object_offset(typesize_in_bytes/HeapWordSize)
- : typesize_in_bytes/HeapWordSize);
- }
-
- // Return the maximum length of an array of BasicType. The length can passed
+ // Return the maximum length of an array of BasicType. The length can be passed
// to typeArrayOop::object_size(scale, length, header_size) without causing an
// overflow. We also need to make sure that this will not overflow a size_t on
// 32 bit platforms when we convert it to a byte size.
@@ -141,8 +134,12 @@ class arrayOopDesc : public oopDesc {
assert(type < T_CONFLICT, "wrong type");
assert(type2aelembytes(type) != 0, "wrong type");
+ size_t hdr_size_in_bytes = base_offset_in_bytes(type);
+ // This is rounded-up and may overlap with the first array elements.
+ size_t hdr_size_in_words = align_up(hdr_size_in_bytes, HeapWordSize) / HeapWordSize;
+
const size_t max_element_words_per_size_t =
- align_down((SIZE_MAX/HeapWordSize - header_size(type)), MinObjAlignment);
+ align_down((SIZE_MAX/HeapWordSize - hdr_size_in_words), MinObjAlignment);
const size_t max_elements_per_size_t =
HeapWordSize * max_element_words_per_size_t / type2aelembytes(type);
if ((size_t)max_jint < max_elements_per_size_t) {
@@ -150,7 +147,7 @@ class arrayOopDesc : public oopDesc {
// (CollectedHeap, Klass::oop_oop_iterate(), and more) uses an int for
// passing around the size (in words) of an object. So, we need to avoid
// overflowing an int when we add the header. See CRs 4718400 and 7110613.
- return align_down(max_jint - header_size(type), MinObjAlignment);
+ return align_down(max_jint - hdr_size_in_words, MinObjAlignment);
}
return (int32_t)max_elements_per_size_t;
}
diff --git a/src/hotspot/share/oops/compiledICHolder.cpp b/src/hotspot/share/oops/compiledICHolder.cpp
deleted file mode 100644
index 8bfa55bcce7a5..0000000000000
--- a/src/hotspot/share/oops/compiledICHolder.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright (c) 1998, 2022, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#include "precompiled.hpp"
-#include "oops/compiledICHolder.hpp"
-#include "runtime/atomic.hpp"
-
-#ifdef ASSERT
-volatile int CompiledICHolder::_live_count;
-volatile int CompiledICHolder::_live_not_claimed_count;
-#endif
-
-CompiledICHolder::CompiledICHolder(Metadata* metadata, Klass* klass, bool is_method)
- : _holder_metadata(metadata), _holder_klass(klass), _next(nullptr), _is_metadata_method(is_method) {
-#ifdef ASSERT
- Atomic::inc(&_live_count);
- Atomic::inc(&_live_not_claimed_count);
-#endif // ASSERT
-}
-
-#ifdef ASSERT
-CompiledICHolder::~CompiledICHolder() {
- assert(_live_count > 0, "underflow");
- Atomic::dec(&_live_count);
-}
-#endif // ASSERT
-
-// Printing
-
-void CompiledICHolder::print_on(outputStream* st) const {
- st->print("%s", internal_name());
- st->print(" - metadata: "); holder_metadata()->print_value_on(st); st->cr();
- st->print(" - klass: "); holder_klass()->print_value_on(st); st->cr();
-}
-
-void CompiledICHolder::print_value_on(outputStream* st) const {
- st->print("%s", internal_name());
-}
-
-
-// Verification
-
-void CompiledICHolder::verify_on(outputStream* st) {
- guarantee(holder_metadata()->is_method() || holder_metadata()->is_klass(), "should be method or klass");
- guarantee(holder_klass()->is_klass(), "should be klass");
-}
-
-#ifdef ASSERT
-
-void CompiledICHolder::claim() {
- Atomic::dec(&_live_not_claimed_count);
-}
-
-#endif // ASSERT
diff --git a/src/hotspot/share/oops/compiledICHolder.hpp b/src/hotspot/share/oops/compiledICHolder.hpp
deleted file mode 100644
index 4509c8f578b77..0000000000000
--- a/src/hotspot/share/oops/compiledICHolder.hpp
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_OOPS_COMPILEDICHOLDER_HPP
-#define SHARE_OOPS_COMPILEDICHOLDER_HPP
-
-#include "oops/oop.hpp"
-#include "utilities/macros.hpp"
-#include "oops/klass.hpp"
-#include "oops/method.hpp"
-
-// A CompiledICHolder* is a helper object for the inline cache implementation.
-// It holds:
-// (1) (method+klass pair) when converting from compiled to an interpreted call
-// (2) (klass+klass pair) when calling itable stub from megamorphic compiled call
-//
-// These are always allocated in the C heap and are freed during a
-// safepoint by the ICBuffer logic. It's unsafe to free them earlier
-// since they might be in use.
-//
-
-
-class CompiledICHolder : public CHeapObj {
- friend class VMStructs;
- private:
-#ifdef ASSERT
- static volatile int _live_count; // allocated
- static volatile int _live_not_claimed_count; // allocated but not yet in use so not
- // reachable by iterating over nmethods
-#endif
-
- Metadata* _holder_metadata;
- Klass* _holder_klass; // to avoid name conflict with oopDesc::_klass
- CompiledICHolder* _next;
- bool _is_metadata_method;
-
- public:
- // Constructor
- CompiledICHolder(Metadata* metadata, Klass* klass, bool is_method = true);
- ~CompiledICHolder() NOT_DEBUG_RETURN;
-
-#ifdef ASSERT
- static int live_count() { return _live_count; }
- static int live_not_claimed_count() { return _live_not_claimed_count; }
-#endif
-
- // accessors
- Klass* holder_klass() const { return _holder_klass; }
- Metadata* holder_metadata() const { return _holder_metadata; }
-
- static ByteSize holder_metadata_offset() { return byte_offset_of(CompiledICHolder, _holder_metadata); }
- static ByteSize holder_klass_offset() { return byte_offset_of(CompiledICHolder, _holder_klass); }
-
- CompiledICHolder* next() { return _next; }
- void set_next(CompiledICHolder* n) { _next = n; }
-
- inline bool is_loader_alive();
-
- // Verify
- void verify_on(outputStream* st);
-
- // Printing
- void print_on(outputStream* st) const;
- void print_value_on(outputStream* st) const;
-
- const char* internal_name() const { return "{compiledICHolder}"; }
-
- void claim() NOT_DEBUG_RETURN;
-};
-
-#endif // SHARE_OOPS_COMPILEDICHOLDER_HPP
diff --git a/src/hotspot/share/oops/compiledICHolder.inline.hpp b/src/hotspot/share/oops/compiledICHolder.inline.hpp
deleted file mode 100644
index efbfcec064762..0000000000000
--- a/src/hotspot/share/oops/compiledICHolder.inline.hpp
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
-#define SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
-
-#include "oops/compiledICHolder.hpp"
-
-#include "oops/klass.inline.hpp"
-
-inline bool CompiledICHolder::is_loader_alive() {
- Klass* k = _is_metadata_method ? ((Method*)_holder_metadata)->method_holder() : (Klass*)_holder_metadata;
- if (!k->is_loader_alive()) {
- return false;
- }
- if (!_holder_klass->is_loader_alive()) {
- return false;
- }
- return true;
-}
-
-#endif // SHARE_OOPS_COMPILEDICHOLDER_INLINE_HPP
diff --git a/src/hotspot/share/oops/constMethod.cpp b/src/hotspot/share/oops/constMethod.cpp
index 8b4b509323f3c..a6a93d6f48a94 100644
--- a/src/hotspot/share/oops/constMethod.cpp
+++ b/src/hotspot/share/oops/constMethod.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -384,7 +384,7 @@ AnnotationArray** ConstMethod::default_annotations_addr() const {
return (AnnotationArray**)constMethod_end() - offset;
}
-Array* copy_annotations(ClassLoaderData* loader_data, AnnotationArray* from, TRAPS) {
+static Array* copy_annotations(ClassLoaderData* loader_data, AnnotationArray* from, TRAPS) {
int length = from->length();
Array* a = MetadataFactory::new_array(loader_data, length, 0, CHECK_NULL);
memcpy((void*)a->adr_at(0), (void*)from->adr_at(0), length);
diff --git a/src/hotspot/share/oops/cpCache.cpp b/src/hotspot/share/oops/cpCache.cpp
index daa094baa7eed..43e30a9ca3c5f 100644
--- a/src/hotspot/share/oops/cpCache.cpp
+++ b/src/hotspot/share/oops/cpCache.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -453,7 +453,7 @@ void ConstantPoolCache::set_archived_references(int root_index) {
#endif
#if INCLUDE_JVMTI
-void log_adjust(const char* entry_type, Method* old_method, Method* new_method, bool* trace_name_printed) {
+static void log_adjust(const char* entry_type, Method* old_method, Method* new_method, bool* trace_name_printed) {
ResourceMark rm;
if (!(*trace_name_printed)) {
diff --git a/src/hotspot/share/oops/klassVtable.cpp b/src/hotspot/share/oops/klassVtable.cpp
index ece1ad3c6de10..cbc379709f1f4 100644
--- a/src/hotspot/share/oops/klassVtable.cpp
+++ b/src/hotspot/share/oops/klassVtable.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1442,7 +1442,7 @@ class InterfaceVisiterClosure : public StackObj {
};
// Visit all interfaces with at least one itable method
-void visit_all_interfaces(Array* transitive_intf, InterfaceVisiterClosure *blk) {
+static void visit_all_interfaces(Array* transitive_intf, InterfaceVisiterClosure *blk) {
// Handle array argument
for(int i = 0; i < transitive_intf->length(); i++) {
InstanceKlass* intf = transitive_intf->at(i);
diff --git a/src/hotspot/share/oops/objArrayOop.hpp b/src/hotspot/share/oops/objArrayOop.hpp
index de6d4d3d042ba..20e2953fee9f5 100644
--- a/src/hotspot/share/oops/objArrayOop.hpp
+++ b/src/hotspot/share/oops/objArrayOop.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -51,32 +51,6 @@ class objArrayOopDesc : public arrayOopDesc {
return base_offset_in_bytes() + sizeof(T) * index;
}
-private:
- // Give size of objArrayOop in HeapWords minus the header
- static int array_size(int length) {
- const uint OopsPerHeapWord = HeapWordSize/heapOopSize;
- assert(OopsPerHeapWord >= 1 && (HeapWordSize % heapOopSize == 0),
- "Else the following (new) computation would be in error");
- uint res = ((uint)length + OopsPerHeapWord - 1)/OopsPerHeapWord;
-#ifdef ASSERT
- // The old code is left in for sanity-checking; it'll
- // go away pretty soon. XXX
- // Without UseCompressedOops, this is simply:
- // oop->length() * HeapWordsPerOop;
- // With narrowOops, HeapWordsPerOop is 1/2 or equal 0 as an integer.
- // The oop elements are aligned up to wordSize
- const uint HeapWordsPerOop = heapOopSize/HeapWordSize;
- uint old_res;
- if (HeapWordsPerOop > 0) {
- old_res = length * HeapWordsPerOop;
- } else {
- old_res = align_up((uint)length, OopsPerHeapWord)/OopsPerHeapWord;
- }
- assert(res == old_res, "Inconsistency between old and new.");
-#endif // ASSERT
- return res;
- }
-
public:
// Returns the offset of the first element.
static int base_offset_in_bytes() {
@@ -94,16 +68,15 @@ class objArrayOopDesc : public arrayOopDesc {
oop replace_if_null(int index, oop exchange_value);
// Sizing
- static int header_size() { return arrayOopDesc::header_size(T_OBJECT); }
size_t object_size() { return object_size(length()); }
static size_t object_size(int length) {
// This returns the object size in HeapWords.
- uint asz = array_size(length);
- uint osz = align_object_size(header_size() + asz);
- assert(osz >= asz, "no overflow");
- assert((int)osz > 0, "no overflow");
- return (size_t)osz;
+ size_t asz = (size_t)length * heapOopSize;
+ size_t size_words = heap_word_size(base_offset_in_bytes() + asz);
+ size_t osz = align_object_size(size_words);
+ assert(osz < max_jint, "no overflow");
+ return osz;
}
Klass* element_klass();
diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp
index 17a9bf9c41296..65727374df872 100644
--- a/src/hotspot/share/oops/oop.cpp
+++ b/src/hotspot/share/oops/oop.cpp
@@ -43,8 +43,6 @@
void oopDesc::print_on(outputStream* st) const {
if (*((juint*)this) == badHeapWordVal) {
st->print_cr("BAD WORD");
- } else if (*((juint*)this) == badMetaWordVal) {
- st->print_cr("BAD META WORD");
} else {
klass()->oop_print_on(cast_to_oop(this), st);
}
@@ -58,8 +56,6 @@ void oopDesc::print_address_on(outputStream* st) const {
void oopDesc::print_name_on(outputStream* st) const {
if (*((juint*)this) == badHeapWordVal) {
st->print_cr("BAD WORD");
- } else if (*((juint*)this) == badMetaWordVal) {
- st->print_cr("BAD META WORD");
} else {
st->print_cr("%s", klass()->external_name());
}
diff --git a/src/hotspot/share/oops/oopsHierarchy.hpp b/src/hotspot/share/oops/oopsHierarchy.hpp
index b01153da46d54..4bc5a7d4c174c 100644
--- a/src/hotspot/share/oops/oopsHierarchy.hpp
+++ b/src/hotspot/share/oops/oopsHierarchy.hpp
@@ -179,9 +179,6 @@ class MethodData;
// class Metadata
class Method;
class ConstantPool;
-// class CHeapObj
-class CompiledICHolder;
-
// The klass hierarchy is separate from the oop hierarchy.
diff --git a/src/hotspot/share/oops/stackChunkOop.cpp b/src/hotspot/share/oops/stackChunkOop.cpp
index 4e771939dc941..e114161625b66 100644
--- a/src/hotspot/share/oops/stackChunkOop.cpp
+++ b/src/hotspot/share/oops/stackChunkOop.cpp
@@ -411,8 +411,6 @@ template void stackChunkOopDesc::fix_thawed_frame(const frame& f, const SmallReg
void stackChunkOopDesc::print_on(bool verbose, outputStream* st) const {
if (*((juint*)this) == badHeapWordVal) {
st->print_cr("BAD WORD");
- } else if (*((juint*)this) == badMetaWordVal) {
- st->print_cr("BAD META WORD");
} else {
InstanceStackChunkKlass::print_chunk(const_cast(this), verbose, st);
}
diff --git a/src/hotspot/share/opto/addnode.cpp b/src/hotspot/share/opto/addnode.cpp
index 1b92193300449..443086b6737d9 100644
--- a/src/hotspot/share/opto/addnode.cpp
+++ b/src/hotspot/share/opto/addnode.cpp
@@ -764,7 +764,7 @@ Node* OrINode::Identity(PhaseGVN* phase) {
}
// Find shift value for Integer or Long OR.
-Node* rotate_shift(PhaseGVN* phase, Node* lshift, Node* rshift, int mask) {
+static Node* rotate_shift(PhaseGVN* phase, Node* lshift, Node* rshift, int mask) {
// val << norm_con_shift | val >> ({32|64} - norm_con_shift) => rotate_left val, norm_con_shift
const TypeInt* lshift_t = phase->type(lshift)->isa_int();
const TypeInt* rshift_t = phase->type(rshift)->isa_int();
@@ -1080,7 +1080,7 @@ const Type* XorLNode::Value(PhaseGVN* phase) const {
return AddNode::Value(phase);
}
-Node* build_min_max_int(Node* a, Node* b, bool is_max) {
+static Node* build_min_max_int(Node* a, Node* b, bool is_max) {
if (is_max) {
return new MaxINode(a, b);
} else {
@@ -1312,7 +1312,7 @@ const Type *MinINode::add_ring( const Type *t0, const Type *t1 ) const {
//
// Note: we assume that SubL was already replaced by an AddL, and that the stride
// has its sign flipped: SubL(limit, stride) -> AddL(limit, -stride).
-Node* fold_subI_no_underflow_pattern(Node* n, PhaseGVN* phase) {
+static Node* fold_subI_no_underflow_pattern(Node* n, PhaseGVN* phase) {
assert(n->Opcode() == Op_MaxL || n->Opcode() == Op_MinL, "sanity");
// Check that the two clamps have the correct values.
jlong clamp = (n->Opcode() == Op_MaxL) ? min_jint : max_jint;
diff --git a/src/hotspot/share/opto/arraycopynode.cpp b/src/hotspot/share/opto/arraycopynode.cpp
index 71b7a7e5024d9..4e2b159bf8e14 100644
--- a/src/hotspot/share/opto/arraycopynode.cpp
+++ b/src/hotspot/share/opto/arraycopynode.cpp
@@ -152,7 +152,10 @@ int ArrayCopyNode::get_count(PhaseGVN *phase) const {
}
Node* ArrayCopyNode::load(BarrierSetC2* bs, PhaseGVN *phase, Node*& ctl, MergeMemNode* mem, Node* adr, const TypePtr* adr_type, const Type *type, BasicType bt) {
- DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY;
+ // Pin the load: if this is an array load, it's going to be dependent on a condition that's not a range check for that
+ // access. If that condition is replaced by an identical dominating one, then an unpinned load would risk floating
+ // above runtime checks that guarantee it is within bounds.
+ DecoratorSet decorators = C2_READ_ACCESS | C2_CONTROL_DEPENDENT_LOAD | IN_HEAP | C2_ARRAY_COPY | C2_UNKNOWN_CONTROL_LOAD;
C2AccessValuePtr addr(adr, adr_type);
C2OptAccess access(*phase, ctl, mem, decorators, bt, adr->in(AddPNode::Base), addr);
Node* res = bs->load_at(access, type);
diff --git a/src/hotspot/share/opto/c2_CodeStubs.hpp b/src/hotspot/share/opto/c2_CodeStubs.hpp
index 3df1e4b726d07..83d170810703d 100644
--- a/src/hotspot/share/opto/c2_CodeStubs.hpp
+++ b/src/hotspot/share/opto/c2_CodeStubs.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -97,6 +97,26 @@ class C2EntryBarrierStub: public C2CodeStub {
void emit(C2_MacroAssembler& masm);
};
+class C2FastUnlockLightweightStub : public C2CodeStub {
+private:
+ Register _obj;
+ Register _mark;
+ Register _t;
+ Register _thread;
+ Label _push_and_slow_path;
+ Label _check_successor;
+ Label _unlocked_continuation;
+public:
+ C2FastUnlockLightweightStub(Register obj, Register mark, Register t, Register thread) : C2CodeStub(),
+ _obj(obj), _mark(mark), _t(t), _thread(thread) {}
+ int max_size() const;
+ void emit(C2_MacroAssembler& masm);
+ Label& push_and_slow_path() { return _push_and_slow_path; }
+ Label& check_successor() { return _check_successor; }
+ Label& unlocked_continuation() { return _unlocked_continuation; }
+ Label& slow_path_continuation() { return continuation(); }
+};
+
#ifdef _LP64
class C2HandleAnonOMOwnerStub : public C2CodeStub {
private:
diff --git a/src/hotspot/share/opto/callnode.cpp b/src/hotspot/share/opto/callnode.cpp
index 3379a9b01e00d..98837e5e046ad 100644
--- a/src/hotspot/share/opto/callnode.cpp
+++ b/src/hotspot/share/opto/callnode.cpp
@@ -2002,7 +2002,7 @@ Node *LockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
// If we are locking an non-escaped object, the lock/unlock is unnecessary
//
ConnectionGraph *cgr = phase->C->congraph();
- if (cgr != nullptr && cgr->not_global_escape(obj_node())) {
+ if (cgr != nullptr && cgr->can_eliminate_lock(this)) {
assert(!is_eliminated() || is_coarsened(), "sanity");
// The lock could be marked eliminated by lock coarsening
// code during first IGVN before EA. Replace coarsened flag
@@ -2165,6 +2165,7 @@ bool LockNode::is_nested_lock_region(Compile * c) {
obj_node = bs->step_over_gc_barrier(obj_node);
BoxLockNode* box_node = sfn->monitor_box(jvms, idx)->as_BoxLock();
if ((box_node->stack_slot() < stk_slot) && obj_node->eqv_uncast(obj)) {
+ box->set_nested();
return true;
}
}
@@ -2198,7 +2199,7 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) {
// If we are unlocking an non-escaped object, the lock/unlock is unnecessary.
//
ConnectionGraph *cgr = phase->C->congraph();
- if (cgr != nullptr && cgr->not_global_escape(obj_node())) {
+ if (cgr != nullptr && cgr->can_eliminate_lock(this)) {
assert(!is_eliminated() || is_coarsened(), "sanity");
// The lock could be marked eliminated by lock coarsening
// code during first IGVN before EA. Replace coarsened flag
diff --git a/src/hotspot/share/opto/cfgnode.hpp b/src/hotspot/share/opto/cfgnode.hpp
index d72e9bce21381..4bc4962ccf000 100644
--- a/src/hotspot/share/opto/cfgnode.hpp
+++ b/src/hotspot/share/opto/cfgnode.hpp
@@ -421,6 +421,9 @@ class IfNode : public MultiBranchNode {
init_req(0,control);
init_req(1,b);
}
+
+ static IfNode* make_with_same_profile(IfNode* if_node_profile, Node* ctrl, BoolNode* bol);
+
virtual int Opcode() const;
virtual bool pinned() const { return true; }
virtual const Type *bottom_type() const { return TypeTuple::IFBOTH; }
diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp
index 83b922492b339..1e4c8fc51b7af 100644
--- a/src/hotspot/share/opto/compile.cpp
+++ b/src/hotspot/share/opto/compile.cpp
@@ -56,6 +56,7 @@
#include "opto/divnode.hpp"
#include "opto/escape.hpp"
#include "opto/idealGraphPrinter.hpp"
+#include "opto/locknode.hpp"
#include "opto/loopnode.hpp"
#include "opto/machnode.hpp"
#include "opto/macro.hpp"
@@ -4867,10 +4868,25 @@ void Compile::add_coarsened_locks(GrowableArray& locks) {
if (length > 0) {
// Have to keep this list until locks elimination during Macro nodes elimination.
Lock_List* locks_list = new (comp_arena()) Lock_List(comp_arena(), length);
+ AbstractLockNode* alock = locks.at(0);
+ BoxLockNode* box = alock->box_node()->as_BoxLock();
for (int i = 0; i < length; i++) {
AbstractLockNode* lock = locks.at(i);
assert(lock->is_coarsened(), "expecting only coarsened AbstractLock nodes, but got '%s'[%d] node", lock->Name(), lock->_idx);
locks_list->push(lock);
+ BoxLockNode* this_box = lock->box_node()->as_BoxLock();
+ if (this_box != box) {
+ // Locking regions (BoxLock) could be Unbalanced here:
+ // - its coarsened locks were eliminated in earlier
+ // macro nodes elimination followed by loop unroll
+ // Preserve Unbalanced status in such cases.
+ if (!this_box->is_unbalanced()) {
+ this_box->set_coarsened();
+ }
+ if (!box->is_unbalanced()) {
+ box->set_coarsened();
+ }
+ }
}
_coarsened_locks.append(locks_list);
}
@@ -4948,6 +4964,38 @@ bool Compile::coarsened_locks_consistent() {
return true;
}
+// Mark locking regions (identified by BoxLockNode) as unbalanced if
+// locks coarsening optimization removed Lock/Unlock nodes from them.
+// Such regions become unbalanced because coarsening only removes part
+// of Lock/Unlock nodes in region. As result we can't execute other
+// locks elimination optimizations which assume all code paths have
+// corresponding pair of Lock/Unlock nodes - they are balanced.
+void Compile::mark_unbalanced_boxes() const {
+ int count = coarsened_count();
+ for (int i = 0; i < count; i++) {
+ Node_List* locks_list = _coarsened_locks.at(i);
+ uint size = locks_list->size();
+ if (size > 0) {
+ AbstractLockNode* alock = locks_list->at(0)->as_AbstractLock();
+ BoxLockNode* box = alock->box_node()->as_BoxLock();
+ if (alock->is_coarsened()) {
+ // coarsened_locks_consistent(), which is called before this method, verifies
+ // that the rest of Lock/Unlock nodes on locks_list are also coarsened.
+ assert(!box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
+ for (uint j = 1; j < size; j++) {
+ assert(locks_list->at(j)->as_AbstractLock()->is_coarsened(), "only coarsened locks are expected here");
+ BoxLockNode* this_box = locks_list->at(j)->as_AbstractLock()->box_node()->as_BoxLock();
+ if (box != this_box) {
+ assert(!this_box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
+ box->set_unbalanced();
+ this_box->set_unbalanced();
+ }
+ }
+ }
+ }
+ }
+}
+
/**
* Remove the speculative part of types and clean up the graph
*/
diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp
index a5e4b69d26330..917ceb52d9ab3 100644
--- a/src/hotspot/share/opto/compile.hpp
+++ b/src/hotspot/share/opto/compile.hpp
@@ -780,6 +780,7 @@ class Compile : public Phase {
void add_coarsened_locks(GrowableArray& locks);
void remove_coarsened_lock(Node* n);
bool coarsened_locks_consistent();
+ void mark_unbalanced_boxes() const;
bool post_loop_opts_phase() { return _post_loop_opts_phase; }
void set_post_loop_opts_phase() { _post_loop_opts_phase = true; }
diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp
index 122da9b447c5e..0a5e27ed5b13f 100644
--- a/src/hotspot/share/opto/doCall.cpp
+++ b/src/hotspot/share/opto/doCall.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -50,15 +50,15 @@
#include "jfr/jfr.hpp"
#endif
-void print_trace_type_profile(outputStream* out, int depth, ciKlass* prof_klass, int site_count, int receiver_count) {
+static void print_trace_type_profile(outputStream* out, int depth, ciKlass* prof_klass, int site_count, int receiver_count) {
CompileTask::print_inline_indent(depth, out);
out->print(" \\-> TypeProfile (%d/%d counts) = ", receiver_count, site_count);
prof_klass->name()->print_symbol_on(out);
out->cr();
}
-void trace_type_profile(Compile* C, ciMethod* method, int depth, int bci, ciMethod* prof_method,
- ciKlass* prof_klass, int site_count, int receiver_count) {
+static void trace_type_profile(Compile* C, ciMethod* method, int depth, int bci, ciMethod* prof_method,
+ ciKlass* prof_klass, int site_count, int receiver_count) {
if (TraceTypeProfile || C->print_inlining()) {
outputStream* out = tty;
if (!C->print_inlining()) {
diff --git a/src/hotspot/share/opto/escape.cpp b/src/hotspot/share/opto/escape.cpp
index ec9c0b79fb0c1..99a96a45be6a7 100644
--- a/src/hotspot/share/opto/escape.cpp
+++ b/src/hotspot/share/opto/escape.cpp
@@ -37,6 +37,7 @@
#include "opto/compile.hpp"
#include "opto/escape.hpp"
#include "opto/macro.hpp"
+#include "opto/locknode.hpp"
#include "opto/phaseX.hpp"
#include "opto/movenode.hpp"
#include "opto/rootnode.hpp"
@@ -2547,7 +2548,7 @@ void ConnectionGraph::optimize_ideal_graph(GrowableArray& ptr_cmp_worklis
if (n->is_AbstractLock()) { // Lock and Unlock nodes
AbstractLockNode* alock = n->as_AbstractLock();
if (!alock->is_non_esc_obj()) {
- if (not_global_escape(alock->obj_node())) {
+ if (can_eliminate_lock(alock)) {
assert(!alock->is_eliminated() || alock->is_coarsened(), "sanity");
// The lock could be marked eliminated by lock coarsening
// code during first IGVN before EA. Replace coarsened flag
@@ -2880,6 +2881,21 @@ bool ConnectionGraph::not_global_escape(Node *n) {
return true;
}
+// Return true if locked object does not escape globally
+// and locked code region (identified by BoxLockNode) is balanced:
+// all compiled code paths have corresponding Lock/Unlock pairs.
+bool ConnectionGraph::can_eliminate_lock(AbstractLockNode* alock) {
+ BoxLockNode* box = alock->box_node()->as_BoxLock();
+ if (!box->is_unbalanced() && not_global_escape(alock->obj_node())) {
+ if (EliminateNestedLocks) {
+ // We can mark whole locking region as Local only when only
+ // one object is used for locking.
+ box->set_local();
+ }
+ return true;
+ }
+ return false;
+}
// Helper functions
diff --git a/src/hotspot/share/opto/escape.hpp b/src/hotspot/share/opto/escape.hpp
index 7e6bd87d4931f..4ed7fb7f054d8 100644
--- a/src/hotspot/share/opto/escape.hpp
+++ b/src/hotspot/share/opto/escape.hpp
@@ -112,6 +112,7 @@
class Compile;
class Node;
+class AbstractLockNode;
class CallNode;
class PhiNode;
class PhaseTransform;
@@ -630,6 +631,8 @@ class ConnectionGraph: public ArenaObj {
bool not_global_escape(Node *n);
+ bool can_eliminate_lock(AbstractLockNode* alock);
+
// To be used by, e.g., BarrierSetC2 impls
Node* get_addp_base(Node* addp);
diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp
index 6ae62b24b3c8d..91e1c40076e2a 100644
--- a/src/hotspot/share/opto/ifnode.cpp
+++ b/src/hotspot/share/opto/ifnode.cpp
@@ -454,6 +454,22 @@ static Node* split_if(IfNode *iff, PhaseIterGVN *igvn) {
return new ConINode(TypeInt::ZERO);
}
+IfNode* IfNode::make_with_same_profile(IfNode* if_node_profile, Node* ctrl, BoolNode* bol) {
+ // Assert here that we only try to create a clone from an If node with the same profiling if that actually makes sense.
+ // Some If node subtypes should not be cloned in this way. In theory, we should not clone BaseCountedLoopEndNodes.
+ // But they can end up being used as normal If nodes when peeling a loop - they serve as zero-trip guard.
+ // Allow them as well.
+ assert(if_node_profile->Opcode() == Op_If || if_node_profile->is_RangeCheck()
+ || if_node_profile->is_BaseCountedLoopEnd(), "should not clone other nodes");
+ if (if_node_profile->is_RangeCheck()) {
+ // RangeCheck nodes could be further optimized.
+ return new RangeCheckNode(ctrl, bol, if_node_profile->_prob, if_node_profile->_fcnt);
+ } else {
+ // Not a RangeCheckNode? Fall back to IfNode.
+ return new IfNode(ctrl, bol, if_node_profile->_prob, if_node_profile->_fcnt);
+ }
+}
+
// if this IfNode follows a range check pattern return the projection
// for the failed path
ProjNode* IfNode::range_check_trap_proj(int& flip_test, Node*& l, Node*& r) {
diff --git a/src/hotspot/share/opto/intrinsicnode.cpp b/src/hotspot/share/opto/intrinsicnode.cpp
index 7d8894c8bdfa1..010579722bb10 100644
--- a/src/hotspot/share/opto/intrinsicnode.cpp
+++ b/src/hotspot/share/opto/intrinsicnode.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -174,7 +174,7 @@ Node* CompressBitsNode::Ideal(PhaseGVN* phase, bool can_reshape) {
return nullptr;
}
-Node* compress_expand_identity(PhaseGVN* phase, Node* n) {
+static Node* compress_expand_identity(PhaseGVN* phase, Node* n) {
BasicType bt = n->bottom_type()->basic_type();
// compress(x, 0) == 0, expand(x, 0) == 0
if(phase->type(n->in(2))->higher_equal(TypeInteger::zero(bt))) return n->in(2);
diff --git a/src/hotspot/share/opto/locknode.cpp b/src/hotspot/share/opto/locknode.cpp
index d73150684a71d..17d26d620e5b0 100644
--- a/src/hotspot/share/opto/locknode.cpp
+++ b/src/hotspot/share/opto/locknode.cpp
@@ -40,7 +40,7 @@ const RegMask &BoxLockNode::out_RegMask() const {
uint BoxLockNode::size_of() const { return sizeof(*this); }
BoxLockNode::BoxLockNode( int slot ) : Node( Compile::current()->root() ),
- _slot(slot), _is_eliminated(false) {
+ _slot(slot), _kind(BoxLockNode::Regular) {
init_class_id(Class_BoxLock);
init_flags(Flag_rematerialize);
OptoReg::Name reg = OptoReg::stack2reg(_slot);
@@ -51,19 +51,48 @@ BoxLockNode::BoxLockNode( int slot ) : Node( Compile::current()->root() ),
_inmask.Insert(reg);
}
-//-----------------------------hash--------------------------------------------
uint BoxLockNode::hash() const {
- if (EliminateNestedLocks)
+ if (EliminateNestedLocks) {
return NO_HASH; // Each locked region has own BoxLock node
- return Node::hash() + _slot + (_is_eliminated ? Compile::current()->fixed_slots() : 0);
+ }
+ return Node::hash() + _slot + (is_eliminated() ? Compile::current()->fixed_slots() : 0);
}
-//------------------------------cmp--------------------------------------------
bool BoxLockNode::cmp( const Node &n ) const {
- if (EliminateNestedLocks)
+ if (EliminateNestedLocks) {
return (&n == this); // Always fail except on self
+ }
const BoxLockNode &bn = (const BoxLockNode &)n;
- return bn._slot == _slot && bn._is_eliminated == _is_eliminated;
+ return (bn._slot == _slot) && (bn.is_eliminated() == is_eliminated());
+}
+
+Node* BoxLockNode::Identity(PhaseGVN* phase) {
+ if (!EliminateNestedLocks && !this->is_eliminated()) {
+ Node* n = phase->hash_find(this);
+ if (n == nullptr || n == this) {
+ return this;
+ }
+ BoxLockNode* old_box = n->as_BoxLock();
+ // Set corresponding status (_kind) when commoning BoxLock nodes.
+ if (this->_kind != old_box->_kind) {
+ if (this->is_unbalanced()) {
+ old_box->set_unbalanced();
+ }
+ if (!old_box->is_unbalanced()) {
+ // Only Regular or Coarsened status should be here:
+ // Nested and Local are set only when EliminateNestedLocks is on.
+ if (old_box->is_regular()) {
+ assert(this->is_coarsened(),"unexpected kind: %s", _kind_name[(int)this->_kind]);
+ old_box->set_coarsened();
+ } else {
+ assert(this->is_regular(),"unexpected kind: %s", _kind_name[(int)this->_kind]);
+ assert(old_box->is_coarsened(),"unexpected kind: %s", _kind_name[(int)old_box->_kind]);
+ }
+ }
+ }
+ return old_box;
+ }
+ return this;
}
BoxLockNode* BoxLockNode::box_node(Node* box) {
@@ -90,6 +119,9 @@ OptoReg::Name BoxLockNode::reg(Node* box) {
// Is BoxLock node used for one simple lock region (same box and obj)?
bool BoxLockNode::is_simple_lock_region(LockNode** unique_lock, Node* obj, Node** bad_lock) {
+ if (is_unbalanced()) {
+ return false;
+ }
LockNode* lock = nullptr;
bool has_one_lock = false;
for (uint i = 0; i < this->outcnt(); i++) {
diff --git a/src/hotspot/share/opto/locknode.hpp b/src/hotspot/share/opto/locknode.hpp
index 4a74e50425f17..3bc684c40a9da 100644
--- a/src/hotspot/share/opto/locknode.hpp
+++ b/src/hotspot/share/opto/locknode.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,9 +33,37 @@ class RTMLockingCounters;
//------------------------------BoxLockNode------------------------------------
class BoxLockNode : public Node {
+private:
const int _slot; // stack slot
RegMask _inmask; // OptoReg corresponding to stack slot
- bool _is_eliminated; // Associated locks were safely eliminated
+ enum {
+ Regular = 0, // Normal locking region
+ Local, // EA found that local not escaping object is used for locking
+ Nested, // This region is inside other region which use the same object
+ Coarsened, // Some lock/unlock in region were marked as coarsened
+ Unbalanced, // This region become unbalanced after coarsened lock/unlock were eliminated
+ // or it is locking region from OSR when locking is done in Interpreter
+ Eliminated // All lock/unlock in region were eliminated
+ } _kind;
+
+#ifdef ASSERT
+ const char* _kind_name[6] = {
+ "Regular",
+ "Local",
+ "Nested",
+ "Coarsened",
+ "Unbalanced",
+ "Eliminated"
+ };
+#endif
+
+ // Allowed transitions of _kind:
+ // Regular -> Local, Nested, Coarsened
+ // Local -> Eliminated
+ // Nested -> Eliminated
+ // Coarsened -> Local, Nested, Unbalanced
+ // EA and nested lock elimination can overwrite Coarsened kind.
+ // Also allow transition to the same kind.
public:
BoxLockNode( int lock );
@@ -49,6 +77,7 @@ class BoxLockNode : public Node {
virtual bool cmp( const Node &n ) const;
virtual const class Type *bottom_type() const { return TypeRawPtr::BOTTOM; }
virtual uint ideal_reg() const { return Op_RegP; }
+ virtual Node* Identity(PhaseGVN* phase);
static OptoReg::Name reg(Node* box_node);
static BoxLockNode* box_node(Node* box_node);
@@ -57,9 +86,38 @@ class BoxLockNode : public Node {
}
int stack_slot() const { return _slot; }
- bool is_eliminated() const { return _is_eliminated; }
- // mark lock as eliminated.
- void set_eliminated() { _is_eliminated = true; }
+ bool is_regular() const { return _kind == Regular; }
+ bool is_local() const { return _kind == Local; }
+ bool is_nested() const { return _kind == Nested; }
+ bool is_coarsened() const { return _kind == Coarsened; }
+ bool is_eliminated() const { return _kind == Eliminated; }
+ bool is_unbalanced() const { return _kind == Unbalanced; }
+
+ void set_local() {
+ assert((_kind == Regular || _kind == Local || _kind == Coarsened),
+ "incorrect kind for Local transitioni: %s", _kind_name[(int)_kind]);
+ _kind = Local;
+ }
+ void set_nested() {
+ assert((_kind == Regular || _kind == Nested || _kind == Coarsened),
+ "incorrect kind for Nested transition: %s", _kind_name[(int)_kind]);
+ _kind = Nested;
+ }
+ void set_coarsened() {
+ assert((_kind == Regular || _kind == Coarsened),
+ "incorrect kind for Coarsened transition: %s", _kind_name[(int)_kind]);
+ _kind = Coarsened;
+ }
+ void set_eliminated() {
+ assert((_kind == Local || _kind == Nested),
+ "incorrect kind for Eliminated transition: %s", _kind_name[(int)_kind]);
+ _kind = Eliminated;
+ }
+ void set_unbalanced() {
+ assert((_kind == Coarsened || _kind == Unbalanced),
+ "incorrect kind for Unbalanced transition: %s", _kind_name[(int)_kind]);
+ _kind = Unbalanced;
+ }
// Is BoxLock node used for one simple lock region?
bool is_simple_lock_region(LockNode** unique_lock, Node* obj, Node** bad_lock);
diff --git a/src/hotspot/share/opto/loopUnswitch.cpp b/src/hotspot/share/opto/loopUnswitch.cpp
index ae657b55fbb23..05623aefaa94a 100644
--- a/src/hotspot/share/opto/loopUnswitch.cpp
+++ b/src/hotspot/share/opto/loopUnswitch.cpp
@@ -24,6 +24,8 @@
#include "precompiled.hpp"
#include "memory/allocation.inline.hpp"
+#include "opto/castnode.hpp"
+#include "opto/cfgnode.hpp"
#include "opto/connode.hpp"
#include "opto/convertnode.hpp"
#include "opto/loopnode.hpp"
@@ -31,31 +33,63 @@
#include "opto/predicates.hpp"
#include "opto/rootnode.hpp"
-//================= Loop Unswitching =====================
+// Loop Unswitching is a loop optimization to move an invariant, non-loop-exiting test in the loop body before the loop.
+// Such a test is either always true or always false in all loop iterations and could therefore only be executed once.
+// To achieve that, we duplicate the loop and change the original and cloned loop as follows:
+// - Original loop -> true-path-loop:
+// The true-path of the invariant, non-loop-exiting test in the original loop
+// is kept while the false-path is killed. We call this unswitched loop version
+// the true-path-loop.
+// - Cloned loop -> false-path-loop:
+// The false-path of the invariant, non-loop-exiting test in the cloned loop
+// is kept while the true-path is killed. We call this unswitched loop version
+// the false-path loop.
//
-// orig: transformed:
-// if (invariant-test) then
-// predicates predicates
-// loop loop
-// stmt1 stmt1
-// if (invariant-test) then stmt2
-// stmt2 stmt4
-// else endloop
-// stmt3 else
-// endif predicates [clone]
-// stmt4 loop [clone]
-// endloop stmt1 [clone]
-// stmt3
-// stmt4 [clone]
-// endloop
-// endif
+// The invariant, non-loop-exiting test can now be moved before both loops (to only execute it once) and turned into a
+// loop selector If node to select at runtime which unswitched loop version should be executed.
+// - Loop selector true? Execute the true-path-loop.
+// - Loop selector false? Execute the false-path-loop.
//
-// Note: the "else" clause may be empty
-
-//------------------------------policy_unswitching-----------------------------
-// Return TRUE or FALSE if the loop should be unswitched
-// (ie. clone loop with an invariant test that does not exit the loop)
-bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
+// Note that even though an invariant test that exits the loop could also be optimized with Loop Unswitching, it is more
+// efficient to simply peel the loop which achieves the same result in a simpler manner (also see policy_peeling()).
+//
+// The following graphs summarizes the Loop Unswitching optimization.
+// We start with the original loop:
+//
+// [Predicates]
+// |
+// Original Loop
+// stmt1
+// if (invariant-test)
+// if-path
+// else
+// else-path
+// stmt2
+// Endloop
+//
+//
+// which is unswitched into a true-path-loop and a false-path-loop together with a loop selector:
+//
+//
+// [Initialized Assertion Predicates]
+// |
+// loop selector If (invariant-test)
+// / \
+// true? false?
+// / \
+// [Cloned Parse Predicates] [Cloned Parse Predicates]
+// [Cloned Template [Cloned Template
+// Assertion Predicates] Assertion Predicates]
+// | |
+// True-Path-Loop False-Path-Loop
+// cloned stmt1 cloned stmt1
+// cloned if-path cloned else-path
+// cloned stmt2 cloned stmt2
+// Endloop Endloop
+
+
+// Return true if the loop should be unswitched or false otherwise.
+bool IdealLoopTree::policy_unswitching(PhaseIdealLoop* phase) const {
if (!LoopUnswitching) {
return false;
}
@@ -75,7 +109,7 @@ bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
if (head->unswitch_count() + 1 > head->unswitch_max()) {
return false;
}
- if (phase->find_unswitching_candidate(this) == nullptr) {
+ if (phase->find_unswitch_candidate(this) == nullptr) {
return false;
}
@@ -83,13 +117,11 @@ bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const {
return phase->may_require_nodes(est_loop_clone_sz(2));
}
-//------------------------------find_unswitching_candidate-----------------------------
-// Find candidate "if" for unswitching
-IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const {
-
- // Find first invariant test that doesn't exit the loop
- LoopNode *head = loop->_head->as_Loop();
- IfNode* unswitch_iff = nullptr;
+// Find an invariant test in the loop body that does not exit the loop. If multiple tests are found, we pick the first
+// one in the loop body. Return the "unswitch candidate" If to apply Loop Unswitching on.
+IfNode* PhaseIdealLoop::find_unswitch_candidate(const IdealLoopTree* loop) const {
+ LoopNode* head = loop->_head->as_Loop();
+ IfNode* unswitch_candidate = nullptr;
Node* n = head->in(LoopNode::LoopBackControl);
while (n != head) {
Node* n_dom = idom(n);
@@ -102,7 +134,8 @@ IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) co
// If condition is invariant and not a loop exit,
// then found reason to unswitch.
if (loop->is_invariant(bol) && !loop->is_loop_exit(iff)) {
- unswitch_iff = iff;
+ assert(iff->Opcode() == Op_If || iff->is_RangeCheck() || iff->is_BaseCountedLoopEnd(), "valid ifs");
+ unswitch_candidate = iff;
}
}
}
@@ -110,106 +143,196 @@ IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) co
}
n = n_dom;
}
- return unswitch_iff;
+ return unswitch_candidate;
}
-//------------------------------do_unswitching-----------------------------
-// Clone loop with an invariant test (that does not exit) and
-// insert a clone of the test that selects which version to
-// execute.
-void PhaseIdealLoop::do_unswitching(IdealLoopTree *loop, Node_List &old_new) {
- LoopNode* head = loop->_head->as_Loop();
- if (has_control_dependencies_from_predicates(head)) {
- return;
+// This class creates an If node (i.e. loop selector) that selects if the true-path-loop or the false-path-loop should be
+// executed at runtime. This is done by finding an invariant and non-loop-exiting unswitch candidate If node (guaranteed
+// to exist at this point) to perform Loop Unswitching on.
+class UnswitchedLoopSelector : public StackObj {
+ PhaseIdealLoop* const _phase;
+ IdealLoopTree* const _outer_loop;
+ Node* const _original_loop_entry;
+ IfNode* const _unswitch_candidate;
+ IfNode* const _selector;
+ IfTrueNode* const _true_path_loop_proj;
+ IfFalseNode* const _false_path_loop_proj;
+
+ enum PathToLoop { TRUE_PATH, FALSE_PATH };
+
+ public:
+ UnswitchedLoopSelector(IdealLoopTree* loop)
+ : _phase(loop->_phase),
+ _outer_loop(loop->skip_strip_mined()->_parent),
+ _original_loop_entry(loop->_head->as_Loop()->skip_strip_mined()->in(LoopNode::EntryControl)),
+ _unswitch_candidate(find_unswitch_candidate(loop)),
+ _selector(create_selector_if()),
+ _true_path_loop_proj(create_proj_to_loop(TRUE_PATH)->as_IfTrue()),
+ _false_path_loop_proj(create_proj_to_loop(FALSE_PATH)->as_IfFalse()) {
+ }
+ NONCOPYABLE(UnswitchedLoopSelector);
+
+ private:
+ IfNode* find_unswitch_candidate(IdealLoopTree* loop) {
+ IfNode* unswitch_candidate = _phase->find_unswitch_candidate(loop);
+ assert(unswitch_candidate != nullptr, "guaranteed to exist by policy_unswitching");
+ assert(_phase->is_member(loop, unswitch_candidate), "must be inside original loop");
+ return unswitch_candidate;
}
- // Find first invariant test that doesn't exit the loop
- IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop);
- assert(unswitch_iff != nullptr, "should be at least one");
+ IfNode* create_selector_if() const {
+ const uint dom_depth = _phase->dom_depth(_original_loop_entry);
+ _phase->igvn().rehash_node_delayed(_original_loop_entry);
+ BoolNode* unswitch_candidate_bool = _unswitch_candidate->in(1)->as_Bool();
+ IfNode* selector_if = IfNode::make_with_same_profile(_unswitch_candidate, _original_loop_entry,
+ unswitch_candidate_bool);
+ _phase->register_node(selector_if, _outer_loop, _original_loop_entry, dom_depth);
+ return selector_if;
+ }
-#ifndef PRODUCT
- if (TraceLoopOpts) {
- tty->print("Unswitch %d ", head->unswitch_count()+1);
- loop->dump_head();
+ IfProjNode* create_proj_to_loop(const PathToLoop path_to_loop) {
+ const uint dom_depth = _phase->dom_depth(_original_loop_entry);
+ IfProjNode* proj_to_loop;
+ if (path_to_loop == TRUE_PATH) {
+ proj_to_loop = new IfTrueNode(_selector);
+ } else {
+ proj_to_loop = new IfFalseNode(_selector);
+ }
+ _phase->register_node(proj_to_loop, _outer_loop, _selector, dom_depth);
+ return proj_to_loop;
}
-#endif
- C->print_method(PHASE_BEFORE_LOOP_UNSWITCHING, 4, head);
+ public:
+ IfNode* unswitch_candidate() const {
+ return _unswitch_candidate;
+ }
- // Need to revert back to normal loop
- if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) {
- head->as_CountedLoop()->set_normal_loop();
+ IfNode* selector() const {
+ return _selector;
}
- IfNode* invar_iff = create_slow_version_of_loop(loop, old_new, unswitch_iff, CloneIncludesStripMined);
- ProjNode* proj_true = invar_iff->proj_out(1);
- verify_fast_loop(head, proj_true);
+ IfTrueNode* true_path_loop_proj() const {
+ return _true_path_loop_proj;
+ }
- // Increment unswitch count
- LoopNode* head_clone = old_new[head->_idx]->as_Loop();
- int nct = head->unswitch_count() + 1;
- head->set_unswitch_count(nct);
- head_clone->set_unswitch_count(nct);
+ IfFalseNode* false_path_loop_proj() const {
+ return _false_path_loop_proj;
+ }
+};
+
+// Class to unswitch the original loop and create Predicates at the new unswitched loop versions. The newly cloned loop
+// becomes the false-path-loop while original loop becomes the true-path-loop.
+class OriginalLoop : public StackObj {
+ LoopNode* const _loop_head; // OuterStripMinedLoopNode if loop strip mined, else just the loop head.
+ IdealLoopTree* const _loop;
+ Node_List& _old_new;
+ PhaseIdealLoop* const _phase;
+
+ public:
+ OriginalLoop(IdealLoopTree* loop, Node_List& old_new)
+ : _loop_head(loop->_head->as_Loop()->skip_strip_mined()),
+ _loop(loop),
+ _old_new(old_new),
+ _phase(loop->_phase) {}
+ NONCOPYABLE(OriginalLoop);
+
+ private:
+ void fix_loop_entries(IfProjNode* true_path_loop_entry, IfProjNode* false_path_loop_entry) {
+ _phase->replace_loop_entry(_loop_head, true_path_loop_entry);
+ LoopNode* false_path_loop_strip_mined_head = old_to_new(_loop_head)->as_Loop();
+ _phase->replace_loop_entry(false_path_loop_strip_mined_head, false_path_loop_entry);
+ }
- // Hoist invariant casts out of each loop to the appropriate
- // control projection.
+ Node* old_to_new(const Node* old) const {
+ return _old_new[old->_idx];
+ }
- Node_List worklist;
- for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) {
- ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj();
- // Copy to a worklist for easier manipulation
- for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
- Node* use = proj->fast_out(j);
- if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) {
- worklist.push(use);
- }
- }
- ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj();
- while (worklist.size() > 0) {
- Node* use = worklist.pop();
- Node* nuse = use->clone();
- nuse->set_req(0, invar_proj);
- _igvn.replace_input_of(use, 1, nuse);
- register_new_node(nuse, invar_proj);
- // Same for the clone
- Node* use_clone = old_new[use->_idx];
- _igvn.replace_input_of(use_clone, 1, nuse);
- }
+#ifdef ASSERT
+ void verify_unswitched_loop_versions(LoopNode* true_path_loop_head,
+ const UnswitchedLoopSelector& unswitched_loop_selector) const {
+ verify_unswitched_loop_version(true_path_loop_head, unswitched_loop_selector.true_path_loop_proj());
+ verify_unswitched_loop_version(old_to_new(true_path_loop_head)->as_Loop(),
+ unswitched_loop_selector.false_path_loop_proj());
+ }
+
+ static void verify_unswitched_loop_version(LoopNode* loop_head, IfProjNode* loop_selector_if_proj) {
+ Node* entry = loop_head->skip_strip_mined()->in(LoopNode::EntryControl);
+ const Predicates predicates(entry);
+ // When skipping all predicates, we should end up at 'loop_selector_if_proj'.
+ assert(loop_selector_if_proj == predicates.entry(), "should end up at loop selector If");
}
+#endif // ASSERT
- // Hardwire the control paths in the loops into if(true) and if(false)
- _igvn.rehash_node_delayed(unswitch_iff);
- dominated_by(proj_true->as_IfProj(), unswitch_iff);
+ // Remove the unswitch candidate If nodes in both unswitched loop versions which are now dominated by the loop selector
+ // If node. Keep the true-path-path in the true-path-loop and the false-path-path in the false-path-loop by setting
+ // the bool input accordingly. The unswitch candidate If nodes are folded in the next IGVN round.
+ void remove_unswitch_candidate_from_loops(const UnswitchedLoopSelector& unswitched_loop_selector) {
+ IfNode* unswitching_candidate = unswitched_loop_selector.unswitch_candidate();
+ _phase->igvn().rehash_node_delayed(unswitching_candidate);
+ _phase->dominated_by(unswitched_loop_selector.true_path_loop_proj(), unswitching_candidate);
+
+ IfNode* unswitching_candidate_clone = _old_new[unswitching_candidate->_idx]->as_If();
+ _phase->igvn().rehash_node_delayed(unswitching_candidate_clone);
+ _phase->dominated_by(unswitched_loop_selector.false_path_loop_proj(), unswitching_candidate_clone);
+ }
- IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If();
- _igvn.rehash_node_delayed(unswitch_iff_clone);
- ProjNode* proj_false = invar_iff->proj_out(0);
- dominated_by(proj_false->as_IfProj(), unswitch_iff_clone);
+ public:
+ // Unswitch the original loop on the invariant loop selector by creating a true-path-loop and a false-path-loop.
+ // Remove the unswitch candidate If from both unswitched loop versions which are now covered by the loop selector If.
+ void unswitch(const UnswitchedLoopSelector& unswitched_loop_selector) {
+ _phase->clone_loop(_loop, _old_new, _phase->dom_depth(_loop_head),
+ PhaseIdealLoop::CloneIncludesStripMined, unswitched_loop_selector.selector());
- // Reoptimize loops
- loop->record_for_igvn();
- for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
- Node *n = loop->_body[i];
- Node *n_clone = old_new[n->_idx];
- _igvn._worklist.push(n_clone);
+ // At this point, the selector If projections are the corresponding loop entries.
+ // clone_parse_and_assertion_predicates_to_unswitched_loop() could clone additional predicates after the selector
+ // If projections. The loop entries are updated accordingly.
+ IfProjNode* true_path_loop_entry = unswitched_loop_selector.true_path_loop_proj();
+ IfProjNode* false_path_loop_entry = unswitched_loop_selector.false_path_loop_proj();
+ _phase->clone_parse_and_assertion_predicates_to_unswitched_loop(_loop, _old_new,
+ true_path_loop_entry, false_path_loop_entry);
+
+ fix_loop_entries(true_path_loop_entry, false_path_loop_entry);
+
+ DEBUG_ONLY(verify_unswitched_loop_versions(_loop->_head->as_Loop(), unswitched_loop_selector);)
+
+ _phase->recompute_dom_depth();
+ remove_unswitch_candidate_from_loops(unswitched_loop_selector);
}
+};
-#ifndef PRODUCT
- if (TraceLoopUnswitching) {
- tty->print_cr("Loop unswitching orig: %d @ %d new: %d @ %d",
- head->_idx, unswitch_iff->_idx,
- old_new[head->_idx]->_idx, unswitch_iff_clone->_idx);
+// See comments below file header for more information about Loop Unswitching.
+void PhaseIdealLoop::do_unswitching(IdealLoopTree* loop, Node_List& old_new) {
+ assert(LoopUnswitching, "LoopUnswitching must be enabled");
+
+ LoopNode* original_head = loop->_head->as_Loop();
+ if (has_control_dependencies_from_predicates(original_head)) {
+ NOT_PRODUCT(trace_loop_unswitching_impossible(original_head);)
+ return;
}
-#endif
- C->print_method(PHASE_AFTER_LOOP_UNSWITCHING, 4, head_clone);
+ NOT_PRODUCT(trace_loop_unswitching_count(loop, original_head);)
+ C->print_method(PHASE_BEFORE_LOOP_UNSWITCHING, 4, original_head);
+
+ revert_to_normal_loop(original_head);
+
+ const UnswitchedLoopSelector unswitched_loop_selector(loop);
+ OriginalLoop original_loop(loop, old_new);
+ original_loop.unswitch(unswitched_loop_selector);
+ hoist_invariant_check_casts(loop, old_new, unswitched_loop_selector);
+ add_unswitched_loop_version_bodies_to_igvn(loop, old_new);
+
+ LoopNode* new_head = old_new[original_head->_idx]->as_Loop();
+ increment_unswitch_counts(original_head, new_head);
+
+ NOT_PRODUCT(trace_loop_unswitching_result(unswitched_loop_selector, original_head, new_head);)
+ C->print_method(PHASE_AFTER_LOOP_UNSWITCHING, 4, new_head);
C->set_major_progress();
}
-bool PhaseIdealLoop::has_control_dependencies_from_predicates(LoopNode* head) const {
+bool PhaseIdealLoop::has_control_dependencies_from_predicates(LoopNode* head) {
Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
- Predicates predicates(entry);
+ const Predicates predicates(entry);
if (predicates.has_any()) {
assert(entry->is_IfProj(), "sanity - must be ifProj since there is at least one predicate");
if (entry->outcnt() > 1) {
@@ -222,69 +345,87 @@ bool PhaseIdealLoop::has_control_dependencies_from_predicates(LoopNode* head) co
return false;
}
-//-------------------------create_slow_version_of_loop------------------------
-// Create a slow version of the loop by cloning the loop
-// and inserting an if to select fast-slow versions.
-// Return the inserted if.
-IfNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop,
- Node_List &old_new,
- IfNode* unswitch_iff,
- CloneLoopMode mode) {
- LoopNode* head = loop->_head->as_Loop();
- Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
- _igvn.rehash_node_delayed(entry);
- IdealLoopTree* outer_loop = loop->_parent;
-
- head->verify_strip_mined(1);
-
- // Add test to new "if" outside of loop
- Node *bol = unswitch_iff->in(1)->as_Bool();
- IfNode* iff = (unswitch_iff->Opcode() == Op_RangeCheck) ? new RangeCheckNode(entry, bol, unswitch_iff->_prob, unswitch_iff->_fcnt) :
- new IfNode(entry, bol, unswitch_iff->_prob, unswitch_iff->_fcnt);
- register_node(iff, outer_loop, entry, dom_depth(entry));
- IfProjNode* iffast = new IfTrueNode(iff);
- register_node(iffast, outer_loop, iff, dom_depth(iff));
- IfProjNode* ifslow = new IfFalseNode(iff);
- register_node(ifslow, outer_loop, iff, dom_depth(iff));
-
- // Clone the loop body. The clone becomes the slow loop. The
- // original pre-header will (illegally) have 3 control users
- // (old & new loops & new if).
- clone_loop(loop, old_new, dom_depth(head->skip_strip_mined()), mode, iff);
- assert(old_new[head->_idx]->is_Loop(), "" );
-
- // Fast (true) and Slow (false) control
- IfProjNode* iffast_pred = iffast;
- IfProjNode* ifslow_pred = ifslow;
- clone_parse_and_assertion_predicates_to_unswitched_loop(loop, old_new, iffast_pred, ifslow_pred);
-
- Node* l = head->skip_strip_mined();
- _igvn.replace_input_of(l, LoopNode::EntryControl, iffast_pred);
- set_idom(l, iffast_pred, dom_depth(l));
- LoopNode* slow_l = old_new[head->_idx]->as_Loop()->skip_strip_mined();
- _igvn.replace_input_of(slow_l, LoopNode::EntryControl, ifslow_pred);
- set_idom(slow_l, ifslow_pred, dom_depth(l));
-
- recompute_dom_depth();
-
- return iff;
+#ifndef PRODUCT
+void PhaseIdealLoop::trace_loop_unswitching_impossible(const LoopNode* original_head) {
+ if (TraceLoopUnswitching) {
+ tty->print_cr("Loop Unswitching \"%d %s\" not possible due to control dependencies",
+ original_head->_idx, original_head->Name());
+ }
}
-#ifdef ASSERT
-void PhaseIdealLoop::verify_fast_loop(LoopNode* head, const ProjNode* proj_true) const {
- assert(proj_true->is_IfTrue(), "must be true projection");
- Node* entry = head->skip_strip_mined()->in(LoopNode::EntryControl);
- Predicates predicates(entry);
- if (!predicates.has_any()) {
- // No Parse Predicate.
- Node* uniqc = proj_true->unique_ctrl_out();
- assert((uniqc == head && !head->is_strip_mined()) || (uniqc == head->in(LoopNode::EntryControl)
- && head->is_strip_mined()), "must hold by construction if no predicates");
- } else {
- // There is at least one Parse Predicate. When skipping all predicates/predicate blocks, we should end up
- // at 'proj_true'.
- assert(proj_true == predicates.entry(), "must hold by construction if at least one Parse Predicate");
+void PhaseIdealLoop::trace_loop_unswitching_count(IdealLoopTree* loop, LoopNode* original_head) {
+ if (TraceLoopOpts) {
+ tty->print("Unswitch %d ", original_head->unswitch_count() + 1);
+ loop->dump_head();
}
}
-#endif // ASSERT
+
+void PhaseIdealLoop::trace_loop_unswitching_result(const UnswitchedLoopSelector& unswitched_loop_selector,
+ const LoopNode* original_head, const LoopNode* new_head) {
+ if (TraceLoopUnswitching) {
+ IfNode* unswitch_candidate = unswitched_loop_selector.unswitch_candidate();
+ IfNode* loop_selector = unswitched_loop_selector.selector();
+ tty->print_cr("Loop Unswitching:");
+ tty->print_cr("- Unswitch-Candidate-If: %d %s", unswitch_candidate->_idx, unswitch_candidate->Name());
+ tty->print_cr("- Loop-Selector-If: %d %s", loop_selector->_idx, loop_selector->Name());
+ tty->print_cr("- True-Path-Loop (=Orig): %d %s", original_head->_idx, original_head->Name());
+ tty->print_cr("- False-Path-Loop (=Clone): %d %s", new_head->_idx, new_head->Name());
+ }
+}
+#endif
+
+// When unswitching a counted loop, we need to convert it back to a normal loop since it's not a proper pre, main or,
+// post loop anymore after loop unswitching.
+void PhaseIdealLoop::revert_to_normal_loop(const LoopNode* loop_head) {
+ CountedLoopNode* cl = loop_head->isa_CountedLoop();
+ if (cl != nullptr && !cl->is_normal_loop()) {
+ cl->set_normal_loop();
+ }
+}
+
+// Hoist invariant CheckCastPPNodes out of each unswitched loop version to the appropriate loop selector If projection.
+void PhaseIdealLoop::hoist_invariant_check_casts(const IdealLoopTree* loop, const Node_List& old_new,
+ const UnswitchedLoopSelector& unswitched_loop_selector) {
+ IfNode* unswitch_candidate = unswitched_loop_selector.unswitch_candidate();
+ IfNode* loop_selector = unswitched_loop_selector.selector();
+ ResourceMark rm;
+ GrowableArray loop_invariant_check_casts;
+ for (DUIterator_Fast imax, i = unswitch_candidate->fast_outs(imax); i < imax; i++) {
+ IfProjNode* proj = unswitch_candidate->fast_out(i)->as_IfProj();
+ // Copy to a worklist for easier manipulation
+ for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) {
+ CheckCastPPNode* check_cast = proj->fast_out(j)->isa_CheckCastPP();
+ if (check_cast != nullptr && loop->is_invariant(check_cast->in(1))) {
+ loop_invariant_check_casts.push(check_cast);
+ }
+ }
+ IfProjNode* loop_selector_if_proj = loop_selector->proj_out(proj->_con)->as_IfProj();
+ while (loop_invariant_check_casts.length() > 0) {
+ CheckCastPPNode* cast = loop_invariant_check_casts.pop();
+ Node* cast_clone = cast->clone();
+ cast_clone->set_req(0, loop_selector_if_proj);
+ _igvn.replace_input_of(cast, 1, cast_clone);
+ register_new_node(cast_clone, loop_selector_if_proj);
+ // Same for the clone
+ Node* use_clone = old_new[cast->_idx];
+ _igvn.replace_input_of(use_clone, 1, cast_clone);
+ }
+ }
+}
+
+// Enable more optimizations possibilities in the next IGVN round.
+void PhaseIdealLoop::add_unswitched_loop_version_bodies_to_igvn(IdealLoopTree* loop, const Node_List& old_new) {
+ loop->record_for_igvn();
+ for(int i = loop->_body.size() - 1; i >= 0 ; i--) {
+ Node* n = loop->_body[i];
+ Node* n_clone = old_new[n->_idx];
+ _igvn._worklist.push(n_clone);
+ }
+}
+
+void PhaseIdealLoop::increment_unswitch_counts(LoopNode* original_head, LoopNode* new_head) {
+ const int unswitch_count = original_head->unswitch_count() + 1;
+ original_head->set_unswitch_count(unswitch_count);
+ new_head->set_unswitch_count(unswitch_count);
+}
diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp
index 38fc36cec06bf..fdfbab09b8fe8 100644
--- a/src/hotspot/share/opto/loopnode.cpp
+++ b/src/hotspot/share/opto/loopnode.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -5071,7 +5071,7 @@ bool PhaseIdealLoop::verify_loop_ctrl(Node* n, const PhaseIdealLoop* phase_verif
}
}
-int compare_tree(IdealLoopTree* const& a, IdealLoopTree* const& b) {
+static int compare_tree(IdealLoopTree* const& a, IdealLoopTree* const& b) {
assert(a != nullptr && b != nullptr, "must be");
return a->_head->_idx - b->_head->_idx;
}
diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp
index 3b281e0f77de4..3d96ce3e0e27e 100644
--- a/src/hotspot/share/opto/loopnode.hpp
+++ b/src/hotspot/share/opto/loopnode.hpp
@@ -42,6 +42,7 @@ class OuterStripMinedLoopEndNode;
class PredicateBlock;
class PathFrequency;
class PhaseIdealLoop;
+class UnswitchedLoopSelector;
class VectorSet;
class VSharedData;
class Invariance;
@@ -770,6 +771,12 @@ class IdealLoopTree : public ResourceObj {
return _has_range_checks;
}
+ // Return the parent's IdealLoopTree for a strip mined loop which is the outer strip mined loop.
+ // In all other cases, return this.
+ IdealLoopTree* skip_strip_mined() {
+ return _head->as_Loop()->is_strip_mined() ? _parent : this;
+ }
+
#ifndef PRODUCT
void dump_head(); // Dump loop head only
void dump(); // Dump this loop recursively
@@ -1098,6 +1105,7 @@ class PhaseIdealLoop : public PhaseTransform {
// Compute the Ideal Node to Loop mapping
PhaseIdealLoop(PhaseIterGVN& igvn, LoopOptsMode mode) :
PhaseTransform(Ideal_Loop),
+ _loop_or_ctrl(igvn.C->comp_arena()),
_igvn(igvn),
_verify_me(nullptr),
_verify_only(false),
@@ -1112,6 +1120,7 @@ class PhaseIdealLoop : public PhaseTransform {
// or only verify that the graph is valid if verify_me is null.
PhaseIdealLoop(PhaseIterGVN& igvn, const PhaseIdealLoop* verify_me = nullptr) :
PhaseTransform(Ideal_Loop),
+ _loop_or_ctrl(igvn.C->comp_arena()),
_igvn(igvn),
_verify_me(verify_me),
_verify_only(verify_me == nullptr),
@@ -1347,6 +1356,11 @@ class PhaseIdealLoop : public PhaseTransform {
public:
void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
+ void replace_loop_entry(LoopNode* loop_head, Node* new_entry) {
+ _igvn.replace_input_of(loop_head, LoopNode::EntryControl, new_entry);
+ set_idom(loop_head, new_entry, dom_depth(new_entry));
+ }
+
// Construct a range check for a predicate if
BoolNode* rc_predicate(IdealLoopTree* loop, Node* ctrl, int scale, Node* offset, Node* init, Node* limit,
jint stride, Node* range, bool upper, bool& overflow);
@@ -1386,8 +1400,6 @@ class PhaseIdealLoop : public PhaseTransform {
void eliminate_useless_zero_trip_guard();
- bool has_control_dependencies_from_predicates(LoopNode* head) const;
- void verify_fast_loop(LoopNode* head, const ProjNode* proj_true) const NOT_DEBUG_RETURN;
public:
// Change the control input of expensive nodes to allow commoning by
// IGVN when it is guaranteed to not result in a more frequent
@@ -1402,21 +1414,30 @@ class PhaseIdealLoop : public PhaseTransform {
// Eliminate range-checks and other trip-counter vs loop-invariant tests.
void do_range_check(IdealLoopTree *loop, Node_List &old_new);
- // Create a slow version of the loop by cloning the loop
- // and inserting an if to select fast-slow versions.
- // Return the inserted if.
- IfNode* create_slow_version_of_loop(IdealLoopTree *loop,
- Node_List &old_new,
- IfNode* unswitch_iff,
- CloneLoopMode mode);
-
// Clone loop with an invariant test (that does not exit) and
// insert a clone of the test that selects which version to
// execute.
- void do_unswitching (IdealLoopTree *loop, Node_List &old_new);
+ void do_unswitching(IdealLoopTree* loop, Node_List& old_new);
+
+ IfNode* find_unswitch_candidate(const IdealLoopTree* loop) const;
+
+ private:
+ static bool has_control_dependencies_from_predicates(LoopNode* head);
+ static void revert_to_normal_loop(const LoopNode* loop_head);
+
+ void hoist_invariant_check_casts(const IdealLoopTree* loop, const Node_List& old_new,
+ const UnswitchedLoopSelector& unswitched_loop_selector);
+ void add_unswitched_loop_version_bodies_to_igvn(IdealLoopTree* loop, const Node_List& old_new);
+ static void increment_unswitch_counts(LoopNode* original_head, LoopNode* new_head);
+ void remove_unswitch_candidate_from_loops(const Node_List& old_new, const UnswitchedLoopSelector& unswitched_loop_selector);
+#ifndef PRODUCT
+ static void trace_loop_unswitching_count(IdealLoopTree* loop, LoopNode* original_head);
+ static void trace_loop_unswitching_impossible(const LoopNode* original_head);
+ static void trace_loop_unswitching_result(const UnswitchedLoopSelector& unswitched_loop_selector,
+ const LoopNode* original_head, const LoopNode* new_head);
+#endif
- // Find candidate "if" for unswitching
- IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const;
+ public:
// Range Check Elimination uses this function!
// Constrain the main loop iterations so the affine function:
@@ -1623,9 +1644,11 @@ class PhaseIdealLoop : public PhaseTransform {
_nodes_required = UINT_MAX;
}
+ public:
// Clone Parse Predicates to slow and fast loop when unswitching a loop
void clone_parse_and_assertion_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new,
IfProjNode*& iffast_pred, IfProjNode*& ifslow_pred);
+ private:
void clone_loop_predication_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new,
const PredicateBlock* predicate_block,
Deoptimization::DeoptReason reason, IfProjNode*& iffast_pred,
@@ -1742,6 +1765,8 @@ class PhaseIdealLoop : public PhaseTransform {
void update_addp_chain_base(Node* x, Node* old_base, Node* new_base);
bool can_move_to_inner_loop(Node* n, LoopNode* n_loop, Node* x);
+
+ void pin_array_access_nodes_dependent_on(Node* ctrl);
};
diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp
index c5d8ed39d9d0c..99b360143eaf9 100644
--- a/src/hotspot/share/opto/loopopts.cpp
+++ b/src/hotspot/share/opto/loopopts.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -571,8 +571,8 @@ Node* PhaseIdealLoop::remix_address_expressions(Node* n) {
}
// Replace ((I1 +p V) +p I2) with ((I1 +p I2) +p V),
- // but not if I2 is a constant.
- if (n_op == Op_AddP) {
+ // but not if I2 is a constant. Skip for irreducible loops.
+ if (n_op == Op_AddP && n_loop->_head->is_Loop()) {
if (n2_loop == n_loop && n3_loop != n_loop) {
if (n->in(2)->Opcode() == Op_AddP && !n->in(3)->is_Con()) {
Node* n22_ctrl = get_ctrl(n->in(2)->in(2));
@@ -1490,7 +1490,16 @@ void PhaseIdealLoop::split_if_with_blocks_post(Node *n) {
// Replace the dominated test with an obvious true or false.
// Place it on the IGVN worklist for later cleanup.
C->set_major_progress();
- dominated_by(prevdom->as_IfProj(), n->as_If());
+ // Split if: pin array accesses that are control dependent on a range check and moved to a regular if,
+ // to prevent an array load from floating above its range check. There are three cases:
+ // 1. Move from RangeCheck "a" to RangeCheck "b": don't need to pin. If we ever remove b, then we pin
+ // all its array accesses at that point.
+ // 2. We move from RangeCheck "a" to regular if "b": need to pin. If we ever remove b, then its array
+ // accesses would start to float, since we don't pin at that point.
+ // 3. If we move from regular if: don't pin. All array accesses are already assumed to be pinned.
+ bool pin_array_access_nodes = n->Opcode() == Op_RangeCheck &&
+ prevdom->in(0)->Opcode() != Op_RangeCheck;
+ dominated_by(prevdom->as_IfProj(), n->as_If(), false, pin_array_access_nodes);
DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } );
return;
}
@@ -1664,7 +1673,20 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
// n has a control input inside a loop but get_ctrl() is member of an outer loop. This could happen, for example,
// for Div nodes inside a loop (control input inside loop) without a use except for an UCT (outside the loop).
// Rewire control of n to right outside of the loop, regardless if its input(s) are later sunk or not.
- _igvn.replace_input_of(n, 0, place_outside_loop(n_ctrl, loop_ctrl));
+ Node* maybe_pinned_n = n;
+ Node* outside_ctrl = place_outside_loop(n_ctrl, loop_ctrl);
+ if (n->depends_only_on_test()) {
+ Node* pinned_clone = n->pin_array_access_node();
+ if (pinned_clone != nullptr) {
+ // Pin array access nodes: if this is an array load, it's going to be dependent on a condition that's not a
+ // range check for that access. If that condition is replaced by an identical dominating one, then an
+ // unpinned load would risk floating above its range check.
+ register_new_node(pinned_clone, n_ctrl);
+ maybe_pinned_n = pinned_clone;
+ _igvn.replace_node(n, pinned_clone);
+ }
+ }
+ _igvn.replace_input_of(maybe_pinned_n, 0, outside_ctrl);
}
}
if (n_loop != _ltree_root && n->outcnt() > 1) {
@@ -1678,7 +1700,16 @@ void PhaseIdealLoop::try_sink_out_of_loop(Node* n) {
for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin;) {
Node* u = n->last_out(j); // Clone private computation per use
_igvn.rehash_node_delayed(u);
- Node* x = n->clone(); // Clone computation
+ Node* x = nullptr;
+ if (n->depends_only_on_test()) {
+ // Pin array access nodes: if this is an array load, it's going to be dependent on a condition that's not a
+ // range check for that access. If that condition is replaced by an identical dominating one, then an
+ // unpinned load would risk floating above its range check.
+ x = n->pin_array_access_node();
+ }
+ if (x == nullptr) {
+ x = n->clone();
+ }
Node* x_ctrl = nullptr;
if (u->is_Phi()) {
// Replace all uses of normal nodes. Replace Phi uses
@@ -2228,6 +2259,20 @@ void PhaseIdealLoop::clone_loop_handle_data_uses(Node* old, Node_List &old_new,
// We notify all uses of old, including use, and the indirect uses,
// that may now be optimized because we have replaced old with phi.
_igvn.add_users_to_worklist(old);
+ if (idx == 0 &&
+ use->depends_only_on_test()) {
+ Node* pinned_clone = use->pin_array_access_node();
+ if (pinned_clone != nullptr) {
+ // Pin array access nodes: control is updated here to a region. If, after some transformations, only one path
+ // into the region is left, an array load could become dependent on a condition that's not a range check for
+ // that access. If that condition is replaced by an identical dominating one, then an unpinned load would risk
+ // floating above its range check.
+ pinned_clone->set_req(0, phi);
+ register_new_node(pinned_clone, get_ctrl(use));
+ _igvn.replace_node(use, pinned_clone);
+ continue;
+ }
+ }
_igvn.replace_input_of(use, idx, phi);
if( use->_idx >= new_counter ) { // If updating new phis
// Not needed for correctness, but prevents a weak assert
@@ -2867,8 +2912,7 @@ ProjNode* PhaseIdealLoop::insert_if_before_proj(Node* left, bool Signed, BoolTes
int opcode = iff->Opcode();
assert(opcode == Op_If || opcode == Op_RangeCheck, "unexpected opcode");
- IfNode* new_if = (opcode == Op_If) ? new IfNode(proj2, bol, iff->_prob, iff->_fcnt):
- new RangeCheckNode(proj2, bol, iff->_prob, iff->_fcnt);
+ IfNode* new_if = IfNode::make_with_same_profile(iff, proj2, bol);
register_node(new_if, loop, proj2, ddepth);
proj->set_req(0, new_if); // reattach
@@ -3863,6 +3907,19 @@ bool PhaseIdealLoop::partial_peel( IdealLoopTree *loop, Node_List &old_new ) {
if (!n->is_CFG() && n->in(0) != nullptr &&
not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) {
Node* n_clone = old_new[n->_idx];
+ if (n_clone->depends_only_on_test()) {
+ // Pin array access nodes: control is updated here to the loop head. If, after some transformations, the
+ // backedge is removed, an array load could become dependent on a condition that's not a range check for that
+ // access. If that condition is replaced by an identical dominating one, then an unpinned load would risk
+ // floating above its range check.
+ Node* pinned_clone = n_clone->pin_array_access_node();
+ if (pinned_clone != nullptr) {
+ register_new_node(pinned_clone, get_ctrl(n_clone));
+ old_new.map(n->_idx, pinned_clone);
+ _igvn.replace_node(n_clone, pinned_clone);
+ n_clone = pinned_clone;
+ }
+ }
_igvn.replace_input_of(n_clone, 0, new_head_clone);
}
}
@@ -4193,7 +4250,8 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old
Node_List *split_if_set = nullptr;
Node_List *split_bool_set = nullptr;
Node_List *split_cex_set = nullptr;
- fix_data_uses(wq, loop, ControlAroundStripMined, head->is_strip_mined() ? loop->_parent : loop, new_counter, old_new, worklist, split_if_set, split_bool_set, split_cex_set);
+ fix_data_uses(wq, loop, ControlAroundStripMined, loop->skip_strip_mined(), new_counter, old_new, worklist,
+ split_if_set, split_bool_set, split_cex_set);
finish_clone_loop(split_if_set, split_bool_set, split_cex_set);
@@ -4232,7 +4290,12 @@ PhaseIdealLoop::auto_vectorize(IdealLoopTree* lpt, VSharedData &vshared) {
// Ensure the shared data is cleared before each use
vshared.clear();
- SuperWord sw(vloop, vshared);
+ const VLoopAnalyzer vloop_analyzer(vloop, vshared);
+ if (!vloop_analyzer.success()) {
+ return AutoVectorizeStatus::TriedAndFailed;
+ }
+
+ SuperWord sw(vloop_analyzer);
if (!sw.transform_loop()) {
return AutoVectorizeStatus::TriedAndFailed;
}
diff --git a/src/hotspot/share/opto/macro.cpp b/src/hotspot/share/opto/macro.cpp
index 262fff7f8e9e6..23928b554e4e0 100644
--- a/src/hotspot/share/opto/macro.cpp
+++ b/src/hotspot/share/opto/macro.cpp
@@ -167,7 +167,8 @@ void PhaseMacroExpand::eliminate_gc_barrier(Node* p2x) {
// Search for a memory operation for the specified memory slice.
static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) {
Node *orig_mem = mem;
- Node *alloc_mem = alloc->in(TypeFunc::Memory);
+ Node *alloc_mem = alloc->as_Allocate()->proj_out_or_null(TypeFunc::Memory, /*io_use:*/false);
+ assert(alloc_mem != nullptr, "Allocation without a memory projection.");
const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr();
while (true) {
if (mem == alloc_mem || mem == start_mem ) {
@@ -371,7 +372,8 @@ Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *
return nullptr; // Give up: phi tree too deep
}
Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
- Node *alloc_mem = alloc->in(TypeFunc::Memory);
+ Node *alloc_mem = alloc->proj_out_or_null(TypeFunc::Memory, /*io_use:*/false);
+ assert(alloc_mem != nullptr, "Allocation without a memory projection.");
uint length = mem->req();
GrowableArray values(length, length, nullptr);
@@ -456,7 +458,8 @@ Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, Node *sfpt_ctl, BasicType
int offset = adr_t->offset();
Node *start_mem = C->start()->proj_out_or_null(TypeFunc::Memory);
Node *alloc_ctrl = alloc->in(TypeFunc::Control);
- Node *alloc_mem = alloc->in(TypeFunc::Memory);
+ Node *alloc_mem = alloc->proj_out_or_null(TypeFunc::Memory, /*io_use:*/false);
+ assert(alloc_mem != nullptr, "Allocation without a memory projection.");
VectorSet visited;
bool done = sfpt_mem == alloc_mem;
@@ -1939,10 +1942,12 @@ void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) {
// marked for elimination since new obj has no escape information.
// Mark all associated (same box and obj) lock and unlock nodes for
// elimination if some of them marked already.
-void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
- if (oldbox->as_BoxLock()->is_eliminated()) {
+void PhaseMacroExpand::mark_eliminated_box(Node* box, Node* obj) {
+ BoxLockNode* oldbox = box->as_BoxLock();
+ if (oldbox->is_eliminated()) {
return; // This BoxLock node was processed already.
}
+ assert(!oldbox->is_unbalanced(), "this should not be called for unbalanced region");
// New implementation (EliminateNestedLocks) has separate BoxLock
// node for each locked region so mark all associated locks/unlocks as
// eliminated even if different objects are referenced in one locked region
@@ -1950,8 +1955,9 @@ void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
if (EliminateNestedLocks ||
oldbox->as_BoxLock()->is_simple_lock_region(nullptr, obj, nullptr)) {
// Box is used only in one lock region. Mark this box as eliminated.
+ oldbox->set_local(); // This verifies correct state of BoxLock
_igvn.hash_delete(oldbox);
- oldbox->as_BoxLock()->set_eliminated(); // This changes box's hash value
+ oldbox->set_eliminated(); // This changes box's hash value
_igvn.hash_insert(oldbox);
for (uint i = 0; i < oldbox->outcnt(); i++) {
@@ -1978,6 +1984,7 @@ void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
// Note: BoxLock node is marked eliminated only here and it is used
// to indicate that all associated lock and unlock nodes are marked
// for elimination.
+ newbox->set_local(); // This verifies correct state of BoxLock
newbox->set_eliminated();
transform_later(newbox);
@@ -2033,6 +2040,9 @@ void PhaseMacroExpand::mark_eliminated_box(Node* oldbox, Node* obj) {
//-----------------------mark_eliminated_locking_nodes-----------------------
void PhaseMacroExpand::mark_eliminated_locking_nodes(AbstractLockNode *alock) {
+ if (alock->box_node()->as_BoxLock()->is_unbalanced()) {
+ return; // Can't do any more elimination for this locking region
+ }
if (EliminateNestedLocks) {
if (alock->is_nested()) {
assert(alock->box_node()->as_BoxLock()->is_eliminated(), "sanity");
@@ -2349,6 +2359,11 @@ void PhaseMacroExpand::eliminate_macro_nodes() {
// Re-marking may break consistency of Coarsened locks.
if (!C->coarsened_locks_consistent()) {
return; // recompile without Coarsened locks if broken
+ } else {
+ // After coarsened locks are eliminated locking regions
+ // become unbalanced. We should not execute any more
+ // locks elimination optimizations on them.
+ C->mark_unbalanced_boxes();
}
// First, attempt to eliminate locks
diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp
index 010a13a07bac9..e0a364d5056b3 100644
--- a/src/hotspot/share/opto/memnode.cpp
+++ b/src/hotspot/share/opto/memnode.cpp
@@ -3408,6 +3408,7 @@ Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) {
my_mem = load_node;
} else {
assert(my_mem->unique_out() == this, "sanity");
+ assert(!trailing_load_store(), "load store node can't be eliminated");
del_req(Precedent);
phase->is_IterGVN()->_worklist.push(my_mem); // remove dead node later
my_mem = nullptr;
diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp
index 44c37768cdc7d..dee4ce80d35d2 100644
--- a/src/hotspot/share/opto/node.cpp
+++ b/src/hotspot/share/opto/node.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -1632,7 +1632,7 @@ void visit_nodes(Node* start, Callback callback, bool traverse_output, bool only
}
// BFS traverse from start, return node with idx
-Node* find_node_by_idx(Node* start, uint idx, bool traverse_output, bool only_ctrl) {
+static Node* find_node_by_idx(Node* start, uint idx, bool traverse_output, bool only_ctrl) {
ResourceMark rm;
Node* result = nullptr;
auto callback = [&] (Node* n) {
@@ -1648,11 +1648,11 @@ Node* find_node_by_idx(Node* start, uint idx, bool traverse_output, bool only_ct
return result;
}
-int node_idx_cmp(const Node** n1, const Node** n2) {
+static int node_idx_cmp(const Node** n1, const Node** n2) {
return (*n1)->_idx - (*n2)->_idx;
}
-void find_nodes_by_name(Node* start, const char* name) {
+static void find_nodes_by_name(Node* start, const char* name) {
ResourceMark rm;
GrowableArray ns;
auto callback = [&] (const Node* n) {
@@ -1667,7 +1667,7 @@ void find_nodes_by_name(Node* start, const char* name) {
}
}
-void find_nodes_by_dump(Node* start, const char* pattern) {
+static void find_nodes_by_dump(Node* start, const char* pattern) {
ResourceMark rm;
GrowableArray ns;
auto callback = [&] (const Node* n) {
diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp
index 9481b91ce39e9..640a24693dedd 100644
--- a/src/hotspot/share/opto/output.cpp
+++ b/src/hotspot/share/opto/output.cpp
@@ -507,8 +507,8 @@ void PhaseOutput::shorten_branches(uint* blk_starts) {
mcall->method_set((intptr_t)mcall->entry_point());
if (mcall->is_MachCallJava() && mcall->as_MachCallJava()->_method) {
- stub_size += CompiledStaticCall::to_interp_stub_size();
- reloc_size += CompiledStaticCall::reloc_to_interp_stub();
+ stub_size += CompiledDirectCall::to_interp_stub_size();
+ reloc_size += CompiledDirectCall::reloc_to_interp_stub();
}
} else if (mach->is_MachSafePoint()) {
// If call/safepoint are adjacent, account for possible
@@ -3412,6 +3412,12 @@ void PhaseOutput::install_code(ciMethod* target,
_code_offsets.set_value(CodeOffsets::Verified_Entry, 0);
_code_offsets.set_value(CodeOffsets::OSR_Entry, _first_block_size);
} else {
+ if (!target->is_static()) {
+ // The UEP of an nmethod ensures that the VEP is padded. However, the padding of the UEP is placed
+ // before the inline cache check, so we don't have to execute any nop instructions when dispatching
+ // through the UEP, yet we can ensure that the VEP is aligned appropriately.
+ _code_offsets.set_value(CodeOffsets::Entry, _first_block_size - MacroAssembler::ic_check_size());
+ }
_code_offsets.set_value(CodeOffsets::Verified_Entry, _first_block_size);
_code_offsets.set_value(CodeOffsets::OSR_Entry, 0);
}
diff --git a/src/hotspot/share/opto/reg_split.cpp b/src/hotspot/share/opto/reg_split.cpp
index 8e5247e93b681..35b2618c64c40 100644
--- a/src/hotspot/share/opto/reg_split.cpp
+++ b/src/hotspot/share/opto/reg_split.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -289,7 +289,7 @@ int PhaseChaitin::split_USE(MachSpillCopyNode::SpillType spill_type, Node *def,
//------------------------------clone_node----------------------------
// Clone node with anti dependence check.
-Node* clone_node(Node* def, Block *b, Compile* C) {
+static Node* clone_node(Node* def, Block *b, Compile* C) {
if (def->needs_anti_dependence_check()) {
#ifdef ASSERT
if (PrintOpto && WizardMode) {
diff --git a/src/hotspot/share/opto/runtime.cpp b/src/hotspot/share/opto/runtime.cpp
index 8f3367ce0003c..7caaca8846c85 100644
--- a/src/hotspot/share/opto/runtime.cpp
+++ b/src/hotspot/share/opto/runtime.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,14 +28,13 @@
#include "code/codeCache.hpp"
#include "code/compiledMethod.inline.hpp"
#include "code/compiledIC.hpp"
-#include "code/icBuffer.hpp"
#include "code/nmethod.hpp"
#include "code/pcDesc.hpp"
#include "code/scopeDesc.hpp"
#include "code/vtableStubs.hpp"
#include "compiler/compileBroker.hpp"
#include "compiler/oopMap.hpp"
-#include "gc/g1/heapRegion.hpp"
+#include "gc/g1/g1HeapRegion.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shared/collectedHeap.hpp"
#include "gc/shared/gcLocker.hpp"
@@ -320,14 +319,17 @@ JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len
// Zero array here if the caller is deoptimized.
const size_t size = TypeArrayKlass::cast(array_type)->oop_size(result);
BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
- const size_t hs = arrayOopDesc::header_size(elem_type);
- // Align to next 8 bytes to avoid trashing arrays's length.
- const size_t aligned_hs = align_object_offset(hs);
+ size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
+ assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
HeapWord* obj = cast_from_oop(result);
- if (aligned_hs > hs) {
- Copy::zero_to_words(obj+hs, aligned_hs-hs);
+ if (!is_aligned(hs_bytes, BytesPerLong)) {
+ *reinterpret_cast(reinterpret_cast(obj) + hs_bytes) = 0;
+ hs_bytes += BytesPerInt;
}
+
// Optimized zeroing.
+ assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
+ const size_t aligned_hs = hs_bytes / BytesPerLong;
Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
}
diff --git a/src/hotspot/share/opto/split_if.cpp b/src/hotspot/share/opto/split_if.cpp
index 3356498ce13f7..dafd5ffdb38ae 100644
--- a/src/hotspot/share/opto/split_if.cpp
+++ b/src/hotspot/share/opto/split_if.cpp
@@ -727,6 +727,14 @@ void PhaseIdealLoop::do_split_if(Node* iff, RegionNode** new_false_region, Regio
} // End of while merge point has phis
_igvn.remove_dead_node(region);
+ if (iff->Opcode() == Op_RangeCheck) {
+ // Pin array access nodes: control is updated here to a region. If, after some transformations, only one path
+ // into the region is left, an array load could become dependent on a condition that's not a range check for
+ // that access. If that condition is replaced by an identical dominating one, then an unpinned load would risk
+ // floating above its range check.
+ pin_array_access_nodes_dependent_on(new_true);
+ pin_array_access_nodes_dependent_on(new_false);
+ }
if (new_false_region != nullptr) {
*new_false_region = new_false;
@@ -737,3 +745,18 @@ void PhaseIdealLoop::do_split_if(Node* iff, RegionNode** new_false_region, Regio
DEBUG_ONLY( if (VerifyLoopOptimizations) { verify(); } );
}
+
+void PhaseIdealLoop::pin_array_access_nodes_dependent_on(Node* ctrl) {
+ for (DUIterator i = ctrl->outs(); ctrl->has_out(i); i++) {
+ Node* use = ctrl->out(i);
+ if (!use->depends_only_on_test()) {
+ continue;
+ }
+ Node* pinned_clone = use->pin_array_access_node();
+ if (pinned_clone != nullptr) {
+ register_new_node(pinned_clone, get_ctrl(use));
+ _igvn.replace_node(use, pinned_clone);
+ --i;
+ }
+ }
+}
diff --git a/src/hotspot/share/opto/superword.cpp b/src/hotspot/share/opto/superword.cpp
index aa1edd01ab19e..75b5e53f2792d 100644
--- a/src/hotspot/share/opto/superword.cpp
+++ b/src/hotspot/share/opto/superword.cpp
@@ -38,20 +38,15 @@
#include "opto/movenode.hpp"
#include "utilities/powerOfTwo.hpp"
-SuperWord::SuperWord(const VLoop &vloop, VSharedData &vshared) :
- _vloop(vloop),
+SuperWord::SuperWord(const VLoopAnalyzer &vloop_analyzer) :
+ _vloop_analyzer(vloop_analyzer),
+ _vloop(vloop_analyzer.vloop()),
_arena(mtCompiler),
_packset(arena(), 8, 0, nullptr), // packs for the current block
- _bb_idx(vshared.node_idx_to_loop_body_idx()), // node idx to index in bb
- _block(arena(), vloop.estimated_body_length(), 0, nullptr), // nodes in current block
- _mem_slice_head(arena(), 8, 0, nullptr), // memory slice heads
- _mem_slice_tail(arena(), 8, 0, nullptr), // memory slice tails
- _node_info(arena(), vloop.estimated_body_length(), 0, SWNodeInfo::initial), // info needed per node
+ _node_info(arena(), _vloop.estimated_body_length(), 0, SWNodeInfo::initial), // info needed per node
_clone_map(phase()->C->clone_map()), // map of nodes created in cloning
_align_to_ref(nullptr), // memory reference to align vectors to
_dg(arena()), // dependence graph
- _nlist(arena(), vloop.estimated_body_length(), 0, nullptr), // scratch list of nodes
- _loop_reductions(arena()), // reduction nodes in the current loop
_race_possible(false), // cases where SDMU is true
_do_vector_loop(phase()->C->do_vector_loop()), // whether to do vectorization/simd style
_num_work_vecs(0), // amount of vector work we have
@@ -255,7 +250,7 @@ void SuperWord::unrolling_analysis(const VLoop &vloop, int &local_loop_unroll_fa
}
}
-bool SuperWord::is_reduction(const Node* n) {
+bool VLoopReductions::is_reduction(const Node* n) {
if (!is_reduction_operator(n)) {
return false;
}
@@ -269,12 +264,12 @@ bool SuperWord::is_reduction(const Node* n) {
return false;
}
-bool SuperWord::is_reduction_operator(const Node* n) {
+bool VLoopReductions::is_reduction_operator(const Node* n) {
int opc = n->Opcode();
return (opc != ReductionNode::opcode(opc, n->bottom_type()->basic_type()));
}
-bool SuperWord::in_reduction_cycle(const Node* n, uint input) {
+bool VLoopReductions::in_reduction_cycle(const Node* n, uint input) {
// First find input reduction path to phi node.
auto has_my_opcode = [&](const Node* m){ return m->Opcode() == n->Opcode(); };
PathEnd path_to_phi = find_in_path(n, input, LoopMaxUnroll, has_my_opcode,
@@ -291,7 +286,7 @@ bool SuperWord::in_reduction_cycle(const Node* n, uint input) {
return path_from_phi.first != nullptr;
}
-Node* SuperWord::original_input(const Node* n, uint i) {
+Node* VLoopReductions::original_input(const Node* n, uint i) {
if (n->has_swapped_edges()) {
assert(n->is_Add() || n->is_Mul(), "n should be commutative");
if (i == 1) {
@@ -303,21 +298,21 @@ Node* SuperWord::original_input(const Node* n, uint i) {
return n->in(i);
}
-void SuperWord::mark_reductions() {
-
- _loop_reductions.clear();
+void VLoopReductions::mark_reductions() {
+ assert(_loop_reductions.is_empty(), "must not yet be computed");
+ CountedLoopNode* cl = _vloop.cl();
// Iterate through all phi nodes associated to the loop and search for
// reduction cycles in the basic block.
- for (DUIterator_Fast imax, i = cl()->fast_outs(imax); i < imax; i++) {
- const Node* phi = cl()->fast_out(i);
+ for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
+ const Node* phi = cl->fast_out(i);
if (!phi->is_Phi()) {
continue;
}
if (phi->outcnt() == 0) {
continue;
}
- if (phi == iv()) {
+ if (phi == _vloop.iv()) {
continue;
}
// The phi's loop-back is considered the first node in the reduction cycle.
@@ -341,8 +336,9 @@ void SuperWord::mark_reductions() {
// to the phi node following edge index 'input'.
PathEnd path =
find_in_path(
- first, input, lpt()->_body.size(),
- [&](const Node* n) { return n->Opcode() == first->Opcode() && in_bb(n); },
+ first, input, _vloop.lpt()->_body.size(),
+ [&](const Node* n) { return n->Opcode() == first->Opcode() &&
+ _vloop.in_bb(n); },
[&](const Node* n) { return n == phi; });
if (path.first != nullptr) {
reduction_input = input;
@@ -361,7 +357,7 @@ void SuperWord::mark_reductions() {
for (int i = 0; i < path_nodes; i++) {
for (DUIterator_Fast jmax, j = current->fast_outs(jmax); j < jmax; j++) {
Node* u = current->fast_out(j);
- if (!in_bb(u)) {
+ if (!_vloop.in_bb(u)) {
continue;
}
if (u == succ) {
@@ -399,16 +395,6 @@ bool SuperWord::transform_loop() {
}
#endif
- // Skip any loop that has not been assigned max unroll by analysis
- if (SuperWordLoopUnrollAnalysis && vloop().cl()->slp_max_unroll() == 0) {
-#ifndef PRODUCT
- if (is_trace_superword_any()) {
- tty->print_cr("\nSuperWord::transform_loop failed: slp max unroll analysis was not already done");
- }
-#endif
- return false;
- }
-
if (!SLP_extract()) {
#ifndef PRODUCT
if (is_trace_superword_any()) {
@@ -463,35 +449,6 @@ bool SuperWord::transform_loop() {
bool SuperWord::SLP_extract() {
assert(cl()->is_main_loop(), "SLP should only work on main loops");
- if (SuperWordReductions) {
- mark_reductions();
- }
-
- // Find memory slices
- find_memory_slices();
-
- if (!is_marked_reduction_loop() &&
- _mem_slice_head.is_empty()) {
-#ifndef PRODUCT
- if (is_trace_superword_any()) {
- tty->print_cr("\nNo reductions or memory slices found, abort SuperWord.");
- tty->cr();
- }
-#endif
- return false;
- }
-
- // Ready the block
- if (!construct_bb()) {
-#ifndef PRODUCT
- if (is_trace_superword_any()) {
- tty->print_cr("\nSuperWord::construct_bb failed: abort SuperWord");
- tty->cr();
- }
-#endif
- return false;
- }
-
// Ensure extra info is allocated.
initialize_node_info();
@@ -501,9 +458,6 @@ bool SuperWord::SLP_extract() {
// compute function depth(Node*)
compute_max_depth();
- // Compute vector element types
- compute_vector_element_type();
-
// Attempt vectorization
find_adjacent_refs();
@@ -517,15 +471,19 @@ bool SuperWord::SLP_extract() {
return false;
}
- extend_packlist();
+ extend_packset_with_more_pairs_by_following_use_and_def();
- combine_packs();
+ combine_pairs_to_longer_packs();
- filter_packs_for_alignment();
+ split_packs_longer_than_max_vector_size();
+ // Now we only remove packs:
construct_my_pack_map();
-
- filter_packs();
+ filter_packs_for_power_of_2_size();
+ filter_packs_for_mutual_independence();
+ filter_packs_for_alignment();
+ filter_packs_for_implemented();
+ filter_packs_for_profitable();
DEBUG_ONLY(verify_packs();)
@@ -542,8 +500,8 @@ bool SuperWord::SLP_extract() {
void SuperWord::find_adjacent_refs() {
// Get list of memory operations
Node_List memops;
- for (int i = 0; i < _block.length(); i++) {
- Node* n = _block.at(i);
+ for (int i = 0; i < body().length(); i++) {
+ Node* n = body().at(i);
if (n->is_Mem() && !n->is_LoadStore() && in_bb(n) &&
is_java_primitive(n->as_Mem()->memory_type())) {
int align = memory_alignment(n->as_Mem(), 0);
@@ -576,13 +534,13 @@ void SuperWord::find_adjacent_refs() {
set_align_to_ref(align_to_mem_ref);
}
- VPointer align_to_ref_p(mem_ref, vloop());
+ VPointer align_to_ref_p(mem_ref, _vloop);
// Set alignment relative to "align_to_ref" for all related memory operations.
for (int i = memops.size() - 1; i >= 0; i--) {
MemNode* s = memops.at(i)->as_Mem();
if (isomorphic(s, mem_ref) &&
(!_do_vector_loop || same_origin_idx(s, mem_ref))) {
- VPointer p2(s, vloop());
+ VPointer p2(s, _vloop);
if (p2.comparable(align_to_ref_p)) {
int align = memory_alignment(s, iv_adjustment);
set_alignment(s, align);
@@ -641,11 +599,11 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) {
// Count number of comparable memory ops
for (uint i = 0; i < memops.size(); i++) {
MemNode* s1 = memops.at(i)->as_Mem();
- VPointer p1(s1, vloop());
+ VPointer p1(s1, _vloop);
for (uint j = i+1; j < memops.size(); j++) {
MemNode* s2 = memops.at(j)->as_Mem();
if (isomorphic(s1, s2)) {
- VPointer p2(s2, vloop());
+ VPointer p2(s2, _vloop);
if (p1.comparable(p2)) {
(*cmp_ct.adr_at(i))++;
(*cmp_ct.adr_at(j))++;
@@ -666,7 +624,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) {
if (s->is_Store()) {
int vw = vector_width_in_bytes(s);
assert(vw > 1, "sanity");
- VPointer p(s, vloop());
+ VPointer p(s, _vloop);
if ( cmp_ct.at(j) > max_ct ||
(cmp_ct.at(j) == max_ct &&
( vw > max_vw ||
@@ -689,7 +647,7 @@ MemNode* SuperWord::find_align_to_ref(Node_List &memops, int &idx) {
if (s->is_Load()) {
int vw = vector_width_in_bytes(s);
assert(vw > 1, "sanity");
- VPointer p(s, vloop());
+ VPointer p(s, _vloop);
if ( cmp_ct.at(j) > max_ct ||
(cmp_ct.at(j) == max_ct &&
( vw > max_vw ||
@@ -762,7 +720,7 @@ int SuperWord::get_vw_bytes_special(MemNode* s) {
//---------------------------get_iv_adjustment---------------------------
// Calculate loop's iv adjustment for this memory ops.
int SuperWord::get_iv_adjustment(MemNode* mem_ref) {
- VPointer align_to_ref_p(mem_ref, vloop());
+ VPointer align_to_ref_p(mem_ref, _vloop);
int offset = align_to_ref_p.offset_in_bytes();
int scale = align_to_ref_p.scale_in_bytes();
int elt_size = align_to_ref_p.memory_size();
@@ -800,23 +758,29 @@ void SuperWord::dependence_graph() {
assert(cl->is_main_loop(), "SLP should only work on main loops");
// First, assign a dependence node to each memory node
- for (int i = 0; i < _block.length(); i++ ) {
- Node *n = _block.at(i);
+ for (int i = 0; i < body().length(); i++ ) {
+ Node* n = body().at(i);
if (n->is_Mem() || n->is_memory_phi()) {
_dg.make_node(n);
}
}
+ const GrowableArray& mem_slice_head = _vloop_analyzer.memory_slices().heads();
+ const GrowableArray& mem_slice_tail = _vloop_analyzer.memory_slices().tails();
+
+ ResourceMark rm;
+ GrowableArray slice_nodes;
+
// For each memory slice, create the dependences
- for (int i = 0; i < _mem_slice_head.length(); i++) {
- Node* n = _mem_slice_head.at(i);
- Node* n_tail = _mem_slice_tail.at(i);
+ for (int i = 0; i < mem_slice_head.length(); i++) {
+ PhiNode* head = mem_slice_head.at(i);
+ MemNode* tail = mem_slice_tail.at(i);
// Get slice in predecessor order (last is first)
- mem_slice_preds(n_tail, n, _nlist);
+ _vloop_analyzer.memory_slices().get_slice_in_reverse_order(head, tail, slice_nodes);
// Make the slice dependent on the root
- DepMem* slice = _dg.dep(n);
+ DepMem* slice = _dg.dep(head);
_dg.make_edge(_dg.root(), slice);
// Create a sink for the slice
@@ -824,20 +788,20 @@ void SuperWord::dependence_graph() {
_dg.make_edge(slice_sink, _dg.tail());
// Now visit each pair of memory ops, creating the edges
- for (int j = _nlist.length() - 1; j >= 0 ; j--) {
- Node* s1 = _nlist.at(j);
+ for (int j = slice_nodes.length() - 1; j >= 0 ; j--) {
+ Node* s1 = slice_nodes.at(j);
// If no dependency yet, use slice
if (_dg.dep(s1)->in_cnt() == 0) {
_dg.make_edge(slice, s1);
}
- VPointer p1(s1->as_Mem(), vloop());
+ VPointer p1(s1->as_Mem(), _vloop);
bool sink_dependent = true;
for (int k = j - 1; k >= 0; k--) {
- Node* s2 = _nlist.at(k);
+ Node* s2 = slice_nodes.at(k);
if (s1->is_Load() && s2->is_Load())
continue;
- VPointer p2(s2->as_Mem(), vloop());
+ VPointer p2(s2->as_Mem(), _vloop);
int cmp = p1.cmp(p2);
if (!VPointer::not_equal(cmp)) {
@@ -853,68 +817,68 @@ void SuperWord::dependence_graph() {
#ifndef PRODUCT
if (is_trace_superword_dependence_graph()) {
- tty->print_cr("\nDependence graph for slice: %d", n->_idx);
- for (int q = 0; q < _nlist.length(); q++) {
- _dg.print(_nlist.at(q));
+ tty->print_cr("\nDependence graph for slice: %d", head->_idx);
+ for (int q = 0; q < slice_nodes.length(); q++) {
+ _dg.print(slice_nodes.at(q));
}
tty->cr();
}
#endif
- _nlist.clear();
+ slice_nodes.clear();
}
}
-void SuperWord::find_memory_slices() {
- assert(_mem_slice_head.length() == 0, "mem_slice_head is empty");
- assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty");
+void VLoopMemorySlices::find_memory_slices() {
+ assert(_heads.is_empty(), "not yet computed");
+ assert(_tails.is_empty(), "not yet computed");
+ CountedLoopNode* cl = _vloop.cl();
// Iterate over all memory phis
- for (DUIterator_Fast imax, i = cl()->fast_outs(imax); i < imax; i++) {
- PhiNode* phi = cl()->fast_out(i)->isa_Phi();
- if (phi != nullptr && in_bb(phi) && phi->is_memory_phi()) {
+ for (DUIterator_Fast imax, i = cl->fast_outs(imax); i < imax; i++) {
+ PhiNode* phi = cl->fast_out(i)->isa_Phi();
+ if (phi != nullptr && _vloop.in_bb(phi) && phi->is_memory_phi()) {
Node* phi_tail = phi->in(LoopNode::LoopBackControl);
if (phi_tail != phi->in(LoopNode::EntryControl)) {
- _mem_slice_head.push(phi);
- _mem_slice_tail.push(phi_tail->as_Mem());
+ _heads.push(phi);
+ _tails.push(phi_tail->as_Mem());
}
}
}
- NOT_PRODUCT( if (is_trace_superword_memory_slices()) { print_memory_slices(); } )
+ NOT_PRODUCT( if (_vloop.is_trace_memory_slices()) { print(); } )
}
#ifndef PRODUCT
-void SuperWord::print_memory_slices() {
- tty->print_cr("\nSuperWord::print_memory_slices: %s",
- _mem_slice_head.length() > 0 ? "" : "NONE");
- for (int m = 0; m < _mem_slice_head.length(); m++) {
- tty->print("%6d ", m); _mem_slice_head.at(m)->dump();
- tty->print(" "); _mem_slice_tail.at(m)->dump();
+void VLoopMemorySlices::print() const {
+ tty->print_cr("\nVLoopMemorySlices::print: %s",
+ heads().length() > 0 ? "" : "NONE");
+ for (int m = 0; m < heads().length(); m++) {
+ tty->print("%6d ", m); heads().at(m)->dump();
+ tty->print(" "); tails().at(m)->dump();
}
}
#endif
-//---------------------------mem_slice_preds---------------------------
-// Return a memory slice (node list) in predecessor order starting at "start"
-void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray &preds) {
- assert(preds.length() == 0, "start empty");
- Node* n = start;
+// Get all memory nodes of a slice, in reverse order
+void VLoopMemorySlices::get_slice_in_reverse_order(PhiNode* head, MemNode* tail, GrowableArray &slice) const {
+ assert(slice.is_empty(), "start empty");
+ Node* n = tail;
Node* prev = nullptr;
while (true) {
- assert(in_bb(n), "must be in block");
+ assert(_vloop.in_bb(n), "must be in block");
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
Node* out = n->fast_out(i);
if (out->is_Load()) {
- if (in_bb(out)) {
- preds.push(out);
+ if (_vloop.in_bb(out)) {
+ slice.push(out);
}
} else {
// FIXME
- if (out->is_MergeMem() && !in_bb(out)) {
+ if (out->is_MergeMem() && !_vloop.in_bb(out)) {
// Either unrolling is causing a memory edge not to disappear,
// or need to run igvn.optimize() again before SLP
- } else if (out->is_memory_phi() && !in_bb(out)) {
+ } else if (out->is_memory_phi() && !_vloop.in_bb(out)) {
// Ditto. Not sure what else to check further.
} else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) {
// StoreCM has an input edge used as a precedence edge.
@@ -924,19 +888,19 @@ void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray &p
}
}//else
}//for
- if (n == stop) break;
- preds.push(n);
+ if (n == head) { break; }
+ slice.push(n);
prev = n;
assert(n->is_Mem(), "unexpected node %s", n->Name());
n = n->in(MemNode::Memory);
}
#ifndef PRODUCT
- if (is_trace_superword_memory_slices()) {
- tty->print_cr("\nSuperWord::mem_slice_preds:");
- stop->dump();
- for (int j = preds.length() - 1; j >= 0 ; j--) {
- preds.at(j)->dump();
+ if (_vloop.is_trace_memory_slices()) {
+ tty->print_cr("\nVLoopMemorySlices::get_slice_in_reverse_order:");
+ head->dump();
+ for (int j = slice.length() - 1; j >= 0 ; j--) {
+ slice.at(j)->dump();
}
}
#endif
@@ -1007,8 +971,8 @@ bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) {
// Adjacent memory references must have the same base, be comparable
// and have the correct distance between them.
- VPointer p1(s1->as_Mem(), vloop());
- VPointer p2(s2->as_Mem(), vloop());
+ VPointer p1(s1->as_Mem(), _vloop);
+ VPointer p2(s2->as_Mem(), _vloop);
if (p1.base() != p2.base() || !p1.comparable(p2)) return false;
int diff = p2.offset_in_bytes() - p1.offset_in_bytes();
return diff == data_size(s1);
@@ -1081,7 +1045,7 @@ bool SuperWord::independent(Node* s1, Node* s2) {
// is the smallest depth of all nodes from the nodes list. Once we have
// traversed all those nodes, and have not found another node from the
// nodes list, we know that all nodes in the nodes list are independent.
-bool SuperWord::mutually_independent(Node_List* nodes) const {
+bool SuperWord::mutually_independent(const Node_List* nodes) const {
ResourceMark rm;
Unique_Node_List worklist;
VectorSet nodes_set;
@@ -1130,26 +1094,19 @@ bool SuperWord::have_similar_inputs(Node* s1, Node* s2) {
return true;
}
-//------------------------------reduction---------------------------
-// Is there a data path between s1 and s2 and the nodes reductions?
-bool SuperWord::reduction(Node* s1, Node* s2) {
- bool retValue = false;
- int d1 = depth(s1);
- int d2 = depth(s2);
- if (d2 > d1) {
- if (is_marked_reduction(s1) && is_marked_reduction(s2)) {
- // This is an ordered set, so s1 should define s2
- for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) {
- Node* t1 = s1->fast_out(i);
- if (t1 == s2) {
- // both nodes are reductions and connected
- retValue = true;
- }
+bool VLoopReductions::is_marked_reduction_pair(Node* s1, Node* s2) const {
+ if (is_marked_reduction(s1) &&
+ is_marked_reduction(s2)) {
+ // This is an ordered set, so s1 should define s2
+ for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) {
+ Node* t1 = s1->fast_out(i);
+ if (t1 == s2) {
+ // both nodes are reductions and connected
+ return true;
}
}
}
-
- return retValue;
+ return false;
}
//------------------------------set_alignment---------------------------
@@ -1162,16 +1119,8 @@ void SuperWord::set_alignment(Node* s1, Node* s2, int align) {
}
}
-//------------------------------data_size---------------------------
-int SuperWord::data_size(Node* s) {
- int bsize = type2aelembytes(velt_basic_type(s));
- assert(bsize != 0, "valid size");
- return bsize;
-}
-
-//------------------------------extend_packlist---------------------------
// Extend packset by following use->def and def->use links from pack members.
-void SuperWord::extend_packlist() {
+void SuperWord::extend_packset_with_more_pairs_by_following_use_and_def() {
bool changed;
do {
packset_sort(_packset.length());
@@ -1192,7 +1141,7 @@ void SuperWord::extend_packlist() {
#ifndef PRODUCT
if (is_trace_superword_packset()) {
- tty->print_cr("\nAfter Superword::extend_packlist");
+ tty->print_cr("\nAfter Superword::extend_packset_with_more_pairs_by_following_use_and_def");
print_packset();
}
#endif
@@ -1478,12 +1427,13 @@ int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; }
int SuperWord::pack_cost(int ct) { return ct; }
int SuperWord::unpack_cost(int ct) { return ct; }
-//------------------------------combine_packs---------------------------
// Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last
-void SuperWord::combine_packs() {
+void SuperWord::combine_pairs_to_longer_packs() {
#ifdef ASSERT
+ assert(!_packset.is_empty(), "packset not empty");
for (int i = 0; i < _packset.length(); i++) {
assert(_packset.at(i) != nullptr, "no nullptr in packset");
+ assert(_packset.at(i)->size() == 2, "all packs are pairs");
}
#endif
@@ -1509,97 +1459,152 @@ void SuperWord::combine_packs() {
}
}
- // Split packs which have size greater then max vector size.
- for (int i = 0; i < _packset.length(); i++) {
- Node_List* p1 = _packset.at(i);
- if (p1 != nullptr) {
- uint max_vlen = max_vector_size_in_def_use_chain(p1->at(0)); // Max elements in vector
- assert(is_power_of_2(max_vlen), "sanity");
- uint psize = p1->size();
- if (!is_power_of_2(psize)) {
- // We currently only support power-of-2 sizes for vectors.
+ // Remove all nullptr from packset
+ compress_packset();
+
+ assert(!_packset.is_empty(), "must have combined some packs");
+
#ifndef PRODUCT
- if (is_trace_superword_rejections()) {
- tty->cr();
- tty->print_cr("WARNING: Removed pack[%d] with size that is not a power of 2:", i);
- print_pack(p1);
- }
+ if (is_trace_superword_packset()) {
+ tty->print_cr("\nAfter Superword::combine_pairs_to_longer_packs");
+ print_packset();
+ }
#endif
- _packset.at_put(i, nullptr);
- continue;
- }
- if (psize > max_vlen) {
- Node_List* pack = new Node_List();
- for (uint j = 0; j < psize; j++) {
- pack->push(p1->at(j));
- if (pack->size() >= max_vlen) {
- assert(is_power_of_2(pack->size()), "sanity");
- _packset.append(pack);
- pack = new Node_List();
- }
- }
- _packset.at_put(i, nullptr);
+}
+
+void SuperWord::split_packs_longer_than_max_vector_size() {
+ assert(!_packset.is_empty(), "packset not empty");
+ DEBUG_ONLY( int old_packset_length = _packset.length(); )
+
+ for (int i = 0; i < _packset.length(); i++) {
+ Node_List* pack = _packset.at(i);
+ assert(pack != nullptr, "no nullptr in packset");
+ uint max_vlen = max_vector_size_in_def_use_chain(pack->at(0));
+ assert(is_power_of_2(max_vlen), "sanity");
+ uint pack_size = pack->size();
+ if (pack_size <= max_vlen) {
+ continue;
+ }
+ // Split off the "upper" nodes into new packs
+ Node_List* new_pack = new Node_List();
+ for (uint j = max_vlen; j < pack_size; j++) {
+ Node* n = pack->at(j);
+ // is new_pack full?
+ if (new_pack->size() >= max_vlen) {
+ assert(is_power_of_2(new_pack->size()), "sanity %d", new_pack->size());
+ _packset.append(new_pack);
+ new_pack = new Node_List();
+ }
+ new_pack->push(n);
+ }
+ // remaining new_pack
+ if (new_pack->size() > 1) {
+ _packset.append(new_pack);
+ } else {
+#ifndef PRODUCT
+ if (is_trace_superword_rejections()) {
+ tty->cr();
+ tty->print_cr("WARNING: Node dropped out of odd size pack:");
+ new_pack->at(0)->dump();
+ print_pack(pack);
}
+#endif
+ }
+ // truncate
+ while (pack->size() > max_vlen) {
+ pack->pop();
}
}
- // We know that the nodes in a pair pack were independent - this gives us independence
- // at distance 1. But now that we may have more than 2 nodes in a pack, we need to check
- // if they are all mutually independent. If there is a dependence we remove the pack.
- // This is better than giving up completely - we can have partial vectorization if some
- // are rejected and others still accepted.
- //
- // Examples with dependence at distance 1 (pack pairs are not created):
- // for (int i ...) { v[i + 1] = v[i] + 5; }
- // for (int i ...) { v[i] = v[i - 1] + 5; }
- //
- // Example with independence at distance 1, but dependence at distance 2 (pack pairs are
- // created and we need to filter them out now):
- // for (int i ...) { v[i + 2] = v[i] + 5; }
- // for (int i ...) { v[i] = v[i - 2] + 5; }
- //
- // Note: dependencies are created when a later load may reference the same memory location
- // as an earlier store. This happens in "read backward" or "store forward" cases. On the
- // other hand, "read forward" or "store backward" cases do not have such dependencies:
- // for (int i ...) { v[i] = v[i + 1] + 5; }
- // for (int i ...) { v[i - 1] = v[i] + 5; }
- for (int i = 0; i < _packset.length(); i++) {
- Node_List* p = _packset.at(i);
- if (p != nullptr) {
- // reductions are trivially connected
- if (!is_marked_reduction(p->at(0)) &&
- !mutually_independent(p)) {
+ assert(old_packset_length <= _packset.length(), "we only increased the number of packs");
+
#ifndef PRODUCT
- if (is_trace_superword_rejections()) {
- tty->cr();
- tty->print_cr("WARNING: Found dependency at distance greater than 1.");
- tty->print_cr("In pack[%d]", i);
- print_pack(p);
- }
+ if (is_trace_superword_packset()) {
+ tty->print_cr("\nAfter Superword::split_packs_longer_than_max_vector_size");
+ print_packset();
+ }
#endif
- _packset.at_put(i, nullptr);
+}
+
+template
+void SuperWord::filter_packs(const char* filter_name,
+ const char* error_message,
+ FilterPredicate filter) {
+ int new_packset_length = 0;
+ for (int i = 0; i < _packset.length(); i++) {
+ Node_List* pack = _packset.at(i);
+ assert(pack != nullptr, "no nullptr in packset");
+ if (filter(pack)) {
+ assert(i >= new_packset_length, "only move packs down");
+ _packset.at_put(new_packset_length++, pack);
+ } else {
+ remove_pack_at(i);
+#ifndef PRODUCT
+ if (is_trace_superword_rejections()) {
+ tty->cr();
+ tty->print_cr("WARNING: Removed pack: %s:", error_message);
+ print_pack(pack);
}
+#endif
}
}
- // Remove all nullptr from packset
- compress_packset();
+ assert(_packset.length() >= new_packset_length, "filter only reduces number of packs");
+ _packset.trunc_to(new_packset_length);
#ifndef PRODUCT
- if (is_trace_superword_packset()) {
- tty->print_cr("\nAfter Superword::combine_packs");
+ if (is_trace_superword_packset() && filter_name != nullptr) {
+ tty->print_cr("\nAfter %s:", filter_name);
print_packset();
}
#endif
}
+void SuperWord::filter_packs_for_power_of_2_size() {
+ filter_packs("SuperWord::filter_packs_for_power_of_2_size",
+ "size is not a power of 2",
+ [&](const Node_List* pack) {
+ return is_power_of_2(pack->size());
+ });
+}
+
+// We know that the nodes in a pair pack were independent - this gives us independence
+// at distance 1. But now that we may have more than 2 nodes in a pack, we need to check
+// if they are all mutually independent. If there is a dependence we remove the pack.
+// This is better than giving up completely - we can have partial vectorization if some
+// are rejected and others still accepted.
+//
+// Examples with dependence at distance 1 (pack pairs are not created):
+// for (int i ...) { v[i + 1] = v[i] + 5; }
+// for (int i ...) { v[i] = v[i - 1] + 5; }
+//
+// Example with independence at distance 1, but dependence at distance 2 (pack pairs are
+// created and we need to filter them out now):
+// for (int i ...) { v[i + 2] = v[i] + 5; }
+// for (int i ...) { v[i] = v[i - 2] + 5; }
+//
+// Note: dependencies are created when a later load may reference the same memory location
+// as an earlier store. This happens in "read backward" or "store forward" cases. On the
+// other hand, "read forward" or "store backward" cases do not have such dependencies:
+// for (int i ...) { v[i] = v[i + 1] + 5; }
+// for (int i ...) { v[i - 1] = v[i] + 5; }
+void SuperWord::filter_packs_for_mutual_independence() {
+ filter_packs("SuperWord::filter_packs_for_mutual_independence",
+ "found dependency between nodes at distance greater than 1",
+ [&](const Node_List* pack) {
+ // reductions are trivially connected
+ return is_marked_reduction(pack->at(0)) ||
+ mutually_independent(pack);
+ });
+}
+
// Find the set of alignment solutions for load/store pack.
-const AlignmentSolution* SuperWord::pack_alignment_solution(Node_List* pack) {
+const AlignmentSolution* SuperWord::pack_alignment_solution(const Node_List* pack) {
assert(pack != nullptr && (pack->at(0)->is_Load() || pack->at(0)->is_Store()), "only load/store packs");
const MemNode* mem_ref = pack->at(0)->as_Mem();
- VPointer mem_ref_p(mem_ref, vloop());
- const CountedLoopEndNode* pre_end = vloop().pre_loop_end();
+ VPointer mem_ref_p(mem_ref, _vloop);
+ const CountedLoopEndNode* pre_end = _vloop.pre_loop_end();
assert(pre_end->stride_is_con(), "pre loop stride is constant");
AlignmentSolver solver(pack->at(0)->as_Mem(),
@@ -1638,41 +1643,36 @@ void SuperWord::filter_packs_for_alignment() {
AlignmentSolution const* current = new TrivialAlignmentSolution();
int mem_ops_count = 0;
int mem_ops_rejected = 0;
- for (int i = 0; i < _packset.length(); i++) {
- Node_List* p = _packset.at(i);
- if (p != nullptr) {
- if (p->at(0)->is_Load() || p->at(0)->is_Store()) {
- mem_ops_count++;
- // Find solution for pack p, and filter with current solution.
- const AlignmentSolution* s = pack_alignment_solution(p);
- const AlignmentSolution* intersect = current->filter(s);
-#ifndef PRODUCT
- if (is_trace_align_vector()) {
- tty->print(" solution for pack: ");
- s->print();
- tty->print(" intersection with current: ");
- intersect->print();
- }
-#endif
+ filter_packs("SuperWord::filter_packs_for_alignment",
+ "rejected by AlignVector (strict alignment requirement)",
+ [&](const Node_List* pack) {
+ // Only memops need to be aligned.
+ if (!pack->at(0)->is_Load() &&
+ !pack->at(0)->is_Store()) {
+ return true; // accept all non memops
+ }
+
+ mem_ops_count++;
+ const AlignmentSolution* s = pack_alignment_solution(pack);
+ const AlignmentSolution* intersect = current->filter(s);
- if (intersect->is_empty()) {
- // Solution failed or is not compatible, remove pack i.
#ifndef PRODUCT
- if (is_trace_superword_rejections() || is_trace_align_vector()) {
- tty->print_cr("Rejected by AlignVector:");
- p->at(0)->dump();
- }
+ if (is_trace_align_vector()) {
+ tty->print(" solution for pack: ");
+ s->print();
+ tty->print(" intersection with current: ");
+ intersect->print();
+ }
#endif
- _packset.at_put(i, nullptr);
- mem_ops_rejected++;
- } else {
- // Solution is compatible.
- current = intersect;
- }
- }
- }
- }
+ if (intersect->is_empty()) {
+ mem_ops_rejected++;
+ return false; // reject because of empty solution
+ }
+
+ current = intersect;
+ return true; // accept because of non-empty solution
+ });
#ifndef PRODUCT
if (is_trace_superword_info() || is_trace_align_vector()) {
@@ -1689,16 +1689,6 @@ void SuperWord::filter_packs_for_alignment() {
// -> must change pre-limit to achieve alignment
set_align_to_ref(current->as_constrained()->mem_ref());
}
-
- // Remove all nullptr from packset
- compress_packset();
-
-#ifndef PRODUCT
- if (is_trace_superword_packset() || is_trace_align_vector()) {
- tty->print_cr("\nAfter Superword::filter_packs_for_alignment");
- print_packset();
- }
-#endif
}
// Compress packset, such that it has no nullptr entries
@@ -1716,7 +1706,7 @@ void SuperWord::compress_packset() {
//-----------------------------construct_my_pack_map--------------------------
// Construct the map from nodes to packs. Only valid after the
-// point where a node is only in one pack (after combine_packs).
+// point where a node is only in one pack (after combine_pairs_to_longer_packs).
void SuperWord::construct_my_pack_map() {
for (int i = 0; i < _packset.length(); i++) {
Node_List* p = _packset.at(i);
@@ -1735,23 +1725,22 @@ void SuperWord::construct_my_pack_map() {
}
}
-//------------------------------filter_packs---------------------------
-// Remove packs that are not implemented or not profitable.
-void SuperWord::filter_packs() {
- // Remove packs that are not implemented
- for (int i = _packset.length() - 1; i >= 0; i--) {
- Node_List* pk = _packset.at(i);
- bool impl = implemented(pk);
- if (!impl) {
-#ifndef PRODUCT
- if (is_trace_superword_rejections()) {
- tty->print_cr("Unimplemented");
- pk->at(0)->dump();
- }
-#endif
- remove_pack_at(i);
- }
- Node *n = pk->at(0);
+// Remove packs that are not implemented
+void SuperWord::filter_packs_for_implemented() {
+ filter_packs("SuperWord::filter_packs_for_implemented",
+ "Unimplemented",
+ [&](const Node_List* pack) {
+ return implemented(pack);
+ });
+}
+
+// Remove packs that are not profitable.
+void SuperWord::filter_packs_for_profitable() {
+ // Count the number of reductions vs other vector ops, for the
+ // reduction profitability heuristic.
+ for (int i = 0; i < _packset.length(); i++) {
+ Node_List* pack = _packset.at(i);
+ Node* n = pack->at(0);
if (is_marked_reduction(n)) {
_num_reductions++;
} else {
@@ -1760,28 +1749,22 @@ void SuperWord::filter_packs() {
}
// Remove packs that are not profitable
- bool changed;
- do {
- changed = false;
- for (int i = _packset.length() - 1; i >= 0; i--) {
- Node_List* pk = _packset.at(i);
- bool prof = profitable(pk);
- if (!prof) {
-#ifndef PRODUCT
- if (is_trace_superword_rejections()) {
- tty->print_cr("Unprofitable");
- pk->at(0)->dump();
- }
-#endif
- remove_pack_at(i);
- changed = true;
- }
+ while (true) {
+ int old_packset_length = _packset.length();
+ filter_packs(nullptr, // don't dump each time
+ "size is not a power of 2",
+ [&](const Node_List* pack) {
+ return profitable(pack);
+ });
+ // Repeat until stable
+ if (old_packset_length == _packset.length()) {
+ break;
}
- } while (changed);
+ }
#ifndef PRODUCT
if (is_trace_superword_packset()) {
- tty->print_cr("\nAfter Superword::filter_packs");
+ tty->print_cr("\nAfter Superword::filter_packs_for_profitable");
print_packset();
tty->cr();
}
@@ -1790,7 +1773,7 @@ void SuperWord::filter_packs() {
//------------------------------implemented---------------------------
// Can code be generated for pack p?
-bool SuperWord::implemented(Node_List* p) {
+bool SuperWord::implemented(const Node_List* p) {
bool retValue = false;
Node* p0 = p->at(0);
if (p0 != nullptr) {
@@ -1850,7 +1833,7 @@ bool SuperWord::requires_long_to_int_conversion(int opc) {
//------------------------------same_inputs--------------------------
// For pack p, are all idx operands the same?
-bool SuperWord::same_inputs(Node_List* p, int idx) {
+bool SuperWord::same_inputs(const Node_List* p, int idx) {
Node* p0 = p->at(0);
uint vlen = p->size();
Node* p0_def = p0->in(idx);
@@ -1866,7 +1849,7 @@ bool SuperWord::same_inputs(Node_List* p, int idx) {
//------------------------------profitable---------------------------
// For pack p, are all operands and all uses (with in the block) vector?
-bool SuperWord::profitable(Node_List* p) {
+bool SuperWord::profitable(const Node_List* p) {
Node* p0 = p->at(0);
uint start, end;
VectorNode::vector_operands(p0, &start, &end);
@@ -1886,9 +1869,8 @@ bool SuperWord::profitable(Node_List* p) {
Node* second_in = p0->in(2);
Node_List* second_pk = my_pack(second_in);
if ((second_pk == nullptr) || (_num_work_vecs == _num_reductions)) {
- // Unmark reduction if no parent pack or if not enough work
+ // No parent pack or not enough work
// to cover reduction expansion overhead
- _loop_reductions.remove(p0->_idx);
return false;
} else if (second_pk->size() != p->size()) {
return false;
@@ -1991,8 +1973,8 @@ void SuperWord::verify_packs() {
}
// Check that no other node has my_pack set.
- for (int i = 0; i < _block.length(); i++) {
- Node* n = _block.at(i);
+ for (int i = 0; i < body().length(); i++) {
+ Node* n = body().at(i);
if (!processed.member(n)) {
assert(my_pack(n) == nullptr, "should not have pack if not in packset");
}
@@ -2075,9 +2057,9 @@ class PacksetGraph {
// Create nodes (from packs and scalar-nodes), and add edges, based on DepPreds.
void build() {
- const GrowableArray &packset = _slp->packset();
- const GrowableArray &block = _slp->block();
- const DepGraph &dg = _slp->dg();
+ const GrowableArray& packset = _slp->packset();
+ const GrowableArray& body = _slp->body();
+ const DepGraph& dg = _slp->dg();
// Map nodes in packsets
for (int i = 0; i < packset.length(); i++) {
Node_List* p = packset.at(i);
@@ -2092,8 +2074,8 @@ class PacksetGraph {
int max_pid_packset = _max_pid;
// Map nodes not in packset
- for (int i = 0; i < block.length(); i++) {
- Node* n = block.at(i);
+ for (int i = 0; i < body.length(); i++) {
+ Node* n = body.at(i);
if (n->is_Phi() || n->is_CFG()) {
continue; // ignore control flow
}
@@ -2120,7 +2102,7 @@ class PacksetGraph {
if (pred_pid == pid && _slp->is_marked_reduction(n)) {
continue; // reduction -> self-cycle is not a cyclic dependency
}
- // Only add edges once, and only for mapped nodes (in block)
+ // Only add edges once, and only for mapped nodes (in body)
if (pred_pid > 0 && !set.test_set(pred_pid)) {
incnt_set(pid, incnt(pid) + 1); // increment
out(pred_pid).push(pid);
@@ -2130,8 +2112,8 @@ class PacksetGraph {
}
// Map edges for nodes not in packset
- for (int i = 0; i < block.length(); i++) {
- Node* n = block.at(i);
+ for (int i = 0; i < body.length(); i++) {
+ Node* n = body.at(i);
int pid = get_pid_or_zero(n); // zero for Phi or CFG
if (pid <= max_pid_packset) {
continue; // Only scalar-nodes
@@ -2139,7 +2121,7 @@ class PacksetGraph {
for (DepPreds preds(n, dg); !preds.done(); preds.next()) {
Node* pred = preds.current();
int pred_pid = get_pid_or_zero(pred);
- // Only add edges for mapped nodes (in block)
+ // Only add edges for mapped nodes (in body)
if (pred_pid > 0) {
incnt_set(pid, incnt(pid) + 1); // increment
out(pred_pid).push(pid);
@@ -2200,7 +2182,7 @@ class PacksetGraph {
// print_nodes = true: print all C2 nodes beloning to PacksetGrahp node.
// print_zero_incnt = false: do not print nodes that have no in-edges (any more).
void print(bool print_nodes, bool print_zero_incnt) {
- const GrowableArray &block = _slp->block();
+ const GrowableArray &body = _slp->body();
tty->print_cr("PacksetGraph");
for (int pid = 1; pid <= _max_pid; pid++) {
if (incnt(pid) == 0 && !print_zero_incnt) {
@@ -2213,8 +2195,8 @@ class PacksetGraph {
tty->print_cr("]");
#ifndef PRODUCT
if (print_nodes) {
- for (int i = 0; i < block.length(); i++) {
- Node* n = block.at(i);
+ for (int i = 0; i < body.length(); i++) {
+ Node* n = body.at(i);
if (get_pid_or_zero(n) == pid) {
tty->print(" ");
n->dump();
@@ -2292,9 +2274,11 @@ void SuperWord::schedule_reorder_memops(Node_List &memops_schedule) {
// loop we may have a different last store, and we need to adjust the uses accordingly.
GrowableArray