From 83c6d7b3eba28de2fd862bd3c5579a4a84e8bc01 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 7 Jan 2025 18:10:28 +0100 Subject: [PATCH 001/101] KLUTE --- .../share/classfile/classFileParser.cpp | 5 + .../share/classfile/classLoaderData.cpp | 13 +- .../share/classfile/classLoaderData.hpp | 2 +- .../share/classfile/classLoaderDataGraph.cpp | 2 +- .../share/classfile/classLoaderDataGraph.hpp | 2 +- .../share/classfile/fieldLayoutBuilder.cpp | 1 - .../gc/parallel/psPromotionManager.inline.hpp | 2 +- src/hotspot/share/logging/logTag.hpp | 1 + .../share/memory/classLoaderMetaspace.cpp | 2 +- src/hotspot/share/memory/iterator.hpp | 9 + src/hotspot/share/memory/iterator.inline.hpp | 215 +++++++++++++ .../metaspace/metaspaceArenaGrowthPolicy.cpp | 2 +- src/hotspot/share/memory/universe.cpp | 6 + src/hotspot/share/oops/compressedKlass.cpp | 15 +- src/hotspot/share/oops/compressedKlass.hpp | 3 - .../share/oops/instanceClassLoaderKlass.hpp | 22 +- .../oops/instanceClassLoaderKlass.inline.hpp | 64 +++- src/hotspot/share/oops/instanceKlass.cpp | 21 +- src/hotspot/share/oops/instanceKlass.hpp | 34 +- .../share/oops/instanceKlass.inline.hpp | 115 ++++++- .../share/oops/instanceMirrorKlass.hpp | 31 +- .../share/oops/instanceMirrorKlass.inline.hpp | 102 ++++-- src/hotspot/share/oops/instanceRefKlass.hpp | 15 +- .../share/oops/instanceRefKlass.inline.hpp | 26 ++ .../share/oops/instanceStackChunkKlass.hpp | 17 +- .../oops/instanceStackChunkKlass.inline.hpp | 22 ++ src/hotspot/share/oops/klass.cpp | 5 +- src/hotspot/share/oops/klass.hpp | 54 +++- src/hotspot/share/oops/klass.inline.hpp | 1 + src/hotspot/share/oops/klassInfoLUT.cpp | 246 +++++++++++++++ src/hotspot/share/oops/klassInfoLUT.hpp | 106 +++++++ .../share/oops/klassInfoLUT.inline.hpp | 63 ++++ src/hotspot/share/oops/klassInfoLUTEntry.cpp | 298 ++++++++++++++++++ src/hotspot/share/oops/klassInfoLUTEntry.hpp | 246 +++++++++++++++ .../share/oops/klassInfoLUTEntry.inline.hpp | 100 ++++++ src/hotspot/share/oops/objArrayKlass.cpp | 17 +- src/hotspot/share/oops/objArrayKlass.hpp | 15 +- .../share/oops/objArrayKlass.inline.hpp | 22 ++ src/hotspot/share/oops/oop.hpp | 1 + src/hotspot/share/oops/oop.inline.hpp | 62 +++- src/hotspot/share/oops/typeArrayKlass.cpp | 16 + src/hotspot/share/oops/typeArrayKlass.hpp | 17 +- .../share/oops/typeArrayKlass.inline.hpp | 20 ++ src/hotspot/share/runtime/arguments.cpp | 12 + src/hotspot/share/runtime/globals.hpp | 6 + src/hotspot/share/runtime/java.cpp | 5 + src/hotspot/share/utilities/devirtualizer.hpp | 1 + .../share/utilities/devirtualizer.inline.hpp | 1 + .../share/utilities/globalDefinitions.hpp | 5 +- 49 files changed, 1950 insertions(+), 118 deletions(-) create mode 100644 src/hotspot/share/oops/klassInfoLUT.cpp create mode 100644 src/hotspot/share/oops/klassInfoLUT.hpp create mode 100644 src/hotspot/share/oops/klassInfoLUT.inline.hpp create mode 100644 src/hotspot/share/oops/klassInfoLUTEntry.cpp create mode 100644 src/hotspot/share/oops/klassInfoLUTEntry.hpp create mode 100644 src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp index 3e6246d4aeedc..f88a3769af15d 100644 --- a/src/hotspot/share/classfile/classFileParser.cpp +++ b/src/hotspot/share/classfile/classFileParser.cpp @@ -53,6 +53,7 @@ #include "oops/instanceKlass.inline.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/klassVtable.hpp" #include "oops/metadata.hpp" #include "oops/method.inline.hpp" @@ -5240,6 +5241,10 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik, // in order for it to not be destroyed in the ClassFileParser destructor. set_klass_to_deallocate(nullptr); + if (UseKLUT) { + KlassInfoLUT::register_klass(ik); + } + // it's official set_klass(ik); diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index 825072cb13bef..7a6201f0274a2 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -985,6 +985,7 @@ class PrintKlassClosure: public KlassClosure { _out->print("%s,", k->external_name()); } }; +#endif // PRODUCT void ClassLoaderData::print_on(outputStream* out) const { ResourceMark rm; @@ -1018,14 +1019,11 @@ void ClassLoaderData::print_on(outputStream* out) const { default: ShouldNotReachHere(); } out->print_cr(" - handles %d", _handles.count()); - out->print_cr(" - dependency count %d", _dependency_count); + DEBUG_ONLY(out->print_cr(" - dependency count %d", _dependency_count);) out->print (" - klasses { "); - if (Verbose) { - PrintKlassClosure closure(out); - ((ClassLoaderData*)this)->classes_do(&closure); - } else { - out->print("..."); - } + + { PrintClassClosure closure(out, true); + ((ClassLoaderData*)this)->classes_do(&closure); } out->print_cr(" }"); out->print_cr(" - packages " INTPTR_FORMAT, p2i(_packages)); out->print_cr(" - module " INTPTR_FORMAT, p2i(_modules)); @@ -1044,7 +1042,6 @@ void ClassLoaderData::print_on(outputStream* out) const { out->print_cr(" - deallocate list " INTPTR_FORMAT, p2i(_deallocate_list)); out->print_cr(" - next CLD " INTPTR_FORMAT, p2i(_next)); } -#endif // PRODUCT void ClassLoaderData::print() const { print_on(tty); } diff --git a/src/hotspot/share/classfile/classLoaderData.hpp b/src/hotspot/share/classfile/classLoaderData.hpp index 19dbb0b1b3650..f6ca17cc8af50 100644 --- a/src/hotspot/share/classfile/classLoaderData.hpp +++ b/src/hotspot/share/classfile/classLoaderData.hpp @@ -320,7 +320,7 @@ class ClassLoaderData : public CHeapObj { void set_jmethod_ids(JNIMethodBlock* new_block) { _jmethod_ids = new_block; } void print() const; - void print_on(outputStream* out) const PRODUCT_RETURN; + void print_on(outputStream* out) const ; void print_value() const; void print_value_on(outputStream* out) const; void verify(); diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.cpp b/src/hotspot/share/classfile/classLoaderDataGraph.cpp index fca6a9e74ad31..b50477d2ffc48 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.cpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.cpp @@ -562,6 +562,7 @@ extern "C" int print_loader_data_graph() { ClassLoaderDataGraph::print_on(tty); return 0; } +#endif // PRODUCT void ClassLoaderDataGraph::print_on(outputStream * const out) { ClassLoaderDataGraphIterator iter; @@ -569,6 +570,5 @@ void ClassLoaderDataGraph::print_on(outputStream * const out) { cld->print_on(out); } } -#endif // PRODUCT void ClassLoaderDataGraph::print() { print_on(tty); } diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.hpp b/src/hotspot/share/classfile/classLoaderDataGraph.hpp index 49f15e8af33df..72aefc1cfd8db 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.hpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.hpp @@ -107,7 +107,7 @@ class ClassLoaderDataGraph : public AllStatic { static bool has_metaspace_oom() { return _metaspace_oom; } static void set_metaspace_oom(bool value) { _metaspace_oom = value; } - static void print_on(outputStream * const out) PRODUCT_RETURN; + static void print_on(outputStream * const out) ; static void print(); static void verify(); diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp index b6007923907c8..0e4d1279e3630 100644 --- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp +++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp @@ -603,7 +603,6 @@ void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) { } } -// Computation of regular classes layout is an evolution of the previous default layout // (FieldAllocationStyle 1): // - primitive fields are allocated first (from the biggest to the smallest) // - oop fields are allocated, either in existing gaps or at the end of diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index 4f3d135c91998..2747516e824a1 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -126,7 +126,7 @@ inline void InstanceRefKlass::oop_oop_iterate_reverseklass()->is_typeArray_klass()) { + if (!obj->is_typeArray()) { PSPushContentsClosure pcc(this); obj->oop_iterate_backwards(&pcc); } diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp index d61e2461e8da6..bae786aff4310 100644 --- a/src/hotspot/share/logging/logTag.hpp +++ b/src/hotspot/share/logging/logTag.hpp @@ -110,6 +110,7 @@ class outputStream; LOG_TAG(jni) \ LOG_TAG(jvmci) \ LOG_TAG(jvmti) \ + LOG_TAG(klut) \ LOG_TAG(lambda) \ LOG_TAG(library) \ LOG_TAG(link) \ diff --git a/src/hotspot/share/memory/classLoaderMetaspace.cpp b/src/hotspot/share/memory/classLoaderMetaspace.cpp index efa6adc7a7bf5..dce01af34b679 100644 --- a/src/hotspot/share/memory/classLoaderMetaspace.cpp +++ b/src/hotspot/share/memory/classLoaderMetaspace.cpp @@ -85,7 +85,7 @@ ClassLoaderMetaspace::ClassLoaderMetaspace(Mutex* lock, Metaspace::MetaspaceType "class arena"); } - UL2(debug, "born (nonclass arena: " PTR_FORMAT ", class arena: " PTR_FORMAT ".", + UL2(debug, "born (nonclass arena: " PTR_FORMAT ", class arena: " PTR_FORMAT ").", p2i(_non_class_space_arena), p2i(_class_space_arena)); } diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 044951142b0ac..96aa1af544730 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -28,6 +28,7 @@ #include "memory/allocation.hpp" #include "memory/memRegion.hpp" #include "oops/oopsHierarchy.hpp" +#include "oops/klassInfoLUT.hpp" class CodeBlob; class nmethod; @@ -317,6 +318,14 @@ class OopIteratorClosureDispatch { template static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass); template static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass, MemRegion mr); template static void oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* klass); + + // Note: we only need Klass* for later, in the Klass, to optionally do metadata iteration. However, this may hopefully change in the future. + template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); + template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); + template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); + template static size_t oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + }; #endif // SHARE_MEMORY_ITERATOR_HPP diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 7ed2b9b3faae1..14ddc3bd0ff74 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -38,6 +38,7 @@ #include "oops/instanceClassLoaderKlass.inline.hpp" #include "oops/instanceRefKlass.inline.hpp" #include "oops/instanceStackChunkKlass.inline.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/typeArrayKlass.inline.hpp" #include "utilities/debug.hpp" @@ -310,4 +311,218 @@ void OopIteratorClosureDispatch::oop_oop_iterate_backwards(OopClosureType* cl, o OopOopIterateBackwardsDispatch::function(klass)(cl, obj, klass); } + +////////////// KLUTE variants ///////////////////// + +// Macro arguments: +// ITERATION_FUNCTION - name of target iteration function inside Klass, e.g., oop_oop_iterate_bounded +// ARGUMENT_DEFINITION - argument definition including brackets +// ARGUMENTS - arguments, including brackets + +#define DEFINE_DISPATCH_CLASS(CLASSNAME, ITERATION_FUNCTION, ARGUMENT_DEFINITION, ARGUMENTS) \ +template \ +class CLASSNAME { \ + typedef void (*FunctionType) ARGUMENT_DEFINITION; \ + \ + class Table { \ + \ + FunctionType _function [Klass::KLASS_KIND_COUNT]; \ + \ + template \ + static void invoke_real ARGUMENT_DEFINITION { \ + KlassType::template ITERATION_FUNCTION ARGUMENTS; \ + } \ + \ + template \ + static void init_and_execute ARGUMENT_DEFINITION { \ + CLASSNAME::_table.set_resolve_function_and_execute ARGUMENTS; \ + } \ + \ + template \ + void set_resolve_function_and_execute ARGUMENT_DEFINITION { \ + set_resolve_function(); \ + _function[KlassType::Kind] ARGUMENTS; \ + } \ + \ + template \ + void set_init_function() { \ + _function[KlassType::Kind] = &init_and_execute; \ + } \ + \ + template \ + void set_resolve_function() { \ + _function[KlassType::Kind] = UseCompressedOops ? \ + &invoke_real : \ + &invoke_real; \ + } \ + \ + public: \ + \ + Table(){ \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + } \ + \ + void invoke ARGUMENT_DEFINITION { \ + const int slot = klute.kind(); \ + _function[slot] ARGUMENTS; \ + } \ + \ + }; \ + \ + static Table _table; \ + \ +public: \ + \ + static void invoke ARGUMENT_DEFINITION { _table.invoke ARGUMENTS; } \ + \ +}; \ + \ +template \ +typename CLASSNAME::Table CLASSNAME::_table; \ + \ +template \ +void OopIteratorClosureDispatch::ITERATION_FUNCTION ARGUMENT_DEFINITION { \ + CLASSNAME::invoke ARGUMENTS; \ +} + +DEFINE_DISPATCH_CLASS( + OopOopIterateDispatchWithKlute, + oop_oop_iterate, + (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, klute, nk) +) + +DEFINE_DISPATCH_CLASS( + OopOopIterateDispatchWithKluteReverse, + oop_oop_iterate_reverse, + (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, klute, nk) +) + +DEFINE_DISPATCH_CLASS( + OopOopIterateDispatchWithKluteBounded, + oop_oop_iterate_bounded, + (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, mr, klute, nk) +) + +// Same, but returns object size + +template +static inline size_t calculate_size_for_object(narrowKlass nk, KlassLUTEntry klute, oop obj) { + if (KlassType::Kind < Klass::TypeArrayKlassKind) { + assert(klute.is_instance(), "Sanity"); + if (klute.ik_carries_infos()) { + return klute.ik_wordsize(); + } + // Size not statically computable (e.g. MirrorKlass); calculate using Klass + Klass* k = CompressedKlassPointers::decode_not_null(nk); + return obj->size_given_klass(k); + } else { + assert(klute.is_array(), "Sanity"); + return klute.ak_calculate_wordsize_given_oop(obj); + } +} + +#define DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE(CLASSNAME, ITERATION_FUNCTION, ARGUMENT_DEFINITION, ARGUMENTS) \ +template \ +class CLASSNAME { \ + typedef size_t (*FunctionType) ARGUMENT_DEFINITION; \ + \ + class Table { \ + \ + FunctionType _function [Klass::KLASS_KIND_COUNT]; \ + \ + template \ + static size_t invoke_real ARGUMENT_DEFINITION { \ + KlassType::template ITERATION_FUNCTION ARGUMENTS; \ + return calculate_size_for_object(nk, klute, obj); \ + } \ + \ + template \ + static size_t init_and_execute ARGUMENT_DEFINITION { \ + return CLASSNAME::_table.set_resolve_function_and_execute ARGUMENTS; \ + } \ + \ + template \ + size_t set_resolve_function_and_execute ARGUMENT_DEFINITION { \ + set_resolve_function(); \ + return _function[KlassType::Kind] ARGUMENTS; \ + } \ + \ + template \ + void set_init_function() { \ + _function[KlassType::Kind] = &init_and_execute; \ + } \ + \ + template \ + void set_resolve_function() { \ + _function[KlassType::Kind] = UseCompressedOops ? \ + &invoke_real : \ + &invoke_real; \ + } \ + \ + public: \ + \ + Table(){ \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + set_init_function(); \ + } \ + \ + size_t invoke ARGUMENT_DEFINITION { \ + const int slot = klute.kind(); \ + return _function[slot] ARGUMENTS; \ + } \ + \ + }; \ + \ + static Table _table; \ + \ +public: \ + \ + static size_t invoke ARGUMENT_DEFINITION { return _table.invoke ARGUMENTS; } \ + \ +}; \ + \ +template \ +typename CLASSNAME::Table CLASSNAME::_table; \ + \ +template \ +size_t OopIteratorClosureDispatch::ITERATION_FUNCTION ## _size ARGUMENT_DEFINITION { \ + return CLASSNAME::invoke ARGUMENTS; \ +} + +DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( + OopOopIterateDispatchWithKluteReturnSize, + oop_oop_iterate, + (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, klute, nk) +) + +/* +DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( + OopOopIterateDispatchWithKluteReverseReturnSize, + oop_oop_iterate_reverse, + (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, klute, nk) +)*/ + +DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( + OopOopIterateDispatchWithKluteBoundedReturnSize, + oop_oop_iterate_bounded, + (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk), + (obj, cl, mr, klute, nk) +) + #endif // SHARE_MEMORY_ITERATOR_INLINE_HPP diff --git a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp index b16f2d216103c..cd171579c043b 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp @@ -70,7 +70,7 @@ static const chunklevel_t g_sequ_boot_non_class[] = { }; static const chunklevel_t g_sequ_boot_class[] = { - chunklevel::CHUNK_LEVEL_256K + chunklevel::CHUNK_LEVEL_16M // .. repeat last }; diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index e846eb3ddde1b..4c218edd2e19b 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -61,6 +61,7 @@ #include "oops/instanceKlass.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/objLayout.hpp" #include "oops/oop.inline.hpp" @@ -408,6 +409,11 @@ void Universe::genesis(TRAPS) { assert(arrayOopDesc::length_offset_in_bytes() < static_cast(os::vm_page_size()), "Array length offset is expected to be less than the page size"); + // Initialize KLUT before starting to create any Klass + if (UseKLUT) { + KlassInfoLUT::initialize(); + } + { AutoModifyRestore temporarily(_bootstrapping, true); java_lang_Class::allocate_fixup_lists(); diff --git a/src/hotspot/share/oops/compressedKlass.cpp b/src/hotspot/share/oops/compressedKlass.cpp index df639388a5ee0..d1ba7de8cfcbc 100644 --- a/src/hotspot/share/oops/compressedKlass.cpp +++ b/src/hotspot/share/oops/compressedKlass.cpp @@ -225,13 +225,16 @@ void CompressedKlassPointers::initialize(address addr, size_t len) { // a cacheline size. _base = addr; - const int log_cacheline = exact_log2(DEFAULT_CACHE_LINE_SIZE); - int s = max_shift(); - while (s > log_cacheline && ((size_t)nth_bit(narrow_klass_pointer_bits() + s - 1) > len)) { - s--; + if (UseKLUT) { + _shift = max_shift(); + } else { + const int log_cacheline = exact_log2(DEFAULT_CACHE_LINE_SIZE); + int s = max_shift(); + while (s > log_cacheline && ((size_t)nth_bit(narrow_klass_pointer_bits() + s - 1) > len)) { + s--; + } + _shift = s; } - _shift = s; - } else { // Traditional (non-compact) header mode diff --git a/src/hotspot/share/oops/compressedKlass.hpp b/src/hotspot/share/oops/compressedKlass.hpp index 4ce644d9cef69..8e39e4c05397f 100644 --- a/src/hotspot/share/oops/compressedKlass.hpp +++ b/src/hotspot/share/oops/compressedKlass.hpp @@ -95,9 +95,6 @@ class Klass; // 0x8_0000_0000 0x8_4800_0000 0x9_0000_0000 // -// If compressed klass pointers then use narrowKlass. -typedef juint narrowKlass; - // For UseCompressedClassPointers. class CompressedKlassPointers : public AllStatic { friend class VMStructs; diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index 985c81c3cd662..f1d8c4c28c855 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_INSTANCECLASSLOADERKLASS_HPP #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "utilities/macros.hpp" class ClassFileParser; @@ -40,11 +41,16 @@ class InstanceClassLoaderKlass: public InstanceKlass { friend class VMStructs; friend class InstanceKlass; public: - static const KlassKind Kind = InstanceClassLoaderKlassKind; + static constexpr KlassKind Kind = InstanceClassLoaderKlassKind; private: InstanceClassLoaderKlass(const ClassFileParser& parser) : InstanceKlass(parser, Kind) {} + template + static inline void oop_oop_iterate_metadata(oop obj, OopClosureType* closure); + template + static inline void oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr); + public: InstanceClassLoaderKlass(); @@ -66,6 +72,20 @@ class InstanceClassLoaderKlass: public InstanceKlass { // Iterate over the oop fields and metadata. template inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + + + + DECLARE_EXACT_CAST_FUNCTIONS(InstanceClassLoaderKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceClassLoaderKlass) + }; #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_HPP diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index 22eb72816544b..3f6ff2a1feeca 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -25,9 +25,8 @@ #ifndef SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP #define SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP -#include "oops/instanceClassLoaderKlass.hpp" - #include "classfile/javaClasses.hpp" +#include "oops/instanceClassLoaderKlass.hpp" #include "oops/instanceKlass.inline.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" @@ -36,9 +35,7 @@ #include "utilities/macros.hpp" template -inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate(obj, closure); - +inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata(oop obj, OopClosureType* closure) { if (Devirtualizer::do_metadata(closure)) { ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); // cld can be null if we have a non-registered class loader. @@ -49,17 +46,7 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* c } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure); - - assert(!Devirtualizer::do_metadata(closure), - "Code to handle metadata is not implemented"); -} - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); - +inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr) { if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); @@ -71,4 +58,49 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosur } } +#define NO_METADATA_ITERATION assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); + +// External entries implementation: klute variants + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { + InstanceKlass::oop_oop_iterate(obj, closure); + oop_oop_iterate_metadata(obj, closure); +} + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure); + NO_METADATA_ITERATION +} + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); + oop_oop_iterate_metadata_bounded(obj, closure, mr); +} + +// External entries implementation: klute variants + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); + oop_oop_iterate_metadata(obj, closure); +} + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); + NO_METADATA_ITERATION +} + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); + oop_oop_iterate_metadata_bounded(obj, closure, mr); +} + +DEFINE_EXACT_CAST_FUNCTIONS(InstanceClassLoaderKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceClassLoaderKlass) + #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index 715c8f473d087..4d596e898f4a1 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -472,7 +472,7 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par assert(loader_data != nullptr, "invariant"); InstanceKlass* ik; - const bool use_class_space = parser.klass_needs_narrow_id(); + const bool use_class_space = true; // parser.klass_needs_narrow_id(); // Allocation if (parser.is_instance_ref_klass()) { @@ -503,6 +503,14 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par return nullptr; } + { + char tmp[1024]; + log_debug(metaspace)("Returning new IK @" PTR_FORMAT " for %s (use_class_space: %d), nKlass=%u, word size=%d", + p2i(ik), + parser.class_name()->as_C_string(tmp, sizeof(tmp)), use_class_space, + CompressedKlassPointers::encode(ik), size); + } + return ik; } @@ -3629,12 +3637,19 @@ void InstanceKlass::print_on(outputStream* st) const { assert(is_klass(), "must be klass"); Klass::print_on(st); + if (UseCompressedClassPointers) { + st->print(BULLET"narrow Klass: %x", CompressedKlassPointers::encode(const_cast(this))); st->cr(); + } st->print(BULLET"instance size: %d", size_helper()); st->cr(); st->print(BULLET"klass size: %d", size()); st->cr(); + st->print(BULLET"klass header size: %d", header_size()); st->cr(); + st->print(BULLET"vtable size: %d", itable_length()); st->cr(); + st->print(BULLET"itable size: %d", vtable_length()); st->cr(); + st->print(BULLET"nonstatic_oopmap size: %d", nonstatic_oop_map_size()); st->cr(); + st->print(BULLET"access: "); access_flags().print_on(st); st->cr(); st->print(BULLET"flags: "); _misc_flags.print_on(st); st->cr(); st->print(BULLET"state: "); st->print_cr("%s", init_state_name()); - st->print(BULLET"name: "); name()->print_value_on(st); st->cr(); st->print(BULLET"super: "); Metadata::print_value_on_maybe_null(st, super()); st->cr(); st->print(BULLET"sub: "); Klass* sub = subklass(); @@ -3758,7 +3773,7 @@ void InstanceKlass::print_on(outputStream* st) const { OopMapBlock* map = start_of_nonstatic_oop_maps(); OopMapBlock* end_map = map + nonstatic_oop_map_count(); while (map < end_map) { - st->print("%d-%d ", map->offset(), map->offset() + heapOopSize*(map->count() - 1)); + st->print_cr("[@%d-@%d) (%d oops) ", map->offset(), map->offset() + heapOopSize * map->count(), map->count()); map++; } st->cr(); diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index cedc17e9bafde..559ddfc73bfe6 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -32,6 +32,7 @@ #include "oops/fieldInfo.hpp" #include "oops/instanceKlassFlags.hpp" #include "oops/instanceOop.hpp" +#include "oops/klassInfoLUT.hpp" #include "runtime/handles.hpp" #include "runtime/javaThread.hpp" #include "utilities/accessFlags.hpp" @@ -138,7 +139,7 @@ class InstanceKlass: public Klass { friend class CompileReplay; public: - static const KlassKind Kind = InstanceKlassKind; + static constexpr KlassKind Kind = InstanceKlassKind; protected: InstanceKlass(const ClassFileParser& parser, KlassKind kind = Kind, ReferenceType reference_type = REF_NONE); @@ -337,7 +338,6 @@ class InstanceKlass: public Klass { void set_nonstatic_field_size(int size) { _nonstatic_field_size = size; } int static_field_size() const { return _static_field_size; } - void set_static_field_size(int size) { _static_field_size = size; } int static_oop_field_count() const { return (int)_static_oop_field_count; } void set_static_oop_field_count(u2 size) { _static_oop_field_count = size; } @@ -909,6 +909,9 @@ class InstanceKlass: public Klass { return static_cast(k); } + DECLARE_EXACT_CAST_FUNCTIONS(InstanceKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceKlass) + virtual InstanceKlass* java_super() const { return (super() == nullptr) ? nullptr : cast(super()); } @@ -1009,10 +1012,9 @@ class InstanceKlass: public Klass { template inline void oop_oop_iterate(oop obj, OopClosureType* closure); - // Iterate over all oop fields in one oop map. + // Iterate over all oop fields in a single oop map. template - inline void oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure); - + static inline void oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure); // Reverse iteration // Iterate over all oop fields and metadata. @@ -1026,8 +1028,7 @@ class InstanceKlass: public Klass { // Iterate over all oop fields in one oop map. template - inline void oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure); - + static inline void oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure); // Bounded range iteration public: @@ -1042,8 +1043,25 @@ class InstanceKlass: public Klass { private: // Iterate over all oop fields in one oop map. template - inline void oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr); + // Single oop map iteration given by count and offset + template + static inline void oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count); + template + static inline void oop_oop_iterate_single_oop_map_reverse(oop obj, OopClosureType* closure, unsigned offset, unsigned count); + template + static inline void oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count); + + public: + + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); public: u2 idnum_allocated_count() const { return _idnum_allocated_count; } diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 1b4664f5a4bd5..be28fced1aa01 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -29,6 +29,7 @@ #include "memory/memRegion.hpp" #include "oops/fieldInfo.inline.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/klass.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" @@ -85,8 +86,26 @@ inline void InstanceKlass::release_set_methods_jmethod_ids(jmethodID* jmeths) { template ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure) { - T* p = obj->field_addr(map->offset()); - T* const end = p + map->count(); + assert(map->offset() > 0, "must be"); + oop_oop_iterate_single_oop_map(obj, closure, (unsigned)map->offset(), map->count()); +} + +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure) { + assert(map->offset() > 0, "must be"); + oop_oop_iterate_single_oop_map_reverse(obj, closure, (unsigned)map->offset(), map->count()); +} + +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr) { + assert(map->offset() > 0, "must be"); + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, (unsigned)map->offset(), map->count()); +} + +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count) { + T* p = obj->field_addr(offset); + T* const end = p + count; for (; p < end; ++p) { Devirtualizer::do_oop(closure, p); @@ -94,9 +113,9 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map(OopMapBlock* map, oop o } template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure) { - T* const start = obj->field_addr(map->offset()); - T* p = start + map->count(); +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_single_oop_map_reverse(oop obj, OopClosureType* closure, unsigned offset, unsigned count) { + T* const start = obj->field_addr(offset); + T* p = start + count; while (start < p) { --p; @@ -105,9 +124,9 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_reverse(OopMapBlock* ma } template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr) { - T* p = obj->field_addr(map->offset()); - T* end = p + map->count(); +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count) { + T* p = obj->field_addr(offset); + T* end = p + count; T* const l = (T*)mr.start(); T* const h = (T*)mr.end(); @@ -186,4 +205,84 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType oop_oop_iterate_oop_maps_bounded(obj, closure, mr); } +// Klute variants + +// Iterate over all oop fields and metadata. +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + if (Devirtualizer::do_metadata(closure)) { + const unsigned perma_cld_index = klute.loader_index(); + ClassLoaderData* const cld = perma_cld_index > 0 ? KlassInfoLUT::get_perma_cld(perma_cld_index) : + CompressedKlassPointers::decode_not_null(nk)->class_loader_data(); + Devirtualizer::do_cld(closure, cld); + } + + if (klute.ik_carries_infos()) { + // klute may encode 0, 1 or 2 oop maps. Iterate those. + if (klute.ik_omb_count_1() > 0) { + oop_oop_iterate_single_oop_map(obj, closure, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); + if (klute.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map(obj, closure, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + } + } + } else { + // Fall back to normal iteration: read OopMapBlocks from Klass + InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); + assert(this_ik == obj->klass(), "sanity"); + this_ik->oop_oop_iterate_oop_maps(obj, closure); + } +} + +// Iterate over all oop fields in the oop maps (no metadata traversal) +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + assert(!Devirtualizer::do_metadata(closure), + "Code to handle metadata is not implemented"); + + if (klute.ik_carries_infos()) { + // klute may encode 0, 1 or 2 oop maps. Iterate those (reversely). + if (klute.ik_omb_count_1() > 0) { + if (klute.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map_reverse(obj, closure, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + } + oop_oop_iterate_single_oop_map_reverse(obj, closure, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); + } + } else { + // Fall back to normal iteration: read OopMapBlocks from Klass + InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); + assert(this_ik == obj->klass(), "sanity"); + this_ik->oop_oop_iterate_oop_maps_reverse(obj, closure); + } +} + +// Iterate over all oop fields and metadata. +template +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + if (Devirtualizer::do_metadata(closure)) { + if (mr.contains(obj)) { + const unsigned perma_cld_index = klute.loader_index(); + ClassLoaderData* const cld = perma_cld_index > 0 ? KlassInfoLUT::get_perma_cld(perma_cld_index) : + CompressedKlassPointers::decode_not_null(nk)->class_loader_data(); + Devirtualizer::do_cld(closure, cld); + } + } + + if (klute.ik_carries_infos()) { + // klute may encode 0, 1 or 2 oop maps. Iterate those (bounded). + if (klute.ik_omb_count_1() > 0) { + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); + if (klute.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + } + } + } else { + InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); + assert(this_ik == obj->klass(), "sanity"); + this_ik->oop_oop_iterate_oop_maps_bounded(obj, closure, mr); + } +} + +DEFINE_EXACT_CAST_FUNCTIONS(InstanceKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceKlass) + #endif // SHARE_OOPS_INSTANCEKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index 9783d416a1d1b..b2c3201450c6a 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -27,6 +27,7 @@ #include "classfile/vmClasses.hpp" #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "runtime/handles.hpp" #include "utilities/macros.hpp" @@ -45,7 +46,7 @@ class InstanceMirrorKlass: public InstanceKlass { friend class InstanceKlass; public: - static const KlassKind Kind = InstanceMirrorKlassKind; + static constexpr KlassKind Kind = InstanceMirrorKlassKind; private: static int _offset_of_static_fields; @@ -97,10 +98,6 @@ class InstanceMirrorKlass: public InstanceKlass { // // The InstanceMirrorKlass iterators also visit the hidden Klass pointer. - // Iterate over the static fields. - template - inline void oop_oop_iterate_statics(oop obj, OopClosureType* closure); - // Forward iteration // Iterate over the oop fields and metadata. template @@ -116,11 +113,33 @@ class InstanceMirrorKlass: public InstanceKlass { template inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + private: // Iterate over the static fields. template - inline void oop_oop_iterate_statics_bounded(oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_statics(oop obj, OopClosureType* closure); + template + static inline void oop_oop_iterate_statics_bounded(oop obj, OopClosureType* closure, MemRegion mr); + + // Iterate over the metadata + template + static inline void oop_oop_iterate_metadata(oop obj, OopClosureType* closure); + template + static inline void oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr); + + public: + + DECLARE_EXACT_CAST_FUNCTIONS(InstanceMirrorKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceMirrorKlass) + }; #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_HPP diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index 867a0580a126d..3ae82e8437859 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -47,9 +47,32 @@ void InstanceMirrorKlass::oop_oop_iterate_statics(oop obj, OopClosureType* closu } template -void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate(obj, closure); +void InstanceMirrorKlass::oop_oop_iterate_statics_bounded(oop obj, + OopClosureType* closure, + MemRegion mr) { + T* p = (T*)start_of_static_fields(obj); + T* end = p + java_lang_Class::static_oop_field_count(obj); + + T* const l = (T*)mr.start(); + T* const h = (T*)mr.end(); + assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && + mask_bits((intptr_t)h, sizeof(T)-1) == 0, + "bounded region must be properly aligned"); + if (p < l) { + p = l; + } + if (end > h) { + end = h; + } + + for (;p < end; ++p) { + Devirtualizer::do_oop(closure, p); + } +} + +template +void InstanceMirrorKlass::oop_oop_iterate_metadata(oop obj, OopClosureType* closure) { if (Devirtualizer::do_metadata(closure)) { Klass* klass = java_lang_Class::as_Klass(obj); // We'll get null for primitive mirrors. @@ -79,8 +102,29 @@ void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { // assert(java_lang_Class::is_primitive(obj), "Sanity check"); } } +} +template +void InstanceMirrorKlass::oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr) { + if (Devirtualizer::do_metadata(closure)) { + if (mr.contains(obj)) { + Klass* klass = java_lang_Class::as_Klass(obj); + // We'll get null for primitive mirrors. + if (klass != nullptr) { + Devirtualizer::do_klass(closure, klass); + } + } + } +} + +// Externals + +template +void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { + InstanceKlass::oop_oop_iterate(obj, closure); + oop_oop_iterate_metadata(obj, closure); oop_oop_iterate_statics(obj, closure); + } template @@ -90,46 +134,40 @@ void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closu InstanceMirrorKlass::oop_oop_iterate_statics(obj, closure); } + template -void InstanceMirrorKlass::oop_oop_iterate_statics_bounded(oop obj, - OopClosureType* closure, - MemRegion mr) { - T* p = (T*)start_of_static_fields(obj); - T* end = p + java_lang_Class::static_oop_field_count(obj); +void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); + oop_oop_iterate_metadata_bounded(obj, closure, mr); + oop_oop_iterate_statics_bounded(obj, closure, mr); +} - T* const l = (T*)mr.start(); - T* const h = (T*)mr.end(); - assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && - mask_bits((intptr_t)h, sizeof(T)-1) == 0, - "bounded region must be properly aligned"); +// Externals, klute variants - if (p < l) { - p = l; - } - if (end > h) { - end = h; - } +template +void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); + oop_oop_iterate_metadata(obj, closure); + oop_oop_iterate_statics(obj, closure); - for (;p < end; ++p) { - Devirtualizer::do_oop(closure, p); - } } template -void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); +void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); + + InstanceMirrorKlass::oop_oop_iterate_statics(obj, closure); +} - if (Devirtualizer::do_metadata(closure)) { - if (mr.contains(obj)) { - Klass* klass = java_lang_Class::as_Klass(obj); - // We'll get null for primitive mirrors. - if (klass != nullptr) { - Devirtualizer::do_klass(closure, klass); - } - } - } +template +void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); + oop_oop_iterate_metadata_bounded(obj, closure, mr); oop_oop_iterate_statics_bounded(obj, closure, mr); } +DEFINE_EXACT_CAST_FUNCTIONS(InstanceMirrorKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceMirrorKlass) + #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index c372b83a26702..d9f03e226a690 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_INSTANCEREFKLASS_HPP #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "utilities/macros.hpp" class ClassFileParser; @@ -50,7 +51,7 @@ class ClassFileParser; class InstanceRefKlass: public InstanceKlass { friend class InstanceKlass; public: - static const KlassKind Kind = InstanceRefKlassKind; + static constexpr KlassKind Kind = InstanceRefKlassKind; private: InstanceRefKlass(const ClassFileParser& parser); @@ -78,6 +79,14 @@ class InstanceRefKlass: public InstanceKlass { template inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + private: // Reference processing part of the iterators. @@ -127,6 +136,10 @@ class InstanceRefKlass: public InstanceKlass { public: // Verification void oop_verify_on(oop obj, outputStream* st); + + DECLARE_EXACT_CAST_FUNCTIONS(InstanceRefKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceRefKlass) + }; #endif // SHARE_OOPS_INSTANCEREFKLASS_HPP diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index 72c532828e62f..2804280fc418c 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -154,6 +154,7 @@ void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { oop_oop_iterate_ref_processing(obj, closure); } + template void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { InstanceKlass::oop_oop_iterate_reverse(obj, closure); @@ -168,6 +169,28 @@ void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, oop_oop_iterate_ref_processing_bounded(obj, closure, mr); } +// klute variants +template +void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing(obj, closure); +} + +template +void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing(obj, closure); +} + +template +void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing_bounded(obj, closure, mr); +} + #ifdef ASSERT template void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) { @@ -190,4 +213,7 @@ void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) { } #endif +DEFINE_EXACT_CAST_FUNCTIONS(InstanceRefKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceRefKlass) + #endif // SHARE_OOPS_INSTANCEREFKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index 19ce37c4ddddf..bf2144250f4ef 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/macros.hpp" #include "utilities/ostream.hpp" @@ -105,7 +106,7 @@ class InstanceStackChunkKlass: public InstanceKlass { friend class Continuations; public: - static const KlassKind Kind = InstanceStackChunkKlassKind; + static constexpr KlassKind Kind = InstanceStackChunkKlassKind; private: static int _offset_of_stack; @@ -161,6 +162,14 @@ class InstanceStackChunkKlass: public InstanceKlass { template inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + private: template inline void oop_oop_iterate_header(stackChunkOop chunk, OopClosureType* closure); @@ -183,6 +192,12 @@ class InstanceStackChunkKlass: public InstanceKlass { void do_methods(stackChunkOop chunk, OopIterateClosure* cl); void oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); + + public: + + DECLARE_EXACT_CAST_FUNCTIONS(InstanceStackChunkKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceStackChunkKlass) + }; #endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 3374895c6a147..9841ac8fdbb51 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -90,6 +90,25 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c oop_oop_iterate_lockstack(chunk, closure, mr); } +// Klute variants don't do anything else for now. Just exist to make Dispatch happy. +template +void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate(obj, closure); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_reverse(obj, closure); +} + +template +void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_bounded(obj, closure, mr); +} + template void InstanceStackChunkKlass::oop_oop_iterate_header(stackChunkOop chunk, OopClosureType* closure) { T* parent_addr = chunk->field_addr(jdk_internal_vm_StackChunk::parent_offset()); @@ -164,4 +183,7 @@ void InstanceStackChunkKlass::oop_oop_iterate_stack_with_bitmap(stackChunkOop ch } } +DEFINE_EXACT_CAST_FUNCTIONS(InstanceStackChunkKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceStackChunkKlass) + #endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index ac0e125ab4c8a..32d0bdb3de552 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -1011,9 +1011,12 @@ jint Klass::jvmti_class_status() const { void Klass::print_on(outputStream* st) const { ResourceMark rm; // print title - st->print("%s", internal_name()); + st->print("Klass: %s", internal_name()); print_address_on(st); st->cr(); + st->print(" - kind: %d", kind()); st->cr(); + st->print(" - layouthelper raw 0x%x", layout_helper()); st->cr(); + st->print(" - name: "); name()->print_value_on(st); st->cr(); } #define BULLET " - " diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 87f85dab956f3..be98d1742fac4 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -65,19 +65,55 @@ class Klass : public Metadata { friend class VMStructs; friend class JVMCIVMStructs; public: - // Klass Kinds for all subclasses of Klass + +#define KLASS_ALL_KINDS_DO(what) \ + what(InstanceKlass, IK) \ + what(InstanceRefKlass, IRK) \ + what(InstanceMirrorKlass, IMK) \ + what(InstanceClassLoaderKlass, ICLK) \ + what(InstanceStackChunkKlass, ISCK) \ + what(TypeArrayKlass, TAK) \ + what(ObjArrayKlass, OAK) + enum KlassKind : u2 { - InstanceKlassKind, - InstanceRefKlassKind, - InstanceMirrorKlassKind, - InstanceClassLoaderKlassKind, - InstanceStackChunkKlassKind, - TypeArrayKlassKind, - ObjArrayKlassKind, +#define WHAT(name, shortname) name ## Kind, + KLASS_ALL_KINDS_DO(WHAT) +#undef WHAT UnknownKlassKind }; static const uint KLASS_KIND_COUNT = ObjArrayKlassKind + 1; + +#define DECLARE_EXACT_CAST_FUNCTIONS(TYPE) \ + static inline const TYPE* cast_exact(const Klass* k); \ + static inline TYPE* cast_exact( Klass* k); + +#define DEFINE_EXACT_CAST_FUNCTIONS(TYPE) \ + inline const TYPE* TYPE::cast_exact(const Klass* k) { \ + assert(k != nullptr, "klass null"); \ + assert(k->kind() == Klass::TYPE ## Kind, \ + "Klass @" PTR_FORMAT ": wrong kind %d", p2i(k), k->kind()) ; \ + return static_cast(k); \ + } \ + inline TYPE* TYPE::cast_exact(Klass* k) { \ + const TYPE* const ck = TYPE::cast_exact((const Klass*)k); \ + return const_cast(ck); \ + } + +#define DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(TYPE) \ + static inline const TYPE* narrow_klass_to_const_klass(narrowKlass nk); \ + static inline TYPE* narrow_klass_to_klass(narrowKlass nk); + +#define DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(TYPE) \ + inline const TYPE* TYPE::narrow_klass_to_const_klass(narrowKlass nk) { \ + const Klass* const k = CompressedKlassPointers::decode_not_null(nk); \ + return cast_exact(k); \ + } \ + inline TYPE* TYPE::narrow_klass_to_klass(narrowKlass nk) { \ + Klass* const k = CompressedKlassPointers::decode_not_null(nk); \ + return cast_exact(k); \ + } \ + protected: // If you add a new field that points to any metaspace object, you @@ -209,7 +245,7 @@ class Klass : public Metadata { Klass(); public: - int kind() { return _kind; } + int kind() const { return _kind; } enum class DefaultsLookupMode { find, skip }; enum class OverpassLookupMode { find, skip }; diff --git a/src/hotspot/share/oops/klass.inline.hpp b/src/hotspot/share/oops/klass.inline.hpp index 4f9401f1709f0..e7282edc92574 100644 --- a/src/hotspot/share/oops/klass.inline.hpp +++ b/src/hotspot/share/oops/klass.inline.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_KLASS_INLINE_HPP #define SHARE_OOPS_KLASS_INLINE_HPP + #include "oops/klass.hpp" #include "classfile/classLoaderData.inline.hpp" diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp new file mode 100644 index 0000000000000..2439d1f23666d --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "logging/log.hpp" +#include "memory/allocation.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/klass.hpp" +#include "oops/klassInfoLUT.inline.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" +#include "runtime/atomic.hpp" +#include "utilities/debug.hpp" +#include "utilities/ostream.hpp" + +ClassLoaderData* KlassInfoLUT::_common_loaders[4] = { nullptr }; +uint32_t* KlassInfoLUT::_entries = nullptr; + +void KlassInfoLUT::initialize() { + assert(UseKLUT, "?"); + assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "sanity"); + const size_t memory_needed = sizeof(uint32_t) * num_entries(); + if (UseLargePages) { + const size_t large_page_size = os::large_page_size(); + if (is_aligned(memory_needed, large_page_size)) { + char* memory = os::reserve_memory_special(memory_needed, large_page_size, large_page_size, nullptr, false); + if (memory != nullptr) { + _entries = (uint32_t*)memory; + log_info(klut)("KLUT initialized (large pages): " RANGEFMT, RANGEFMTARGS(_entries, memory_needed)); + } + } + } + if (_entries == nullptr) { + // Fallback, just use C-heap. + _entries = NEW_C_HEAP_ARRAY(uint32_t, num_entries(), mtClass); + log_info(klut)("KLUT initialized (normal pages): " RANGEFMT, RANGEFMTARGS(_entries, memory_needed)); + } + for (unsigned i = 0; i < num_entries(); i++) { + _entries[i] = KlassLUTEntry::invalid_entry; + } +} + +int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { + int index = 0; + if (cld->is_permanent_class_loader_data()) { + if (cld->is_the_null_class_loader_data()) { + index = 1; + } else if (cld->is_system_class_loader_data()) { + index = 2; + } else if (cld->is_platform_class_loader_data()) { + index = 3; + } + } + if (index > 0) { + ClassLoaderData* old_cld = Atomic::load(_common_loaders + index); + if (old_cld == nullptr) { + old_cld = Atomic::cmpxchg(&_common_loaders[index], (ClassLoaderData*)nullptr, cld); + if (old_cld == nullptr || old_cld == cld) { + return index; + } + } else if (old_cld == cld) { + return index; + } + } + return 0; +} + +void KlassInfoLUT::register_klass(const Klass* k) { + assert(UseKLUT, "?"); + const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // grr why is this nonconst + assert(nk < num_entries(), "oob"); + KlassLUTEntry e(k); + _entries[nk] = e.value(); + char tmp[1024]; + log_debug(klut)("registered Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " + "Klute: " INT32_FORMAT_X_0 ", %s", + p2i(k), nk, e.value(), k->name()->as_C_string(tmp, sizeof(tmp))); +#ifdef ASSERT + // sanity checks + { + // We use at(), not get_entry(), since we don't want to log or count stats + KlassLUTEntry e2(at(nk)); + assert(e2.value() == e.value(), "Sanity"); + e2.verify_against(k); + } +#endif // ASSERT + // update register stats + switch (k->kind()) { + case Klass::InstanceKlassKind: inc_registered_IK(); break; + case Klass::InstanceRefKlassKind: inc_registered_IRK(); break; + case Klass::InstanceMirrorKlassKind: inc_registered_IMK(); break; + case Klass::InstanceClassLoaderKlassKind: inc_registered_ICLK(); break; + case Klass::InstanceStackChunkKlassKind: inc_registered_ISCK(); break; + case Klass::TypeArrayKlassKind: inc_registered_TAK(); break; + case Klass::ObjArrayKlassKind: inc_registered_OAK(); break; + default: ShouldNotReachHere(); + }; + if (k->is_abstract() || k->is_interface()) { + inc_registered_IK_for_abstract_or_interface(); + } +} + +// Counters and incrementors +#define XX(xx) \ +volatile uint64_t counter_##xx = 0; \ +void KlassInfoLUT::inc_##xx() { \ + Atomic::inc(&counter_##xx); \ +} +REGISTER_STATS_DO(XX) +#ifdef KLUT_ENABLE_EXPENSIVE_STATS +HIT_STATS_DO(XX) +#endif // KLUT_ENABLE_EXPENSIVE_STATS +#undef XX + +void KlassInfoLUT::print_statistics(outputStream* st) { + + assert(UseKLUT, "?"); + st->print_cr("KLUT stats:"); + + const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + + counter_registered_ICLK + counter_registered_ISCK + counter_registered_TAK + counter_registered_OAK; + +#define PERCENTAGE_OF(x, x100) ( ((double)x * 100.0f) / x100 ) +#define PRINT_WITH_PERCENTAGE(title, x, x100) \ + st->print(" " title ": "); \ + st->fill_to(24); \ + st->print_cr(" " UINT64_FORMAT " (%.2f%%)", x, PERCENTAGE_OF(x, x100)); + + st->print_cr(" Registered classes, total: " UINT64_FORMAT, registered_all); +#define XX(name, shortname) PRINT_WITH_PERCENTAGE("Registered, " #shortname, counter_registered_##shortname, registered_all); + ALL_KLASS_KINDS_DO(XX) +#undef XX + + const uint64_t registered_AK = counter_registered_OAK - counter_registered_TAK; + const uint64_t registered_IK = registered_all - registered_AK; + PRINT_WITH_PERCENTAGE("Registered classes, IK (all)", registered_IK, registered_all); + PRINT_WITH_PERCENTAGE("Registered classes, AK (all)", registered_AK, registered_all); + + PRINT_WITH_PERCENTAGE("Registered classes, IK, for abstract/interface", counter_registered_IK_for_abstract_or_interface, registered_all); + +#ifdef KLUT_ENABLE_EXPENSIVE_STATS + const uint64_t hits = counter_hits_IK + counter_hits_IRK + counter_hits_IMK + + counter_hits_ICLK + counter_hits_ISCK + counter_hits_TAK + counter_hits_OAK; + + st->print_cr(" Hits, total: " UINT64_FORMAT, hits); +#define XX(name, shortname) PRINT_WITH_PERCENTAGE("Hits, " #shortname, counter_hits_##shortname, hits); + ALL_KLASS_KINDS_DO(XX) +#undef XX + + const uint64_t hits_ak = counter_hits_OAK + counter_hits_TAK; + const uint64_t hits_ik = hits - hits_ak; + const uint64_t no_info_hits = counter_noinfo_ICLK + counter_noinfo_IMK + counter_noinfo_IK_other; + + PRINT_WITH_PERCENTAGE("Hits, IK (all)", hits_ik, hits); + PRINT_WITH_PERCENTAGE("Hits, AK (all)", hits_ak, hits); + + PRINT_WITH_PERCENTAGE("Hits, all for bootloader", counter_hits_bootloader, hits); + PRINT_WITH_PERCENTAGE("Hits, all for systemloader", counter_hits_sysloader, hits); + PRINT_WITH_PERCENTAGE("Hits, all for platformloader", counter_hits_platformloader, hits); + + st->print_cr(" IK details missing for " UINT64_FORMAT " hits (%.2f%%) due to: " + "IMK " UINT64_FORMAT " (%.2f%%) " + "ICLK " UINT64_FORMAT " (%.2f%%) " + "other " UINT64_FORMAT " (%.2f%%)", + no_info_hits, PERCENTAGE_OF(no_info_hits, hits), + counter_noinfo_IMK, PERCENTAGE_OF(counter_noinfo_IMK, hits), + counter_noinfo_ICLK, PERCENTAGE_OF(counter_noinfo_ICLK, hits), + counter_noinfo_IK_other, PERCENTAGE_OF(counter_noinfo_IK_other, hits) + ); +#endif // KLUT_ENABLE_EXPENSIVE_STATS + + // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) + constexpr int chacheline_size = 64; + constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); + const int num_cachelines = num_entries() / slots_per_cacheline; + int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; + for (int i = 0; i < num_cachelines; i++) { + int n = 0; + for (int j = 0; j < slots_per_cacheline; j++) { + KlassLUTEntry e(at((i * slots_per_cacheline) + j)); + const bool valid = !e.is_invalid() && (e.is_array() || e.ik_carries_infos()); + if (valid) { + n++; + } + } + assert(n <= slots_per_cacheline, "Sanity"); + valid_hits_per_cacheline_distribution[n]++; + } + st->print_cr("LUT valid hit density over cacheline size:"); + for (int i = 0; i <= slots_per_cacheline; i++) { + st->print_cr("%d valid entries per cacheline: %d", i, valid_hits_per_cacheline_distribution[i]); + } + // Just for info, print limits + KlassLUTEntry::print_limits(st); +} + +#ifdef KLUT_ENABLE_EXPENSIVE_STATS +void KlassInfoLUT::update_hit_stats(KlassLUTEntry klute) { + switch (klute.kind()) { +#define XX(name, shortname) case Klass::name ## Kind: inc_hits_ ## shortname(); break; + ALL_KLASS_KINDS_DO(XX) +#undef XX + default: ShouldNotReachHere(); + }; + if (klute.is_instance() && !klute.ik_carries_infos()) { + switch (klute.kind()) { + case Klass::InstanceClassLoaderKlassKind: inc_noinfo_ICLK(); break; + case Klass::InstanceMirrorKlassKind: inc_noinfo_IMK(); break; + default: inc_noinfo_IK_other(); break; + } + } + switch (klute.loader_index()) { + case 1: inc_hits_bootloader(); break; + case 2: inc_hits_sysloader(); break; + case 3: inc_hits_platformloader(); break; + }; +} +#endif // KLUT_ENABLE_EXPENSIVE_STATS + +#ifdef KLUT_ENABLE_EXPENSIVE_LOG +void KlassInfoLUT::log_hit(KlassLUTEntry klute) { + //log_debug(klut)("retrieval: klute: name: %s kind: %d", k->name()->as_C_string(), k->kind()); +} +#endif + diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp new file mode 100644 index 0000000000000..32f0254e13f9c --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_KLASSINFOLUT_HPP +#define SHARE_OOPS_KLASSINFOLUT_HPP + +#include "memory/allStatic.hpp" +#include "oops/compressedKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" +#include "utilities/globalDefinitions.hpp" + +class Klass; +class ClassLoaderData; + +#ifdef ASSERT +#define KLUT_ENABLE_EXPENSIVE_STATS +//#define KLUT_ENABLE_EXPENSIVE_LOG +#endif + +class KlassInfoLUT : public AllStatic { + + static ClassLoaderData* _common_loaders[4]; // See "loader" bits in Klute + static uint32_t* _entries; + + static inline unsigned num_entries(); + + static inline uint32_t at(unsigned index); + + // register stats are not expensive +#define REGISTER_STATS_DO(f) \ + f(registered_IK) \ + f(registered_IRK) \ + f(registered_IMK) \ + f(registered_ICLK) \ + f(registered_ISCK) \ + f(registered_TAK) \ + f(registered_OAK) \ + f(registered_IK_for_abstract_or_interface) +#define XX(xx) static void inc_##xx(); + REGISTER_STATS_DO(XX) +#undef XX + + // hit stats are expensive +#ifdef KLUT_ENABLE_EXPENSIVE_STATS +#define HIT_STATS_DO(f) \ + f(hits_IK) \ + f(hits_IRK) \ + f(hits_IMK) \ + f(hits_ICLK) \ + f(hits_ISCK) \ + f(hits_TAK) \ + f(hits_OAK) \ + f(hits_bootloader) \ + f(hits_sysloader) \ + f(hits_platformloader) \ + f(noinfo_IMK) \ + f(noinfo_ICLK) \ + f(noinfo_IK_other) +#define XX(xx) static void inc_##xx(); + HIT_STATS_DO(XX) +#undef XX + static void update_hit_stats(KlassLUTEntry klute); +#endif // KLUT_ENABLE_EXPENSIVE_STATS + +#ifdef KLUT_ENABLE_EXPENSIVE_LOG + static void log_hit(KlassLUTEntry klute); +#endif + +public: + + static void initialize(); + + static void register_klass(const Klass* k); + + static inline KlassLUTEntry get_entry(narrowKlass k); + + static int try_register_perma_cld(ClassLoaderData* cld); + static inline ClassLoaderData* get_perma_cld(int index); + + static void print_statistics(outputStream* out); + +}; + +#endif // SHARE_OOPS_KLASSINFOLUT_HPP diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp new file mode 100644 index 0000000000000..5da9a42d2f062 --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_KLASSINFOLUT_INLINE_HPP +#define SHARE_OOPS_KLASSINFOLUT_INLINE_HPP + +#include "oops/compressedKlass.inline.hpp" +#include "oops/klassInfoLUT.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/debug.hpp" + +inline unsigned KlassInfoLUT::num_entries() { + return nth_bit(CompressedKlassPointers::narrow_klass_pointer_bits()); +} + +ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { + assert(index < num_entries(), "oob (%x vs %x)", index, num_entries()); + return _entries[index]; +} + +ALWAYSINLINE KlassLUTEntry KlassInfoLUT::get_entry(narrowKlass nk) { + + const uint32_t v = at(nk); + KlassLUTEntry e(v); + assert(!e.is_invalid(), "invalid?"); +#ifdef KLUT_ENABLE_EXPENSIVE_STATS + update_hit_stats(e); +#endif +#ifdef KLUT_ENABLE_EXPENSIVE_LOG + log_hit(e); +#endif + return e; +} + +ALWAYSINLINE ClassLoaderData* KlassInfoLUT::get_perma_cld(int index) { + assert(index >= 1 && index <= 3, "Sanity"); + return _common_loaders[index]; +} + +#endif // SHARE_OOPS_KLASSINFOLUT_INLINE_HPP diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp new file mode 100644 index 0000000000000..803d4a345aac1 --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "logging/log.hpp" +#include "memory/resourceArea.hpp" +#include "oops/instanceKlass.inline.hpp" +#include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" +#include "utilities/debug.hpp" + +// See klass.hpp +union LayoutHelperHelper { + unsigned raw; + struct { + // lsb + unsigned lh_esz : 8; // element size + unsigned lh_ebt : 8; // element BasicType (currently unused) + unsigned lh_hsz : 8; // header size (offset to first element) + unsigned lh_tag : 8; // 0x80 or 0xc0 + // msb + } bytes; +}; + +// small helper +static void read_and_check_omb_values(const OopMapBlock* omb, unsigned& omb_offset, unsigned& omb_count) { + const int offset_bytes = omb->offset(); + assert(offset_bytes > 0 && is_aligned(offset_bytes, BytesPerHeapOop), + "weird or misaligned oop map block offset (%d)", offset_bytes); + omb_offset = (unsigned)offset_bytes / BytesPerHeapOop; + + const unsigned count = omb->count(); + assert(count > 0, "omb count zero?"); + omb_count = count; +} + +uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_encodable_reason) { + + assert(ik->is_instance_klass(), "sanity"); + + const int kind = ik->kind(); + const int lh = ik->layout_helper(); + const int loader_index = KlassInfoLUT::try_register_perma_cld(ik->class_loader_data()); + + U value(0); + + // Set common bits, these are always present + assert(kind < 0b111, "sanity"); + value.common.kind = kind; + value.common.loader = loader_index; + + // We may not be able to encode the IK-specific info; if we can't, those bits are left zero + // and we return an error string for logging + #define NOPE(s) { not_encodable_reason = s; return value.raw; } + + if (Klass::layout_helper_needs_slow_path(lh)) { + if (ik->is_abstract() || ik->is_interface()) { + NOPE("Size not trivially computable (klass abstract or interface)"); + // Please note that we could represent abstract or interface classes, but atm there is not + // much of a point. + } else { + NOPE("Size not trivially computable"); + } + } + + const size_t wordsize = Klass::layout_helper_to_size_helper(lh); + if (wordsize >= ik_wordsize_limit) { + NOPE("Size too large"); + } + + // Has more than one nonstatic OopMapBlock? + const int oop_map_count = ik->nonstatic_oop_map_count(); + + unsigned omb_offset_1 = 0, omb_count_1 = 0, omb_offset_2 = 0, omb_count_2 = 0; + + switch(oop_map_count) { + case 0: // nothing to do + break; + case 1: + read_and_check_omb_values(ik->start_of_nonstatic_oop_maps(), omb_offset_1, omb_count_1); + break; + case 2: + read_and_check_omb_values(ik->start_of_nonstatic_oop_maps(), omb_offset_1, omb_count_1); + read_and_check_omb_values(ik->start_of_nonstatic_oop_maps() + 1, omb_offset_2, omb_count_2); + break; + default: + NOPE("More than 2 oop map blocks"); + } + + if (omb_offset_1 >= ik_omb_offset_1_limit) { + NOPE("omb offset 1 overflow"); + } + + if (omb_count_1 >= ik_omb_count_1_limit) { + NOPE("omb count 1 overflow"); + } + + if (omb_offset_2 >= ik_omb_offset_2_limit) { + NOPE("omb offset 2 overflow"); + } + + if (omb_count_2 >= ik_omb_count_2_limit) { + NOPE("omb count 2 overflow"); + } + +#undef NOPE + // Okay, we are good. + + value.ike.wordsize = wordsize; + value.ike.omb_count_1 = omb_count_1; + value.ike.omb_offset_1 = omb_offset_1; + value.ike.omb_count_2 = omb_count_2; + value.ike.omb_offset_2 = omb_offset_2; + + return value.raw; + +} + +uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { + + assert(ak->is_array_klass(), "sanity"); + + const int kind = ak->kind(); + const int lh = ak->layout_helper(); + const int loader_index = KlassInfoLUT::try_register_perma_cld(ak->class_loader_data()); + + assert(Klass::layout_helper_is_objArray(lh) || Klass::layout_helper_is_typeArray(lh), "unexpected"); + + LayoutHelperHelper lhu = { (unsigned) lh }; + U value(0); + value.common.kind = kind; + value.common.loader = loader_index; + value.ake.lh_ebt = lhu.bytes.lh_ebt; + value.ake.lh_esz = lhu.bytes.lh_esz; + value.ake.lh_hsz = lhu.bytes.lh_hsz; + return value.raw; + +} + +uint32_t KlassLUTEntry::build_from(const Klass* k) { + + uint32_t value = invalid_entry; + if (k->is_array_klass()) { + value = build_from_ak(ArrayKlass::cast(k)); + } else { + assert(k->is_instance_klass(), "sanity"); + const char* not_encodable_reason = nullptr; + value = build_from_ik(InstanceKlass::cast(k), not_encodable_reason); + if (not_encodable_reason != nullptr) { + log_debug(klut)("klass klute register: %s cannot be encoded because: %s.", k->name()->as_C_string(), not_encodable_reason); + } + } + return value; + +} + +#ifdef ASSERT + +void KlassLUTEntry::verify_against(const Klass* k) const { + + // General static asserts that need access to private members, but I don't want + // to place them in a header + STATIC_ASSERT(bits_common + bits_specific == bits_total); + STATIC_ASSERT(32 == bits_total); + STATIC_ASSERT(bits_ak_lh <= bits_specific); + + STATIC_ASSERT(sizeof(KE) == sizeof(uint32_t)); + STATIC_ASSERT(sizeof(AKE) == sizeof(uint32_t)); + STATIC_ASSERT(sizeof(IKE) == sizeof(uint32_t)); + STATIC_ASSERT(sizeof(U) == sizeof(uint32_t)); + + // Kind must fit into 3 bits + STATIC_ASSERT(Klass::KLASS_KIND_COUNT < nth_bit(bits_kind)); + +#define XX(name, ignored) STATIC_ASSERT((int)name ## Kind == (int)Klass::name ## Kind); + ALL_KLASS_KINDS_DO(XX) +#undef XX + + assert(!is_invalid(), "Entry should be valid (%x)", _v.raw); + + // kind + const unsigned real_kind = (unsigned)k->kind(); + const unsigned our_kind = kind(); + const int real_lh = k->layout_helper(); + + assert(our_kind == real_kind, "kind mismatch (%d vs %d) (%x)", real_kind, our_kind, _v.raw); + + const ClassLoaderData* const real_cld = k->class_loader_data(); + const unsigned loader = loader_index(); + assert(loader < 4, "invalid loader index"); + if (loader > 0) { + assert(KlassInfoLUT::get_perma_cld(loader) == real_cld, "Different CLD?"); + } else { + assert(!real_cld->is_permanent_class_loader_data(), "perma cld?"); + } + + if (k->is_array_klass()) { + + // compare our (truncated) lh with the real one + const LayoutHelperHelper lhu = { (unsigned) real_lh }; + assert(lhu.bytes.lh_ebt == ak_layouthelper_ebt() && + lhu.bytes.lh_esz == ak_layouthelper_esz() && + lhu.bytes.lh_hsz == ak_layouthelper_hsz() && + ( (lhu.bytes.lh_tag == 0xC0 && real_kind == Klass::TypeArrayKlassKind) || + (lhu.bytes.lh_tag == 0x80 && real_kind == Klass::ObjArrayKlassKind) ), + "layouthelper mismatch (0x%x vs 0x%x)", real_lh, _v.raw); + + } else { + + assert(k->is_instance_klass(), "unexpected"); + const InstanceKlass* const ik = InstanceKlass::cast(k); + + const int real_oop_map_count = ik->nonstatic_oop_map_count(); + const unsigned omb_offset_1 = (real_oop_map_count >= 1) ? (unsigned)ik->start_of_nonstatic_oop_maps()[0].offset() : max_uintx; + const unsigned omb_count_1 = (real_oop_map_count >= 1) ? ik->start_of_nonstatic_oop_maps()[0].count() : max_uintx; + const unsigned omb_offset_2 = (real_oop_map_count >= 2) ? (unsigned)ik->start_of_nonstatic_oop_maps()[1].offset() : max_uintx; + const unsigned omb_count_2 = (real_oop_map_count >= 2) ? ik->start_of_nonstatic_oop_maps()[1].count() : max_uintx; + + const int real_wordsize = Klass::layout_helper_needs_slow_path(real_lh) ? + -1 : Klass::layout_helper_to_size_helper(real_lh); + + if (ik_carries_infos()) { + + // check wordsize + assert(real_wordsize == ik_wordsize(), "wordsize mismatch? (%d vs %d) (%x)", real_wordsize, ik_wordsize(), _v.raw); + + // check omb info + switch (real_oop_map_count) { + case 0: { + assert(ik_omb_offset_1() == 0 && ik_omb_count_1() == 0 && + ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, "omb should not be present (0x%x)", _v.raw); + } + break; + case 1: { + assert(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch (0x%x)", _v.raw); + assert(ik_omb_count_1() == omb_count_1, "first omb count mismatch (0x%x)", _v.raw); + assert(ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, "second omb should not be present (0x%x)", _v.raw); + } + break; + case 2: { + assert(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch (0x%x)", _v.raw); + assert(ik_omb_count_1() == omb_count_1, "first omb count mismatch (0x%x)", _v.raw); + assert(ik_omb_offset_2() * BytesPerHeapOop == omb_offset_2, "second omb offset mismatch (0x%x)", _v.raw); + assert(ik_omb_count_2() == omb_count_2, "second omb count mismatch (0x%x)", _v.raw); + } + break; + default: fatal("More than one oop maps, IKE should not be encodable"); + } + } else { + // Check if this Klass should, in fact, have been encodable + assert( Klass::layout_helper_needs_slow_path(real_lh) || + (real_wordsize >= (int)ik_wordsize_limit) || + (real_oop_map_count > 2) || + ((size_t) omb_offset_1 >= ik_omb_offset_1_limit) || + ((size_t) omb_count_1 >= ik_omb_count_1_limit) || + ((size_t) omb_offset_2 >= ik_omb_offset_2_limit) || + ((size_t) omb_count_2 >= ik_omb_count_2_limit), + "Klass should have been encodable" ); + } + } + +} // KlassLUTEntry::verify_against + +#endif // ASSERT + +KlassLUTEntry::KlassLUTEntry(const Klass* k) : _v(build_from(k)) { +} + +// Helper function, prints current limits +void KlassLUTEntry::print_limits(outputStream* st) { + st->print_cr("IKE Limits: instance byte size %zu, omb1 count: %zu, omb1 byte offset: %zu, omb2 oop count: %zu, omb2 byte offset: %zu", + ik_wordsize_limit * BytesPerWord, + ik_omb_count_1_limit, ik_omb_offset_1_limit * BytesPerHeapOop, + ik_omb_count_2_limit, ik_omb_offset_2_limit * BytesPerHeapOop); +} diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp new file mode 100644 index 0000000000000..fd05a68e69c67 --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_KLASSINFOLUTENTRY_HPP +#define SHARE_OOPS_KLASSINFOLUTENTRY_HPP + +// Included by oop.hpp, keep it short and sweet here +#include "memory/allStatic.hpp" +#include "utilities/globalDefinitions.hpp" + +class OopMapBlock; +class Klass; +class outputStream; + +// msb lsb +// +// invalid_entry (debug zap): 1111 1111 1111 1111 1111 1111 1111 1111 (relies on kind == 0b111 == 7 being invalid) +// +// All valid entries: KKKL L... .... .... .... .... .... .... +// +// InstanceKlass: KKKL LSSS SSSO OOOO CCCC CCOO OOCC CCCC +// 2 2222 2222 2211 1111 1111 +// InstanceKlass, has_no_addinfo: KKKL L000 0000 0000 0000 0000 0000 0000 (all IK specific bits 0) (note: means that "0" is a valid IK entry with no add. info) +// InstanceKlass, has no oopmap entries: KKKL L... .... .... .... .... 0000 0000 (omb count bits are 0) (only valid if !has_no_addinfo) +// +// ArrayKlass: KKKL L--- tttt tttt hhhh hhhh eeee eeee +// +// +// IK specific bits: +// C1 : Count of first OMB (6 bits) +// O1 : Offset, in number-of-oops, of first OMB (4 bits) +// C2 : Count of second OMB (6 bits) +// O2 : Offset, in number-of-oops, of second OMB (5 bits) +// S : Object instance size in words (6 bits) +// +// AK specific bits: +// e : log2 element size (8 bits) +// h : header size (8 bits) +// t : element basic type (8 bits) +// +// Common bits: +// L : Loader (2 bits) +// (0 unknown, 1 boot loader, +// 2 system loader, 3 platform loader) +// K : KlassKind (3 bits) +// +// - : unused + +class KlassLUTEntry { + +#define ALL_KLASS_KINDS_DO(what) \ + what(InstanceKlass, IK) \ + what(InstanceRefKlass, IRK) \ + what(InstanceMirrorKlass, IMK) \ + what(InstanceClassLoaderKlass, ICLK) \ + what(InstanceStackChunkKlass, ISCK) \ + what(TypeArrayKlass, TAK) \ + what(ObjArrayKlass, OAK) \ + + // Todo: move KlassKind out of Klass + // Don't want to include it here for all the baggage it brings + enum LocalKlassKind { +#define XX(name, ignored) name ## Kind, + ALL_KLASS_KINDS_DO(XX) +#undef XX + UnknownKlassKind, + LastKlassKind = ObjArrayKlassKind, + FirstArrayKlassKind = TypeArrayKlassKind + }; + + static constexpr int bits_total = 32; + + // All valid entries: KKKB ---- ---- ---- ---- ---- ---- ---- + static constexpr int bits_kind = 3; + static constexpr int bits_loader = 2; + static constexpr int bits_common = bits_kind + bits_loader; + static constexpr int bits_specific = bits_total - bits_common; + + // Bits valid for all entries, regardless of Klass kind + struct KE { + // lsb + unsigned kind_specific_bits : bits_specific; + unsigned loader : bits_loader; + unsigned kind : bits_kind; + // msb + }; + + // Bits only valid for InstanceKlass + static constexpr int bits_ik_omb_count_1 = 6; + static constexpr int bits_ik_omb_offset_1 = 4; + static constexpr int bits_ik_omb_count_2 = 6; + static constexpr int bits_ik_omb_offset_2 = 5; + static constexpr int bits_ik_omb_bits = bits_ik_omb_count_1 + bits_ik_omb_offset_1 + bits_ik_omb_count_2 + bits_ik_omb_offset_2; + static constexpr int bits_ik_wordsize = bits_specific - bits_ik_omb_bits; + struct IKE { + // lsb + unsigned omb_count_1 : bits_ik_omb_count_1; + unsigned omb_offset_1 : bits_ik_omb_offset_1; + unsigned omb_count_2 : bits_ik_omb_count_2; + unsigned omb_offset_2 : bits_ik_omb_offset_2; + unsigned wordsize : bits_ik_wordsize; + unsigned other : bits_common; + // msb + }; + + // Bits only valid for ArrayKlass + static constexpr int bits_ak_lh_esz = 8; + static constexpr int bits_ak_lh_ebt = 8; + static constexpr int bits_ak_lh_hsz = 8; + static constexpr int bits_ak_lh = bits_ak_lh_esz + bits_ak_lh_ebt + bits_ak_lh_hsz; + struct AKE { + // lsb + // see klass.hpp + unsigned lh_esz : bits_ak_lh_esz; // element size + unsigned lh_ebt : bits_ak_lh_ebt; // element BasicType (currently unused) + unsigned lh_hsz : bits_ak_lh_hsz; // header size (offset to first element) + unsigned unused : bits_specific - bits_ak_lh_esz - bits_ak_lh_ebt - bits_ak_lh_hsz; + unsigned other : bits_common; + // msb + }; + + union U { + uint32_t raw; + KE common; + IKE ike; + AKE ake; + U(uint32_t v) : raw(v) {} + }; + + const U _v; + + // The limits to what we can numerically represent in an (InstanceKlass) Entry + static constexpr size_t ik_wordsize_limit = nth_bit(bits_ik_wordsize); + static constexpr size_t ik_omb_offset_1_limit = nth_bit(bits_ik_omb_offset_1); + static constexpr size_t ik_omb_count_1_limit = nth_bit(bits_ik_omb_count_1); + static constexpr size_t ik_omb_offset_2_limit = nth_bit(bits_ik_omb_offset_2); + static constexpr size_t ik_omb_count_2_limit = nth_bit(bits_ik_omb_count_2); + + static uint32_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); + static uint32_t build_from_ak(const ArrayKlass* k); + + static uint32_t build_from(const Klass* k); + +public: + + // Invalid entries are entries that have not been set yet. + // Note: cannot use "0" as invalid_entry, since 0 is valid (interface or abstract InstanceKlass, size = 0 and has no oop map) + // We use kind=7=0b111 (invalid), and set the rest of the bits also to 1 + static constexpr uint32_t invalid_entry = 0xFFFFFFFF; + +// inline KlassLUTEntry() : _v(invalid_entry) {} + inline KlassLUTEntry(uint32_t v) : _v(v) {} + inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} + + KlassLUTEntry(const Klass* k); + + // Note: all entries should be valid. An invalid entry indicates + // an error somewhere. + bool is_invalid() const { return _v.raw == invalid_entry; } + +#ifdef ASSERT + void verify_against(const Klass* k) const; +#endif + + uint32_t value() const { return _v.raw; } + + inline unsigned kind() const { return _v.common.kind; } + + // returns loader index (0 for unknown) + inline int loader_index() const { return _v.common.loader; } + + bool is_array() const { return _v.common.kind >= FirstArrayKlassKind; } + bool is_instance() const { return !is_array(); } + + bool is_obj_array() const { return _v.common.kind == ObjArrayKlassKind; } + bool is_type_array() const { return _v.common.kind == TypeArrayKlassKind; } + + // Calculates the object size. Note, introduces a branch (is_array or not). + // If possible, use either ik_wordsize() or ak_calculate_wordsize_given_oop() instead. + inline unsigned calculate_wordsize_given_oop(oop obj) const; + + // Following methods only if IK: + + // Returns true if entry carries IK-specific info (oop map block info + size). + // If false, caller needs to look up these infor Klass*. + inline bool ik_carries_infos() const; + + // Following methods only if IK *and* ik_carries_infos() == true: + + // Returns size, in words, of oops of this class + inline int ik_wordsize() const; + + // Returns count of first OopMapBlock, 0 if there is no oop map block. + inline unsigned ik_omb_count_1() const; + + // Returns offset of first OopMapBlock in number-of-oops (so, scaled by BytesPerHeapOop). + inline unsigned ik_omb_offset_1() const; + + // Returns count of second OopMapBlock, 0 if there is no second oop map block. + inline unsigned ik_omb_count_2() const; + + // Returns offset of second OopMapBlock in number-of-oops (so, scaled by BytesPerHeapOop). + inline unsigned ik_omb_offset_2() const; + + // Following methods only if AK: + + // returns log2 element size + inline unsigned ak_layouthelper_esz() const { return _v.ake.lh_esz; } + + // returns ebt byte + inline unsigned ak_layouthelper_ebt() const { return _v.ake.lh_ebt; } + + // returns distance to first element + inline unsigned ak_layouthelper_hsz() const { return _v.ake.lh_hsz; } + + // calculates word size given header size, element size, and array length + inline unsigned ak_calculate_wordsize_given_oop(oop obj) const; + + // Helper function, prints current limits + static void print_limits(outputStream* st); + +}; // KlassInfoLUEntryIK + +#endif // SHARE_OOPS_KLASSINFOLUTENTRY_HPP diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp new file mode 100644 index 0000000000000..abf313f04f31e --- /dev/null +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024, Red Hat, Inc. All rights reserved. + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP +#define SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP + +#include "oops/instanceKlass.hpp" +#include "oops/klass.hpp" // for KlassKind +#include "oops/klassInfoLUTEntry.hpp" +#include "oops/oop.hpp" +#include "utilities/debug.hpp" + +// Returns true if entry carries IK-specific info (oop map block info + size). +// If false, caller needs to look up these infor Klass*. +inline bool KlassLUTEntry::ik_carries_infos() const { + assert(is_instance(), "sanity"); + return _v.common.kind_specific_bits > 0; +} + +// Returns size, in words, of oops of this class +inline int KlassLUTEntry::ik_wordsize() const { + assert(is_instance() && ik_carries_infos(), "only for valid ik entries"); + return _v.ike.wordsize; +} + +// Returns count of first OopMapBlock. Returns 0 if there is no OopMapBlock. +inline unsigned KlassLUTEntry::ik_omb_count_1() const { + assert(is_instance() && ik_carries_infos(), "only for valid ik entries"); + return _v.ike.omb_count_1; +} + +// Returns offset of first OopMapBlock. Only call if count is > 0 +inline unsigned KlassLUTEntry::ik_omb_offset_1() const { + assert(is_instance() && ik_carries_infos(), "only for valid ik entries"); + return _v.ike.omb_offset_1; +} + +// Returns count of second OopMapBlock. Returns 0 if there is no second OopMapBlock. +inline unsigned KlassLUTEntry::ik_omb_count_2() const { + assert(is_instance() && ik_carries_infos(), "only for valid ik entries"); + return _v.ike.omb_count_2; +} + +// Returns offset of second OopMapBlock. Only call if count is > 0 +inline unsigned KlassLUTEntry::ik_omb_offset_2() const { + assert(is_instance() && ik_carries_infos(), "only for valid ik entries"); + return _v.ike.omb_offset_2; +} + +// calculates word size given header size, element size, and array length +inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop(oop obj) const { + assert(is_array(), "only for ak entries"); + assert(UseCompactObjectHeaders, "+COH only"); + assert(UseKLUT, "+KLUT only"); + + // See oopDesc::size_given_klass + const unsigned l2esz = ak_layouthelper_esz(); + const unsigned hsz = ak_layouthelper_hsz(); + + // In +COH, array length is always at offset 8 + STATIC_ASSERT(sizeof(markWord) == 8); + const int* const array_len_addr = (int*)(obj->field_addr(8)); + const size_t array_length = (size_t) (*array_len_addr); + const size_t size_in_bytes = (array_length << l2esz) + hsz; + + // Note: for UseKLUT, we require a standard object alignment (see argument.cpp) + constexpr int HardCodedObjectAlignmentInBytes = BytesPerWord; + assert(MinObjAlignmentInBytes == HardCodedObjectAlignmentInBytes, "Sanity"); + + return align_up(size_in_bytes, HardCodedObjectAlignmentInBytes) / HeapWordSize; +} + +inline unsigned KlassLUTEntry::calculate_wordsize_given_oop(oop obj) const { + size_t rc = 0; + return is_array() ? ak_calculate_wordsize_given_oop(obj) : ik_wordsize(); +} + +#endif // SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index 7af6b963052f1..afe4daba867ac 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -36,6 +36,7 @@ #include "oops/arrayKlass.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" @@ -50,7 +51,16 @@ ObjArrayKlass* ObjArrayKlass::allocate(ClassLoaderData* loader_data, int n, Klas int size = ArrayKlass::static_size(ObjArrayKlass::header_size()); - return new (loader_data, size, THREAD) ObjArrayKlass(n, k, name); + ObjArrayKlass* oak = new (loader_data, size, THREAD) ObjArrayKlass(n, k, name); + + { + char tmp[1024]; + log_debug(metaspace)("Returning new OAK @" PTR_FORMAT " for %s, nKlass=%u, word size=%d", + p2i(oak), + oak->name()->as_C_string(tmp, sizeof(tmp)), + CompressedKlassPointers::encode(oak), size); + } + return oak; } ObjArrayKlass* ObjArrayKlass::allocate_objArray_klass(ClassLoaderData* loader_data, @@ -139,6 +149,11 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK set_layout_helper(array_layout_helper(T_OBJECT)); assert(is_array_klass(), "sanity"); assert(is_objArray_klass(), "sanity"); + + // Add to KLUT + if (UseKLUT) { + KlassInfoLUT::register_klass(this); + } } size_t ObjArrayKlass::oop_size(oop obj) const { diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index c44c8d28f6247..9f19d5197558f 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_OBJARRAYKLASS_HPP #include "oops/arrayKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "utilities/macros.hpp" class ClassLoaderData; @@ -37,7 +38,7 @@ class ObjArrayKlass : public ArrayKlass { friend class JVMCIVMStructs; public: - static const KlassKind Kind = ObjArrayKlassKind; + static constexpr KlassKind Kind = ObjArrayKlassKind; private: // If you add a new field that points to any metaspace object, you @@ -132,6 +133,14 @@ class ObjArrayKlass : public ArrayKlass { template inline void oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end); + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + public: // Iterate over all oop elements. template @@ -161,6 +170,10 @@ class ObjArrayKlass : public ArrayKlass { void verify_on(outputStream* st); void oop_verify_on(oop obj, outputStream* st); + + DECLARE_EXACT_CAST_FUNCTIONS(ObjArrayKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(ObjArrayKlass) + }; #endif // SHARE_OOPS_OBJARRAYKLASS_HPP diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index a92c42d21a8ed..844d5adb0150a 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -98,6 +98,25 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me oop_oop_iterate_elements_bounded(a, closure, mr.start(), mr.end()); } +// Klute variants +template +void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate(obj, closure); +} + +template +void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_bounded(obj, closure, mr); +} + +template +void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + // Todo: for now just resolve the Klass. Maybe more parts can be made static. + narrow_klass_to_klass(nk)->oop_oop_iterate_reverse(obj, closure); +} + // Like oop_oop_iterate but only iterates over a specified range and only used // for objArrayOops. template @@ -118,4 +137,7 @@ void objArrayOopDesc::oop_iterate_range(OopClosureType* blk, int start, int end) } } +DEFINE_EXACT_CAST_FUNCTIONS(ObjArrayKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(ObjArrayKlass) + #endif // SHARE_OOPS_OBJARRAYKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 8048c8770c2ba..7fe3558d69774 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -29,6 +29,7 @@ #include "memory/memRegion.hpp" #include "oops/compressedKlass.hpp" #include "oops/accessDecorators.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/markWord.hpp" #include "oops/metadata.hpp" #include "oops/objLayout.hpp" diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 3dad778a73a47..b7b2834b22761 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -34,6 +34,8 @@ #include "oops/arrayOop.hpp" #include "oops/compressedKlass.inline.hpp" #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUT.inline.hpp" +#include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/objLayout.inline.hpp" #include "oops/markWord.inline.hpp" #include "oops/oopsHierarchy.hpp" @@ -229,11 +231,25 @@ bool oopDesc::is_instance() const { return klass()->is_instance_klass(); bool oopDesc::is_instanceRef() const { return klass()->is_reference_instance_klass(); } bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_klass(); } bool oopDesc::is_array() const { return klass()->is_array_klass(); } -bool oopDesc::is_objArray() const { return klass()->is_objArray_klass(); } -bool oopDesc::is_typeArray() const { return klass()->is_typeArray_klass(); } + +bool oopDesc::is_objArray() const { + if (UseKLUT) { + const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + return klute.is_obj_array(); + } + return klass()->is_objArray_klass(); +} + +bool oopDesc::is_typeArray() const { + if (UseKLUT) { + const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + return klute.is_type_array(); + } + return klass()->is_typeArray_klass(); +} template -T* oopDesc::field_addr(int offset) const { return reinterpret_cast(cast_from_oop(as_oop()) + offset); } +T* oopDesc::field_addr(int offset_bytes) const { return reinterpret_cast(cast_from_oop(as_oop()) + offset_bytes); } template size_t oopDesc::field_offset(T* p) const { return pointer_delta((void*)p, (void*)this, 1); } @@ -374,16 +390,39 @@ void oopDesc::incr_age() { template void oopDesc::oop_iterate(OopClosureType* cl) { + + if (UseKLUT) { + const narrowKlass nk = mark().narrow_klass(); + const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute, nk); + return; + } + OopIteratorClosureDispatch::oop_oop_iterate(cl, this, klass()); } template void oopDesc::oop_iterate(OopClosureType* cl, MemRegion mr) { + + if (UseKLUT) { + const narrowKlass nk = mark().narrow_klass(); + const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute, nk); + return; + } + OopIteratorClosureDispatch::oop_oop_iterate(cl, this, klass(), mr); } template size_t oopDesc::oop_iterate_size(OopClosureType* cl) { + + if (UseKLUT) { + const narrowKlass nk = mark().narrow_klass(); + const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute, nk); + } + Klass* k = klass(); size_t size = size_given_klass(k); OopIteratorClosureDispatch::oop_oop_iterate(cl, this, k); @@ -392,6 +431,13 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl) { template size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { + + if (UseKLUT) { + const narrowKlass nk = mark().narrow_klass(); + const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute, nk); + } + Klass* k = klass(); size_t size = size_given_klass(k); OopIteratorClosureDispatch::oop_oop_iterate(cl, this, k, mr); @@ -407,6 +453,16 @@ template void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { // In this assert, we cannot safely access the Klass* with compact headers. assert(k == klass(), "wrong klass"); + + if (UseKLUT) { + const narrowKlass nk = CompressedKlassPointers::encode_not_null(k); + const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + if (!klute.is_type_array()) { // no need to iterate TAK + OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute, nk); + } + return; + } + OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k); } diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index 39385bb0184e7..d3d2ae2888fd0 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -34,6 +34,7 @@ #include "oops/arrayKlass.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" #include "oops/typeArrayKlass.inline.hpp" @@ -52,6 +53,15 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type, TypeArrayKlass* ak = TypeArrayKlass::allocate(null_loader_data, type, sym, CHECK_NULL); + { + char tmp[1024]; + const int size = ArrayKlass::static_size(TypeArrayKlass::header_size()); + log_debug(metaspace)("Returning new TAK @" PTR_FORMAT " for %s, nKlass=%u, word size=%d", + p2i(ak), + ak->name()->as_C_string(tmp, sizeof(tmp)), + CompressedKlassPointers::encode(ak), size); + } + // Call complete_create_array_klass after all instance variables have been initialized. complete_create_array_klass(ak, ak->super(), ModuleEntryTable::javabase_moduleEntry(), CHECK_NULL); @@ -62,6 +72,12 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type, // an array class without a mirror. null_loader_data->add_class(ak); JFR_ONLY(ASSIGN_PRIMITIVE_CLASS_ID(ak);) + + // Add to KLUT + if (UseKLUT) { + KlassInfoLUT::register_klass(ak); + } + return ak; } diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index fa4e301e3e446..0da17cdba48bc 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_TYPEARRAYKLASS_HPP #include "oops/arrayKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" class ClassLoaderData; @@ -36,7 +37,7 @@ class TypeArrayKlass : public ArrayKlass { friend class VMStructs; public: - static const KlassKind Kind = TypeArrayKlassKind; + static constexpr KlassKind Kind = TypeArrayKlassKind; private: jint _max_length; // maximum number of elements allowed in an array @@ -80,7 +81,7 @@ class TypeArrayKlass : public ArrayKlass { private: // The implementation used by all oop_oop_iterate functions in TypeArrayKlasses. - inline void oop_oop_iterate_impl(oop obj, OopIterateClosure* closure); + static inline void oop_oop_iterate_impl(oop obj, OopIterateClosure* closure); public: // Wraps oop_oop_iterate_impl to conform to macros. @@ -95,6 +96,14 @@ class TypeArrayKlass : public ArrayKlass { template inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + // klute variants + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + public: static TypeArrayKlass* cast(Klass* k) { return const_cast(cast(const_cast(k))); @@ -127,6 +136,10 @@ class TypeArrayKlass : public ArrayKlass { ModuleEntry* module() const; PackageEntry* package() const; + + DECLARE_EXACT_CAST_FUNCTIONS(TypeArrayKlass) + DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(TypeArrayKlass) + }; #endif // SHARE_OOPS_TYPEARRAYKLASS_HPP diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index 098f9e7399343..d4e4b9fade966 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -55,4 +55,24 @@ void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { oop_oop_iterate_impl(obj, closure); } +// Klute variant does nothing special, since there is nothing in the klute that would help +// us here. It only exists to make the dispatcher happy. +template +inline void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + oop_oop_iterate_impl(obj, closure); +} + +template +inline void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { + oop_oop_iterate_impl(obj, closure); +} + +template +void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { + oop_oop_iterate_impl(obj, closure); +} + +DEFINE_EXACT_CAST_FUNCTIONS(TypeArrayKlass) +DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(TypeArrayKlass) + #endif // SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 69d37a11e45ab..2578a09465f16 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -3686,6 +3686,18 @@ void Arguments::set_compact_headers_flags() { if (UseCompactObjectHeaders && !UseCompressedClassPointers) { FLAG_SET_DEFAULT(UseCompressedClassPointers, true); } + // KLUT + if (!UseCompactObjectHeaders) { + FLAG_SET_DEFAULT(UseKLUT, false); + } + if (UseKLUT && ObjectAlignmentInBytes != BytesPerWord) { + warning("UseKLUT requires default object alignment."); + FLAG_SET_DEFAULT(UseKLUT, false); + } + if (UseKLUT && UseSharedSpaces) { + warning("UseKLUT, for the moment, won't work with CDS enabled. Please specify -Xshare:off."); + FLAG_SET_DEFAULT(UseKLUT, false); + } #endif } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index 29f0948276ada..d19e1a0d926e0 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -1399,6 +1399,12 @@ const int ObjectAlignmentInBytes = 8; "Force the class space to be allocated at this address or " \ "fails VM initialization (requires -Xshare=off.") \ \ + product(bool, UseKLUT, false, EXPERIMENTAL, \ + "Only for coh") \ + \ + product(bool, PrintKLUTStatistics, false, \ + "Print KLUT statistics at exit") \ + \ develop(bool, RandomizeClassSpaceLocation, true, \ "Randomize location of class space.") \ \ diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index b3ad15ae3ff68..f4d1df5329e11 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -54,6 +54,7 @@ #include "oops/generateOopMap.hpp" #include "oops/instanceKlass.hpp" #include "oops/instanceOop.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/klassVtable.hpp" #include "oops/method.inline.hpp" #include "oops/objArrayOop.hpp" @@ -359,6 +360,10 @@ void print_statistics() { ThreadsSMRSupport::log_statistics(); + if (UseKLUT && PrintKLUTStatistics) { + KlassInfoLUT::print_statistics(tty); + } + if (log_is_enabled(Info, perf, class, link)) { LogStreamHandle(Info, perf, class, link) log; log.print_cr("At VM exit:"); diff --git a/src/hotspot/share/utilities/devirtualizer.hpp b/src/hotspot/share/utilities/devirtualizer.hpp index b4d444dc5a85a..40b0c978f3f79 100644 --- a/src/hotspot/share/utilities/devirtualizer.hpp +++ b/src/hotspot/share/utilities/devirtualizer.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_UTILITIES_DEVIRTUALIZER_HPP #define SHARE_UTILITIES_DEVIRTUALIZER_HPP +#include "oops/klassInfoLUTEntry.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/bitMap.hpp" diff --git a/src/hotspot/share/utilities/devirtualizer.inline.hpp b/src/hotspot/share/utilities/devirtualizer.inline.hpp index 0cae27f87e95b..a1dbdc01cdcc4 100644 --- a/src/hotspot/share/utilities/devirtualizer.inline.hpp +++ b/src/hotspot/share/utilities/devirtualizer.inline.hpp @@ -28,6 +28,7 @@ #include "utilities/devirtualizer.hpp" #include "classfile/classLoaderData.hpp" +#include "oops/compressedKlass.inline.hpp" #include "oops/access.inline.hpp" #include "utilities/debug.hpp" diff --git a/src/hotspot/share/utilities/globalDefinitions.hpp b/src/hotspot/share/utilities/globalDefinitions.hpp index cc5f3ebb29126..ab4053922b9e1 100644 --- a/src/hotspot/share/utilities/globalDefinitions.hpp +++ b/src/hotspot/share/utilities/globalDefinitions.hpp @@ -241,7 +241,7 @@ class MetaWordImpl; // Opaque, never defined. typedef MetaWordImpl* MetaWord; // HeapWordSize must be 2^LogHeapWordSize. -const int HeapWordSize = sizeof(HeapWord); +constexpr int HeapWordSize = sizeof(HeapWord); #ifdef _LP64 const int LogHeapWordSize = 3; #else @@ -1340,4 +1340,7 @@ std::add_rvalue_reference_t declval() noexcept; // handled. bool IEEE_subnormal_handling_OK(); +// If compressed klass pointers then use narrowKlass. +typedef juint narrowKlass; + #endif // SHARE_UTILITIES_GLOBALDEFINITIONS_HPP From d43cfa183ba9a6b0d8254daf192e62732edf66e8 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 24 Mar 2025 15:33:49 +0100 Subject: [PATCH 002/101] revert printouts --- src/hotspot/share/classfile/classLoaderData.cpp | 13 ++++++++----- src/hotspot/share/classfile/classLoaderData.hpp | 2 +- .../share/classfile/classLoaderDataGraph.cpp | 2 +- .../share/classfile/classLoaderDataGraph.hpp | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index 7a6201f0274a2..825072cb13bef 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -985,7 +985,6 @@ class PrintKlassClosure: public KlassClosure { _out->print("%s,", k->external_name()); } }; -#endif // PRODUCT void ClassLoaderData::print_on(outputStream* out) const { ResourceMark rm; @@ -1019,11 +1018,14 @@ void ClassLoaderData::print_on(outputStream* out) const { default: ShouldNotReachHere(); } out->print_cr(" - handles %d", _handles.count()); - DEBUG_ONLY(out->print_cr(" - dependency count %d", _dependency_count);) + out->print_cr(" - dependency count %d", _dependency_count); out->print (" - klasses { "); - - { PrintClassClosure closure(out, true); - ((ClassLoaderData*)this)->classes_do(&closure); } + if (Verbose) { + PrintKlassClosure closure(out); + ((ClassLoaderData*)this)->classes_do(&closure); + } else { + out->print("..."); + } out->print_cr(" }"); out->print_cr(" - packages " INTPTR_FORMAT, p2i(_packages)); out->print_cr(" - module " INTPTR_FORMAT, p2i(_modules)); @@ -1042,6 +1044,7 @@ void ClassLoaderData::print_on(outputStream* out) const { out->print_cr(" - deallocate list " INTPTR_FORMAT, p2i(_deallocate_list)); out->print_cr(" - next CLD " INTPTR_FORMAT, p2i(_next)); } +#endif // PRODUCT void ClassLoaderData::print() const { print_on(tty); } diff --git a/src/hotspot/share/classfile/classLoaderData.hpp b/src/hotspot/share/classfile/classLoaderData.hpp index f6ca17cc8af50..19dbb0b1b3650 100644 --- a/src/hotspot/share/classfile/classLoaderData.hpp +++ b/src/hotspot/share/classfile/classLoaderData.hpp @@ -320,7 +320,7 @@ class ClassLoaderData : public CHeapObj { void set_jmethod_ids(JNIMethodBlock* new_block) { _jmethod_ids = new_block; } void print() const; - void print_on(outputStream* out) const ; + void print_on(outputStream* out) const PRODUCT_RETURN; void print_value() const; void print_value_on(outputStream* out) const; void verify(); diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.cpp b/src/hotspot/share/classfile/classLoaderDataGraph.cpp index b50477d2ffc48..fca6a9e74ad31 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.cpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.cpp @@ -562,7 +562,6 @@ extern "C" int print_loader_data_graph() { ClassLoaderDataGraph::print_on(tty); return 0; } -#endif // PRODUCT void ClassLoaderDataGraph::print_on(outputStream * const out) { ClassLoaderDataGraphIterator iter; @@ -570,5 +569,6 @@ void ClassLoaderDataGraph::print_on(outputStream * const out) { cld->print_on(out); } } +#endif // PRODUCT void ClassLoaderDataGraph::print() { print_on(tty); } diff --git a/src/hotspot/share/classfile/classLoaderDataGraph.hpp b/src/hotspot/share/classfile/classLoaderDataGraph.hpp index 72aefc1cfd8db..49f15e8af33df 100644 --- a/src/hotspot/share/classfile/classLoaderDataGraph.hpp +++ b/src/hotspot/share/classfile/classLoaderDataGraph.hpp @@ -107,7 +107,7 @@ class ClassLoaderDataGraph : public AllStatic { static bool has_metaspace_oom() { return _metaspace_oom; } static void set_metaspace_oom(bool value) { _metaspace_oom = value; } - static void print_on(outputStream * const out) ; + static void print_on(outputStream * const out) PRODUCT_RETURN; static void print(); static void verify(); From 8a204309d1064184d59b1c2c6139b77563f39bb8 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 24 Mar 2025 16:03:23 +0100 Subject: [PATCH 003/101] remove precompiled --- src/hotspot/share/oops/klassInfoLUT.cpp | 1 - src/hotspot/share/oops/klassInfoLUTEntry.cpp | 1 - src/hotspot/share/oops/klassInfoLUTEntry.hpp | 4 ++-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 2439d1f23666d..b5ecb70de2720 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -23,7 +23,6 @@ * */ -#include "precompiled.hpp" #include "logging/log.hpp" #include "memory/allocation.hpp" #include "oops/instanceKlass.inline.hpp" diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 803d4a345aac1..efdf49688cfea 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -23,7 +23,6 @@ * */ -#include "precompiled.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" #include "oops/instanceKlass.inline.hpp" diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index fd05a68e69c67..fb0e15eafbeb9 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2024, Red Hat, Inc. All rights reserved. - * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2025, Red Hat, Inc. All rights reserved. + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 26b8c3a651832bf0ce4b00579603903b04edaa49 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 2 Apr 2025 13:22:37 +0200 Subject: [PATCH 004/101] move KLUT init forward to init_globals --- src/hotspot/share/memory/universe.cpp | 10 ++++----- .../share/oops/klassInfoLUT.inline.hpp | 1 + src/hotspot/share/runtime/arguments.cpp | 21 ++++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 4c218edd2e19b..7dbc8e8758e34 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -409,11 +409,6 @@ void Universe::genesis(TRAPS) { assert(arrayOopDesc::length_offset_in_bytes() < static_cast(os::vm_page_size()), "Array length offset is expected to be less than the page size"); - // Initialize KLUT before starting to create any Klass - if (UseKLUT) { - KlassInfoLUT::initialize(); - } - { AutoModifyRestore temporarily(_bootstrapping, true); java_lang_Class::allocate_fixup_lists(); @@ -868,6 +863,11 @@ jint universe_init() { TraceTime timer("Genesis", TRACETIME_LOG(Info, startuptime)); + // Initialize KLUT before starting to create any Klass + if (UseKLUT) { + KlassInfoLUT::initialize(); + } + initialize_global_behaviours(); GCLogPrecious::initialize(); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 5da9a42d2f062..f26ce2ff3bf6c 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -37,6 +37,7 @@ inline unsigned KlassInfoLUT::num_entries() { } ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { + assert(_entries != nullptr, "LUT table does not exist"); assert(index < num_entries(), "oob (%x vs %x)", index, num_entries()); return _entries[index]; } diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 2578a09465f16..9848d07a00278 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -3687,16 +3687,17 @@ void Arguments::set_compact_headers_flags() { FLAG_SET_DEFAULT(UseCompressedClassPointers, true); } // KLUT - if (!UseCompactObjectHeaders) { - FLAG_SET_DEFAULT(UseKLUT, false); - } - if (UseKLUT && ObjectAlignmentInBytes != BytesPerWord) { - warning("UseKLUT requires default object alignment."); - FLAG_SET_DEFAULT(UseKLUT, false); - } - if (UseKLUT && UseSharedSpaces) { - warning("UseKLUT, for the moment, won't work with CDS enabled. Please specify -Xshare:off."); - FLAG_SET_DEFAULT(UseKLUT, false); + if (UseKLUT) { + if (!UseCompactObjectHeaders) { + warning("UseKLUT requires COH."); + FLAG_SET_DEFAULT(UseKLUT, false); + } else if (ObjectAlignmentInBytes != BytesPerWord) { + warning("UseKLUT requires default object alignment."); + FLAG_SET_DEFAULT(UseKLUT, false); + } else if (UseSharedSpaces) { + warning("UseKLUT, for the moment, won't work with CDS enabled. Please specify -Xshare:off."); + // FLAG_SET_DEFAULT(UseKLUT, false); + } } #endif } From 7264f16f7df1fedc9429e6899ed8c7cd97c6448d Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 2 Apr 2025 16:39:28 +0200 Subject: [PATCH 005/101] make KLUT work with CDS (late init for now :( ) --- src/hotspot/share/cds/archiveBuilder.cpp | 6 +++++ src/hotspot/share/oops/klass.cpp | 5 +++-- src/hotspot/share/oops/klass.hpp | 6 +++++ src/hotspot/share/oops/klassInfoLUT.cpp | 22 +++++++++++++++++-- src/hotspot/share/oops/klassInfoLUT.hpp | 4 +++- .../share/oops/klassInfoLUT.inline.hpp | 7 +++++- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 4 ++++ src/hotspot/share/oops/klassInfoLUTEntry.hpp | 9 ++++---- src/hotspot/share/runtime/arguments.cpp | 3 --- 9 files changed, 53 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index c1ffd60370b42..5f20d40ff405b 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -52,6 +52,7 @@ #include "memory/resourceArea.hpp" #include "oops/compressedKlass.inline.hpp" #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oopHandle.inline.hpp" @@ -778,6 +779,11 @@ void ArchiveBuilder::make_klasses_shareable() { if (k->is_instance_klass()) { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } +#ifdef ASSERT + // every archived Klass shall carry a valid KLUTE. + KlassLUTEntry e(k->klute()); + e.verify_against(k); +#endif // ASSERT } for (int i = 0; i < klasses()->length(); i++) { diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 32d0bdb3de552..3ddfecd7aaec4 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -291,7 +291,7 @@ static markWord make_prototype(const Klass* kls) { return prototype; } -Klass::Klass() : _kind(UnknownKlassKind) { +Klass::Klass() : _klute(KlassLUTEntry::invalid_entry), _kind(UnknownKlassKind) { assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for cds"); } @@ -299,7 +299,8 @@ Klass::Klass() : _kind(UnknownKlassKind) { // which zeros out memory - calloc equivalent. // The constructor is also used from CppVtableCloner, // which doesn't zero out the memory before calling the constructor. -Klass::Klass(KlassKind kind) : _kind(kind), +Klass::Klass(KlassKind kind) : _klute(KlassLUTEntry::invalid_entry), + _kind(kind), _prototype_header(make_prototype(this)), _shared_class_path_index(-1) { CDS_ONLY(_shared_class_flags = 0;) diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index be98d1742fac4..776e2176b38df 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -151,6 +151,9 @@ class Klass : public Metadata { // because it is frequently queried. jint _layout_helper; + // KLUT entry + uint32_t _klute; + // Klass kind used to resolve the runtime type of the instance. // - Used to implement devirtualized oop closure dispatching. // - Various type checking in the JVM @@ -252,6 +255,9 @@ class Klass : public Metadata { enum class StaticLookupMode { find, skip }; enum class PrivateLookupMode { find, skip }; + uint32_t klute() const { return _klute; } + void set_klute(uint32_t v) { _klute = v; } + virtual bool is_klass() const { return true; } // super() cannot be InstanceKlass* -- Java arrays are covariant, and _super is used diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index b5ecb70de2720..165bf4fe41737 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -85,12 +85,13 @@ int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { return 0; } -void KlassInfoLUT::register_klass(const Klass* k) { +void KlassInfoLUT::register_klass(Klass* k) { assert(UseKLUT, "?"); const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // grr why is this nonconst assert(nk < num_entries(), "oob"); - KlassLUTEntry e(k); + KlassLUTEntry e(k); // This calculates the klute entry from Klass properties _entries[nk] = e.value(); + k->set_klute(e.value()); // also store in class itself char tmp[1024]; log_debug(klut)("registered Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " "Klute: " INT32_FORMAT_X_0 ", %s", @@ -120,6 +121,23 @@ void KlassInfoLUT::register_klass(const Klass* k) { } } +KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { + const Klass* k = CompressedKlassPointers::decode(nk); + // We only tolerate this for CDS. In that case, we expect the original class - during dumptime - + // to be registered already, so it should have a valid KLUTE entry set which we only need to copy. + // Note: we cannot calculate the klute here, since at this point the Klass has no associated + // class loader data... + assert(k->is_shared(), "Only for CDS classes"); + KlassLUTEntry e(k->klute()); + char tmp[1024]; + log_debug(klut)("late-register Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " + "Klute: " INT32_FORMAT_X_0 ", %s", + p2i(k), nk, e.value(), k->name()->as_C_string(tmp, sizeof(tmp))); + assert(!e.is_invalid(), "Cannot be"); + _entries[nk] = e.value(); + return e; +} + // Counters and incrementors #define XX(xx) \ volatile uint64_t counter_##xx = 0; \ diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 32f0254e13f9c..9b1631991b8be 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -88,11 +88,13 @@ class KlassInfoLUT : public AllStatic { static void log_hit(KlassLUTEntry klute); #endif + static KlassLUTEntry late_register_klass(narrowKlass nk); + public: static void initialize(); - static void register_klass(const Klass* k); + static void register_klass(Klass* k); static inline KlassLUTEntry get_entry(narrowKlass k); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index f26ce2ff3bf6c..ba6d0c8afc7ff 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -46,7 +46,12 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::get_entry(narrowKlass nk) { const uint32_t v = at(nk); KlassLUTEntry e(v); - assert(!e.is_invalid(), "invalid?"); + if (e.is_invalid()) { + // This path only exists because it is so very difficult to iterate CDS classes after loading + // CDS archives. See discussion surrounding 8353225. Hopefully we can remove this in the future. + // And equally hopefully branch prediction takes the sting out of this branch in real oop iteration. + return late_register_klass(nk); + } #ifdef KLUT_ENABLE_EXPENSIVE_STATS update_hit_stats(e); #endif diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index efdf49688cfea..53fcb46f8f42e 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -288,6 +288,10 @@ void KlassLUTEntry::verify_against(const Klass* k) const { KlassLUTEntry::KlassLUTEntry(const Klass* k) : _v(build_from(k)) { } +void KlassLUTEntry::print(outputStream* st) const { + st->print("%X (Kind: %d Loader: %d)", value(), kind(), loader_index()); +} + // Helper function, prints current limits void KlassLUTEntry::print_limits(outputStream* st) { st->print_cr("IKE Limits: instance byte size %zu, omb1 count: %zu, omb1 byte offset: %zu, omb2 oop count: %zu, omb2 byte offset: %zu", diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index fb0e15eafbeb9..4fd817fb0cec5 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -36,14 +36,14 @@ class outputStream; // msb lsb // -// invalid_entry (debug zap): 1111 1111 1111 1111 1111 1111 1111 1111 (relies on kind == 0b111 == 7 being invalid) +// invalid_entry (debug zap): 1111 1111 1111 1111 1111 1111 1111 1111 (relies on ClassKind == 0b111 == 7 being invalid) // // All valid entries: KKKL L... .... .... .... .... .... .... // // InstanceKlass: KKKL LSSS SSSO OOOO CCCC CCOO OOCC CCCC // 2 2222 2222 2211 1111 1111 // InstanceKlass, has_no_addinfo: KKKL L000 0000 0000 0000 0000 0000 0000 (all IK specific bits 0) (note: means that "0" is a valid IK entry with no add. info) -// InstanceKlass, has no oopmap entries: KKKL L... .... .... .... .... 0000 0000 (omb count bits are 0) (only valid if !has_no_addinfo) +// InstanceKlass, has no oopmap entries: KKKL LSSS SSS. .... .... .... 0000 0000 (omb count bits are 0) (only valid if !has_no_addinfo) // // ArrayKlass: KKKL L--- tttt tttt hhhh hhhh eeee eeee // @@ -170,7 +170,6 @@ class KlassLUTEntry { // We use kind=7=0b111 (invalid), and set the rest of the bits also to 1 static constexpr uint32_t invalid_entry = 0xFFFFFFFF; -// inline KlassLUTEntry() : _v(invalid_entry) {} inline KlassLUTEntry(uint32_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} @@ -241,6 +240,8 @@ class KlassLUTEntry { // Helper function, prints current limits static void print_limits(outputStream* st); -}; // KlassInfoLUEntryIK + void print(outputStream* st) const; + +}; // KlassInfoLUEntry #endif // SHARE_OOPS_KLASSINFOLUTENTRY_HPP diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 9848d07a00278..3c2d741a50d39 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -3694,9 +3694,6 @@ void Arguments::set_compact_headers_flags() { } else if (ObjectAlignmentInBytes != BytesPerWord) { warning("UseKLUT requires default object alignment."); FLAG_SET_DEFAULT(UseKLUT, false); - } else if (UseSharedSpaces) { - warning("UseKLUT, for the moment, won't work with CDS enabled. Please specify -Xshare:off."); - // FLAG_SET_DEFAULT(UseKLUT, false); } } #endif From a669cdd4030a2ecd7f27f96e334de23b141605e0 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 3 Apr 2025 12:10:28 +0200 Subject: [PATCH 006/101] wip --- src/hotspot/share/cds/archiveBuilder.cpp | 8 ++- .../share/classfile/classFileParser.cpp | 5 +- .../classfile/systemDictionaryShared.cpp | 1 + src/hotspot/share/oops/klass.cpp | 10 +++ src/hotspot/share/oops/klass.hpp | 3 +- src/hotspot/share/oops/klassInfoLUT.cpp | 63 ++++++++++++++----- src/hotspot/share/oops/klassInfoLUT.hpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 14 ++--- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 8 +-- src/hotspot/share/oops/objArrayKlass.cpp | 5 +- src/hotspot/share/oops/typeArrayKlass.cpp | 5 +- 11 files changed, 80 insertions(+), 44 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 5f20d40ff405b..b2f4bd529918a 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -780,9 +780,11 @@ void ArchiveBuilder::make_klasses_shareable() { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } #ifdef ASSERT - // every archived Klass shall carry a valid KLUTE. - KlassLUTEntry e(k->klute()); - e.verify_against(k); + if (UseKLUT) { + // every archived Klass shall carry a valid KLUTE. + KlassLUTEntry e(k->klute()); + e.verify_against_klass(k); + } #endif // ASSERT } diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp index f88a3769af15d..efe72706bd6cb 100644 --- a/src/hotspot/share/classfile/classFileParser.cpp +++ b/src/hotspot/share/classfile/classFileParser.cpp @@ -53,7 +53,6 @@ #include "oops/instanceKlass.inline.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/klass.inline.hpp" -#include "oops/klassInfoLUT.hpp" #include "oops/klassVtable.hpp" #include "oops/metadata.hpp" #include "oops/method.inline.hpp" @@ -5241,9 +5240,7 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik, // in order for it to not be destroyed in the ClassFileParser destructor. set_klass_to_deallocate(nullptr); - if (UseKLUT) { - KlassInfoLUT::register_klass(ik); - } + ik->register_with_klut(); // it's official set_klass(ik); diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp index 5c4ee3f9452ad..521481889528b 100644 --- a/src/hotspot/share/classfile/systemDictionaryShared.cpp +++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp @@ -430,6 +430,7 @@ InstanceKlass* SystemDictionaryShared::find_or_load_shared_class( if (k != nullptr) { SharedClassLoadingMark slm(THREAD, k); k = find_or_define_instance_class(name, class_loader, k, CHECK_NULL); + k->register_with_klut(); } } } diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 3ddfecd7aaec4..9d0c17a6ec402 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -46,6 +46,8 @@ #include "oops/compressedOops.inline.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" #include "oops/oopHandle.inline.hpp" @@ -1352,3 +1354,11 @@ void Klass::on_secondary_supers_verification_failure(Klass* super, Klass* sub, b fatal("%s: %s implements %s: linear_search: %d; table_lookup: %d", msg, sub->external_name(), super->external_name(), linear_result, table_result); } + +void Klass::register_with_klut() { + if (UseKLUT) { + _klute = KlassLUTEntry::invalid_entry; + KlassLUTEntry e = KlassInfoLUT::register_klass(this); + _klute = e.value(); + } +} diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 776e2176b38df..95930a2df5a33 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -255,8 +255,9 @@ class Klass : public Metadata { enum class StaticLookupMode { find, skip }; enum class PrivateLookupMode { find, skip }; + // Klute handling uint32_t klute() const { return _klute; } - void set_klute(uint32_t v) { _klute = v; } + void register_with_klut(); virtual bool is_klass() const { return true; } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 165bf4fe41737..9c59fc7d6edbe 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -25,6 +25,7 @@ #include "logging/log.hpp" #include "memory/allocation.hpp" +#include "cds/cdsConfig.hpp" #include "oops/instanceKlass.inline.hpp" #include "oops/klass.hpp" #include "oops/klassInfoLUT.inline.hpp" @@ -55,6 +56,10 @@ void KlassInfoLUT::initialize() { _entries = NEW_C_HEAP_ARRAY(uint32_t, num_entries(), mtClass); log_info(klut)("KLUT initialized (normal pages): " RANGEFMT, RANGEFMTARGS(_entries, memory_needed)); } + // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. + // We also do this in debug builds, unconditionally. + const bool do_zap = DEBUG_ONLY(true ||) + CDSConfig::is_using_archive() || CDSConfig::is_dumping_archive(); for (unsigned i = 0; i < num_entries(); i++) { _entries[i] = KlassLUTEntry::invalid_entry; } @@ -85,26 +90,52 @@ int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { return 0; } -void KlassInfoLUT::register_klass(Klass* k) { +KlassLUTEntry KlassInfoLUT::register_klass(Klass* k) { assert(UseKLUT, "?"); - const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // grr why is this nonconst - assert(nk < num_entries(), "oob"); - KlassLUTEntry e(k); // This calculates the klute entry from Klass properties - _entries[nk] = e.value(); - k->set_klute(e.value()); // also store in class itself + const narrowKlass nk = CompressedKlassPointers::encode(k); + assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); + uint32_t klute; char tmp[1024]; - log_debug(klut)("registered Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " - "Klute: " INT32_FORMAT_X_0 ", %s", - p2i(k), nk, e.value(), k->name()->as_C_string(tmp, sizeof(tmp))); + + bool already_registered = false; + const KlassLUTEntry e_stored_in_klass(k->klute()); + if (!e_stored_in_klass.is_invalid()) { + // The Klass may already carry the pre-computed klute if it was loaded + // from a shared archive; in that case, it would have been calculated when the + // Klass was originally (dynamically) loaded at dump time. In that case avoid + // recalculating it; just update the table value from that stored value. + + if (e_stored_in_klass.value() == _entries[nk]) { + log_debug(klut)("Klass " PTR_FORMAT " (%s) already registered, nothing to do.", + p2i(k), k->name()->as_C_string(tmp, sizeof(tmp))); + } else { + log_debug(klut)("Klass " PTR_FORMAT " (%s): updating table value already registered, nothing to do.", + p2i(k), k->name()->as_C_string(tmp, sizeof(tmp))); + } + + already_registered = (e_stored_in_klass.value() == _entries[nk]); + // Store Klute into table + _entries[nk] = e_stored_in_klass.value(); + + if () + log_debug(klut)("registered Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " + "Klute: " INT32_FORMAT_X_0 ",%s%s %s", + p2i(k), nk, klute, + (k->is_shared() ? " shared," : ""), + (already_registered ? " already registered," : ""), + k->name()->as_C_string(tmp, sizeof(tmp))); + } + + #ifdef ASSERT // sanity checks { - // We use at(), not get_entry(), since we don't want to log or count stats KlassLUTEntry e2(at(nk)); - assert(e2.value() == e.value(), "Sanity"); - e2.verify_against(k); + assert(e2.value() == klute, "Sanity"); + e2.verify_against_klass(k); } #endif // ASSERT + // update register stats switch (k->kind()) { case Klass::InstanceKlassKind: inc_registered_IK(); break; @@ -119,6 +150,8 @@ void KlassInfoLUT::register_klass(Klass* k) { if (k->is_abstract() || k->is_interface()) { inc_registered_IK_for_abstract_or_interface(); } + + return KlassLUTEntry(klute); } KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { @@ -131,8 +164,10 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { KlassLUTEntry e(k->klute()); char tmp[1024]; log_debug(klut)("late-register Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " - "Klute: " INT32_FORMAT_X_0 ", %s", - p2i(k), nk, e.value(), k->name()->as_C_string(tmp, sizeof(tmp))); + "Klute: " INT32_FORMAT_X_0 ",%s %s", + p2i(k), nk, e.value(), + (k->is_shared() ? " shared," : ""), + k->name()->as_C_string(tmp, sizeof(tmp))); assert(!e.is_invalid(), "Cannot be"); _entries[nk] = e.value(); return e; diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 9b1631991b8be..9d6e46fa67b1f 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -94,7 +94,7 @@ class KlassInfoLUT : public AllStatic { static void initialize(); - static void register_klass(Klass* k); + static KlassLUTEntry register_klass(Klass* k); static inline KlassLUTEntry get_entry(narrowKlass k); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 53fcb46f8f42e..3884ed53f9ff6 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -77,7 +77,7 @@ uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_ if (Klass::layout_helper_needs_slow_path(lh)) { if (ik->is_abstract() || ik->is_interface()) { - NOPE("Size not trivially computable (klass abstract or interface)"); + NOPE("klass is abstract or interface"); // Please note that we could represent abstract or interface classes, but atm there is not // much of a point. } else { @@ -159,7 +159,7 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { } -uint32_t KlassLUTEntry::build_from(const Klass* k) { +KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { uint32_t value = invalid_entry; if (k->is_array_klass()) { @@ -169,16 +169,17 @@ uint32_t KlassLUTEntry::build_from(const Klass* k) { const char* not_encodable_reason = nullptr; value = build_from_ik(InstanceKlass::cast(k), not_encodable_reason); if (not_encodable_reason != nullptr) { - log_debug(klut)("klass klute register: %s cannot be encoded because: %s.", k->name()->as_C_string(), not_encodable_reason); + log_debug(klut)("InstanceKlass " PTR_FORMAT ": (%s) cannot encoded details: %s.", p2i(k), + k->name()->as_C_string(), not_encodable_reason); } } - return value; + return KlassLUTEntry(value); } #ifdef ASSERT -void KlassLUTEntry::verify_against(const Klass* k) const { +void KlassLUTEntry::verify_against_klass(const Klass* k) const { // General static asserts that need access to private members, but I don't want // to place them in a header @@ -285,9 +286,6 @@ void KlassLUTEntry::verify_against(const Klass* k) const { #endif // ASSERT -KlassLUTEntry::KlassLUTEntry(const Klass* k) : _v(build_from(k)) { -} - void KlassLUTEntry::print(outputStream* st) const { st->print("%X (Kind: %d Loader: %d)", value(), kind(), loader_index()); } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 4fd817fb0cec5..2c70c15070c8e 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -161,8 +161,6 @@ class KlassLUTEntry { static uint32_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); static uint32_t build_from_ak(const ArrayKlass* k); - static uint32_t build_from(const Klass* k); - public: // Invalid entries are entries that have not been set yet. @@ -173,14 +171,14 @@ class KlassLUTEntry { inline KlassLUTEntry(uint32_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} - KlassLUTEntry(const Klass* k); - // Note: all entries should be valid. An invalid entry indicates // an error somewhere. bool is_invalid() const { return _v.raw == invalid_entry; } + static KlassLUTEntry build_from_klass(const Klass* k); + #ifdef ASSERT - void verify_against(const Klass* k) const; + void verify_against_klass(const Klass* k) const; #endif uint32_t value() const { return _v.raw; } diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index afe4daba867ac..bd8dfafe33b43 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -36,7 +36,6 @@ #include "oops/arrayKlass.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" -#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" @@ -151,9 +150,7 @@ ObjArrayKlass::ObjArrayKlass(int n, Klass* element_klass, Symbol* name) : ArrayK assert(is_objArray_klass(), "sanity"); // Add to KLUT - if (UseKLUT) { - KlassInfoLUT::register_klass(this); - } + register_with_klut(); } size_t ObjArrayKlass::oop_size(oop obj) const { diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index d3d2ae2888fd0..8609f7a022684 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -34,7 +34,6 @@ #include "oops/arrayKlass.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" -#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" #include "oops/typeArrayKlass.inline.hpp" @@ -74,9 +73,7 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type, JFR_ONLY(ASSIGN_PRIMITIVE_CLASS_ID(ak);) // Add to KLUT - if (UseKLUT) { - KlassInfoLUT::register_klass(ak); - } + ak->register_with_klut(); return ak; } From e23d21e0173b8727534a2fb6e384cd08fd0a9cc3 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 3 Apr 2025 17:35:21 +0200 Subject: [PATCH 007/101] wip --- src/hotspot/share/memory/universe.cpp | 10 +- src/hotspot/share/oops/compressedKlass.hpp | 3 + src/hotspot/share/oops/klassInfoLUT.cpp | 93 +++++++++---------- src/hotspot/share/oops/klassInfoLUT.hpp | 4 +- .../share/oops/klassInfoLUT.inline.hpp | 4 - src/hotspot/share/oops/klassInfoLUTEntry.cpp | 7 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 2 +- 7 files changed, 60 insertions(+), 63 deletions(-) diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 7dbc8e8758e34..2d0f1607d442c 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -863,11 +863,6 @@ jint universe_init() { TraceTime timer("Genesis", TRACETIME_LOG(Info, startuptime)); - // Initialize KLUT before starting to create any Klass - if (UseKLUT) { - KlassInfoLUT::initialize(); - } - initialize_global_behaviours(); GCLogPrecious::initialize(); @@ -892,6 +887,11 @@ jint universe_init() { Metaspace::global_initialize(); + // Initialize KLUT before starting to create any Klass + if (UseKLUT) { + KlassInfoLUT::initialize(); + } + // Initialize performance counters for metaspaces MetaspaceCounters::initialize_performance_counters(); diff --git a/src/hotspot/share/oops/compressedKlass.hpp b/src/hotspot/share/oops/compressedKlass.hpp index 8e39e4c05397f..5cd2a3431b72b 100644 --- a/src/hotspot/share/oops/compressedKlass.hpp +++ b/src/hotspot/share/oops/compressedKlass.hpp @@ -206,6 +206,9 @@ class CompressedKlassPointers : public AllStatic { // structures outside this range). static void initialize(address addr, size_t len); + static bool pre_initialized() { return _narrow_klass_pointer_bits != -1; } + static bool fully_initialized() { return _base != (address)-1; } + static void print_mode(outputStream* st); // Can only be used after initialization diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 9c59fc7d6edbe..85f7730d01979 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -36,33 +36,39 @@ ClassLoaderData* KlassInfoLUT::_common_loaders[4] = { nullptr }; uint32_t* KlassInfoLUT::_entries = nullptr; +unsigned KlassInfoLUT::_num_entries = -1; void KlassInfoLUT::initialize() { assert(UseKLUT, "?"); - assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "sanity"); - const size_t memory_needed = sizeof(uint32_t) * num_entries(); + assert(CompressedKlassPointers::fully_initialized(), "Too early"); + assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "Use only for COH"); + assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); + + const narrowKlass highest_nk = CompressedKlassPointers::highest_valid_narrow_klass_id(); + _num_entries = highest_nk; + + const size_t memory_needed = sizeof(uint32_t) * _num_entries; if (UseLargePages) { const size_t large_page_size = os::large_page_size(); if (is_aligned(memory_needed, large_page_size)) { char* memory = os::reserve_memory_special(memory_needed, large_page_size, large_page_size, nullptr, false); if (memory != nullptr) { _entries = (uint32_t*)memory; - log_info(klut)("KLUT initialized (large pages): " RANGEFMT, RANGEFMTARGS(_entries, memory_needed)); + log_info(klut)("KLUT initialized (%u entries, using large pages): " RANGEFMT, + _num_entries, RANGEFMTARGS(_entries, memory_needed)); } } } if (_entries == nullptr) { // Fallback, just use C-heap. _entries = NEW_C_HEAP_ARRAY(uint32_t, num_entries(), mtClass); - log_info(klut)("KLUT initialized (normal pages): " RANGEFMT, RANGEFMTARGS(_entries, memory_needed)); + log_info(klut)("KLUT initialized (%u entries, using normal pages): " RANGEFMT, + _num_entries, RANGEFMTARGS(_entries, memory_needed)); } + // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. - // We also do this in debug builds, unconditionally. - const bool do_zap = DEBUG_ONLY(true ||) - CDSConfig::is_using_archive() || CDSConfig::is_dumping_archive(); - for (unsigned i = 0; i < num_entries(); i++) { - _entries[i] = KlassLUTEntry::invalid_entry; - } + memset(_entries, 0xff, _num_entries * sizeof(uint32_t)); + assert(_entries[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff } int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { @@ -90,46 +96,42 @@ int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { return 0; } +static void log_klass_registration(const Klass* k, narrowKlass nk, uint32_t klute, const char* message) { + char tmp[1024]; + log_debug(klut)("Klass " PTR_FORMAT ", nk %u, klute: " INT32_FORMAT_X_0 ": %s %s%s", + p2i(k), nk, klute, + message, + (k->is_shared() ? "(shared) " : ""), + k->name()->as_C_string(tmp, sizeof(tmp))); +} + KlassLUTEntry KlassInfoLUT::register_klass(Klass* k) { assert(UseKLUT, "?"); const narrowKlass nk = CompressedKlassPointers::encode(k); assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); - uint32_t klute; - char tmp[1024]; - bool already_registered = false; - const KlassLUTEntry e_stored_in_klass(k->klute()); - if (!e_stored_in_klass.is_invalid()) { - // The Klass may already carry the pre-computed klute if it was loaded - // from a shared archive; in that case, it would have been calculated when the - // Klass was originally (dynamically) loaded at dump time. In that case avoid - // recalculating it; just update the table value from that stored value. - - if (e_stored_in_klass.value() == _entries[nk]) { - log_debug(klut)("Klass " PTR_FORMAT " (%s) already registered, nothing to do.", - p2i(k), k->name()->as_C_string(tmp, sizeof(tmp))); + uint32_t klute = k->klute(); + if (KlassLUTEntry(klute).is_invalid()) { + // Calculate klute from Klass properties and update the table value. + klute = KlassLUTEntry::build_from_klass(k); + _entries[nk] = klute; + log_klass_registration(k, nk, klute, "registered"); + } else { + // The Klass may already carry the pre-computed klute. That can happen if it was loaded from a shared + // archive, in which case it contains the klute computed at (dynamic) load time when dumping. In that + // case just reuse that value. + if (klute == _entries[nk]) { + log_klass_registration(k, nk, klute, "already registered"); } else { - log_debug(klut)("Klass " PTR_FORMAT " (%s): updating table value already registered, nothing to do.", - p2i(k), k->name()->as_C_string(tmp, sizeof(tmp))); + _entries[nk] = klute; + log_klass_registration(k, nk, klute, "updated table value for"); } - already_registered = (e_stored_in_klass.value() == _entries[nk]); - // Store Klute into table - _entries[nk] = e_stored_in_klass.value(); - - if () - log_debug(klut)("registered Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " - "Klute: " INT32_FORMAT_X_0 ",%s%s %s", - p2i(k), nk, klute, - (k->is_shared() ? " shared," : ""), - (already_registered ? " already registered," : ""), - k->name()->as_C_string(tmp, sizeof(tmp))); } - #ifdef ASSERT - // sanity checks { + // sanity checks KlassLUTEntry e2(at(nk)); assert(e2.value() == klute, "Sanity"); e2.verify_against_klass(k); @@ -161,16 +163,11 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { // Note: we cannot calculate the klute here, since at this point the Klass has no associated // class loader data... assert(k->is_shared(), "Only for CDS classes"); - KlassLUTEntry e(k->klute()); - char tmp[1024]; - log_debug(klut)("late-register Klass with KLUT: Klass*=" PTR_FORMAT ", nKlass %u, " - "Klute: " INT32_FORMAT_X_0 ",%s %s", - p2i(k), nk, e.value(), - (k->is_shared() ? " shared," : ""), - k->name()->as_C_string(tmp, sizeof(tmp))); - assert(!e.is_invalid(), "Cannot be"); - _entries[nk] = e.value(); - return e; + const uint32_t klute = k->klute(); + assert(!KlassLUTEntry(klute).is_invalid(), "Must be a valid klute"); + _entries[nk] = klute; + log_klass_registration(k, nk, klute, "late-registered"); + return KlassLUTEntry(klute); } // Counters and incrementors diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 9d6e46fa67b1f..d7b507580a98b 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -44,7 +44,9 @@ class KlassInfoLUT : public AllStatic { static ClassLoaderData* _common_loaders[4]; // See "loader" bits in Klute static uint32_t* _entries; - static inline unsigned num_entries(); + static unsigned _num_entries; + + static inline unsigned num_entries() { return _num_entries; } static inline uint32_t at(unsigned index); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index ba6d0c8afc7ff..ea981998960d4 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -32,10 +32,6 @@ #include "utilities/globalDefinitions.hpp" #include "utilities/debug.hpp" -inline unsigned KlassInfoLUT::num_entries() { - return nth_bit(CompressedKlassPointers::narrow_klass_pointer_bits()); -} - ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { assert(_entries != nullptr, "LUT table does not exist"); assert(index < num_entries(), "oob (%x vs %x)", index, num_entries()); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 3884ed53f9ff6..945559870c2ff 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -159,7 +159,7 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { } -KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { +uint32_t KlassLUTEntry::build_from_klass(const Klass* k) { uint32_t value = invalid_entry; if (k->is_array_klass()) { @@ -169,12 +169,11 @@ KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { const char* not_encodable_reason = nullptr; value = build_from_ik(InstanceKlass::cast(k), not_encodable_reason); if (not_encodable_reason != nullptr) { - log_debug(klut)("InstanceKlass " PTR_FORMAT ": (%s) cannot encoded details: %s.", p2i(k), + log_debug(klut)("InstanceKlass " PTR_FORMAT ": (%s) cannot encode details: %s.", p2i(k), k->name()->as_C_string(), not_encodable_reason); } } - return KlassLUTEntry(value); - + return value; } #ifdef ASSERT diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 2c70c15070c8e..758d280112bea 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -175,7 +175,7 @@ class KlassLUTEntry { // an error somewhere. bool is_invalid() const { return _v.raw == invalid_entry; } - static KlassLUTEntry build_from_klass(const Klass* k); + static uint32_t build_from_klass(const Klass* k); #ifdef ASSERT void verify_against_klass(const Klass* k) const; From c08234f522a014d51b3b34e2b60262c1e8a42239 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 3 Apr 2025 17:46:04 +0200 Subject: [PATCH 008/101] allow-latereg-only-when-built-with-cds --- src/hotspot/share/oops/klassInfoLUT.cpp | 20 ++++++++++--------- src/hotspot/share/oops/klassInfoLUT.hpp | 3 +++ .../share/oops/klassInfoLUT.inline.hpp | 10 +++++++++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 85f7730d01979..3876d31092372 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -156,19 +156,21 @@ KlassLUTEntry KlassInfoLUT::register_klass(Klass* k) { return KlassLUTEntry(klute); } +#if INCLUDE_CDS +// We only tolerate this for CDS. In that case, we expect the original class - during dumptime - +// to be registered already, so it should have a valid KLUTE entry set which we only need to copy. +// Note: we cannot calculate the klute here, since at this point the Klass has no associated +// class loader data... KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { const Klass* k = CompressedKlassPointers::decode(nk); - // We only tolerate this for CDS. In that case, we expect the original class - during dumptime - - // to be registered already, so it should have a valid KLUTE entry set which we only need to copy. - // Note: we cannot calculate the klute here, since at this point the Klass has no associated - // class loader data... assert(k->is_shared(), "Only for CDS classes"); - const uint32_t klute = k->klute(); - assert(!KlassLUTEntry(klute).is_invalid(), "Must be a valid klute"); - _entries[nk] = klute; - log_klass_registration(k, nk, klute, "late-registered"); - return KlassLUTEntry(klute); + const KlassLUTEntry klute(k->klute()); + assert(!klute.is_invalid(), "Must be a valid klute"); + _entries[nk] = klute.value(); + log_klass_registration(k, nk, klute.value(), "late-registered"); + return klute; } +#endif // INCLUDE_CDS // Counters and incrementors #define XX(xx) \ diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index d7b507580a98b..f8b48584f77cd 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -90,7 +90,10 @@ class KlassInfoLUT : public AllStatic { static void log_hit(KlassLUTEntry klute); #endif +#if INCLUDE_CDS + // only allowed (grudgingly) when built with CDS, until I figure out a better solution. static KlassLUTEntry late_register_klass(narrowKlass nk); +#endif public: diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index ea981998960d4..2bc2e5d6ad615 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -42,18 +42,26 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::get_entry(narrowKlass nk) { const uint32_t v = at(nk); KlassLUTEntry e(v); +#if INCLUDE_CDS if (e.is_invalid()) { - // This path only exists because it is so very difficult to iterate CDS classes after loading + // This branch only exists because it is so very difficult to iterate CDS classes after loading // CDS archives. See discussion surrounding 8353225. Hopefully we can remove this in the future. // And equally hopefully branch prediction takes the sting out of this branch in real oop iteration. + // In order to measure without this branch, build without CDS. return late_register_klass(nk); } +#else + assert(!e.is_invalid(), "must never be invalid"); +#endif + #ifdef KLUT_ENABLE_EXPENSIVE_STATS update_hit_stats(e); #endif + #ifdef KLUT_ENABLE_EXPENSIVE_LOG log_hit(e); #endif + return e; } From 3cc05ea80399b18a649dcaa8c1ce9b2a6cf623c6 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 4 Apr 2025 07:08:12 +0200 Subject: [PATCH 009/101] WIP --- src/hotspot/share/oops/klassInfoLUT.cpp | 5 ++++- src/hotspot/share/oops/klassInfoLUT.hpp | 2 ++ src/hotspot/share/oops/klassInfoLUTEntry.hpp | 5 ++++- src/hotspot/share/oops/oop.inline.hpp | 18 ++++++++---------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 3876d31092372..307914d455f25 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -39,8 +39,11 @@ uint32_t* KlassInfoLUT::_entries = nullptr; unsigned KlassInfoLUT::_num_entries = -1; void KlassInfoLUT::initialize() { - assert(UseKLUT, "?"); assert(CompressedKlassPointers::fully_initialized(), "Too early"); + + // We allocate a lookup table only if we can use the narrowKlass for a lookup reasonably well. + // We can do this only if the nKlass is small enough - we allow it for COH (22 bit nKlass with + // 10 bit shift means we have a small and condensed table). We don't bother for -COH, assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "Use only for COH"); assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index f8b48584f77cd..280beef636758 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -50,6 +50,8 @@ class KlassInfoLUT : public AllStatic { static inline uint32_t at(unsigned index); + static void allocate_lookup_table(); + // register stats are not expensive #define REGISTER_STATS_DO(f) \ f(registered_IK) \ diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 758d280112bea..489b711142375 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -149,7 +149,7 @@ class KlassLUTEntry { U(uint32_t v) : raw(v) {} }; - const U _v; + U _v; // The limits to what we can numerically represent in an (InstanceKlass) Entry static constexpr size_t ik_wordsize_limit = nth_bit(bits_ik_wordsize); @@ -171,6 +171,9 @@ class KlassLUTEntry { inline KlassLUTEntry(uint32_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} + inline KlassLUTEntry& operator=(uint32_t v) { _v = v; return *this; } + inline KlassLUTEntry& KlassLUTEntry(const KlassLUTEntry& other) { _v = other._v; return *this; } + // Note: all entries should be valid. An invalid entry indicates // an error somewhere. bool is_invalid() const { return _v.raw == invalid_entry; } diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index b7b2834b22761..0de102d9bfecc 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -233,19 +233,17 @@ bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_k bool oopDesc::is_array() const { return klass()->is_array_klass(); } bool oopDesc::is_objArray() const { - if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); - return klute.is_obj_array(); - } - return klass()->is_objArray_klass(); + const KlassLUTEntry klute = UseKLUT ? + KlassInfoLUT::get_entry(mark().narrow_klass()) : + KlassLUTEntry(klass()->klute()); + return klute.is_obj_array(); } bool oopDesc::is_typeArray() const { - if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); - return klute.is_type_array(); - } - return klass()->is_typeArray_klass(); + const KlassLUTEntry klute = UseKLUT ? + KlassInfoLUT::get_entry(mark().narrow_klass()) : + KlassLUTEntry(klass()->klute()); + return klute.is_type_array(); } template From 8280ceeb70f000bd16217e7944723a848e4aab0d Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 5 Apr 2025 08:15:14 +0200 Subject: [PATCH 010/101] wip --- src/hotspot/share/oops/klassInfoLUT.cpp | 48 ++++++++++---------- src/hotspot/share/oops/klassInfoLUT.hpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 4 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 9 ++-- src/hotspot/share/oops/oop.inline.hpp | 23 ++++++---- 5 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 307914d455f25..edb15bf8d2b36 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -99,44 +99,45 @@ int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { return 0; } -static void log_klass_registration(const Klass* k, narrowKlass nk, uint32_t klute, const char* message) { +static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry klute, const char* message) { char tmp[1024]; log_debug(klut)("Klass " PTR_FORMAT ", nk %u, klute: " INT32_FORMAT_X_0 ": %s %s%s", - p2i(k), nk, klute, + p2i(k), nk, klute.value(), message, (k->is_shared() ? "(shared) " : ""), k->name()->as_C_string(tmp, sizeof(tmp))); } -KlassLUTEntry KlassInfoLUT::register_klass(Klass* k) { +KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { assert(UseKLUT, "?"); - const narrowKlass nk = CompressedKlassPointers::encode(k); + const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // TODO CompressedKlassPointers should support correct constness assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); - uint32_t klute = k->klute(); - if (KlassLUTEntry(klute).is_invalid()) { - // Calculate klute from Klass properties and update the table value. - klute = KlassLUTEntry::build_from_klass(k); - _entries[nk] = klute; - log_klass_registration(k, nk, klute, "registered"); - } else { - // The Klass may already carry the pre-computed klute. That can happen if it was loaded from a shared - // archive, in which case it contains the klute computed at (dynamic) load time when dumping. In that - // case just reuse that value. - if (klute == _entries[nk]) { + KlassLUTEntry klute(k->klute()); + if (!klute.is_invalid()) { + // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared + // archive, in which case it contains the klute computed at (dynamic) load time when dumping. + if (klute.value() == _entries[nk]) { log_klass_registration(k, nk, klute, "already registered"); } else { - _entries[nk] = klute; + // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, + // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS + // initialization we encounter Klasses with no associated CLD, for instance). + _entries[nk] = klute.value(); log_klass_registration(k, nk, klute, "updated table value for"); } - + } else { + // Calculate klute from Klass properties and update the table value. + klute = KlassLUTEntry::build_from_klass(k); + _entries[nk] = klute.value(); + log_klass_registration(k, nk, klute, "registered"); } #ifdef ASSERT { // sanity checks KlassLUTEntry e2(at(nk)); - assert(e2.value() == klute, "Sanity"); + assert(e2 == klute, "Sanity"); e2.verify_against_klass(k); } #endif // ASSERT @@ -156,17 +157,18 @@ KlassLUTEntry KlassInfoLUT::register_klass(Klass* k) { inc_registered_IK_for_abstract_or_interface(); } - return KlassLUTEntry(klute); + return klute; } #if INCLUDE_CDS -// We only tolerate this for CDS. In that case, we expect the original class - during dumptime - -// to be registered already, so it should have a valid KLUTE entry set which we only need to copy. -// Note: we cannot calculate the klute here, since at this point the Klass has no associated -// class loader data... +// We only tolerate this for CDS. KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); + // In the CDS case, we expect the original class to have been registered during dumptime; so + // it should carry a valid KLUTE entry. We cannot calculate the KLUTE from the Klass here, since + // the Klass may not yet been fully initialized (CDS calls functions like oopDesc methods on + // oops that have Klasses that are not loaded yet, and therefore, e.g., have no associated CLD). const KlassLUTEntry klute(k->klute()); assert(!klute.is_invalid(), "Must be a valid klute"); _entries[nk] = klute.value(); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 280beef636758..7137621820f52 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -101,7 +101,7 @@ class KlassInfoLUT : public AllStatic { static void initialize(); - static KlassLUTEntry register_klass(Klass* k); + static KlassLUTEntry register_klass(const Klass* k); static inline KlassLUTEntry get_entry(narrowKlass k); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 945559870c2ff..3c3ff8c7456b8 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -159,7 +159,7 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { } -uint32_t KlassLUTEntry::build_from_klass(const Klass* k) { +KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { uint32_t value = invalid_entry; if (k->is_array_klass()) { @@ -173,7 +173,7 @@ uint32_t KlassLUTEntry::build_from_klass(const Klass* k) { k->name()->as_C_string(), not_encodable_reason); } } - return value; + return KlassLUTEntry(value); } #ifdef ASSERT diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 489b711142375..b733308e080f7 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -171,14 +171,17 @@ class KlassLUTEntry { inline KlassLUTEntry(uint32_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} - inline KlassLUTEntry& operator=(uint32_t v) { _v = v; return *this; } - inline KlassLUTEntry& KlassLUTEntry(const KlassLUTEntry& other) { _v = other._v; return *this; } + inline KlassLUTEntry& operator=(uint32_t v) { _v = v; return *this; } + inline KlassLUTEntry& operator=(const KlassLUTEntry& other) { _v = other._v; return *this; } + + inline bool operator==(const KlassLUTEntry& other) const { return _v.raw == other._v.raw; } + inline bool operator!=(const KlassLUTEntry& other) const { return _v.raw != other._v.raw; } // Note: all entries should be valid. An invalid entry indicates // an error somewhere. bool is_invalid() const { return _v.raw == invalid_entry; } - static uint32_t build_from_klass(const Klass* k); + static KlassLUTEntry build_from_klass(const Klass* k); #ifdef ASSERT void verify_against_klass(const Klass* k) const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 0de102d9bfecc..29607e393f700 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -233,17 +233,24 @@ bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_k bool oopDesc::is_array() const { return klass()->is_array_klass(); } bool oopDesc::is_objArray() const { - const KlassLUTEntry klute = UseKLUT ? - KlassInfoLUT::get_entry(mark().narrow_klass()) : - KlassLUTEntry(klass()->klute()); - return klute.is_obj_array(); + if (UseKLUT) { + const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + return klute.is_obj_array(); + } + return klass()->is_objArray_klass(); + +// const KlassLUTEntry klute = UseKLUT ? +// KlassInfoLUT::get_entry(mark().narrow_klass()) : +// KlassLUTEntry(klass()->klute()); +// return klute.is_obj_array(); } bool oopDesc::is_typeArray() const { - const KlassLUTEntry klute = UseKLUT ? - KlassInfoLUT::get_entry(mark().narrow_klass()) : - KlassLUTEntry(klass()->klute()); - return klute.is_type_array(); + if (UseKLUT) { + const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + return klute.is_type_array(); + } + return klass()->is_typeArray_klass(); } template From b1bef162b44c1c59a9e246ef7398a7ca208fb870 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 10:53:28 +0200 Subject: [PATCH 011/101] remove narrowKlass argument --- src/hotspot/share/memory/iterator.hpp | 10 +++--- src/hotspot/share/memory/iterator.inline.hpp | 32 ++++++++--------- .../share/oops/instanceClassLoaderKlass.hpp | 6 ++-- .../oops/instanceClassLoaderKlass.inline.hpp | 12 +++---- src/hotspot/share/oops/instanceKlass.hpp | 6 ++-- .../share/oops/instanceKlass.inline.hpp | 35 ++++++++++--------- .../share/oops/instanceMirrorKlass.hpp | 6 ++-- .../share/oops/instanceMirrorKlass.inline.hpp | 12 +++---- src/hotspot/share/oops/instanceRefKlass.hpp | 6 ++-- .../share/oops/instanceRefKlass.inline.hpp | 23 ++++++------ .../share/oops/instanceStackChunkKlass.hpp | 6 ++-- .../oops/instanceStackChunkKlass.inline.hpp | 18 +++++----- src/hotspot/share/oops/klass.hpp | 2 ++ src/hotspot/share/oops/objArrayKlass.hpp | 6 ++-- .../share/oops/objArrayKlass.inline.hpp | 20 +++++------ src/hotspot/share/oops/oop.inline.hpp | 10 +++--- src/hotspot/share/oops/typeArrayKlass.hpp | 6 ++-- .../share/oops/typeArrayKlass.inline.hpp | 6 ++-- 18 files changed, 115 insertions(+), 107 deletions(-) diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 96aa1af544730..efda467acc807 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -320,11 +320,11 @@ class OopIteratorClosureDispatch { template static void oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* klass); // Note: we only need Klass* for later, in the Klass, to optionally do metadata iteration. However, this may hopefully change in the future. - template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); - template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); - template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); - template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk); - template static size_t oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static size_t oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); }; diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 14ddc3bd0ff74..23326945de579 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -394,36 +394,36 @@ void OopIteratorClosureDispatch::ITERATION_FUNCTION ARGUMENT_DEFINITION { DEFINE_DISPATCH_CLASS( OopOopIterateDispatchWithKlute, oop_oop_iterate, - (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), - (obj, cl, klute, nk) + (oop obj, OopClosureType* cl, KlassLUTEntry klute), + (obj, cl, klute) ) DEFINE_DISPATCH_CLASS( OopOopIterateDispatchWithKluteReverse, oop_oop_iterate_reverse, - (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), - (obj, cl, klute, nk) + (oop obj, OopClosureType* cl, KlassLUTEntry klute), + (obj, cl, klute) ) DEFINE_DISPATCH_CLASS( OopOopIterateDispatchWithKluteBounded, oop_oop_iterate_bounded, - (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk), - (obj, cl, mr, klute, nk) + (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), + (obj, cl, mr, klute) ) // Same, but returns object size template -static inline size_t calculate_size_for_object(narrowKlass nk, KlassLUTEntry klute, oop obj) { - if (KlassType::Kind < Klass::TypeArrayKlassKind) { +static inline size_t calculate_size_for_object(KlassLUTEntry klute, oop obj) { + if (KlassType::Kind < Klass::TypeArrayKlassKind) { // note: this is constexpr assert(klute.is_instance(), "Sanity"); if (klute.ik_carries_infos()) { return klute.ik_wordsize(); } + // Rare path. // Size not statically computable (e.g. MirrorKlass); calculate using Klass - Klass* k = CompressedKlassPointers::decode_not_null(nk); - return obj->size_given_klass(k); + return obj->size_given_klass(obj->klass()); } else { assert(klute.is_array(), "Sanity"); return klute.ak_calculate_wordsize_given_oop(obj); @@ -442,7 +442,7 @@ class CLASSNAME { template \ static size_t invoke_real ARGUMENT_DEFINITION { \ KlassType::template ITERATION_FUNCTION ARGUMENTS; \ - return calculate_size_for_object(nk, klute, obj); \ + return calculate_size_for_object(klute, obj); \ } \ \ template \ @@ -506,23 +506,23 @@ size_t OopIteratorClosureDispatch::ITERATION_FUNCTION ## _size ARGUMENT_DEFINITI DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( OopOopIterateDispatchWithKluteReturnSize, oop_oop_iterate, - (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), - (obj, cl, klute, nk) + (oop obj, OopClosureType* cl, KlassLUTEntry klute), + (obj, cl, klute) ) /* DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( OopOopIterateDispatchWithKluteReverseReturnSize, oop_oop_iterate_reverse, - (oop obj, OopClosureType* cl, KlassLUTEntry klute, narrowKlass nk), + (oop obj, OopClosureType* cl, KlassLUTEntry klute), (obj, cl, klute, nk) )*/ DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( OopOopIterateDispatchWithKluteBoundedReturnSize, oop_oop_iterate_bounded, - (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute, narrowKlass nk), - (obj, cl, mr, klute, nk) + (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), + (obj, cl, mr, klute) ) #endif // SHARE_MEMORY_ITERATOR_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index f1d8c4c28c855..6caeb78d7fd0c 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -75,11 +75,11 @@ class InstanceClassLoaderKlass: public InstanceKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index 3f6ff2a1feeca..c42c632539f89 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -83,20 +83,20 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosur // External entries implementation: klute variants template -inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); +inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate(obj, closure, klute); oop_oop_iterate_metadata(obj, closure); } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); +inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); NO_METADATA_ITERATION } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); +inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); oop_oop_iterate_metadata_bounded(obj, closure, mr); } diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 559ddfc73bfe6..2f39286925508 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1057,11 +1057,11 @@ class InstanceKlass: public Klass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); public: u2 idnum_allocated_count() const { return _idnum_allocated_count; } diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index be28fced1aa01..fd2aa35da3bf8 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -209,11 +209,12 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType // Iterate over all oop fields and metadata. template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { if (Devirtualizer::do_metadata(closure)) { const unsigned perma_cld_index = klute.loader_index(); - ClassLoaderData* const cld = perma_cld_index > 0 ? KlassInfoLUT::get_perma_cld(perma_cld_index) : - CompressedKlassPointers::decode_not_null(nk)->class_loader_data(); + ClassLoaderData* const cld = perma_cld_index > 0 ? + KlassInfoLUT::get_perma_cld(perma_cld_index) : + obj->klass()->class_loader_data(); // Rare path Devirtualizer::do_cld(closure, cld); } @@ -226,16 +227,16 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closur } } } else { + // Rare path // Fall back to normal iteration: read OopMapBlocks from Klass - InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); - assert(this_ik == obj->klass(), "sanity"); - this_ik->oop_oop_iterate_oop_maps(obj, closure); + InstanceKlass* const ik = InstanceKlass::cast(obj->klass()); + ik->oop_oop_iterate_oop_maps(obj, closure); } } // Iterate over all oop fields in the oop maps (no metadata traversal) template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); @@ -248,21 +249,22 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType oop_oop_iterate_single_oop_map_reverse(obj, closure, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); } } else { + // Rare path // Fall back to normal iteration: read OopMapBlocks from Klass - InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); - assert(this_ik == obj->klass(), "sanity"); - this_ik->oop_oop_iterate_oop_maps_reverse(obj, closure); + InstanceKlass* const ik = InstanceKlass::cast(obj->klass()); + ik->oop_oop_iterate_oop_maps_reverse(obj, closure); } } // Iterate over all oop fields and metadata. template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { const unsigned perma_cld_index = klute.loader_index(); - ClassLoaderData* const cld = perma_cld_index > 0 ? KlassInfoLUT::get_perma_cld(perma_cld_index) : - CompressedKlassPointers::decode_not_null(nk)->class_loader_data(); + ClassLoaderData* const cld = perma_cld_index > 0 ? + KlassInfoLUT::get_perma_cld(perma_cld_index) : + obj->klass()->class_loader_data(); // Rare path Devirtualizer::do_cld(closure, cld); } } @@ -276,9 +278,10 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType } } } else { - InstanceKlass* const this_ik = InstanceKlass::cast(CompressedKlassPointers::decode(nk)); - assert(this_ik == obj->klass(), "sanity"); - this_ik->oop_oop_iterate_oop_maps_bounded(obj, closure, mr); + // Rare path + // Fall back to normal iteration: read OopMapBlocks from Klass + InstanceKlass* const ik = InstanceKlass::cast(obj->klass()); + ik->oop_oop_iterate_oop_maps_bounded(obj, closure, mr); } } diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index b2c3201450c6a..780f3f6ee3132 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -115,11 +115,11 @@ class InstanceMirrorKlass: public InstanceKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index 3ae82e8437859..46193928ace72 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -145,24 +145,24 @@ void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closu // Externals, klute variants template -void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); +void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate(obj, closure, klute); oop_oop_iterate_metadata(obj, closure); oop_oop_iterate_statics(obj, closure); } template -void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); +void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); InstanceMirrorKlass::oop_oop_iterate_statics(obj, closure); } template -void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); +void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); oop_oop_iterate_metadata_bounded(obj, closure, mr); oop_oop_iterate_statics_bounded(obj, closure, mr); } diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index d9f03e226a690..822124a6635cf 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -81,11 +81,11 @@ class InstanceRefKlass: public InstanceKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index 2804280fc418c..e7a334ffbd474 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -171,24 +171,27 @@ void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, // klute variants template -void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate(obj, closure, klute, nk); - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing(obj, closure); +void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate(obj, closure, klute); + // Todo: could oop_oop_iterate_ref_processing not be static? + InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); + k->oop_oop_iterate_ref_processing(obj, closure); } template -void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute, nk); +void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing(obj, closure); + InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); + k->oop_oop_iterate_ref_processing(obj, closure); } template -void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute, nk); +void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_ref_processing_bounded(obj, closure, mr); + InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); + k->oop_oop_iterate_ref_processing_bounded(obj, closure, mr); } #ifdef ASSERT diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index bf2144250f4ef..24bb6483580df 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -164,11 +164,11 @@ class InstanceStackChunkKlass: public InstanceKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: template diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 9841ac8fdbb51..1f4e5f6eeccb8 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -92,21 +92,21 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c // Klute variants don't do anything else for now. Just exist to make Dispatch happy. template -void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate(obj, closure); +void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); + k->oop_oop_iterate(obj, closure); } template -void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_reverse(obj, closure); +void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); + k->oop_oop_iterate_reverse(obj, closure); } template -void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_bounded(obj, closure, mr); +void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); + k->oop_oop_iterate_bounded(obj, closure, mr); } template diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 95930a2df5a33..a9fa766e23807 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -84,6 +84,8 @@ class Klass : public Metadata { static const uint KLASS_KIND_COUNT = ObjArrayKlassKind + 1; + // TODO simplify cleanup + #define DECLARE_EXACT_CAST_FUNCTIONS(TYPE) \ static inline const TYPE* cast_exact(const Klass* k); \ static inline TYPE* cast_exact( Klass* k); diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 9f19d5197558f..3746fbe569f7d 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -135,11 +135,11 @@ class ObjArrayKlass : public ArrayKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); public: // Iterate over all oop elements. diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 844d5adb0150a..5f383d960ba2a 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -92,7 +92,7 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me objArrayOop a = objArrayOop(obj); if (Devirtualizer::do_metadata(closure)) { - Devirtualizer::do_klass(closure, a->klass()); + Devirtualizer::do_klass(closure, a->klass()); // Todo: why not "this" ?? } oop_oop_iterate_elements_bounded(a, closure, mr.start(), mr.end()); @@ -100,21 +100,21 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me // Klute variants template -void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate(obj, closure); +void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? + oak->oop_oop_iterate(obj, closure); } template -void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_bounded(obj, closure, mr); +void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? + oak->oop_oop_iterate_bounded(obj, closure, mr); } template -void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - narrow_klass_to_klass(nk)->oop_oop_iterate_reverse(obj, closure); +void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? + oak->oop_oop_iterate_reverse(obj, closure); } // Like oop_oop_iterate but only iterates over a specified range and only used diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 29607e393f700..7755b9d26b5d3 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -399,7 +399,7 @@ void oopDesc::oop_iterate(OopClosureType* cl) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); - OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute, nk); + OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute); return; } @@ -412,7 +412,7 @@ void oopDesc::oop_iterate(OopClosureType* cl, MemRegion mr) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); - OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute, nk); + OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute); return; } @@ -425,7 +425,7 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); - return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute, nk); + return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute); } Klass* k = klass(); @@ -440,7 +440,7 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); - return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute, nk); + return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute); } Klass* k = klass(); @@ -463,7 +463,7 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { const narrowKlass nk = CompressedKlassPointers::encode_not_null(k); const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); if (!klute.is_type_array()) { // no need to iterate TAK - OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute, nk); + OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); } return; } diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index 0da17cdba48bc..2667b4f75cfae 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -98,11 +98,11 @@ class TypeArrayKlass : public ArrayKlass { // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); public: static TypeArrayKlass* cast(Klass* k) { diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index d4e4b9fade966..82e7fec25de95 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -58,17 +58,17 @@ void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { // Klute variant does nothing special, since there is nothing in the klute that would help // us here. It only exists to make the dispatcher happy. template -inline void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { +inline void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); } template -inline void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute, narrowKlass nk) { +inline void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); } template -void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute, narrowKlass nk) { +void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); } From 217345f9330dc9b63424e40f4dddcacb18a2f6cc Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 11:16:34 +0200 Subject: [PATCH 012/101] ObjArrayKlass iteration functions can be static --- src/hotspot/share/oops/objArrayKlass.hpp | 12 ++++++------ src/hotspot/share/oops/objArrayKlass.inline.hpp | 9 +++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 3746fbe569f7d..5908a0581fe70 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -119,19 +119,19 @@ class ObjArrayKlass : public ArrayKlass { // Iterate over oop elements and metadata. template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure); // Iterate over oop elements and metadata. template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); // Iterate over oop elements within mr, and metadata. template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); // Iterate over oop elements within [start, end), and metadata. template - inline void oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end); + static inline void oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end); // klute variants template @@ -144,12 +144,12 @@ class ObjArrayKlass : public ArrayKlass { public: // Iterate over all oop elements. template - inline void oop_oop_iterate_elements(objArrayOop a, OopClosureType* closure); + static inline void oop_oop_iterate_elements(objArrayOop a, OopClosureType* closure); private: // Iterate over all oop elements with indices within mr. template - inline void oop_oop_iterate_elements_bounded(objArrayOop a, OopClosureType* closure, void* low, void* high); + static inline void oop_oop_iterate_elements_bounded(objArrayOop a, OopClosureType* closure, void* low, void* high); public: u2 compute_modifier_flags() const; diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 5f383d960ba2a..a0e37cb66c431 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -101,20 +101,17 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me // Klute variants template void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? - oak->oop_oop_iterate(obj, closure); + oop_oop_iterate(obj, closure); } template void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? - oak->oop_oop_iterate_bounded(obj, closure, mr); + oop_oop_iterate_bounded(obj, closure, mr); } template void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - ObjArrayKlass* const oak = ObjArrayKlass::cast_exact(obj->klass()); // TODO can this be made static? Why do we need Klass instance? - oak->oop_oop_iterate_reverse(obj, closure); + oop_oop_iterate_reverse(obj, closure); } // Like oop_oop_iterate but only iterates over a specified range and only used From 62a63ab368d52f27f1312c0b2524fa4ea9d4e023 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 11:35:36 +0200 Subject: [PATCH 013/101] get_entry->lookup --- src/hotspot/share/oops/klassInfoLUT.hpp | 2 +- src/hotspot/share/oops/klassInfoLUT.inline.hpp | 2 +- src/hotspot/share/oops/oop.inline.hpp | 14 +++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 7137621820f52..555908eae3e0f 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -103,7 +103,7 @@ class KlassInfoLUT : public AllStatic { static KlassLUTEntry register_klass(const Klass* k); - static inline KlassLUTEntry get_entry(narrowKlass k); + static inline KlassLUTEntry lookup(narrowKlass k); static int try_register_perma_cld(ClassLoaderData* cld); static inline ClassLoaderData* get_perma_cld(int index); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 2bc2e5d6ad615..34198da8a4a83 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -38,7 +38,7 @@ ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { return _entries[index]; } -ALWAYSINLINE KlassLUTEntry KlassInfoLUT::get_entry(narrowKlass nk) { +ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { const uint32_t v = at(nk); KlassLUTEntry e(v); diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 7755b9d26b5d3..4a8146d67ab9a 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -234,7 +234,7 @@ bool oopDesc::is_array() const { return klass()->is_array_klass(); bool oopDesc::is_objArray() const { if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + const KlassLUTEntry klute = KlassInfoLUT::lookup(mark().narrow_klass()); return klute.is_obj_array(); } return klass()->is_objArray_klass(); @@ -247,7 +247,7 @@ bool oopDesc::is_objArray() const { bool oopDesc::is_typeArray() const { if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::get_entry(mark().narrow_klass()); + const KlassLUTEntry klute = KlassInfoLUT::lookup(mark().narrow_klass()); return klute.is_type_array(); } return klass()->is_typeArray_klass(); @@ -398,7 +398,7 @@ void oopDesc::oop_iterate(OopClosureType* cl) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute); return; } @@ -411,7 +411,7 @@ void oopDesc::oop_iterate(OopClosureType* cl, MemRegion mr) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute); return; } @@ -424,7 +424,7 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute); } @@ -439,7 +439,7 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { if (UseKLUT) { const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute); } @@ -461,7 +461,7 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { if (UseKLUT) { const narrowKlass nk = CompressedKlassPointers::encode_not_null(k); - const KlassLUTEntry klute = KlassInfoLUT::get_entry(nk); + const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); if (!klute.is_type_array()) { // no need to iterate TAK OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); } From 4ea5556546a2aaee98c7285e49f30634e38188d2 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 13:35:41 +0200 Subject: [PATCH 014/101] klute() returns class not uint32 --- src/hotspot/share/cds/archiveBuilder.cpp | 3 +-- src/hotspot/share/oops/klass.cpp | 10 ++++------ src/hotspot/share/oops/klass.hpp | 5 +++-- src/hotspot/share/oops/klassInfoLUT.cpp | 12 ++++++------ src/hotspot/share/oops/klassInfoLUT.inline.hpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 8 ++++++-- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index b2f4bd529918a..b67899113111c 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -782,8 +782,7 @@ void ArchiveBuilder::make_klasses_shareable() { #ifdef ASSERT if (UseKLUT) { // every archived Klass shall carry a valid KLUTE. - KlassLUTEntry e(k->klute()); - e.verify_against_klass(k); + k->klute().verify_against_klass(k); } #endif // ASSERT } diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 9d0c17a6ec402..78855e3126bcf 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -293,7 +293,7 @@ static markWord make_prototype(const Klass* kls) { return prototype; } -Klass::Klass() : _klute(KlassLUTEntry::invalid_entry), _kind(UnknownKlassKind) { +Klass::Klass() : _kind(UnknownKlassKind) { assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for cds"); } @@ -301,8 +301,7 @@ Klass::Klass() : _klute(KlassLUTEntry::invalid_entry), _kind(UnknownKlassKind) { // which zeros out memory - calloc equivalent. // The constructor is also used from CppVtableCloner, // which doesn't zero out the memory before calling the constructor. -Klass::Klass(KlassKind kind) : _klute(KlassLUTEntry::invalid_entry), - _kind(kind), +Klass::Klass(KlassKind kind) : _kind(kind), _prototype_header(make_prototype(this)), _shared_class_path_index(-1) { CDS_ONLY(_shared_class_flags = 0;) @@ -1357,8 +1356,7 @@ void Klass::on_secondary_supers_verification_failure(Klass* super, Klass* sub, b void Klass::register_with_klut() { if (UseKLUT) { - _klute = KlassLUTEntry::invalid_entry; - KlassLUTEntry e = KlassInfoLUT::register_klass(this); - _klute = e.value(); + _klute = KlassInfoLUT::register_klass(this); + assert(_klute.is_valid(), "Must be valid"); } } diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index a9fa766e23807..540a839e0973a 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_KLASS_HPP #include "oops/klassFlags.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/markWord.hpp" #include "oops/metadata.hpp" #include "oops/oop.hpp" @@ -154,7 +155,7 @@ class Klass : public Metadata { jint _layout_helper; // KLUT entry - uint32_t _klute; + KlassLUTEntry _klute; // Klass kind used to resolve the runtime type of the instance. // - Used to implement devirtualized oop closure dispatching. @@ -258,7 +259,7 @@ class Klass : public Metadata { enum class PrivateLookupMode { find, skip }; // Klute handling - uint32_t klute() const { return _klute; } + KlassLUTEntry klute() const { return _klute; } void register_with_klut(); virtual bool is_klass() const { return true; } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index edb15bf8d2b36..1f6ad384abd6e 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -113,8 +113,8 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // TODO CompressedKlassPointers should support correct constness assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); - KlassLUTEntry klute(k->klute()); - if (!klute.is_invalid()) { + KlassLUTEntry klute = k->klute(); + if (klute.is_valid()) { // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared // archive, in which case it contains the klute computed at (dynamic) load time when dumping. if (klute.value() == _entries[nk]) { @@ -169,8 +169,8 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { // it should carry a valid KLUTE entry. We cannot calculate the KLUTE from the Klass here, since // the Klass may not yet been fully initialized (CDS calls functions like oopDesc methods on // oops that have Klasses that are not loaded yet, and therefore, e.g., have no associated CLD). - const KlassLUTEntry klute(k->klute()); - assert(!klute.is_invalid(), "Must be a valid klute"); + const KlassLUTEntry klute = k->klute(); + assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); log_klass_registration(k, nk, klute.value(), "late-registered"); return klute; @@ -255,8 +255,8 @@ void KlassInfoLUT::print_statistics(outputStream* st) { int n = 0; for (int j = 0; j < slots_per_cacheline; j++) { KlassLUTEntry e(at((i * slots_per_cacheline) + j)); - const bool valid = !e.is_invalid() && (e.is_array() || e.ik_carries_infos()); - if (valid) { + const bool fully_valid = e.is_valid() && (e.is_array() || e.ik_carries_infos()); + if (fully_valid) { n++; } } diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 34198da8a4a83..0ccf0a226364f 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -43,7 +43,7 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { const uint32_t v = at(nk); KlassLUTEntry e(v); #if INCLUDE_CDS - if (e.is_invalid()) { + if (!e.is_valid()) { // This branch only exists because it is so very difficult to iterate CDS classes after loading // CDS archives. See discussion surrounding 8353225. Hopefully we can remove this in the future. // And equally hopefully branch prediction takes the sting out of this branch in real oop iteration. diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 3c3ff8c7456b8..947f7d27a215a 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -198,7 +198,7 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { ALL_KLASS_KINDS_DO(XX) #undef XX - assert(!is_invalid(), "Entry should be valid (%x)", _v.raw); + assert(is_valid(), "klute should be valid (%x)", _v.raw); // kind const unsigned real_kind = (unsigned)k->kind(); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index b733308e080f7..3a40444a4b036 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -26,13 +26,16 @@ #ifndef SHARE_OOPS_KLASSINFOLUTENTRY_HPP #define SHARE_OOPS_KLASSINFOLUTENTRY_HPP -// Included by oop.hpp, keep it short and sweet here +// Included by oop.hpp and klass.hpp, keep includes short #include "memory/allStatic.hpp" #include "utilities/globalDefinitions.hpp" class OopMapBlock; class Klass; +class InstanceKlass; +class ArrayKlass; class outputStream; +class oop; // msb lsb // @@ -168,6 +171,7 @@ class KlassLUTEntry { // We use kind=7=0b111 (invalid), and set the rest of the bits also to 1 static constexpr uint32_t invalid_entry = 0xFFFFFFFF; + inline KlassLUTEntry() : _v(invalid_entry) {} inline KlassLUTEntry(uint32_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} @@ -179,7 +183,7 @@ class KlassLUTEntry { // Note: all entries should be valid. An invalid entry indicates // an error somewhere. - bool is_invalid() const { return _v.raw == invalid_entry; } + bool is_valid() const { return _v.raw != invalid_entry; } static KlassLUTEntry build_from_klass(const Klass* k); From 6e601b6344c47e890e1291216ad9264821ebe60a Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 13:49:25 +0200 Subject: [PATCH 015/101] statistics, also during error report and VM.info --- src/hotspot/share/oops/klassInfoLUT.cpp | 23 +++++++++++++++++------ src/hotspot/share/oops/klassInfoLUT.hpp | 7 ++++--- src/hotspot/share/runtime/java.cpp | 2 +- src/hotspot/share/utilities/vmError.cpp | 11 +++++++++++ 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 1f6ad384abd6e..e119f4d5beb4a 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -161,14 +161,19 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { } #if INCLUDE_CDS -// We only tolerate this for CDS. +// We only tolerate this for CDS. I hope to find a better solution that allows me to +// safely register all Klass structures *before* anyone calls any methods on them. CDS +// makes that very difficult. +// The problem with late_register_class is that it imposes an overhead on every lookup, since +// a lookup table entry may potentially be still invalid. KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); // In the CDS case, we expect the original class to have been registered during dumptime; so - // it should carry a valid KLUTE entry. We cannot calculate the KLUTE from the Klass here, since - // the Klass may not yet been fully initialized (CDS calls functions like oopDesc methods on - // oops that have Klasses that are not loaded yet, and therefore, e.g., have no associated CLD). + // it should carry a valid KLUTE entry. We just copy the klute into the lookup table. Note that + // we cannot calculate the KLUTE from the Klass here even if we wanted, since the Klass may not + // yet been fully initialized (CDS calls Klass functions on Klass structures that are not yet + // fully initialized, e.g. have no associated CLD). const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); @@ -191,8 +196,14 @@ HIT_STATS_DO(XX) void KlassInfoLUT::print_statistics(outputStream* st) { - assert(UseKLUT, "?"); - st->print_cr("KLUT stats:"); + if (!UseKLUT) { + st->print_cr("KLUT is not used."); + return; + } + + st->print_cr("KLUT statistics:"); + + st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _num_entries, _num_entries * sizeof(uint32_t)); const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + counter_registered_ICLK + counter_registered_ISCK + counter_registered_TAK + counter_registered_OAK; diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 555908eae3e0f..b490ce6796699 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -52,7 +52,8 @@ class KlassInfoLUT : public AllStatic { static void allocate_lookup_table(); - // register stats are not expensive + // Statistics about the distribution of Klasses registered. These + // are not expensive (counter hit per class) #define REGISTER_STATS_DO(f) \ f(registered_IK) \ f(registered_IRK) \ @@ -66,7 +67,8 @@ class KlassInfoLUT : public AllStatic { REGISTER_STATS_DO(XX) #undef XX - // hit stats are expensive + // Statistics about the hit rate of the lookup table. These are + // very expensive. #ifdef KLUT_ENABLE_EXPENSIVE_STATS #define HIT_STATS_DO(f) \ f(hits_IK) \ @@ -93,7 +95,6 @@ class KlassInfoLUT : public AllStatic { #endif #if INCLUDE_CDS - // only allowed (grudgingly) when built with CDS, until I figure out a better solution. static KlassLUTEntry late_register_klass(narrowKlass nk); #endif diff --git a/src/hotspot/share/runtime/java.cpp b/src/hotspot/share/runtime/java.cpp index f4d1df5329e11..128db849802d1 100644 --- a/src/hotspot/share/runtime/java.cpp +++ b/src/hotspot/share/runtime/java.cpp @@ -360,7 +360,7 @@ void print_statistics() { ThreadsSMRSupport::log_statistics(); - if (UseKLUT && PrintKLUTStatistics) { + if (PrintKLUTStatistics) { KlassInfoLUT::print_statistics(tty); } diff --git a/src/hotspot/share/utilities/vmError.cpp b/src/hotspot/share/utilities/vmError.cpp index ee402b8b894f5..1f0fecffb03b2 100644 --- a/src/hotspot/share/utilities/vmError.cpp +++ b/src/hotspot/share/utilities/vmError.cpp @@ -1221,6 +1221,11 @@ void VMError::report(outputStream* st, bool _verbose) { st->print_cr("Metaspace:"); MetaspaceUtils::print_basic_report(st, 0); + STEP_IF("printing KLUT information", _verbose && Universe::is_fully_initialized()) + if (Universe::is_fully_initialized()) { + KlassInfoLUT::print_statistics(st); + } + STEP_IF("printing code cache information", _verbose && Universe::is_fully_initialized()) // print code cache information before vm abort CodeCache::print_summary(st); @@ -1425,6 +1430,12 @@ void VMError::print_vm_info(outputStream* st) { MetaspaceUtils::print_basic_report(st, 0); } + // STEP("printing KLUT information") + + if (Universe::is_fully_initialized()) { + KlassInfoLUT::print_statistics(st); + } + // STEP("printing code cache information") if (Universe::is_fully_initialized()) { From 4b65b77386ba2d81afee69c8d30691c712e7944f Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 14:09:51 +0200 Subject: [PATCH 016/101] table allocation --- src/hotspot/share/nmt/memTag.hpp | 1 + src/hotspot/share/oops/klassInfoLUT.cpp | 28 +++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/nmt/memTag.hpp b/src/hotspot/share/nmt/memTag.hpp index 9255645638d83..f1fc548f5eebb 100644 --- a/src/hotspot/share/nmt/memTag.hpp +++ b/src/hotspot/share/nmt/memTag.hpp @@ -58,6 +58,7 @@ f(mtMetaspace, "Metaspace") \ f(mtStringDedup, "String Deduplication") \ f(mtObjectMonitor, "Object Monitors") \ + f(mtKLUT, "Class Metadata lookup table (KLUT)") \ f(mtNone, "Unknown") \ //end diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index e119f4d5beb4a..ac171ff438437 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -48,27 +48,29 @@ void KlassInfoLUT::initialize() { assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); const narrowKlass highest_nk = CompressedKlassPointers::highest_valid_narrow_klass_id(); - _num_entries = highest_nk; - - const size_t memory_needed = sizeof(uint32_t) * _num_entries; + size_t table_size_in_bytes = sizeof(uint32_t) * highest_nk; + bool uses_large_pages = false; if (UseLargePages) { const size_t large_page_size = os::large_page_size(); - if (is_aligned(memory_needed, large_page_size)) { - char* memory = os::reserve_memory_special(memory_needed, large_page_size, large_page_size, nullptr, false); - if (memory != nullptr) { - _entries = (uint32_t*)memory; - log_info(klut)("KLUT initialized (%u entries, using large pages): " RANGEFMT, - _num_entries, RANGEFMTARGS(_entries, memory_needed)); + if (large_page_size < 16 * M) { // not for freakishly large pages + table_size_in_bytes = align_up(table_size_in_bytes, large_page_size); + _entries = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); + if (_entries != nullptr) { + uses_large_pages = true; + _num_entries = table_size_in_bytes / sizeof(uint32_t); } } } if (_entries == nullptr) { - // Fallback, just use C-heap. - _entries = NEW_C_HEAP_ARRAY(uint32_t, num_entries(), mtClass); - log_info(klut)("KLUT initialized (%u entries, using normal pages): " RANGEFMT, - _num_entries, RANGEFMTARGS(_entries, memory_needed)); + table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); + _entries = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); + os::commit_memory_or_exit((char*)_entries, table_size_in_bytes, false, "KLUT"); + _num_entries = table_size_in_bytes / sizeof(uint32_t); } + log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, + _num_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_entries, table_size_in_bytes)); + // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. memset(_entries, 0xff, _num_entries * sizeof(uint32_t)); assert(_entries[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff From 2acacb5bc0ab3b343db436b42815fbf50a463732 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 15:24:34 +0200 Subject: [PATCH 017/101] fix build prob --- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 3a40444a4b036..dbf1481b3c860 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -30,12 +30,12 @@ #include "memory/allStatic.hpp" #include "utilities/globalDefinitions.hpp" -class OopMapBlock; -class Klass; -class InstanceKlass; class ArrayKlass; +class InstanceKlass; +class Klass; +class oopDesc; +class OopMapBlock; class outputStream; -class oop; // msb lsb // @@ -206,7 +206,7 @@ class KlassLUTEntry { // Calculates the object size. Note, introduces a branch (is_array or not). // If possible, use either ik_wordsize() or ak_calculate_wordsize_given_oop() instead. - inline unsigned calculate_wordsize_given_oop(oop obj) const; + inline unsigned calculate_wordsize_given_oop(oopDesc* obj) const; // Following methods only if IK: @@ -243,7 +243,7 @@ class KlassLUTEntry { inline unsigned ak_layouthelper_hsz() const { return _v.ake.lh_hsz; } // calculates word size given header size, element size, and array length - inline unsigned ak_calculate_wordsize_given_oop(oop obj) const; + inline unsigned ak_calculate_wordsize_given_oop(oopDesc* obj) const; // Helper function, prints current limits static void print_limits(outputStream* st); From 3abace34e66ce0eb0bb8d4a5e0270f533055bb35 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 15:37:12 +0200 Subject: [PATCH 018/101] remove klass arg from iterate_backwards --- src/hotspot/share/oops/oop.hpp | 3 --- src/hotspot/share/oops/oop.inline.hpp | 10 ++-------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 7fe3558d69774..4699decd85fa1 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -302,9 +302,6 @@ class oopDesc { template inline void oop_iterate_backwards(OopClosureType* cl); - template - inline void oop_iterate_backwards(OopClosureType* cl, Klass* klass); - inline static bool is_instanceof_or_null(oop obj, Klass* klass); // identity hash; returns the identity hash key (computes it if necessary) diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 4a8146d67ab9a..6aae6befb7b29 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -451,16 +451,9 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { template void oopDesc::oop_iterate_backwards(OopClosureType* cl) { - oop_iterate_backwards(cl, klass()); -} - -template -void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { - // In this assert, we cannot safely access the Klass* with compact headers. - assert(k == klass(), "wrong klass"); if (UseKLUT) { - const narrowKlass nk = CompressedKlassPointers::encode_not_null(k); + const narrowKlass nk = mark().narrow_klass(); const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); if (!klute.is_type_array()) { // no need to iterate TAK OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); @@ -468,6 +461,7 @@ void oopDesc::oop_iterate_backwards(OopClosureType* cl, Klass* k) { return; } + Klass* k = klass(); OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k); } From 41d5c55db548a6c68419b8514e4ce345ffc1ee36 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 15:49:05 +0200 Subject: [PATCH 019/101] max shift 10 --- src/hotspot/share/oops/compressedKlass.cpp | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/oops/compressedKlass.cpp b/src/hotspot/share/oops/compressedKlass.cpp index d1ba7de8cfcbc..0bbaef1f1aa58 100644 --- a/src/hotspot/share/oops/compressedKlass.cpp +++ b/src/hotspot/share/oops/compressedKlass.cpp @@ -218,23 +218,10 @@ void CompressedKlassPointers::initialize(address addr, size_t len) { if (UseCompactObjectHeaders) { - // In compact object header mode, with 22-bit narrowKlass, we don't attempt for - // zero-based mode. Instead, we set the base to the start of the klass range and - // then try for the smallest shift possible that still covers the whole range. - // The reason is that we want to avoid, if possible, shifts larger than - // a cacheline size. + // TODO: can this now merged with !COH? _base = addr; + _shift = max_shift(); - if (UseKLUT) { - _shift = max_shift(); - } else { - const int log_cacheline = exact_log2(DEFAULT_CACHE_LINE_SIZE); - int s = max_shift(); - while (s > log_cacheline && ((size_t)nth_bit(narrow_klass_pointer_bits() + s - 1) > len)) { - s--; - } - _shift = s; - } } else { // Traditional (non-compact) header mode From d55b558612d7228993dd56fb85f4efa3c698880c Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 7 Apr 2025 16:03:09 +0200 Subject: [PATCH 020/101] always set klute in klass etc --- src/hotspot/share/cds/archiveBuilder.cpp | 8 +- src/hotspot/share/memory/universe.cpp | 4 +- src/hotspot/share/oops/klass.cpp | 5 +- src/hotspot/share/oops/klassInfoLUT.cpp | 146 ++++++++++++----------- src/hotspot/share/oops/klassInfoLUT.hpp | 2 + 5 files changed, 83 insertions(+), 82 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index b67899113111c..2e1f1ef021c71 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -779,12 +779,8 @@ void ArchiveBuilder::make_klasses_shareable() { if (k->is_instance_klass()) { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } -#ifdef ASSERT - if (UseKLUT) { - // every archived Klass shall carry a valid KLUTE. - k->klute().verify_against_klass(k); - } -#endif // ASSERT + // every archived Klass shall carry a valid KLUTE. + DEBUG_ONLY(k->klute().verify_against_klass(k);) } for (int i = 0; i < klasses()->length(); i++) { diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 2d0f1607d442c..00fbce16d419b 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -888,9 +888,7 @@ jint universe_init() { Metaspace::global_initialize(); // Initialize KLUT before starting to create any Klass - if (UseKLUT) { - KlassInfoLUT::initialize(); - } + KlassInfoLUT::initialize(); // Initialize performance counters for metaspaces MetaspaceCounters::initialize_performance_counters(); diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 78855e3126bcf..8cb9559a9c5f1 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -1355,8 +1355,5 @@ void Klass::on_secondary_supers_verification_failure(Klass* super, Klass* sub, b } void Klass::register_with_klut() { - if (UseKLUT) { - _klute = KlassInfoLUT::register_klass(this); - assert(_klute.is_valid(), "Must be valid"); - } + _klute = KlassInfoLUT::register_klass(this); } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index ac171ff438437..a680a47967519 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -41,39 +41,43 @@ unsigned KlassInfoLUT::_num_entries = -1; void KlassInfoLUT::initialize() { assert(CompressedKlassPointers::fully_initialized(), "Too early"); - // We allocate a lookup table only if we can use the narrowKlass for a lookup reasonably well. - // We can do this only if the nKlass is small enough - we allow it for COH (22 bit nKlass with - // 10 bit shift means we have a small and condensed table). We don't bother for -COH, - assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "Use only for COH"); - assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); - - const narrowKlass highest_nk = CompressedKlassPointers::highest_valid_narrow_klass_id(); - size_t table_size_in_bytes = sizeof(uint32_t) * highest_nk; - bool uses_large_pages = false; - if (UseLargePages) { - const size_t large_page_size = os::large_page_size(); - if (large_page_size < 16 * M) { // not for freakishly large pages - table_size_in_bytes = align_up(table_size_in_bytes, large_page_size); - _entries = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); - if (_entries != nullptr) { - uses_large_pages = true; - _num_entries = table_size_in_bytes / sizeof(uint32_t); + if (UseCompactObjectHeaders) { + // Init Lookup Table + // We allocate a lookup table only if we can use the narrowKlass for a lookup reasonably well. + // We can do this only if the nKlass is small enough - we allow it for COH (22 bit nKlass with + // 10 bit shift means we have a small and condensed table). We don't bother for -COH, + assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "Use only for COH"); + assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); + + const narrowKlass highest_nk = CompressedKlassPointers::highest_valid_narrow_klass_id(); + size_t table_size_in_bytes = sizeof(uint32_t) * highest_nk; + bool uses_large_pages = false; + if (UseLargePages) { + const size_t large_page_size = os::large_page_size(); + if (large_page_size < 16 * M) { // not for freakishly large pages + table_size_in_bytes = align_up(table_size_in_bytes, large_page_size); + _entries = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); + if (_entries != nullptr) { + uses_large_pages = true; + _num_entries = table_size_in_bytes / sizeof(uint32_t); + } } } - } - if (_entries == nullptr) { - table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); - _entries = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); - os::commit_memory_or_exit((char*)_entries, table_size_in_bytes, false, "KLUT"); - _num_entries = table_size_in_bytes / sizeof(uint32_t); - } + if (_entries == nullptr) { + table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); + _entries = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); + os::commit_memory_or_exit((char*)_entries, table_size_in_bytes, false, "KLUT"); + _num_entries = table_size_in_bytes / sizeof(uint32_t); + } - log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, - _num_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_entries, table_size_in_bytes)); + log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, + _num_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_entries, table_size_in_bytes)); - // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. - memset(_entries, 0xff, _num_entries * sizeof(uint32_t)); - assert(_entries[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff + // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. + memset(_entries, 0xff, _num_entries * sizeof(uint32_t)); + assert(_entries[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff + + } } int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { @@ -111,36 +115,41 @@ static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry } KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { - assert(UseKLUT, "?"); - const narrowKlass nk = CompressedKlassPointers::encode(const_cast(k)); // TODO CompressedKlassPointers should support correct constness - assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); + + + + const narrowKlass nk = UseCompressedClassPointers ? CompressedKlassPointers::encode(const_cast(k)) : 0; KlassLUTEntry klute = k->klute(); if (klute.is_valid()) { // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared // archive, in which case it contains the klute computed at (dynamic) load time when dumping. - if (klute.value() == _entries[nk]) { - log_klass_registration(k, nk, klute, "already registered"); - } else { - // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, - // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS - // initialization we encounter Klasses with no associated CLD, for instance). - _entries[nk] = klute.value(); - log_klass_registration(k, nk, klute, "updated table value for"); + if (use_lookup_table()) { + assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); + if (klute.value() == _entries[nk]) { + log_klass_registration(k, nk, klute, "already registered"); + } else { + // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, + // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS + // initialization we encounter Klasses with no associated CLD, for instance). + _entries[nk] = klute.value(); + log_klass_registration(k, nk, klute, "updated table value for"); + } } } else { // Calculate klute from Klass properties and update the table value. klute = KlassLUTEntry::build_from_klass(k); - _entries[nk] = klute.value(); + if (use_lookup_table()) { + _entries[nk] = klute.value(); + } log_klass_registration(k, nk, klute, "registered"); } #ifdef ASSERT - { - // sanity checks + klute.verify_against_klass(k); + if (use_lookup_table()) { KlassLUTEntry e2(at(nk)); - assert(e2 == klute, "Sanity"); - e2.verify_against_klass(k); + assert(e2 == klute, "sanity"); } #endif // ASSERT @@ -198,14 +207,11 @@ HIT_STATS_DO(XX) void KlassInfoLUT::print_statistics(outputStream* st) { - if (!UseKLUT) { - st->print_cr("KLUT is not used."); - return; - } - st->print_cr("KLUT statistics:"); - st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _num_entries, _num_entries * sizeof(uint32_t)); + if (use_lookup_table()) { + st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _num_entries, _num_entries * sizeof(uint32_t)); + } const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + counter_registered_ICLK + counter_registered_ISCK + counter_registered_TAK + counter_registered_OAK; @@ -259,26 +265,28 @@ void KlassInfoLUT::print_statistics(outputStream* st) { ); #endif // KLUT_ENABLE_EXPENSIVE_STATS - // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) - constexpr int chacheline_size = 64; - constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); - const int num_cachelines = num_entries() / slots_per_cacheline; - int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; - for (int i = 0; i < num_cachelines; i++) { - int n = 0; - for (int j = 0; j < slots_per_cacheline; j++) { - KlassLUTEntry e(at((i * slots_per_cacheline) + j)); - const bool fully_valid = e.is_valid() && (e.is_array() || e.ik_carries_infos()); - if (fully_valid) { - n++; + if (use_lookup_table()) { + // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) + constexpr int chacheline_size = 64; + constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); + const int num_cachelines = num_entries() / slots_per_cacheline; + int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; + for (int i = 0; i < num_cachelines; i++) { + int n = 0; + for (int j = 0; j < slots_per_cacheline; j++) { + KlassLUTEntry e(at((i * slots_per_cacheline) + j)); + const bool fully_valid = e.is_valid() && (e.is_array() || e.ik_carries_infos()); + if (fully_valid) { + n++; + } } + assert(n <= slots_per_cacheline, "Sanity"); + valid_hits_per_cacheline_distribution[n]++; + } + st->print_cr("LUT valid hit density over cacheline size:"); + for (int i = 0; i <= slots_per_cacheline; i++) { + st->print_cr("%d valid entries per cacheline: %d", i, valid_hits_per_cacheline_distribution[i]); } - assert(n <= slots_per_cacheline, "Sanity"); - valid_hits_per_cacheline_distribution[n]++; - } - st->print_cr("LUT valid hit density over cacheline size:"); - for (int i = 0; i <= slots_per_cacheline; i++) { - st->print_cr("%d valid entries per cacheline: %d", i, valid_hits_per_cacheline_distribution[i]); } // Just for info, print limits KlassLUTEntry::print_limits(st); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index b490ce6796699..edc7d2a4de7e4 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -98,6 +98,8 @@ class KlassInfoLUT : public AllStatic { static KlassLUTEntry late_register_klass(narrowKlass nk); #endif + static bool use_lookup_table() { return _entries != nullptr; } + public: static void initialize(); From 41dda35c40e835e297fa8cc3756db54ce98c29f1 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 8 Apr 2025 07:05:26 +0200 Subject: [PATCH 021/101] wip - CLD kram --- src/hotspot/share/oops/klassInfoLUT.cpp | 59 ++++++++++---------- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 4 +- src/hotspot/share/oops/oop.inline.hpp | 1 + 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index a680a47967519..ba9c5cd638eb3 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -80,6 +80,8 @@ void KlassInfoLUT::initialize() { } } +static const char* common_loader_names[4] = { "other", "null", "system", "platform" }; + int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { int index = 0; if (cld->is_permanent_class_loader_data()) { @@ -95,20 +97,21 @@ int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { ClassLoaderData* old_cld = Atomic::load(_common_loaders + index); if (old_cld == nullptr) { old_cld = Atomic::cmpxchg(&_common_loaders[index], (ClassLoaderData*)nullptr, cld); - if (old_cld == nullptr || old_cld == cld) { - return index; + if (old_cld == nullptr) { + log_debug(klut)("Registered CLD for %s loader at index %d: " PTR_FORMAT, + common_loader_names[index], index, p2i(cld)); + } else { + assert(old_cld == cld, "Different CLD??"); // There should only be one for each } - } else if (old_cld == cld) { - return index; } } - return 0; + return index; } static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry klute, const char* message) { char tmp[1024]; - log_debug(klut)("Klass " PTR_FORMAT ", nk %u, klute: " INT32_FORMAT_X_0 ": %s %s%s", - p2i(k), nk, klute.value(), + log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u, klute: " INT32_FORMAT_X_0 ": %s %s%s", + p2i(k), common_loader_names[klute.loader_index()], nk, klute.value(), message, (k->is_shared() ? "(shared) " : ""), k->name()->as_C_string(tmp, sizeof(tmp))); @@ -116,34 +119,32 @@ static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { - - const narrowKlass nk = UseCompressedClassPointers ? CompressedKlassPointers::encode(const_cast(k)) : 0; - KlassLUTEntry klute = k->klute(); - if (klute.is_valid()) { - // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared - // archive, in which case it contains the klute computed at (dynamic) load time when dumping. - if (use_lookup_table()) { - assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); - if (klute.value() == _entries[nk]) { - log_klass_registration(k, nk, klute, "already registered"); - } else { - // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, - // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS - // initialization we encounter Klasses with no associated CLD, for instance). - _entries[nk] = klute.value(); - log_klass_registration(k, nk, klute, "updated table value for"); - } - } - } else { +// KlassLUTEntry klute = k->klute(); +// if (klute.is_valid()) { +// // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared +// // archive, in which case it contains the klute computed at (dynamic) load time when dumping. +// if (use_lookup_table()) { +// assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); +// if (klute.value() == _entries[nk]) { +// log_klass_registration(k, nk, klute, "already registered"); +// } else { +// // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, +// // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS +// // initialization we encounter Klasses with no associated CLD, for instance). +// _entries[nk] = klute.value(); +// log_klass_registration(k, nk, klute, "updated table value for"); +// } +// } +// } else { // Calculate klute from Klass properties and update the table value. - klute = KlassLUTEntry::build_from_klass(k); + const KlassLUTEntry klute = KlassLUTEntry::build_from_klass(k); if (use_lookup_table()) { _entries[nk] = klute.value(); } - log_klass_registration(k, nk, klute, "registered"); - } +//log_klass_registration(k, nk, klute, "registered"); +// } #ifdef ASSERT klute.verify_against_klass(k); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 947f7d27a215a..b2a2a60041aca 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -62,6 +62,7 @@ uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_ const int kind = ik->kind(); const int lh = ik->layout_helper(); + assert(ik->class_loader_data() != nullptr, "no associated class loader?"); const int loader_index = KlassInfoLUT::try_register_perma_cld(ik->class_loader_data()); U value(0); @@ -211,7 +212,8 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { const unsigned loader = loader_index(); assert(loader < 4, "invalid loader index"); if (loader > 0) { - assert(KlassInfoLUT::get_perma_cld(loader) == real_cld, "Different CLD?"); + assert(KlassInfoLUT::get_perma_cld(loader) == real_cld, "Different CLD (%d, %p vs %p)?", loader, + real_cld, KlassInfoLUT::get_perma_cld(loader)); } else { assert(!real_cld->is_permanent_class_loader_data(), "perma cld?"); } diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 6aae6befb7b29..c544bb33bd856 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -233,6 +233,7 @@ bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_k bool oopDesc::is_array() const { return klass()->is_array_klass(); } bool oopDesc::is_objArray() const { + if (UseKLUT) { const KlassLUTEntry klute = KlassInfoLUT::lookup(mark().narrow_klass()); return klute.is_obj_array(); From 4a5c74ad3323d679ac803963c96a7bee90fd39a6 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 8 Apr 2025 10:32:37 +0200 Subject: [PATCH 022/101] fix merge error --- src/hotspot/share/gc/g1/g1ParScanThreadState.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 89d7e44e6b3d3..6713c62e69358 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -452,7 +452,7 @@ void G1ParScanThreadState::do_iterate_object(oop const obj, } assert(_scanner.skip_card_enqueue_set(), "must be"); - obj->oop_iterate_backwards(&_scanner, klass); + obj->oop_iterate_backwards(&_scanner); } // Private inline function, for direct internal use and providing the From 043a7a09181cbc8020953273cceb068d35b4efb2 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 8 Apr 2025 10:33:54 +0200 Subject: [PATCH 023/101] perma-cld --- src/hotspot/share/oops/instanceKlass.hpp | 2 + .../share/oops/instanceKlass.inline.hpp | 20 ++-- src/hotspot/share/oops/klassInfoLUT.cpp | 98 ++++++++++--------- src/hotspot/share/oops/klassInfoLUT.hpp | 7 +- .../share/oops/klassInfoLUT.inline.hpp | 8 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 20 ++-- 6 files changed, 88 insertions(+), 67 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 2f39286925508..e44f2961fcf70 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1053,6 +1053,8 @@ class InstanceKlass: public Klass { template static inline void oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count); + static inline ClassLoaderData* cld_from_klut_or_klass(oop obj, KlassLUTEntry klute); + public: // klute variants diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index fd2aa35da3bf8..b2d206e6aa217 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -207,14 +207,21 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType // Klute variants +ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, KlassLUTEntry klute) { + const unsigned perma_cld_index = klute.loader_index(); + ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); + if (cld == nullptr) { + // Rare path + cld = obj->klass()->class_loader_data(); + } + return cld; +} + // Iterate over all oop fields and metadata. template ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { if (Devirtualizer::do_metadata(closure)) { - const unsigned perma_cld_index = klute.loader_index(); - ClassLoaderData* const cld = perma_cld_index > 0 ? - KlassInfoLUT::get_perma_cld(perma_cld_index) : - obj->klass()->class_loader_data(); // Rare path + ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); Devirtualizer::do_cld(closure, cld); } @@ -261,10 +268,7 @@ template ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { - const unsigned perma_cld_index = klute.loader_index(); - ClassLoaderData* const cld = perma_cld_index > 0 ? - KlassInfoLUT::get_perma_cld(perma_cld_index) : - obj->klass()->class_loader_data(); // Rare path + ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); Devirtualizer::do_cld(closure, cld); } } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index ba9c5cd638eb3..45ba6cfcae25a 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -80,32 +80,40 @@ void KlassInfoLUT::initialize() { } } -static const char* common_loader_names[4] = { "other", "null", "system", "platform" }; +static const char* common_loader_names[4] = { "other", "boot", "system", "platform" }; -int KlassInfoLUT::try_register_perma_cld(ClassLoaderData* cld) { +void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { + // We remember CLDs for the three permanent class loaders in a lookup array. int index = 0; - if (cld->is_permanent_class_loader_data()) { - if (cld->is_the_null_class_loader_data()) { - index = 1; - } else if (cld->is_system_class_loader_data()) { - index = 2; - } else if (cld->is_platform_class_loader_data()) { - index = 3; - } + if (cld->is_the_null_class_loader_data()) { + index = 1; + } else if (cld->is_system_class_loader_data()) { + index = 2; + } else if (cld->is_platform_class_loader_data()) { + index = 3; + } else { + return; } - if (index > 0) { - ClassLoaderData* old_cld = Atomic::load(_common_loaders + index); + ClassLoaderData* old_cld = Atomic::load(_common_loaders + index); + if (old_cld == nullptr) { + old_cld = Atomic::cmpxchg(&_common_loaders[index], (ClassLoaderData*)nullptr, cld); if (old_cld == nullptr) { - old_cld = Atomic::cmpxchg(&_common_loaders[index], (ClassLoaderData*)nullptr, cld); - if (old_cld == nullptr) { - log_debug(klut)("Registered CLD for %s loader at index %d: " PTR_FORMAT, - common_loader_names[index], index, p2i(cld)); - } else { - assert(old_cld == cld, "Different CLD??"); // There should only be one for each - } + log_debug(klut)("Registered CLD " PTR_FORMAT " (%s loader) CLD at index %d", + p2i(cld), common_loader_names[index], index); + } + } + // There should only be 3 permanent CLDs + assert(old_cld == cld || old_cld == nullptr, "Different CLD??"); +} + +int KlassInfoLUT::index_for_cld(const ClassLoaderData* cld) { + assert(cld != nullptr, "must not be null"); + for (int i = 1; i <= 3; i++) { + if (cld == _common_loaders[i]) { + return i; } } - return index; + return 0; } static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry klute, const char* message) { @@ -119,32 +127,35 @@ static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { + // First register the CLD in case we did not already do that + ClassLoaderData* const cld = k->class_loader_data(); + assert(cld != nullptr, "Require CLD"); + register_cld_if_needed(cld); + const narrowKlass nk = UseCompressedClassPointers ? CompressedKlassPointers::encode(const_cast(k)) : 0; -// KlassLUTEntry klute = k->klute(); -// if (klute.is_valid()) { -// // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared -// // archive, in which case it contains the klute computed at (dynamic) load time when dumping. -// if (use_lookup_table()) { -// assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); -// if (klute.value() == _entries[nk]) { -// log_klass_registration(k, nk, klute, "already registered"); -// } else { -// // Copy the klute value from the Klass to the table slot. This saves some cycles but, more importantly, -// // makes the coding more robust when using it on Klasses that are not fully initialized yet (during CDS -// // initialization we encounter Klasses with no associated CLD, for instance). -// _entries[nk] = klute.value(); -// log_klass_registration(k, nk, klute, "updated table value for"); -// } -// } -// } else { + KlassLUTEntry klute = k->klute(); + if (klute.is_valid()) { + // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared + // archive, in which case it contains the klute computed at (dynamic) load time when dumping. + if (use_lookup_table()) { + assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); + if (klute.value() == _entries[nk]) { + log_klass_registration(k, nk, klute, "already registered"); + } else { + // Copy the klute value from the Klass to the table slot. + _entries[nk] = klute.value(); + log_klass_registration(k, nk, klute, "updated table value for"); + } + } + } else { // Calculate klute from Klass properties and update the table value. - const KlassLUTEntry klute = KlassLUTEntry::build_from_klass(k); + klute = KlassLUTEntry::build_from_klass(k); if (use_lookup_table()) { _entries[nk] = klute.value(); } -//log_klass_registration(k, nk, klute, "registered"); -// } + log_klass_registration(k, nk, klute, "registered"); + } #ifdef ASSERT klute.verify_against_klass(k); @@ -182,10 +193,9 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); // In the CDS case, we expect the original class to have been registered during dumptime; so - // it should carry a valid KLUTE entry. We just copy the klute into the lookup table. Note that - // we cannot calculate the KLUTE from the Klass here even if we wanted, since the Klass may not - // yet been fully initialized (CDS calls Klass functions on Klass structures that are not yet - // fully initialized, e.g. have no associated CLD). + // it should carry a valid klute. We just copy that value into the lookup table. Note that + // we cannot calculate the klute from the Klass here even if we wanted, since the Klass may not + // yet carry a CLD (CDS calls Klass functions on Klass structures that are not yet fully initialized). const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index edc7d2a4de7e4..cd55e89896e30 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -100,16 +100,17 @@ class KlassInfoLUT : public AllStatic { static bool use_lookup_table() { return _entries != nullptr; } + static void register_cld_if_needed(ClassLoaderData* cld); + public: static void initialize(); static KlassLUTEntry register_klass(const Klass* k); - static inline KlassLUTEntry lookup(narrowKlass k); - static int try_register_perma_cld(ClassLoaderData* cld); - static inline ClassLoaderData* get_perma_cld(int index); + static int index_for_cld(const ClassLoaderData* cld); + static inline ClassLoaderData* lookup_cld(int index); static void print_statistics(outputStream* out); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 0ccf0a226364f..7227dcfc09626 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -65,9 +65,11 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { return e; } -ALWAYSINLINE ClassLoaderData* KlassInfoLUT::get_perma_cld(int index) { - assert(index >= 1 && index <= 3, "Sanity"); - return _common_loaders[index]; +ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(int index) { + assert(index >= 0 && index <= 3, "Sanity"); + ClassLoaderData* cld = _common_loaders[index]; + assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); + return cld; } #endif // SHARE_OOPS_KLASSINFOLUT_INLINE_HPP diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index b2a2a60041aca..66192151af13a 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -63,14 +63,14 @@ uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_ const int kind = ik->kind(); const int lh = ik->layout_helper(); assert(ik->class_loader_data() != nullptr, "no associated class loader?"); - const int loader_index = KlassInfoLUT::try_register_perma_cld(ik->class_loader_data()); + const int cld_index = KlassInfoLUT::index_for_cld(ik->class_loader_data()); U value(0); // Set common bits, these are always present assert(kind < 0b111, "sanity"); value.common.kind = kind; - value.common.loader = loader_index; + value.common.loader = cld_index; // We may not be able to encode the IK-specific info; if we can't, those bits are left zero // and we return an error string for logging @@ -145,14 +145,14 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { const int kind = ak->kind(); const int lh = ak->layout_helper(); - const int loader_index = KlassInfoLUT::try_register_perma_cld(ak->class_loader_data()); + const int cld_index = KlassInfoLUT::index_for_cld(ak->class_loader_data()); assert(Klass::layout_helper_is_objArray(lh) || Klass::layout_helper_is_typeArray(lh), "unexpected"); LayoutHelperHelper lhu = { (unsigned) lh }; U value(0); value.common.kind = kind; - value.common.loader = loader_index; + value.common.loader = cld_index; value.ake.lh_ebt = lhu.bytes.lh_ebt; value.ake.lh_esz = lhu.bytes.lh_esz; value.ake.lh_hsz = lhu.bytes.lh_hsz; @@ -209,11 +209,13 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { assert(our_kind == real_kind, "kind mismatch (%d vs %d) (%x)", real_kind, our_kind, _v.raw); const ClassLoaderData* const real_cld = k->class_loader_data(); - const unsigned loader = loader_index(); - assert(loader < 4, "invalid loader index"); - if (loader > 0) { - assert(KlassInfoLUT::get_perma_cld(loader) == real_cld, "Different CLD (%d, %p vs %p)?", loader, - real_cld, KlassInfoLUT::get_perma_cld(loader)); + const unsigned cld_index = loader_index(); + assert(cld_index < 4, "invalid loader index"); + if (cld_index > 0) { + const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cld_index); + assert(cld_from_klute == real_cld, + "Different CLD (loader_index: %d, real CLD: " PTR_FORMAT ", from klute lookup table: " PTR_FORMAT ")?", + cld_index, p2i(real_cld), p2i(cld_from_klute)); } else { assert(!real_cld->is_permanent_class_loader_data(), "perma cld?"); } From 61b497971c4eda62f7f7fce758f5a272908f50f8 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 8 Apr 2025 11:05:15 +0200 Subject: [PATCH 024/101] make KLUT work with Coleens abstract/interface-not-in-ccs-change --- src/hotspot/share/oops/instanceKlass.cpp | 2 +- src/hotspot/share/oops/klassInfoLUT.cpp | 32 ++++++++++++------- .../share/oops/klassInfoLUT.inline.hpp | 2 +- .../share/oops/klassInfoLUTEntry.inline.hpp | 4 +-- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index 4d596e898f4a1..b87c88f7ef830 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -472,7 +472,7 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par assert(loader_data != nullptr, "invariant"); InstanceKlass* ik; - const bool use_class_space = true; // parser.klass_needs_narrow_id(); + const bool use_class_space = parser.klass_needs_narrow_id(); // Allocation if (parser.is_instance_ref_klass()) { diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 45ba6cfcae25a..51d72ab0763d5 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -116,10 +116,13 @@ int KlassInfoLUT::index_for_cld(const ClassLoaderData* cld) { return 0; } -static void log_klass_registration(const Klass* k, narrowKlass nk, KlassLUTEntry klute, const char* message) { +static void log_klass_registration(const Klass* k, narrowKlass nk, bool added_to_table, + KlassLUTEntry klute, const char* message) { char tmp[1024]; - log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u, klute: " INT32_FORMAT_X_0 ": %s %s%s", - p2i(k), common_loader_names[klute.loader_index()], nk, klute.value(), + log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u(%c), klute: " INT32_FORMAT_X_0 ": %s %s%s", + p2i(k), common_loader_names[klute.loader_index()], nk, + (added_to_table ? '+' : '-'), + klute.value(), message, (k->is_shared() ? "(shared) " : ""), k->name()->as_C_string(tmp, sizeof(tmp))); @@ -132,34 +135,40 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { assert(cld != nullptr, "Require CLD"); register_cld_if_needed(cld); - const narrowKlass nk = UseCompressedClassPointers ? CompressedKlassPointers::encode(const_cast(k)) : 0; + // We calculate the klute that will be stored into the Klass. + // + // We also add the klute to the lookup table iff we use a lookup table (we do if COH is enabled) + // and if the Klass is in the narrowKlass encoding range. Interfaces and abstract classes are + // not put there anymore since we don't need narrowKlass lookup for them. + const bool add_to_table = use_lookup_table() ? CompressedKlassPointers::is_encodable(k) : false; + const narrowKlass nk = add_to_table ? CompressedKlassPointers::encode(const_cast(k)) : 0; KlassLUTEntry klute = k->klute(); if (klute.is_valid()) { // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared // archive, in which case it contains the klute computed at (dynamic) load time when dumping. - if (use_lookup_table()) { + if (add_to_table) { assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); if (klute.value() == _entries[nk]) { - log_klass_registration(k, nk, klute, "already registered"); + log_klass_registration(k, nk, false, klute, "already registered"); } else { // Copy the klute value from the Klass to the table slot. _entries[nk] = klute.value(); - log_klass_registration(k, nk, klute, "updated table value for"); + log_klass_registration(k, nk, true, klute, "updated table value for"); } } } else { // Calculate klute from Klass properties and update the table value. klute = KlassLUTEntry::build_from_klass(k); - if (use_lookup_table()) { + if (add_to_table) { _entries[nk] = klute.value(); } - log_klass_registration(k, nk, klute, "registered"); + log_klass_registration(k, nk, add_to_table, klute, "registered"); } #ifdef ASSERT klute.verify_against_klass(k); - if (use_lookup_table()) { + if (add_to_table) { KlassLUTEntry e2(at(nk)); assert(e2 == klute, "sanity"); } @@ -190,6 +199,7 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { // The problem with late_register_class is that it imposes an overhead on every lookup, since // a lookup table entry may potentially be still invalid. KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { + assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); // In the CDS case, we expect the original class to have been registered during dumptime; so @@ -199,7 +209,7 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); - log_klass_registration(k, nk, klute.value(), "late-registered"); + log_klass_registration(k, nk, true, klute.value(), "late-registered"); return klute; } #endif // INCLUDE_CDS diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 7227dcfc09626..bfc60defa268a 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -39,7 +39,7 @@ ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { } ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { - + assert(nk != 0, "null narrow Klass - is this class encodable?"); const uint32_t v = at(nk); KlassLUTEntry e(v); #if INCLUDE_CDS diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index abf313f04f31e..964388029bc79 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -70,7 +70,7 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { } // calculates word size given header size, element size, and array length -inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop(oop obj) const { +inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop(oopDesc* obj) const { assert(is_array(), "only for ak entries"); assert(UseCompactObjectHeaders, "+COH only"); assert(UseKLUT, "+KLUT only"); @@ -92,7 +92,7 @@ inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop(oop obj) const { return align_up(size_in_bytes, HardCodedObjectAlignmentInBytes) / HeapWordSize; } -inline unsigned KlassLUTEntry::calculate_wordsize_given_oop(oop obj) const { +inline unsigned KlassLUTEntry::calculate_wordsize_given_oop(oopDesc* obj) const { size_t rc = 0; return is_array() ? ak_calculate_wordsize_given_oop(obj) : ik_wordsize(); } From 6281f35746053c03d685618efded30ff70a5d381 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 8 Apr 2025 13:02:59 +0200 Subject: [PATCH 025/101] Fix --- src/hotspot/share/oops/klassInfoLUT.cpp | 29 +++++++------------------ 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 51d72ab0763d5..ff5131532cc1a 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -39,13 +39,14 @@ uint32_t* KlassInfoLUT::_entries = nullptr; unsigned KlassInfoLUT::_num_entries = -1; void KlassInfoLUT::initialize() { - assert(CompressedKlassPointers::fully_initialized(), "Too early"); if (UseCompactObjectHeaders) { + // Init Lookup Table // We allocate a lookup table only if we can use the narrowKlass for a lookup reasonably well. // We can do this only if the nKlass is small enough - we allow it for COH (22 bit nKlass with // 10 bit shift means we have a small and condensed table). We don't bother for -COH, + assert(CompressedKlassPointers::fully_initialized(), "Too early"); assert(CompressedKlassPointers::narrow_klass_pointer_bits() <= 22, "Use only for COH"); assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); @@ -144,27 +145,13 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { const narrowKlass nk = add_to_table ? CompressedKlassPointers::encode(const_cast(k)) : 0; KlassLUTEntry klute = k->klute(); - if (klute.is_valid()) { - // The Klass already carries the pre-computed klute. That can happen if it was loaded from a shared - // archive, in which case it contains the klute computed at (dynamic) load time when dumping. - if (add_to_table) { - assert(nk < num_entries(), "narrowKlass %u is OOB for LUT", nk); - if (klute.value() == _entries[nk]) { - log_klass_registration(k, nk, false, klute, "already registered"); - } else { - // Copy the klute value from the Klass to the table slot. - _entries[nk] = klute.value(); - log_klass_registration(k, nk, true, klute, "updated table value for"); - } - } - } else { - // Calculate klute from Klass properties and update the table value. - klute = KlassLUTEntry::build_from_klass(k); - if (add_to_table) { - _entries[nk] = klute.value(); - } - log_klass_registration(k, nk, add_to_table, klute, "registered"); + + // Calculate klute from Klass properties and update the table value. + klute = KlassLUTEntry::build_from_klass(k); + if (add_to_table) { + _entries[nk] = klute.value(); } + log_klass_registration(k, nk, add_to_table, klute, "registered"); #ifdef ASSERT klute.verify_against_klass(k); From b98aa27868783f6c2a1623843464c740ba400a46 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 08:49:27 +0200 Subject: [PATCH 026/101] rework klute for arrays --- src/hotspot/share/memory/iterator.inline.hpp | 310 ++++++++++++------ src/hotspot/share/oops/klassInfoLUTEntry.cpp | 17 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 35 +- .../share/oops/klassInfoLUTEntry.inline.hpp | 46 +-- src/hotspot/share/oops/oop.hpp | 2 + src/hotspot/share/oops/oop.inline.hpp | 46 +-- 6 files changed, 269 insertions(+), 187 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 23326945de579..9c5d4e29c1256 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -412,117 +412,217 @@ DEFINE_DISPATCH_CLASS( (obj, cl, mr, klute) ) -// Same, but returns object size - -template -static inline size_t calculate_size_for_object(KlassLUTEntry klute, oop obj) { - if (KlassType::Kind < Klass::TypeArrayKlassKind) { // note: this is constexpr - assert(klute.is_instance(), "Sanity"); - if (klute.ik_carries_infos()) { - return klute.ik_wordsize(); - } - // Rare path. - // Size not statically computable (e.g. MirrorKlass); calculate using Klass - return obj->size_given_klass(obj->klass()); - } else { - assert(klute.is_array(), "Sanity"); - return klute.ak_calculate_wordsize_given_oop(obj); +class DispatchWithKluteReturnSize_base { +protected: + + // Return the size of an object; uses as much hard-coded information as possible + template + static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { + size_t s; + constexpr bool is_instance = KlassType::Kind < Klass::TypeArrayKlassKind; + if (is_instance) { + if (klute.ik_carries_infos()) { + s = klute.ik_wordsize(); + } else { + // Rare path: size not statically computable (e.g. MirrorKlass); calculate using Klass + s = obj->size(); + } + } else { + constexpr bool is_objarray = KlassType::Kind == Klass::ObjArrayKlassKind; + s = klute.ak_calculate_wordsize_given_oop_fast(obj); + } + assert(s == obj->size(), "Unexpected size (klute %X, %zu vs %zu)", + klute.value(), s, obj->size()); + return s; } + + static inline bool should_use_slowpath_getsize() { + return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; + } +}; + +template +class OopOopIterateDispatchWithKluteReturnSize : public DispatchWithKluteReturnSize_base { + + typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + + struct Table { + + FunctionType _function [Klass::KLASS_KIND_COUNT]; + + template + static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate (obj, cl, klute); + return calculate_size_for_object_fast(klute, obj); + } + + template + static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate (obj, cl, klute); + return obj->size(); + } + + template + static size_t init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + return OopOopIterateDispatchWithKluteReturnSize::_table.resolve_and_execute (obj, cl, klute); + } + + template + void set_init_function() { + _function[KlassType::Kind] = &init_and_execute; + } + + template + size_t resolve_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + resolve(); + return _function[KlassType::Kind] (obj, cl, klute); + } + + template + void resolve() { + if (should_use_slowpath_getsize()) { + _function[KlassType::Kind] = + &invoke_slow; + } else { + if (UseCompressedOops) { + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } + } else { + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } + } + } + } + + public: + + Table() { + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + } + + }; + + static Table _table; + +public: + + static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + const int slot = klute.kind(); + return _table._function[slot](obj, cl, klute); + } + +}; + +template +typename OopOopIterateDispatchWithKluteReturnSize::Table OopOopIterateDispatchWithKluteReturnSize::_table; + +template +size_t OopIteratorClosureDispatch::oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + return OopOopIterateDispatchWithKluteReturnSize::invoke (obj, cl, klute); } -#define DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE(CLASSNAME, ITERATION_FUNCTION, ARGUMENT_DEFINITION, ARGUMENTS) \ -template \ -class CLASSNAME { \ - typedef size_t (*FunctionType) ARGUMENT_DEFINITION; \ - \ - class Table { \ - \ - FunctionType _function [Klass::KLASS_KIND_COUNT]; \ - \ - template \ - static size_t invoke_real ARGUMENT_DEFINITION { \ - KlassType::template ITERATION_FUNCTION ARGUMENTS; \ - return calculate_size_for_object(klute, obj); \ - } \ - \ - template \ - static size_t init_and_execute ARGUMENT_DEFINITION { \ - return CLASSNAME::_table.set_resolve_function_and_execute ARGUMENTS; \ - } \ - \ - template \ - size_t set_resolve_function_and_execute ARGUMENT_DEFINITION { \ - set_resolve_function(); \ - return _function[KlassType::Kind] ARGUMENTS; \ - } \ - \ - template \ - void set_init_function() { \ - _function[KlassType::Kind] = &init_and_execute; \ - } \ - \ - template \ - void set_resolve_function() { \ - _function[KlassType::Kind] = UseCompressedOops ? \ - &invoke_real : \ - &invoke_real; \ - } \ - \ - public: \ - \ - Table(){ \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - } \ - \ - size_t invoke ARGUMENT_DEFINITION { \ - const int slot = klute.kind(); \ - return _function[slot] ARGUMENTS; \ - } \ - \ - }; \ - \ - static Table _table; \ - \ -public: \ - \ - static size_t invoke ARGUMENT_DEFINITION { return _table.invoke ARGUMENTS; } \ - \ -}; \ - \ -template \ -typename CLASSNAME::Table CLASSNAME::_table; \ - \ + + template \ -size_t OopIteratorClosureDispatch::ITERATION_FUNCTION ## _size ARGUMENT_DEFINITION { \ - return CLASSNAME::invoke ARGUMENTS; \ -} +class OopOopIterateDispatchWithKluteBoundedReturnSize : public DispatchWithKluteReturnSize_base { + typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); -DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( - OopOopIterateDispatchWithKluteReturnSize, - oop_oop_iterate, - (oop obj, OopClosureType* cl, KlassLUTEntry klute), - (obj, cl, klute) -) + struct Table { -/* -DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( - OopOopIterateDispatchWithKluteReverseReturnSize, - oop_oop_iterate_reverse, - (oop obj, OopClosureType* cl, KlassLUTEntry klute), - (obj, cl, klute, nk) -)*/ + FunctionType _function [Klass::KLASS_KIND_COUNT]; -DEFINE_DISPATCH_CLASS_RETURN_OBJ_SIZE( - OopOopIterateDispatchWithKluteBoundedReturnSize, - oop_oop_iterate_bounded, - (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), - (obj, cl, mr, klute) -) + template + static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + return calculate_size_for_object_fast(klute, obj); + } + + template + static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + return obj->size(); + } + + template + static size_t init_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + return OopOopIterateDispatchWithKluteBoundedReturnSize::_table.resolve_and_execute (obj, cl, mr, klute); + } + + template + void set_init_function() { + _function[KlassType::Kind] = &init_and_execute; + } + + template + size_t resolve_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + resolve(); + return _function[KlassType::Kind] (obj, cl, mr, klute); + } + + template + void resolve() { + if (should_use_slowpath_getsize()) { + _function[KlassType::Kind] = + &invoke_slow; + } else { + if (UseCompressedOops) { + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } + } else { + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } + } + } + } + + Table() { + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + } + + }; + + static Table _table; + +public: + + static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + const int slot = klute.kind(); + return _table._function[slot](obj, cl, mr, klute); + } + +}; + +template +typename OopOopIterateDispatchWithKluteBoundedReturnSize::Table + OopOopIterateDispatchWithKluteBoundedReturnSize::_table; + +template +size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + return OopOopIterateDispatchWithKluteBoundedReturnSize::invoke(obj, cl, mr, klute); +} #endif // SHARE_MEMORY_ITERATOR_INLINE_HPP diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 66192151af13a..cfa9e0fe6751b 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -153,9 +153,12 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { U value(0); value.common.kind = kind; value.common.loader = cld_index; - value.ake.lh_ebt = lhu.bytes.lh_ebt; - value.ake.lh_esz = lhu.bytes.lh_esz; - value.ake.lh_hsz = lhu.bytes.lh_hsz; + + assert(lhu.bytes.lh_esz <= 3, "Sanity (%X)", lh); + value.ake.l2esz = lhu.bytes.lh_esz; + + assert(lhu.bytes.lh_hsz >= 12 && lhu.bytes.lh_hsz <= 20, "Sanity"); + value.ake.hsz = lhu.bytes.lh_hsz; return value.raw; } @@ -185,7 +188,6 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { // to place them in a header STATIC_ASSERT(bits_common + bits_specific == bits_total); STATIC_ASSERT(32 == bits_total); - STATIC_ASSERT(bits_ak_lh <= bits_specific); STATIC_ASSERT(sizeof(KE) == sizeof(uint32_t)); STATIC_ASSERT(sizeof(AKE) == sizeof(uint32_t)); @@ -224,12 +226,11 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { // compare our (truncated) lh with the real one const LayoutHelperHelper lhu = { (unsigned) real_lh }; - assert(lhu.bytes.lh_ebt == ak_layouthelper_ebt() && - lhu.bytes.lh_esz == ak_layouthelper_esz() && - lhu.bytes.lh_hsz == ak_layouthelper_hsz() && + assert(lhu.bytes.lh_esz == ak_log2_elem_size() && + lhu.bytes.lh_hsz == ak_header_size() && ( (lhu.bytes.lh_tag == 0xC0 && real_kind == Klass::TypeArrayKlassKind) || (lhu.bytes.lh_tag == 0x80 && real_kind == Klass::ObjArrayKlassKind) ), - "layouthelper mismatch (0x%x vs 0x%x)", real_lh, _v.raw); + "layouthelper mismatch (layouthelper: 0x%x, klute: 0x%x)", real_lh, _v.raw); } else { diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index dbf1481b3c860..a8faf8702142f 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -48,7 +48,7 @@ class outputStream; // InstanceKlass, has_no_addinfo: KKKL L000 0000 0000 0000 0000 0000 0000 (all IK specific bits 0) (note: means that "0" is a valid IK entry with no add. info) // InstanceKlass, has no oopmap entries: KKKL LSSS SSS. .... .... .... 0000 0000 (omb count bits are 0) (only valid if !has_no_addinfo) // -// ArrayKlass: KKKL L--- tttt tttt hhhh hhhh eeee eeee +// ArrayKlass: KKKL L--- ---- ---- ---- ---- -eeh hhhh // // // IK specific bits: @@ -59,9 +59,8 @@ class outputStream; // S : Object instance size in words (6 bits) // // AK specific bits: -// e : log2 element size (8 bits) -// h : header size (8 bits) -// t : element basic type (8 bits) +// h : header size (5 bits) +// e : log2 element size (2 bits) // // Common bits: // L : Loader (2 bits) @@ -129,17 +128,13 @@ class KlassLUTEntry { }; // Bits only valid for ArrayKlass - static constexpr int bits_ak_lh_esz = 8; - static constexpr int bits_ak_lh_ebt = 8; - static constexpr int bits_ak_lh_hsz = 8; - static constexpr int bits_ak_lh = bits_ak_lh_esz + bits_ak_lh_ebt + bits_ak_lh_hsz; + static constexpr int bits_ak_l2esz = 2; + static constexpr int bits_ak_hsz = 5; struct AKE { // lsb - // see klass.hpp - unsigned lh_esz : bits_ak_lh_esz; // element size - unsigned lh_ebt : bits_ak_lh_ebt; // element BasicType (currently unused) - unsigned lh_hsz : bits_ak_lh_hsz; // header size (offset to first element) - unsigned unused : bits_specific - bits_ak_lh_esz - bits_ak_lh_ebt - bits_ak_lh_hsz; + unsigned hsz : bits_ak_hsz; // header size (offset to first element) in bytes + unsigned l2esz : bits_ak_l2esz; // log2 elem size + unsigned unused : bits_specific - bits_ak_l2esz - bits_ak_hsz; unsigned other : bits_common; // msb }; @@ -204,10 +199,6 @@ class KlassLUTEntry { bool is_obj_array() const { return _v.common.kind == ObjArrayKlassKind; } bool is_type_array() const { return _v.common.kind == TypeArrayKlassKind; } - // Calculates the object size. Note, introduces a branch (is_array or not). - // If possible, use either ik_wordsize() or ak_calculate_wordsize_given_oop() instead. - inline unsigned calculate_wordsize_given_oop(oopDesc* obj) const; - // Following methods only if IK: // Returns true if entry carries IK-specific info (oop map block info + size). @@ -234,16 +225,14 @@ class KlassLUTEntry { // Following methods only if AK: // returns log2 element size - inline unsigned ak_layouthelper_esz() const { return _v.ake.lh_esz; } - - // returns ebt byte - inline unsigned ak_layouthelper_ebt() const { return _v.ake.lh_ebt; } + inline unsigned ak_log2_elem_size() const { return _v.ake.l2esz; } // returns distance to first element - inline unsigned ak_layouthelper_hsz() const { return _v.ake.lh_hsz; } + inline unsigned ak_header_size() const { return _v.ake.hsz; } // calculates word size given header size, element size, and array length - inline unsigned ak_calculate_wordsize_given_oop(oopDesc* obj) const; + template + inline unsigned ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; // Helper function, prints current limits static void print_limits(outputStream* st); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index 964388029bc79..cedd7af89f5ac 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -70,31 +70,33 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { } // calculates word size given header size, element size, and array length -inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop(oopDesc* obj) const { - assert(is_array(), "only for ak entries"); - assert(UseCompactObjectHeaders, "+COH only"); - assert(UseKLUT, "+KLUT only"); - - // See oopDesc::size_given_klass - const unsigned l2esz = ak_layouthelper_esz(); - const unsigned hsz = ak_layouthelper_hsz(); - - // In +COH, array length is always at offset 8 - STATIC_ASSERT(sizeof(markWord) == 8); - const int* const array_len_addr = (int*)(obj->field_addr(8)); - const size_t array_length = (size_t) (*array_len_addr); - const size_t size_in_bytes = (array_length << l2esz) + hsz; +template +inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { + // The purpose of this function is to be as fast as possible; we hard-code as much + // as we can via template parameters. + assert(is_obj_array() == is_objarray, "Bad call"); + assert(is_type_array() == !is_objarray, "Bad call"); - // Note: for UseKLUT, we require a standard object alignment (see argument.cpp) - constexpr int HardCodedObjectAlignmentInBytes = BytesPerWord; - assert(MinObjAlignmentInBytes == HardCodedObjectAlignmentInBytes, "Sanity"); + assert(sizeof(OopType) == 4 || sizeof(OopType) == 8, "Bad oop type"); + constexpr int log2_oopsize = (sizeof(OopType) == 4 ? 2 : 3); // narrowOop or Oop + assert(UseCompressedOops == (log2_oopsize == 2), + "Bad call - UseCompressedOops mismatch (%d, %zu)", UseCompressedOops, sizeof(OopType)); - return align_up(size_in_bytes, HardCodedObjectAlignmentInBytes) / HeapWordSize; -} + assert(UseCompressedClassPointers, "Bad call"); + assert(UseCompactObjectHeaders == compact_headers, "Bad call - COH mismatch"); + + constexpr int alignment = BytesPerWord; + assert(MinObjAlignmentInBytes == alignment, "Bad call - alignment mismatch"); + + constexpr int length_field_offset = compact_headers ? 8 : 12; + const int first_element_offset = compact_headers ? ak_header_size() : 16; + const unsigned log2_elemsize = is_objarray ? log2_oopsize : ak_log2_elem_size(); + + const int* const array_len_addr = (int*)(obj->field_addr(length_field_offset)); + const size_t array_length = (size_t) (*array_len_addr); -inline unsigned KlassLUTEntry::calculate_wordsize_given_oop(oopDesc* obj) const { - size_t rc = 0; - return is_array() ? ak_calculate_wordsize_given_oop(obj) : ik_wordsize(); + const size_t size_in_bytes = (array_length << log2_elemsize) + first_element_offset; + return align_up(size_in_bytes, alignment) / HeapWordSize; } #endif // SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 4699decd85fa1..9d82c1b21dfff 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -84,6 +84,8 @@ class oopDesc { // objects during a GC) -- requires a valid klass pointer inline void init_mark(); + inline KlassLUTEntry get_klute() const; + inline Klass* klass() const; inline Klass* klass_or_null() const; inline Klass* klass_or_null_acquire() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index c544bb33bd856..ff6473c482502 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -97,6 +97,17 @@ void oopDesc::init_mark() { set_mark(prototype_mark()); } +KlassLUTEntry oopDesc::get_klute() const { + switch (ObjLayout::klass_mode()) { + case ObjLayout::Compact: + return KlassInfoLUT::lookup(mark().narrow_klass()); + case ObjLayout::Compressed: + return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass)->klute(); + default: + return _metadata._klass->klute(); + } +} + Klass* oopDesc::klass() const { switch (ObjLayout::klass_mode()) { case ObjLayout::Compact: @@ -396,43 +407,20 @@ void oopDesc::incr_age() { template void oopDesc::oop_iterate(OopClosureType* cl) { - - if (UseKLUT) { - const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); - OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute); - return; - } - - OopIteratorClosureDispatch::oop_oop_iterate(cl, this, klass()); + const KlassLUTEntry klute = get_klute(); + OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute); } template void oopDesc::oop_iterate(OopClosureType* cl, MemRegion mr) { - - if (UseKLUT) { - const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); - OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute); - return; - } - - OopIteratorClosureDispatch::oop_oop_iterate(cl, this, klass(), mr); + const KlassLUTEntry klute = get_klute(); + OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute); } template size_t oopDesc::oop_iterate_size(OopClosureType* cl) { - - if (UseKLUT) { - const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); - return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute); - } - - Klass* k = klass(); - size_t size = size_given_klass(k); - OopIteratorClosureDispatch::oop_oop_iterate(cl, this, k); - return size; + const KlassLUTEntry klute = get_klute(); + return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute); } template From 81bf9a4ff09e2f5bc383dd6e74c7764f6ce53c29 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 09:17:01 +0200 Subject: [PATCH 027/101] oop iteration fully uses KLUT now --- src/hotspot/share/oops/oop.inline.hpp | 49 ++++++--------------------- 1 file changed, 10 insertions(+), 39 deletions(-) diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index ff6473c482502..81dfa85275741 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -244,25 +244,15 @@ bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_k bool oopDesc::is_array() const { return klass()->is_array_klass(); } bool oopDesc::is_objArray() const { - - if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::lookup(mark().narrow_klass()); - return klute.is_obj_array(); - } - return klass()->is_objArray_klass(); - -// const KlassLUTEntry klute = UseKLUT ? -// KlassInfoLUT::get_entry(mark().narrow_klass()) : -// KlassLUTEntry(klass()->klute()); -// return klute.is_obj_array(); + const bool rc = get_klute().is_obj_array(); + assert(rc == klass()->is_objArray_klass(), "Sanity"); + return rc; } bool oopDesc::is_typeArray() const { - if (UseKLUT) { - const KlassLUTEntry klute = KlassInfoLUT::lookup(mark().narrow_klass()); - return klute.is_type_array(); - } - return klass()->is_typeArray_klass(); + const bool rc = get_klute().is_type_array(); + assert(rc == klass()->is_typeArray_klass(), "Sanity"); + return rc; } template @@ -425,33 +415,14 @@ size_t oopDesc::oop_iterate_size(OopClosureType* cl) { template size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { - - if (UseKLUT) { - const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); - return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute); - } - - Klass* k = klass(); - size_t size = size_given_klass(k); - OopIteratorClosureDispatch::oop_oop_iterate(cl, this, k, mr); - return size; + const KlassLUTEntry klute = get_klute(); + return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute); } template void oopDesc::oop_iterate_backwards(OopClosureType* cl) { - - if (UseKLUT) { - const narrowKlass nk = mark().narrow_klass(); - const KlassLUTEntry klute = KlassInfoLUT::lookup(nk); - if (!klute.is_type_array()) { // no need to iterate TAK - OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); - } - return; - } - - Klass* k = klass(); - OopIteratorClosureDispatch::oop_oop_iterate_backwards(cl, this, k); + const KlassLUTEntry klute = get_klute(); + OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); } bool oopDesc::is_instanceof_or_null(oop obj, Klass* klass) { From 47b34785337e1b840277198ee5a0a9a99825bf24 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 10:48:01 +0200 Subject: [PATCH 028/101] fix macos build error --- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index cfa9e0fe6751b..28612a4fed88f 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -238,10 +238,10 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { const InstanceKlass* const ik = InstanceKlass::cast(k); const int real_oop_map_count = ik->nonstatic_oop_map_count(); - const unsigned omb_offset_1 = (real_oop_map_count >= 1) ? (unsigned)ik->start_of_nonstatic_oop_maps()[0].offset() : max_uintx; - const unsigned omb_count_1 = (real_oop_map_count >= 1) ? ik->start_of_nonstatic_oop_maps()[0].count() : max_uintx; - const unsigned omb_offset_2 = (real_oop_map_count >= 2) ? (unsigned)ik->start_of_nonstatic_oop_maps()[1].offset() : max_uintx; - const unsigned omb_count_2 = (real_oop_map_count >= 2) ? ik->start_of_nonstatic_oop_maps()[1].count() : max_uintx; + const unsigned omb_offset_1 = (real_oop_map_count >= 1) ? (unsigned)ik->start_of_nonstatic_oop_maps()[0].offset() : UINT_MAX; + const unsigned omb_count_1 = (real_oop_map_count >= 1) ? ik->start_of_nonstatic_oop_maps()[0].count() : UINT_MAX; + const unsigned omb_offset_2 = (real_oop_map_count >= 2) ? (unsigned)ik->start_of_nonstatic_oop_maps()[1].offset() : UINT_MAX; + const unsigned omb_count_2 = (real_oop_map_count >= 2) ? ik->start_of_nonstatic_oop_maps()[1].count() : UINT_MAX; const int real_wordsize = Klass::layout_helper_needs_slow_path(real_lh) ? -1 : Klass::layout_helper_to_size_helper(real_lh); From 9559617f171aa6f9069e5eed9d0010b342d86a77 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 10:48:35 +0200 Subject: [PATCH 029/101] Dispatch class: remove non-klut variants --- src/hotspot/share/memory/iterator.hpp | 15 +- src/hotspot/share/memory/iterator.inline.hpp | 236 ++----------------- 2 files changed, 19 insertions(+), 232 deletions(-) diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index efda467acc807..59b39e93a6322 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -315,16 +315,11 @@ class CompareClosure : public Closure { class OopIteratorClosureDispatch { public: - template static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass); - template static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass, MemRegion mr); - template static void oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* klass); - - // Note: we only need Klass* for later, in the Klass, to optionally do metadata iteration. However, this may hopefully change in the future. - template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); - template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static size_t oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute); + template static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); }; diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 9c5d4e29c1256..64d904fbc736d 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -69,6 +69,7 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { m->record_gc_epoch(); } +// TODO update comment // Dispatch table implementation for *Klass::oop_oop_iterate // // It allows for a single call to do a multi-dispatch to an optimized version @@ -103,215 +104,6 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { // oop_oop_iterate function replaces the init function in the table, and // succeeding calls will jump directly to oop_oop_iterate. - -template -class OopOopIterateDispatch : public AllStatic { -private: - typedef void (*FunctionType)(OopClosureType*, oop, Klass*); - - class Table { - private: - template - static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* k) { - ((KlassType*)k)->KlassType::template oop_oop_iterate(obj, cl); - } - - template - static void init(OopClosureType* cl, oop obj, Klass* k) { - OopOopIterateDispatch::_table.set_resolve_function_and_execute(cl, obj, k); - } - - template - void set_init_function() { - _function[KlassType::Kind] = &init; - } - - template - void set_resolve_function() { - // Size requirement to prevent word tearing - // when functions pointers are updated. - STATIC_ASSERT(sizeof(_function[0]) == sizeof(void*)); - if (UseCompressedOops) { - _function[KlassType::Kind] = &oop_oop_iterate; - } else { - _function[KlassType::Kind] = &oop_oop_iterate; - } - } - - template - void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k) { - set_resolve_function(); - _function[KlassType::Kind](cl, obj, k); - } - - public: - FunctionType _function[Klass::KLASS_KIND_COUNT]; - - Table(){ - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - } - }; - - static Table _table; -public: - - static FunctionType function(Klass* klass) { - return _table._function[klass->kind()]; - } -}; - -template -typename OopOopIterateDispatch::Table OopOopIterateDispatch::_table; - - -template -class OopOopIterateBoundedDispatch { -private: - typedef void (*FunctionType)(OopClosureType*, oop, Klass*, MemRegion); - - class Table { - private: - template - static void oop_oop_iterate_bounded(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) { - ((KlassType*)k)->KlassType::template oop_oop_iterate_bounded(obj, cl, mr); - } - - template - static void init(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) { - OopOopIterateBoundedDispatch::_table.set_resolve_function_and_execute(cl, obj, k, mr); - } - - template - void set_init_function() { - _function[KlassType::Kind] = &init; - } - - template - void set_resolve_function() { - if (UseCompressedOops) { - _function[KlassType::Kind] = &oop_oop_iterate_bounded; - } else { - _function[KlassType::Kind] = &oop_oop_iterate_bounded; - } - } - - template - void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) { - set_resolve_function(); - _function[KlassType::Kind](cl, obj, k, mr); - } - - public: - FunctionType _function[Klass::KLASS_KIND_COUNT]; - - Table(){ - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - } - }; - - static Table _table; -public: - - static FunctionType function(Klass* klass) { - return _table._function[klass->kind()]; - } -}; - -template -typename OopOopIterateBoundedDispatch::Table OopOopIterateBoundedDispatch::_table; - - -template -class OopOopIterateBackwardsDispatch { -private: - typedef void (*FunctionType)(OopClosureType*, oop, Klass*); - - class Table { - private: - template - static void oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* k) { - ((KlassType*)k)->KlassType::template oop_oop_iterate_reverse(obj, cl); - } - - template - static void init(OopClosureType* cl, oop obj, Klass* k) { - OopOopIterateBackwardsDispatch::_table.set_resolve_function_and_execute(cl, obj, k); - } - - template - void set_init_function() { - _function[KlassType::Kind] = &init; - } - - template - void set_resolve_function() { - if (UseCompressedOops) { - _function[KlassType::Kind] = &oop_oop_iterate_backwards; - } else { - _function[KlassType::Kind] = &oop_oop_iterate_backwards; - } - } - - template - void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k) { - set_resolve_function(); - _function[KlassType::Kind](cl, obj, k); - } - - public: - FunctionType _function[Klass::KLASS_KIND_COUNT]; - - Table(){ - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - } - }; - - static Table _table; -public: - - static FunctionType function(Klass* klass) { - return _table._function[klass->kind()]; - } -}; - -template -typename OopOopIterateBackwardsDispatch::Table OopOopIterateBackwardsDispatch::_table; - - -template -void OopIteratorClosureDispatch::oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass) { - OopOopIterateDispatch::function(klass)(cl, obj, klass); -} - -template -void OopIteratorClosureDispatch::oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass, MemRegion mr) { - OopOopIterateBoundedDispatch::function(klass)(cl, obj, klass, mr); -} - -template -void OopIteratorClosureDispatch::oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* klass) { - OopOopIterateBackwardsDispatch::function(klass)(cl, obj, klass); -} - - ////////////// KLUTE variants ///////////////////// // Macro arguments: @@ -392,27 +184,27 @@ void OopIteratorClosureDispatch::ITERATION_FUNCTION ARGUMENT_DEFINITION { } DEFINE_DISPATCH_CLASS( - OopOopIterateDispatchWithKlute, + OopOopIterateDispatch, oop_oop_iterate, (oop obj, OopClosureType* cl, KlassLUTEntry klute), (obj, cl, klute) ) DEFINE_DISPATCH_CLASS( - OopOopIterateDispatchWithKluteReverse, + OopOopIterateDispatchReverse, oop_oop_iterate_reverse, (oop obj, OopClosureType* cl, KlassLUTEntry klute), (obj, cl, klute) ) DEFINE_DISPATCH_CLASS( - OopOopIterateDispatchWithKluteBounded, + OopOopIterateDispatchBounded, oop_oop_iterate_bounded, (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), (obj, cl, mr, klute) ) -class DispatchWithKluteReturnSize_base { +class DispatchBase { protected: // Return the size of an object; uses as much hard-coded information as possible @@ -442,7 +234,7 @@ class DispatchWithKluteReturnSize_base { }; template -class OopOopIterateDispatchWithKluteReturnSize : public DispatchWithKluteReturnSize_base { +class OopOopIterateDispatchReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); @@ -464,7 +256,7 @@ class OopOopIterateDispatchWithKluteReturnSize : public DispatchWithKluteReturnS template static size_t init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchWithKluteReturnSize::_table.resolve_and_execute (obj, cl, klute); + return OopOopIterateDispatchReturnSize::_table.resolve_and_execute (obj, cl, klute); } template @@ -526,17 +318,17 @@ class OopOopIterateDispatchWithKluteReturnSize : public DispatchWithKluteReturnS }; template -typename OopOopIterateDispatchWithKluteReturnSize::Table OopOopIterateDispatchWithKluteReturnSize::_table; +typename OopOopIterateDispatchReturnSize::Table OopOopIterateDispatchReturnSize::_table; template size_t OopIteratorClosureDispatch::oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchWithKluteReturnSize::invoke (obj, cl, klute); + return OopOopIterateDispatchReturnSize::invoke (obj, cl, klute); } template \ -class OopOopIterateDispatchWithKluteBoundedReturnSize : public DispatchWithKluteReturnSize_base { +class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); struct Table { @@ -557,7 +349,7 @@ class OopOopIterateDispatchWithKluteBoundedReturnSize : public DispatchWithKlute template static size_t init_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - return OopOopIterateDispatchWithKluteBoundedReturnSize::_table.resolve_and_execute (obj, cl, mr, klute); + return OopOopIterateDispatchBoundedReturnSize::_table.resolve_and_execute (obj, cl, mr, klute); } template @@ -617,12 +409,12 @@ class OopOopIterateDispatchWithKluteBoundedReturnSize : public DispatchWithKlute }; template -typename OopOopIterateDispatchWithKluteBoundedReturnSize::Table - OopOopIterateDispatchWithKluteBoundedReturnSize::_table; +typename OopOopIterateDispatchBoundedReturnSize::Table +OopOopIterateDispatchBoundedReturnSize::_table; template size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - return OopOopIterateDispatchWithKluteBoundedReturnSize::invoke(obj, cl, mr, klute); + return OopOopIterateDispatchBoundedReturnSize::invoke(obj, cl, mr, klute); } #endif // SHARE_MEMORY_ITERATOR_INLINE_HPP From 71bc71fb379b5e91b339f8687c5b0a3b881a98c2 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 11:12:20 +0200 Subject: [PATCH 030/101] fix build errors on windows --- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index a8faf8702142f..2a1a8d047e50e 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -232,7 +232,7 @@ class KlassLUTEntry { // calculates word size given header size, element size, and array length template - inline unsigned ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; + inline size_t ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; // Helper function, prints current limits static void print_limits(outputStream* st); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index cedd7af89f5ac..e5841c7ee2db4 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -71,7 +71,7 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { // calculates word size given header size, element size, and array length template -inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { +inline size_t KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { // The purpose of this function is to be as fast as possible; we hard-code as much // as we can via template parameters. assert(is_obj_array() == is_objarray, "Bad call"); @@ -92,8 +92,8 @@ inline unsigned KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj const int first_element_offset = compact_headers ? ak_header_size() : 16; const unsigned log2_elemsize = is_objarray ? log2_oopsize : ak_log2_elem_size(); - const int* const array_len_addr = (int*)(obj->field_addr(length_field_offset)); - const size_t array_length = (size_t) (*array_len_addr); + const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); + const unsigned array_length = (size_t) (*array_len_addr); const size_t size_in_bytes = (array_length << log2_elemsize) + first_element_offset; return align_up(size_in_bytes, alignment) / HeapWordSize; From 51a5beb0f39c62b096d0764945a8a2141e1c28e0 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 17:06:00 +0200 Subject: [PATCH 031/101] iterator --- src/hotspot/share/memory/iterator.inline.hpp | 302 +++++++++++++++---- 1 file changed, 248 insertions(+), 54 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 64d904fbc736d..fe0929db54da8 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -104,6 +104,242 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { // oop_oop_iterate function replaces the init function in the table, and // succeeding calls will jump directly to oop_oop_iterate. + +class DispatchBase { +protected: + + // Return the size of an object; uses as much hard-coded information as possible + template + static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { + size_t s; + constexpr bool is_instance = KlassType::Kind < Klass::TypeArrayKlassKind; + if (is_instance) { + if (klute.ik_carries_infos()) { + s = klute.ik_wordsize(); + } else { + // Rare path: size not statically computable (e.g. MirrorKlass); calculate using Klass + s = obj->size(); + } + } else { + constexpr bool is_objarray = KlassType::Kind == Klass::ObjArrayKlassKind; + s = klute.ak_calculate_wordsize_given_oop_fast(obj); + } + assert(s == obj->size(), "Unexpected size (klute %X, %zu vs %zu)", + klute.value(), s, obj->size()); + return s; + } + + static inline bool should_use_slowpath_getsize() { + return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; + } +}; + +// Dispatch for normal iteration, unbounded, does not return size +// void XXXKlass::oop_oop_iterate(oop, closure, klute) +template +class OopOopIterateDispatch : public DispatchBase { + typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + + struct Table { + + FunctionType _function [Klass::KLASS_KIND_COUNT]; + + template + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate(obj, cl, klute); + } + + template + static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + OopOopIterateDispatch::_table.resolve_and_execute (obj, cl, klute); + } + + template + void resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + resolve(); + _function[KlassType::Kind] (obj, cl, klute); + } + + template + void set_init_function() { + _function[KlassType::Kind] = &init_and_execute; + } + + template + void resolve() { + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; + } + + Table() { + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + } + + }; + + static Table _table; + +public: + + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + const int slot = klute.kind(); + _table._function[slot](obj, cl, klute); + } + +}; + +template +typename OopOopIterateDispatch::Table OopOopIterateDispatch::_table; + +template +void OopIteratorClosureDispatch::oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + OopOopIterateDispatch::invoke(obj, cl, klute); +} + +// Dispatch for reverse iteration, unbounded, does not return size +// void XXXKlass::oop_oop_iterate_reverse(oop, closure, klute) +template +class OopOopIterateDispatchReverse { + typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + + struct Table { + + FunctionType _function [Klass::KLASS_KIND_COUNT]; + + template + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate_reverse (obj, cl, klute); + } + + template + static void init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + OopOopIterateDispatchReverse::_table.resolve_and_execute(obj, cl, klute); + } + + template + void resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + resolve(); + _function[KlassType::Kind](obj, cl, klute); + } + + template + void set_init_function() { + _function[KlassType::Kind] = &init_and_execute; + } + + template + void resolve() { + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; + } + + Table() { + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + } + + }; + + static Table _table; + +public: + + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + const int slot = klute.kind(); + _table._function[slot] (obj, cl, klute); + } + +}; + +template +typename OopOopIterateDispatchReverse::Table OopOopIterateDispatchReverse::_table; + +template +void OopIteratorClosureDispatch::oop_oop_iterate_reverse(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + OopOopIterateDispatchReverse::invoke (obj, cl, klute); +} + +template +class OopOopIterateDispatchBounded { + typedef void (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + + struct Table { + + FunctionType _function [Klass::KLASS_KIND_COUNT]; + + template + static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + } + + template + static void init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + OopOopIterateDispatchBounded::_table.resolve_and_execute (obj, cl, mr, klute); + } + + template + void resolve_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + resolve(); + _function[KlassType::Kind] (obj, cl, mr, klute); + } + + template + void set_init_function() { + _function[KlassType::Kind] = &init_and_execute; + } + + template + void resolve() { + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; + } + + Table(){ + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + set_init_function(); + } + + }; + + static Table _table; + +public: + + static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + const int slot = klute.kind(); + _table._function[slot] (obj, cl, mr, klute); + } + +}; + +template +typename OopOopIterateDispatchBounded::Table OopOopIterateDispatchBounded::_table; + +template +void OopIteratorClosureDispatch::oop_oop_iterate_bounded(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + OopOopIterateDispatchBounded::invoke(obj, cl, mr, klute); +} + + ////////////// KLUTE variants ///////////////////// // Macro arguments: @@ -183,56 +419,15 @@ void OopIteratorClosureDispatch::ITERATION_FUNCTION ARGUMENT_DEFINITION { CLASSNAME::invoke ARGUMENTS; \ } -DEFINE_DISPATCH_CLASS( - OopOopIterateDispatch, - oop_oop_iterate, - (oop obj, OopClosureType* cl, KlassLUTEntry klute), - (obj, cl, klute) -) - -DEFINE_DISPATCH_CLASS( - OopOopIterateDispatchReverse, - oop_oop_iterate_reverse, - (oop obj, OopClosureType* cl, KlassLUTEntry klute), - (obj, cl, klute) -) - -DEFINE_DISPATCH_CLASS( - OopOopIterateDispatchBounded, - oop_oop_iterate_bounded, - (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), - (obj, cl, mr, klute) -) - -class DispatchBase { -protected: - - // Return the size of an object; uses as much hard-coded information as possible - template - static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { - size_t s; - constexpr bool is_instance = KlassType::Kind < Klass::TypeArrayKlassKind; - if (is_instance) { - if (klute.ik_carries_infos()) { - s = klute.ik_wordsize(); - } else { - // Rare path: size not statically computable (e.g. MirrorKlass); calculate using Klass - s = obj->size(); - } - } else { - constexpr bool is_objarray = KlassType::Kind == Klass::ObjArrayKlassKind; - s = klute.ak_calculate_wordsize_given_oop_fast(obj); - } - assert(s == obj->size(), "Unexpected size (klute %X, %zu vs %zu)", - klute.value(), s, obj->size()); - return s; - } - - static inline bool should_use_slowpath_getsize() { - return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; - } -}; +//DEFINE_DISPATCH_CLASS( +// OopOopIterateDispatchBounded, +// oop_oop_iterate_bounded, +// (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), +// (obj, cl, mr, klute) +//) +// Dispatch for normal iteration, unbounded, returns size +// size_t XXXKlass::oop_oop_iterate(oop, closure, klute) template class OopOopIterateDispatchReturnSize : public DispatchBase { @@ -265,7 +460,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - size_t resolve_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { + size_t resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { resolve(); return _function[KlassType::Kind] (obj, cl, klute); } @@ -292,8 +487,6 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } } - public: - Table() { set_init_function(); set_init_function(); @@ -326,8 +519,9 @@ size_t OopIteratorClosureDispatch::oop_oop_iterate_size (oop obj, OopClosureTyp } - -template \ +// Dispatch for bounded iteration, returns size +// size_t XXXKlass::oop_oop_iterate_bounded(oop, closure, memregion, klute) +template class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); From 3f2efbe16f8d2441a953437d83e8d7bf6ca796f2 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 17:09:13 +0200 Subject: [PATCH 032/101] fix wrong assert for -ccp --- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 28612a4fed88f..3c082a3bd1c6a 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -157,7 +157,7 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { assert(lhu.bytes.lh_esz <= 3, "Sanity (%X)", lh); value.ake.l2esz = lhu.bytes.lh_esz; - assert(lhu.bytes.lh_hsz >= 12 && lhu.bytes.lh_hsz <= 20, "Sanity"); + assert(lhu.bytes.lh_hsz >= 12 && lhu.bytes.lh_hsz <= 24, "Sanity"); value.ake.hsz = lhu.bytes.lh_hsz; return value.raw; From 61612678b7d302f98483f46ad6b505799d22c677 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 9 Apr 2025 17:41:27 +0200 Subject: [PATCH 033/101] fix crashes in InstanceKlass::print_on --- src/hotspot/share/oops/instanceKlass.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index b87c88f7ef830..8f600389cce4b 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -3636,9 +3636,12 @@ const char* InstanceKlass::init_state_name() const { void InstanceKlass::print_on(outputStream* st) const { assert(is_klass(), "must be klass"); Klass::print_on(st); - if (UseCompressedClassPointers) { - st->print(BULLET"narrow Klass: %x", CompressedKlassPointers::encode(const_cast(this))); st->cr(); + if (CompressedKlassPointers::is_encodable(this)) { + st->print(BULLET"narrow Klass: %x", CompressedKlassPointers::encode(const_cast(this))); + } else { + st->print(BULLET"narrow Klass: (not in class space)"); + } } st->print(BULLET"instance size: %d", size_helper()); st->cr(); st->print(BULLET"klass size: %d", size()); st->cr(); From 726adeed1268443914396f3f14124b08d2648028 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 06:34:35 +0200 Subject: [PATCH 034/101] fix for hidden classes clds --- src/hotspot/share/oops/klassInfoLUT.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index ff5131532cc1a..cbb3bb0d03b95 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -81,20 +81,26 @@ void KlassInfoLUT::initialize() { } } -static const char* common_loader_names[4] = { "other", "boot", "system", "platform" }; +static const char* common_loader_names[4] = { "other", "boot", "app", "platform" }; void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { + // We remember CLDs for the three permanent class loaders in a lookup array. - int index = 0; - if (cld->is_the_null_class_loader_data()) { - index = 1; - } else if (cld->is_system_class_loader_data()) { - index = 2; - } else if (cld->is_platform_class_loader_data()) { - index = 3; - } else { + int index = -1; + if (cld->is_permanent_class_loader_data()) { + if (cld->is_the_null_class_loader_data()) { + index = 1; + } else if (cld->is_system_class_loader_data()) { + index = 2; + } else if (cld->is_platform_class_loader_data()) { + index = 3; + } + } + + if (index == -1) { return; } + ClassLoaderData* old_cld = Atomic::load(_common_loaders + index); if (old_cld == nullptr) { old_cld = Atomic::cmpxchg(&_common_loaders[index], (ClassLoaderData*)nullptr, cld); @@ -103,6 +109,7 @@ void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { p2i(cld), common_loader_names[index], index); } } + // There should only be 3 permanent CLDs assert(old_cld == cld || old_cld == nullptr, "Different CLD??"); } From cc010c792d2ae5ff5ff69d109eeeb2b586ea3d43 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 06:34:44 +0200 Subject: [PATCH 035/101] fix for minimal build --- src/hotspot/share/oops/klassInfoLUT.inline.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index bfc60defa268a..897fe7b7a9595 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -51,7 +51,7 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { return late_register_klass(nk); } #else - assert(!e.is_invalid(), "must never be invalid"); + assert(e.is_valid(), "must never be invalid"); #endif #ifdef KLUT_ENABLE_EXPENSIVE_STATS From a08aeb1f87f65d299468e5a6ef57a9bf95cb3e71 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 06:39:52 +0200 Subject: [PATCH 036/101] reduce diff --- src/hotspot/share/classfile/fieldLayoutBuilder.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp index 0e4d1279e3630..b6007923907c8 100644 --- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp +++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp @@ -603,6 +603,7 @@ void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) { } } +// Computation of regular classes layout is an evolution of the previous default layout // (FieldAllocationStyle 1): // - primitive fields are allocated first (from the biggest to the smallest) // - oop fields are allocated, either in existing gaps or at the end of From 602bf832f497a9b919c6ffd573013b22ec81c071 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 07:01:14 +0200 Subject: [PATCH 037/101] remove older iteration dispatch functions --- src/hotspot/share/memory/iterator.inline.hpp | 87 -------------------- 1 file changed, 87 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index fe0929db54da8..bde41c08c6eb9 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -339,93 +339,6 @@ void OopIteratorClosureDispatch::oop_oop_iterate_bounded(oop obj, OopClosureType OopOopIterateDispatchBounded::invoke(obj, cl, mr, klute); } - -////////////// KLUTE variants ///////////////////// - -// Macro arguments: -// ITERATION_FUNCTION - name of target iteration function inside Klass, e.g., oop_oop_iterate_bounded -// ARGUMENT_DEFINITION - argument definition including brackets -// ARGUMENTS - arguments, including brackets - -#define DEFINE_DISPATCH_CLASS(CLASSNAME, ITERATION_FUNCTION, ARGUMENT_DEFINITION, ARGUMENTS) \ -template \ -class CLASSNAME { \ - typedef void (*FunctionType) ARGUMENT_DEFINITION; \ - \ - class Table { \ - \ - FunctionType _function [Klass::KLASS_KIND_COUNT]; \ - \ - template \ - static void invoke_real ARGUMENT_DEFINITION { \ - KlassType::template ITERATION_FUNCTION ARGUMENTS; \ - } \ - \ - template \ - static void init_and_execute ARGUMENT_DEFINITION { \ - CLASSNAME::_table.set_resolve_function_and_execute ARGUMENTS; \ - } \ - \ - template \ - void set_resolve_function_and_execute ARGUMENT_DEFINITION { \ - set_resolve_function(); \ - _function[KlassType::Kind] ARGUMENTS; \ - } \ - \ - template \ - void set_init_function() { \ - _function[KlassType::Kind] = &init_and_execute; \ - } \ - \ - template \ - void set_resolve_function() { \ - _function[KlassType::Kind] = UseCompressedOops ? \ - &invoke_real : \ - &invoke_real; \ - } \ - \ - public: \ - \ - Table(){ \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - set_init_function(); \ - } \ - \ - void invoke ARGUMENT_DEFINITION { \ - const int slot = klute.kind(); \ - _function[slot] ARGUMENTS; \ - } \ - \ - }; \ - \ - static Table _table; \ - \ -public: \ - \ - static void invoke ARGUMENT_DEFINITION { _table.invoke ARGUMENTS; } \ - \ -}; \ - \ -template \ -typename CLASSNAME::Table CLASSNAME::_table; \ - \ -template \ -void OopIteratorClosureDispatch::ITERATION_FUNCTION ARGUMENT_DEFINITION { \ - CLASSNAME::invoke ARGUMENTS; \ -} - -//DEFINE_DISPATCH_CLASS( -// OopOopIterateDispatchBounded, -// oop_oop_iterate_bounded, -// (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute), -// (obj, cl, mr, klute) -//) - // Dispatch for normal iteration, unbounded, returns size // size_t XXXKlass::oop_oop_iterate(oop, closure, klute) template From b11768991c62c7d2b13faefcac41c4fd74b3f77b Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 07:01:24 +0200 Subject: [PATCH 038/101] comment --- src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index e5841c7ee2db4..cfa35a0bdbd3a 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -72,8 +72,7 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { // calculates word size given header size, element size, and array length template inline size_t KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { - // The purpose of this function is to be as fast as possible; we hard-code as much - // as we can via template parameters. + // The purpose of this function is to hard-code as much as we can via template parameters. assert(is_obj_array() == is_objarray, "Bad call"); assert(is_type_array() == !is_objarray, "Bad call"); From 2d47c2e65e8cee2f0f058e11f49897eaa75e6482 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 07:12:24 +0200 Subject: [PATCH 039/101] remove UseKLUT completely --- src/hotspot/share/runtime/arguments.cpp | 10 ---------- src/hotspot/share/runtime/globals.hpp | 3 --- 2 files changed, 13 deletions(-) diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index 3c2d741a50d39..69d37a11e45ab 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -3686,16 +3686,6 @@ void Arguments::set_compact_headers_flags() { if (UseCompactObjectHeaders && !UseCompressedClassPointers) { FLAG_SET_DEFAULT(UseCompressedClassPointers, true); } - // KLUT - if (UseKLUT) { - if (!UseCompactObjectHeaders) { - warning("UseKLUT requires COH."); - FLAG_SET_DEFAULT(UseKLUT, false); - } else if (ObjectAlignmentInBytes != BytesPerWord) { - warning("UseKLUT requires default object alignment."); - FLAG_SET_DEFAULT(UseKLUT, false); - } - } #endif } diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index d19e1a0d926e0..cd8b652e1bf72 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -1399,9 +1399,6 @@ const int ObjectAlignmentInBytes = 8; "Force the class space to be allocated at this address or " \ "fails VM initialization (requires -Xshare=off.") \ \ - product(bool, UseKLUT, false, EXPERIMENTAL, \ - "Only for coh") \ - \ product(bool, PrintKLUTStatistics, false, \ "Print KLUT statistics at exit") \ \ From 2dce1e5368166acaf4310819860e346b00fa932a Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 07:29:04 +0200 Subject: [PATCH 040/101] wip --- src/hotspot/share/oops/klassInfoLUT.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index cbb3bb0d03b95..e4e29dc83b935 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -60,7 +60,7 @@ void KlassInfoLUT::initialize() { _entries = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); if (_entries != nullptr) { uses_large_pages = true; - _num_entries = table_size_in_bytes / sizeof(uint32_t); + _num_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); } } } @@ -68,7 +68,7 @@ void KlassInfoLUT::initialize() { table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); _entries = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); os::commit_memory_or_exit((char*)_entries, table_size_in_bytes, false, "KLUT"); - _num_entries = table_size_in_bytes / sizeof(uint32_t); + _num_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); } log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, From 55f0f4204ab411254443e2fcce961083124b2817 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 10 Apr 2025 07:56:13 +0200 Subject: [PATCH 041/101] Fix crash in logging after InstanceKlass creation --- src/hotspot/share/oops/instanceKlass.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index 8f600389cce4b..cbd5a73ce8430 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -504,11 +504,18 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par } { - char tmp[1024]; - log_debug(metaspace)("Returning new IK @" PTR_FORMAT " for %s (use_class_space: %d), nKlass=%u, word size=%d", - p2i(ik), - parser.class_name()->as_C_string(tmp, sizeof(tmp)), use_class_space, - CompressedKlassPointers::encode(ik), size); + LogTarget(Debug, metaspace) lt; + if (lt.is_enabled()) { + char tmp[1024]; + LogStream ls(lt); + ls.print("Returning new IK @" PTR_FORMAT " for %s (%s)", + p2i(ik), parser.class_name()->as_C_string(tmp, sizeof(tmp)), + (use_class_space ? "in class space" : "outside class space")); + if (use_class_space) { + ls.print(", nKlass=%u", CompressedKlassPointers::encode(ik)); + } + ls.print_cr(", wordsize=%d", size); + } } return ik; From 6e845149d4eac347053fbceee75f8e8f6f76c722 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 07:45:51 +0200 Subject: [PATCH 042/101] Fix mirror klass iteration for -UseCompressedOops --- src/hotspot/share/memory/iterator.inline.hpp | 25 +++++++++++++------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index bde41c08c6eb9..f4ea524185a71 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -357,7 +357,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { KlassType::template oop_oop_iterate (obj, cl, klute); return obj->size(); } @@ -381,8 +381,11 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { template void resolve() { if (should_use_slowpath_getsize()) { - _function[KlassType::Kind] = - &invoke_slow; + if (UseCompressedOops) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } } else { if (UseCompressedOops) { if (UseCompactObjectHeaders) { @@ -427,8 +430,11 @@ template typename OopOopIterateDispatchReturnSize::Table OopOopIterateDispatchReturnSize::_table; template -size_t OopIteratorClosureDispatch::oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchReturnSize::invoke (obj, cl, klute); +size_t OopIteratorClosureDispatch::oop_oop_iterate_size(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + + klute.verify_against_klass(obj->klass()); + + return OopOopIterateDispatchReturnSize::invoke(obj, cl, klute); } @@ -449,7 +455,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } template - static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return obj->size(); } @@ -473,8 +479,11 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template void resolve() { if (should_use_slowpath_getsize()) { - _function[KlassType::Kind] = - &invoke_slow; + if (UseCompressedOops) { + _function[KlassType::Kind] = &invoke; + } else { + _function[KlassType::Kind] = &invoke; + } } else { if (UseCompressedOops) { if (UseCompactObjectHeaders) { From 351b145e5fba71a92df9e32325e1e0fcb271234b Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 08:09:16 +0200 Subject: [PATCH 043/101] reduce diff to master --- src/hotspot/share/memory/iterator.inline.hpp | 40 ++++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index f4ea524185a71..319083b813ed0 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -151,12 +151,12 @@ class OopOopIterateDispatch : public DispatchBase { template static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatch::_table.resolve_and_execute (obj, cl, klute); + OopOopIterateDispatch::_table.set_resolve_function_and_execute (obj, cl, klute); } template - void resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - resolve(); + void set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + set_resolve_function(); _function[KlassType::Kind] (obj, cl, klute); } @@ -166,7 +166,7 @@ class OopOopIterateDispatch : public DispatchBase { } template - void resolve() { + void set_resolve_function() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; @@ -220,12 +220,12 @@ class OopOopIterateDispatchReverse { template static void init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatchReverse::_table.resolve_and_execute(obj, cl, klute); + OopOopIterateDispatchReverse::_table.set_resolve_function_and_execute(obj, cl, klute); } template - void resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - resolve(); + void set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + set_resolve_function(); _function[KlassType::Kind](obj, cl, klute); } @@ -235,7 +235,7 @@ class OopOopIterateDispatchReverse { } template - void resolve() { + void set_resolve_function() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; @@ -287,12 +287,12 @@ class OopOopIterateDispatchBounded { template static void init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - OopOopIterateDispatchBounded::_table.resolve_and_execute (obj, cl, mr, klute); + OopOopIterateDispatchBounded::_table.set_resolve_function_and_execute (obj, cl, mr, klute); } template - void resolve_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - resolve(); + void set_resolve_function_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + set_resolve_function(); _function[KlassType::Kind] (obj, cl, mr, klute); } @@ -302,7 +302,7 @@ class OopOopIterateDispatchBounded { } template - void resolve() { + void set_resolve_function() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; @@ -364,7 +364,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { template static size_t init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchReturnSize::_table.resolve_and_execute (obj, cl, klute); + return OopOopIterateDispatchReturnSize::_table.set_resolve_function_and_execute (obj, cl, klute); } template @@ -373,13 +373,13 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - size_t resolve_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - resolve(); + size_t set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + set_resolve_function(); return _function[KlassType::Kind] (obj, cl, klute); } template - void resolve() { + void set_resolve_function() { if (should_use_slowpath_getsize()) { if (UseCompressedOops) { _function[KlassType::Kind] = &invoke; @@ -462,7 +462,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template static size_t init_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - return OopOopIterateDispatchBoundedReturnSize::_table.resolve_and_execute (obj, cl, mr, klute); + return OopOopIterateDispatchBoundedReturnSize::_table.set_resolve_function_and_execute (obj, cl, mr, klute); } template @@ -471,13 +471,13 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } template - size_t resolve_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - resolve(); + size_t set_resolve_function_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + set_resolve_function(); return _function[KlassType::Kind] (obj, cl, mr, klute); } template - void resolve() { + void set_resolve_function() { if (should_use_slowpath_getsize()) { if (UseCompressedOops) { _function[KlassType::Kind] = &invoke; From c27ddb278c789fffb6add2a06cb9dcff450fc0da Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 11:54:08 +0200 Subject: [PATCH 044/101] remove accidental debug output and fix release build --- src/hotspot/share/memory/iterator.inline.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 319083b813ed0..81d04ccf1e7c7 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -431,9 +431,6 @@ typename OopOopIterateDispatchReturnSize::Table OopOopIterateDis template size_t OopIteratorClosureDispatch::oop_oop_iterate_size(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - - klute.verify_against_klass(obj->klass()); - return OopOopIterateDispatchReturnSize::invoke(obj, cl, klute); } From 5e8b542b35196ab96a3cacd4967be95d9d20c748 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 14:19:20 +0200 Subject: [PATCH 045/101] remove obsolete iteration functions (ICLK, IMK) --- .../share/oops/instanceClassLoaderKlass.hpp | 18 +++---------- .../oops/instanceClassLoaderKlass.inline.hpp | 22 --------------- .../share/oops/instanceMirrorKlass.hpp | 12 ++------- .../share/oops/instanceMirrorKlass.inline.hpp | 27 ------------------- 4 files changed, 6 insertions(+), 73 deletions(-) diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index 6caeb78d7fd0c..59787890bf4e1 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -59,30 +59,20 @@ class InstanceClassLoaderKlass: public InstanceKlass { // The InstanceClassLoaderKlass iterators also visit the CLD pointer // Forward iteration - // Iterate over the oop fields and metadata. + // Iterate over the oop fields and metadata template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration - // Iterate over the oop fields and metadata. + // Iterate over the oop fields and metadata template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // klute variants - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); - - DECLARE_EXACT_CAST_FUNCTIONS(InstanceClassLoaderKlass) DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceClassLoaderKlass) diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index c42c632539f89..6edcfa8d7832a 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -60,28 +60,6 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata_bounded(oop obj, #define NO_METADATA_ITERATION assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); -// External entries implementation: klute variants - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate(obj, closure); - oop_oop_iterate_metadata(obj, closure); -} - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure); - NO_METADATA_ITERATION -} - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); - oop_oop_iterate_metadata_bounded(obj, closure, mr); -} - -// External entries implementation: klute variants - template inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index 780f3f6ee3132..1f5afac9907c7 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -101,24 +101,16 @@ class InstanceMirrorKlass: public InstanceKlass { // Forward iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // klute variants - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index 46193928ace72..f3ad1ee484de8 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -117,33 +117,6 @@ void InstanceMirrorKlass::oop_oop_iterate_metadata_bounded(oop obj, OopClosureTy } } -// Externals - -template -void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate(obj, closure); - oop_oop_iterate_metadata(obj, closure); - oop_oop_iterate_statics(obj, closure); - -} - -template -void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure); - - InstanceMirrorKlass::oop_oop_iterate_statics(obj, closure); -} - - -template -void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); - oop_oop_iterate_metadata_bounded(obj, closure, mr); - oop_oop_iterate_statics_bounded(obj, closure, mr); -} - -// Externals, klute variants - template void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); From be48ca54db8abaf2dd8da3ccedf1d000ae5c3092 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 15:06:43 +0200 Subject: [PATCH 046/101] remove obsolete iteration functions (IRK) --- .../gc/parallel/psPromotionManager.inline.hpp | 9 ++-- src/hotspot/share/oops/instanceRefKlass.hpp | 32 ++++++-------- .../share/oops/instanceRefKlass.inline.hpp | 43 ++++--------------- 3 files changed, 25 insertions(+), 59 deletions(-) diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index 2747516e824a1..65190ce9c8ff0 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -39,6 +39,7 @@ #include "logging/log.hpp" #include "memory/iterator.inline.hpp" #include "oops/access.inline.hpp" +#include "oops/klassInfoLUTEntry.hpp" #include "oops/oop.inline.hpp" #include "runtime/orderAccess.hpp" #include "runtime/prefetch.inline.hpp" @@ -114,15 +115,15 @@ class PSPushContentsClosure: public BasicOopIterateClosure { // order of these function calls. // template <> -inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure) { +inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, KlassLUTEntry klute) { oop_oop_iterate_ref_processing(obj, closure); - InstanceKlass::oop_oop_iterate_reverse(obj, closure); + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); } template <> -inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure) { +inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, KlassLUTEntry klute) { oop_oop_iterate_ref_processing(obj, closure); - InstanceKlass::oop_oop_iterate_reverse(obj, closure); + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); } inline void PSPromotionManager::push_contents(oop obj) { diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index 822124a6635cf..362d67a3c8dfd 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -67,24 +67,16 @@ class InstanceRefKlass: public InstanceKlass { // Forward iteration // Iterate over all oop fields and metadata. template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration // Iterate over all oop fields and metadata. template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over all oop fields and metadata. template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // klute variants - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: @@ -92,41 +84,41 @@ class InstanceRefKlass: public InstanceKlass { // Reference processing part of the iterators. template - inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains); + static inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains); // Only perform reference processing if the referent object is within mr. template - inline void oop_oop_iterate_ref_processing_bounded(oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_ref_processing_bounded(oop obj, OopClosureType* closure, MemRegion mr); // Reference processing template - inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure); // Building blocks for specialized handling. template - static void do_referent(oop obj, OopClosureType* closure, Contains& contains); + static inline void do_referent(oop obj, OopClosureType* closure, Contains& contains); template - static void do_discovered(oop obj, OopClosureType* closure, Contains& contains); + static inline void do_discovered(oop obj, OopClosureType* closure, Contains& contains); template - static bool try_discover(oop obj, ReferenceType type, OopClosureType* closure); + static inline bool try_discover(oop obj, ReferenceType type, OopClosureType* closure); // Do discovery while handling InstanceRefKlasses. Reference discovery // is only done if the closure provides a ReferenceProcessor. template - static void oop_oop_iterate_discovery(oop obj, ReferenceType type, OopClosureType* closure, Contains& contains); + static inline void oop_oop_iterate_discovery(oop obj, ReferenceType type, OopClosureType* closure, Contains& contains); // Apply the closure to all fields. No reference discovery is done. template - static void oop_oop_iterate_fields(oop obj, OopClosureType* closure, Contains& contains); + static inline void oop_oop_iterate_fields(oop obj, OopClosureType* closure, Contains& contains); // Apply the closure to all fields, except the referent field. No reference discovery is done. template - static void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains); + static inline void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains); template - static void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; + static inline void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; public: // Update non-static oop maps so 'referent', 'nextPending' and diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index e7a334ffbd474..2b3efdb3dad46 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -106,10 +106,12 @@ void InstanceRefKlass::oop_oop_iterate_fields_except_referent(oop obj, OopClosur template void InstanceRefKlass::oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains) { switch (closure->reference_iteration_mode()) { - case OopIterateClosure::DO_DISCOVERY: + case OopIterateClosure::DO_DISCOVERY: { + const ReferenceType reftype = InstanceKlass::cast(obj->klass())->reference_type(); trace_reference_gc("do_discovery", obj); - oop_oop_iterate_discovery(obj, reference_type(), closure, contains); - break; + oop_oop_iterate_discovery(obj, reftype, closure, contains); + } + break; case OopIterateClosure::DO_FIELDS: trace_reference_gc("do_fields", obj); oop_oop_iterate_fields(obj, closure, contains); @@ -147,51 +149,22 @@ void InstanceRefKlass::oop_oop_iterate_ref_processing_bounded(oop obj, OopClosur oop_oop_iterate_ref_processing(obj, closure, contains); } -template -void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate(obj, closure); - - oop_oop_iterate_ref_processing(obj, closure); -} - - -template -void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure); - - oop_oop_iterate_ref_processing(obj, closure); -} - -template -void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr); - - oop_oop_iterate_ref_processing_bounded(obj, closure, mr); -} - -// klute variants template void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); - // Todo: could oop_oop_iterate_ref_processing not be static? - InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); - k->oop_oop_iterate_ref_processing(obj, closure); + oop_oop_iterate_ref_processing(obj, closure); } template void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); - k->oop_oop_iterate_ref_processing(obj, closure); + oop_oop_iterate_ref_processing(obj, closure); } template void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); - // Todo: for now just resolve the Klass. Maybe more parts can be made static. - InstanceRefKlass* const k = InstanceRefKlass::cast_exact(obj->klass()); - k->oop_oop_iterate_ref_processing_bounded(obj, closure, mr); + oop_oop_iterate_ref_processing_bounded(obj, closure, mr); } #ifdef ASSERT From be359bfa7b992f8cd15adc43107e84650ae25fc3 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 15:26:57 +0200 Subject: [PATCH 047/101] ISCK --- .../share/oops/instanceStackChunkKlass.hpp | 16 ++++++++-------- .../oops/instanceStackChunkKlass.inline.hpp | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index 24bb6483580df..c6587dcdc4b86 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -150,27 +150,27 @@ class InstanceStackChunkKlass: public InstanceKlass { // Forward iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + +private: - // klute variants template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline void oop_oop_iterate(oop obj, OopClosureType* closure); template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); -private: template inline void oop_oop_iterate_header(stackChunkOop chunk, OopClosureType* closure); diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 1f4e5f6eeccb8..2e43d5ed90422 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -90,7 +90,6 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c oop_oop_iterate_lockstack(chunk, closure, mr); } -// Klute variants don't do anything else for now. Just exist to make Dispatch happy. template void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); From f202089fd0393e7d68be8a1a77804e3232ca37b1 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 15:59:33 +0200 Subject: [PATCH 048/101] Revert experimental class loader chunk size policy change --- .../share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp index cd171579c043b..e9baf6b7f219a 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp @@ -70,7 +70,7 @@ static const chunklevel_t g_sequ_boot_non_class[] = { }; static const chunklevel_t g_sequ_boot_class[] = { - chunklevel::CHUNK_LEVEL_16M + chunklevel::CHUNK_LEVEL_256K, // .. repeat last }; From 3619e736bd370304e0113f7b8736d76ef54ef427 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 11 Apr 2025 16:39:30 +0200 Subject: [PATCH 049/101] fix CompressedClassPointersEncodingScheme --- .../CompressedClassPointersEncodingScheme.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/hotspot/jtreg/runtime/CompressedOops/CompressedClassPointersEncodingScheme.java b/test/hotspot/jtreg/runtime/CompressedOops/CompressedClassPointersEncodingScheme.java index 0e72341e56515..02887a62a6ca2 100644 --- a/test/hotspot/jtreg/runtime/CompressedOops/CompressedClassPointersEncodingScheme.java +++ b/test/hotspot/jtreg/runtime/CompressedOops/CompressedClassPointersEncodingScheme.java @@ -137,19 +137,16 @@ public static void main(String[] args) throws Exception { long forceAddress = 32 * G; long ccsSize = 128 * M; - int expectedShift = 6; + int expectedShift = 10; test(forceAddress, true, ccsSize, forceAddress, expectedShift); ccsSize = 512 * M; - expectedShift = 8; test(forceAddress, true, ccsSize, forceAddress, expectedShift); ccsSize = G; - expectedShift = 9; test(forceAddress, true, ccsSize, forceAddress, expectedShift); ccsSize = 3 * G; - expectedShift = 10; test(forceAddress, true, ccsSize, forceAddress, expectedShift); } From 4169e114aec96536d5d9035a265260b94a29cfbd Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 16 Apr 2025 06:59:06 +0200 Subject: [PATCH 050/101] IK, TAK, OAK --- .../metaspace/metaspaceArenaGrowthPolicy.cpp | 2 +- src/hotspot/share/oops/instanceKlass.hpp | 21 ++----------- .../share/oops/instanceKlass.inline.hpp | 30 ------------------- src/hotspot/share/oops/objArrayKlass.hpp | 20 ++++--------- .../share/oops/objArrayKlass.inline.hpp | 24 +++------------ src/hotspot/share/oops/typeArrayKlass.hpp | 15 ++-------- .../share/oops/typeArrayKlass.inline.hpp | 17 ----------- 7 files changed, 15 insertions(+), 114 deletions(-) diff --git a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp index e9baf6b7f219a..b16f2d216103c 100644 --- a/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp +++ b/src/hotspot/share/memory/metaspace/metaspaceArenaGrowthPolicy.cpp @@ -70,7 +70,7 @@ static const chunklevel_t g_sequ_boot_non_class[] = { }; static const chunklevel_t g_sequ_boot_class[] = { - chunklevel::CHUNK_LEVEL_256K, + chunklevel::CHUNK_LEVEL_256K // .. repeat last }; diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index e44f2961fcf70..1a784a081c4ef 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1003,25 +1003,14 @@ class InstanceKlass: public Klass { // The InstanceKlass iterators also visits the Object's klass. // Forward iteration - public: // Iterate over all oop fields in the oop maps. template inline void oop_oop_iterate_oop_maps(oop obj, OopClosureType* closure); - // Iterate over all oop fields and metadata. - template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); - // Iterate over all oop fields in a single oop map. template static inline void oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure); - // Reverse iteration - // Iterate over all oop fields and metadata. - template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); - - private: // Iterate over all oop fields in the oop maps. template inline void oop_oop_iterate_oop_maps_reverse(oop obj, OopClosureType* closure); @@ -1031,16 +1020,10 @@ class InstanceKlass: public Klass { static inline void oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure); // Bounded range iteration - public: // Iterate over all oop fields in the oop maps. template inline void oop_oop_iterate_oop_maps_bounded(oop obj, OopClosureType* closure, MemRegion mr); - // Iterate over all oop fields and metadata. - template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - private: // Iterate over all oop fields in one oop map. template static inline void oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr); @@ -1057,15 +1040,15 @@ class InstanceKlass: public Klass { public: - // klute variants template static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + template static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); - public: u2 idnum_allocated_count() const { return _idnum_allocated_count; } private: diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index b2d206e6aa217..1679b071dfd01 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -177,36 +177,6 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl } } -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - if (Devirtualizer::do_metadata(closure)) { - Devirtualizer::do_klass(closure, this); - } - - oop_oop_iterate_oop_maps(obj, closure); -} - -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - assert(!Devirtualizer::do_metadata(closure), - "Code to handle metadata is not implemented"); - - oop_oop_iterate_oop_maps_reverse(obj, closure); -} - -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - if (Devirtualizer::do_metadata(closure)) { - if (mr.contains(obj)) { - Devirtualizer::do_klass(closure, this); - } - } - - oop_oop_iterate_oop_maps_bounded(obj, closure, mr); -} - -// Klute variants - ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, KlassLUTEntry klute) { const unsigned perma_cld_index = klute.loader_index(); ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 5908a0581fe70..8b71f016535fe 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -118,29 +118,21 @@ class ObjArrayKlass : public ArrayKlass { // The ObjArrayKlass iterators also visits the Object's klass. // Iterate over oop elements and metadata. - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure); + template + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Iterate over oop elements and metadata. - template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Iterate over oop elements within mr, and metadata. - template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); + template + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); // Iterate over oop elements within [start, end), and metadata. template static inline void oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end); - // klute variants - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); - template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); - public: // Iterate over all oop elements. template diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index a0e37cb66c431..3d49cf2df21c4 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -69,7 +69,7 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded( } template -void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { +void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { assert(obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); @@ -81,13 +81,7 @@ void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { } template -void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - // No reverse implementation ATM. - oop_oop_iterate(obj, closure); -} - -template -void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { +void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { assert(obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); @@ -98,20 +92,10 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me oop_oop_iterate_elements_bounded(a, closure, mr.start(), mr.end()); } -// Klute variants -template -void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - oop_oop_iterate(obj, closure); -} - -template -void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - oop_oop_iterate_bounded(obj, closure, mr); -} - template void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - oop_oop_iterate_reverse(obj, closure); + // No reverse implementation ATM. + oop_oop_iterate(obj, closure, klute); } // Like oop_oop_iterate but only iterates over a specified range and only used diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index 2667b4f75cfae..5b3815cd3a747 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -84,27 +84,16 @@ class TypeArrayKlass : public ArrayKlass { static inline void oop_oop_iterate_impl(oop obj, OopIterateClosure* closure); public: - // Wraps oop_oop_iterate_impl to conform to macros. - template - inline void oop_oop_iterate(oop obj, OopClosureType* closure); - // Wraps oop_oop_iterate_impl to conform to macros. - template - inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // Wraps oop_oop_iterate_impl to conform to macros. - template - inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure); - - // klute variants template static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + template static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); - public: static TypeArrayKlass* cast(Klass* k) { return const_cast(cast(const_cast(k))); } diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index 82e7fec25de95..b14b230dc90a5 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -40,23 +40,6 @@ inline void TypeArrayKlass::oop_oop_iterate_impl(oop obj, OopIterateClosure* clo // TypeArrayKlasses are guaranteed processed via the null class loader. } -template -void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure) { - oop_oop_iterate_impl(obj, closure); -} - -template -void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - oop_oop_iterate_impl(obj, closure); -} - -template -void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure) { - oop_oop_iterate_impl(obj, closure); -} - -// Klute variant does nothing special, since there is nothing in the klute that would help -// us here. It only exists to make the dispatcher happy. template inline void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); From 93bd70328288a519855621545aa556b30e7540b0 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 17 Apr 2025 09:08:55 +0200 Subject: [PATCH 051/101] rename slow variants of invoke --- src/hotspot/share/memory/iterator.inline.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 81d04ccf1e7c7..916b169e9e6ad 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -357,7 +357,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { KlassType::template oop_oop_iterate (obj, cl, klute); return obj->size(); } @@ -382,9 +382,9 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { void set_resolve_function() { if (should_use_slowpath_getsize()) { if (UseCompressedOops) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke_slow; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke_slow; } } else { if (UseCompressedOops) { @@ -452,7 +452,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } template - static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return obj->size(); } @@ -477,9 +477,9 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { void set_resolve_function() { if (should_use_slowpath_getsize()) { if (UseCompressedOops) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke_slow; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke_slow; } } else { if (UseCompressedOops) { From e104bfba7d70e8774a67959601ff1b0a4edd0df2 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 17 Apr 2025 12:49:24 +0200 Subject: [PATCH 052/101] split ak_wordsize_given_oop in TAK and OAK version --- src/hotspot/share/memory/iterator.inline.hpp | 29 +++++---- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 10 +++- .../share/oops/klassInfoLUTEntry.inline.hpp | 59 ++++++++++++++----- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 916b169e9e6ad..34f27680e85b2 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -112,17 +112,26 @@ class DispatchBase { template static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { size_t s; - constexpr bool is_instance = KlassType::Kind < Klass::TypeArrayKlassKind; - if (is_instance) { - if (klute.ik_carries_infos()) { - s = klute.ik_wordsize(); - } else { - // Rare path: size not statically computable (e.g. MirrorKlass); calculate using Klass - s = obj->size(); + constexpr Klass::KlassKind kind = KlassType::Kind; + assert(kind == klute.kind(), "Bad call"); + switch (kind) { + case Klass::ObjArrayKlassKind: { + s = klute.oak_calculate_wordsize_given_oop_fast(obj); + break; + } + case Klass::TypeArrayKlassKind: { + s = klute.tak_calculate_wordsize_given_oop_fast(obj); + break; + } + default: { + // all InstanceKlass variants + if (klute.ik_carries_infos()) { + s = klute.ik_wordsize(); + } else { + // Rare path: size not statically computable (e.g. for MirrorKlass instances); calculate using regular Klass + s = obj->size(); + } } - } else { - constexpr bool is_objarray = KlassType::Kind == Klass::ObjArrayKlassKind; - s = klute.ak_calculate_wordsize_given_oop_fast(obj); } assert(s == obj->size(), "Unexpected size (klute %X, %zu vs %zu)", klute.value(), s, obj->size()); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 2a1a8d047e50e..ac42e1c501b77 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -230,9 +230,13 @@ class KlassLUTEntry { // returns distance to first element inline unsigned ak_header_size() const { return _v.ake.hsz; } - // calculates word size given header size, element size, and array length - template - inline size_t ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; + // for an oak, calculates word size given header size, element size, and array length + template + inline size_t oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; + + // for a tak, calculates word size given header size, element size, and array length + template + inline size_t tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; // Helper function, prints current limits static void print_limits(outputStream* st); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index cfa35a0bdbd3a..ba71632730a54 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -70,32 +70,61 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { } // calculates word size given header size, element size, and array length -template -inline size_t KlassLUTEntry::ak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { +template +inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { // The purpose of this function is to hard-code as much as we can via template parameters. - assert(is_obj_array() == is_objarray, "Bad call"); - assert(is_type_array() == !is_objarray, "Bad call"); - + assert(is_obj_array(), "Bad call"); + assert(obj->is_objArray(), "Bad call"); assert(sizeof(OopType) == 4 || sizeof(OopType) == 8, "Bad oop type"); + + // Only call for +UCCP and for standard ObjectAlignmentInBytes + assert(UseCompressedClassPointers, "Bad call"); + constexpr int obj_alignment = BytesPerWord; + assert(MinObjAlignmentInBytes == obj_alignment, "Bad call"); + constexpr int log2_oopsize = (sizeof(OopType) == 4 ? 2 : 3); // narrowOop or Oop - assert(UseCompressedOops == (log2_oopsize == 2), - "Bad call - UseCompressedOops mismatch (%d, %zu)", UseCompressedOops, sizeof(OopType)); + assert(UseCompressedOops == (log2_oopsize == 2), "Bad call"); + + assert(UseCompactObjectHeaders == compact_headers, "Bad call"); + constexpr unsigned length_field_offset = compact_headers ? 8 : 12; + constexpr unsigned first_element_offset = align_up(length_field_offset + BytesPerInt, sizeof(OopType)); + assert(first_element_offset == ak_header_size(), "sanity"); + + // Load length from object + const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); + const unsigned array_length = (size_t) (*array_len_addr); + + // Calculate size + const unsigned size_in_bytes = (array_length << log2_oopsize) + first_element_offset; + return align_up(size_in_bytes, obj_alignment) / HeapWordSize; +} + +// calculates word size given header size, element size, and array length +template +inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { + // The purpose of this function is to hard-code as much as we can via template parameters. + assert(is_type_array(), "Bad call"); + assert(obj->is_typeArray(), "Bad call"); + // Only call for +UCCP and for standard ObjectAlignmentInBytes assert(UseCompressedClassPointers, "Bad call"); - assert(UseCompactObjectHeaders == compact_headers, "Bad call - COH mismatch"); + constexpr int obj_alignment = BytesPerWord; + assert(MinObjAlignmentInBytes == obj_alignment, "Bad call"); - constexpr int alignment = BytesPerWord; - assert(MinObjAlignmentInBytes == alignment, "Bad call - alignment mismatch"); + const int log2_elemsize = ak_log2_elem_size(); - constexpr int length_field_offset = compact_headers ? 8 : 12; - const int first_element_offset = compact_headers ? ak_header_size() : 16; - const unsigned log2_elemsize = is_objarray ? log2_oopsize : ak_log2_elem_size(); + assert(UseCompactObjectHeaders == compact_headers, "Bad call"); + constexpr unsigned length_field_offset = compact_headers ? 8 : 12; + const unsigned first_element_offset = ak_header_size(); + // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); const unsigned array_length = (size_t) (*array_len_addr); + assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); - const size_t size_in_bytes = (array_length << log2_elemsize) + first_element_offset; - return align_up(size_in_bytes, alignment) / HeapWordSize; + // Calculate size + const unsigned size_in_bytes = (array_length << log2_elemsize) + first_element_offset; + return align_up((size_t)size_in_bytes, obj_alignment) / HeapWordSize; } #endif // SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP From c6ff61eb1e57a984a15db684ebc28643eaa94a09 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 18 Apr 2025 09:25:11 +0200 Subject: [PATCH 053/101] HeaderMode enum for use in templates --- src/hotspot/share/oops/objLayout.cpp | 21 ++++----- src/hotspot/share/oops/objLayout.hpp | 49 ++++++++++++++------- src/hotspot/share/oops/objLayout.inline.hpp | 34 +++++++------- src/hotspot/share/oops/oop.inline.hpp | 20 ++++----- 4 files changed, 72 insertions(+), 52 deletions(-) diff --git a/src/hotspot/share/oops/objLayout.cpp b/src/hotspot/share/oops/objLayout.cpp index 774fe24d376e8..95c1afd394916 100644 --- a/src/hotspot/share/oops/objLayout.cpp +++ b/src/hotspot/share/oops/objLayout.cpp @@ -23,27 +23,24 @@ */ #include "oops/markWord.hpp" -#include "oops/objLayout.hpp" +#include "oops/objLayout.inline.hpp" #include "runtime/globals.hpp" #include "utilities/debug.hpp" -ObjLayout::Mode ObjLayout::_klass_mode = ObjLayout::Undefined; +HeaderMode::Mode ObjLayout::_mode; int ObjLayout::_oop_base_offset_in_bytes = 0; bool ObjLayout::_oop_has_klass_gap = false; void ObjLayout::initialize() { - assert(_klass_mode == Undefined, "ObjLayout initialized twice"); + assert(!is_initialized(), "ObjLayout initialized twice"); if (UseCompactObjectHeaders) { - _klass_mode = Compact; - _oop_base_offset_in_bytes = sizeof(markWord); - _oop_has_klass_gap = false; + _mode = HeaderMode::Compact; } else if (UseCompressedClassPointers) { - _klass_mode = Compressed; - _oop_base_offset_in_bytes = sizeof(markWord) + sizeof(narrowKlass); - _oop_has_klass_gap = true; + _mode = HeaderMode::Compressed; } else { - _klass_mode = Uncompressed; - _oop_base_offset_in_bytes = sizeof(markWord) + sizeof(Klass*); - _oop_has_klass_gap = false; + _mode = HeaderMode::Uncompressed; } + HeaderMode hm(_mode); + _oop_base_offset_in_bytes = hm.base_offset_in_bytes(); + _oop_has_klass_gap = hm.has_klass_gap(); } diff --git a/src/hotspot/share/oops/objLayout.hpp b/src/hotspot/share/oops/objLayout.hpp index e434524d4b0db..9ef41d59a05b7 100644 --- a/src/hotspot/share/oops/objLayout.hpp +++ b/src/hotspot/share/oops/objLayout.hpp @@ -25,6 +25,31 @@ #ifndef SHARE_OOPS_OBJLAYOUT_HPP #define SHARE_OOPS_OBJLAYOUT_HPP +class HeaderMode { +public: + enum Mode { + // +UseCompactObjectHeaders (implies +UseCompressedClassPointers) + Compact = 0, + // +UseCompressedClassPointers (-UseCompactObjectHeaders) + Compressed, + // -UseCompressedClassPointers (-UseCompactObjectHeaders) + Uncompressed + }; +private: + const Mode _mode; +public: + HeaderMode(Mode mode) : _mode(mode) {} + + inline bool has_klass_gap() const; + + // Size of markword, or markword+klassword; offset of length for arrays + inline int base_offset_in_bytes() const; + + // Size of markword, or markword+klassword; offset of length for arrays + template + inline int array_first_element_offset_in_bytes() const; +}; + /* * This class helps to avoid loading more than one flag in some * operations that require checking UseCompressedClassPointers, @@ -34,27 +59,21 @@ * the Klass* is accessed frequently, especially by GC oop iterators * and stack-trace builders. */ -class ObjLayout { -public: - enum Mode { - // +UseCompactObjectHeaders (implies +UseCompressedClassPointers) - Compact, - // +UseCompressedClassPointers (-UseCompactObjectHeaders) - Compressed, - // -UseCompressedClassPointers (-UseCompactObjectHeaders) - Uncompressed, - // Not yet initialized - Undefined - }; +class ObjLayout : public AllStatic { -private: - static Mode _klass_mode; + static HeaderMode::Mode _mode; static int _oop_base_offset_in_bytes; static bool _oop_has_klass_gap; + static bool is_initialized() { + return _oop_base_offset_in_bytes > 0; + } + public: static void initialize(); - static inline Mode klass_mode(); + static inline HeaderMode::Mode klass_mode() { + return _mode; + } static inline int oop_base_offset_in_bytes() { return _oop_base_offset_in_bytes; } diff --git a/src/hotspot/share/oops/objLayout.inline.hpp b/src/hotspot/share/oops/objLayout.inline.hpp index 677c1a933bd8d..7d0cea947bdc9 100644 --- a/src/hotspot/share/oops/objLayout.inline.hpp +++ b/src/hotspot/share/oops/objLayout.inline.hpp @@ -27,22 +27,26 @@ #include "oops/objLayout.hpp" -inline ObjLayout::Mode ObjLayout::klass_mode() { -#ifdef ASSERT - assert(_klass_mode != Undefined, "KlassMode not yet initialized"); - if (UseCompactObjectHeaders) { - assert(_klass_mode == Compact, "Klass mode does not match flags"); - } else if (UseCompressedClassPointers) { - assert(_klass_mode == Compressed, "Klass mode does not match flags"); - } else { - assert(_klass_mode == Uncompressed, "Klass mode does not match flags"); +inline bool HeaderMode::has_klass_gap() const { + return _mode == Compressed; +} + + +// Size of markword, or markword+klassword; offset of length for arrays +inline int HeaderMode::base_offset_in_bytes() const { + switch (_mode) { + case Uncompressed: return sizeof(markWord) + sizeof(Klass*); + case Compressed: return sizeof(markWord) + sizeof(narrowKlass); + case Compact: return sizeof(markWord); } -#endif -#ifdef _LP64 - return _klass_mode; -#else - return Uncompressed; -#endif + ShouldNotReachHere(); + return 0; +} + +// Size of markword, or markword+klassword; offset of length for arrays +template +inline int HeaderMode::array_first_element_offset_in_bytes() const { + return align_up(base_offset_in_bytes() + BytesPerInt, sizeof(T)); } #endif // SHARE_OOPS_OBJLAYOUT_INLINE_HPP diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 81dfa85275741..fe7d510673a38 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -99,9 +99,9 @@ void oopDesc::init_mark() { KlassLUTEntry oopDesc::get_klute() const { switch (ObjLayout::klass_mode()) { - case ObjLayout::Compact: + case HeaderMode::Compact: return KlassInfoLUT::lookup(mark().narrow_klass()); - case ObjLayout::Compressed: + case HeaderMode::Compressed: return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass)->klute(); default: return _metadata._klass->klute(); @@ -110,9 +110,9 @@ KlassLUTEntry oopDesc::get_klute() const { Klass* oopDesc::klass() const { switch (ObjLayout::klass_mode()) { - case ObjLayout::Compact: + case HeaderMode::Compact: return mark().klass(); - case ObjLayout::Compressed: + case HeaderMode::Compressed: return CompressedKlassPointers::decode_not_null(_metadata._compressed_klass); default: return _metadata._klass; @@ -121,9 +121,9 @@ Klass* oopDesc::klass() const { Klass* oopDesc::klass_or_null() const { switch (ObjLayout::klass_mode()) { - case ObjLayout::Compact: + case HeaderMode::Compact: return mark().klass_or_null(); - case ObjLayout::Compressed: + case HeaderMode::Compressed: return CompressedKlassPointers::decode(_metadata._compressed_klass); default: return _metadata._klass; @@ -132,9 +132,9 @@ Klass* oopDesc::klass_or_null() const { Klass* oopDesc::klass_or_null_acquire() const { switch (ObjLayout::klass_mode()) { - case ObjLayout::Compact: + case HeaderMode::Compact: return mark_acquire().klass(); - case ObjLayout::Compressed: { + case HeaderMode::Compressed: { narrowKlass narrow_klass = Atomic::load_acquire(&_metadata._compressed_klass); return CompressedKlassPointers::decode(narrow_klass); } @@ -145,9 +145,9 @@ Klass* oopDesc::klass_or_null_acquire() const { Klass* oopDesc::klass_without_asserts() const { switch (ObjLayout::klass_mode()) { - case ObjLayout::Compact: + case HeaderMode::Compact: return mark().klass_without_asserts(); - case ObjLayout::Compressed: + case HeaderMode::Compressed: return CompressedKlassPointers::decode_without_asserts(_metadata._compressed_klass); default: return _metadata._klass; From 042d164c0b7bbedf4c375a633993782c3e625790 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 18 Apr 2025 10:28:07 +0200 Subject: [PATCH 054/101] New templatized variants of oopDesc::get_klute(), klass() --- src/hotspot/share/memory/iterator.hpp | 10 +- src/hotspot/share/memory/iterator.inline.hpp | 105 ++++++++++--------- src/hotspot/share/oops/oop.hpp | 5 + src/hotspot/share/oops/oop.inline.hpp | 39 ++++--- 4 files changed, 94 insertions(+), 65 deletions(-) diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 59b39e93a6322..03d6510ece64b 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -315,11 +315,11 @@ class CompareClosure : public Closure { class OopIteratorClosureDispatch { public: - template static void oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); - template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl, KlassLUTEntry klute); - template static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + template static void oop_oop_iterate (oop obj, OopClosureType* cl); + template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl); + template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr); + template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl); + template static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr); }; diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 34f27680e85b2..dd206747bac9a 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -143,8 +143,9 @@ class DispatchBase { } }; -// Dispatch for normal iteration, unbounded, does not return size -// void XXXKlass::oop_oop_iterate(oop, closure, klute) +//////////////////////////////////////////////// +// Normal + template class OopOopIterateDispatch : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); @@ -196,24 +197,15 @@ class OopOopIterateDispatch : public DispatchBase { static Table _table; public: - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); _table._function[slot](obj, cl, klute); } - }; -template -typename OopOopIterateDispatch::Table OopOopIterateDispatch::_table; - -template -void OopIteratorClosureDispatch::oop_oop_iterate (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatch::invoke(obj, cl, klute); -} +//////////////////////////////////////////////// +// Reverse -// Dispatch for reverse iteration, unbounded, does not return size -// void XXXKlass::oop_oop_iterate_reverse(oop, closure, klute) template class OopOopIterateDispatchReverse { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); @@ -265,21 +257,14 @@ class OopOopIterateDispatchReverse { static Table _table; public: - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); _table._function[slot] (obj, cl, klute); } - }; -template -typename OopOopIterateDispatchReverse::Table OopOopIterateDispatchReverse::_table; - -template -void OopIteratorClosureDispatch::oop_oop_iterate_reverse(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatchReverse::invoke (obj, cl, klute); -} +//////////////////////////////////////////////// +// Bounded template class OopOopIterateDispatchBounded { @@ -332,24 +317,15 @@ class OopOopIterateDispatchBounded { static Table _table; public: - static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { const int slot = klute.kind(); _table._function[slot] (obj, cl, mr, klute); } - }; -template -typename OopOopIterateDispatchBounded::Table OopOopIterateDispatchBounded::_table; - -template -void OopIteratorClosureDispatch::oop_oop_iterate_bounded(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - OopOopIterateDispatchBounded::invoke(obj, cl, mr, klute); -} +//////////////////////////////////////////////// +// Normal, returns size -// Dispatch for normal iteration, unbounded, returns size -// size_t XXXKlass::oop_oop_iterate(oop, closure, klute) template class OopOopIterateDispatchReturnSize : public DispatchBase { @@ -427,25 +403,15 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { static Table _table; public: - static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); return _table._function[slot](obj, cl, klute); } - }; -template -typename OopOopIterateDispatchReturnSize::Table OopOopIterateDispatchReturnSize::_table; +//////////////////////////////////////////////// +// Bounded, returns size -template -size_t OopIteratorClosureDispatch::oop_oop_iterate_size(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchReturnSize::invoke(obj, cl, klute); -} - - -// Dispatch for bounded iteration, returns size -// size_t XXXKlass::oop_oop_iterate_bounded(oop, closure, memregion, klute) template class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); @@ -530,12 +496,55 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { }; + +//////////////////////////////////////////////// +// Dispatcher tables + +template +typename OopOopIterateDispatch::Table OopOopIterateDispatch::_table; + +template +typename OopOopIterateDispatchReverse::Table OopOopIterateDispatchReverse::_table; + +template +typename OopOopIterateDispatchBounded::Table OopOopIterateDispatchBounded::_table; + +template +typename OopOopIterateDispatchReturnSize::Table OopOopIterateDispatchReturnSize::_table; + +template +typename OopOopIterateDispatchBoundedReturnSize::Table OopOopIterateDispatchBoundedReturnSize::_table; + +//////////////////////////////////////////////// +// Dispatcher entriy points + +template +void OopIteratorClosureDispatch::oop_oop_iterate (oop obj, OopClosureType* cl) { + const KlassLUTEntry klute = obj->get_klute(); + OopOopIterateDispatch::invoke(obj, cl, klute); +} + +template +void OopIteratorClosureDispatch::oop_oop_iterate_reverse(oop obj, OopClosureType* cl) { + const KlassLUTEntry klute = obj->get_klute(); + OopOopIterateDispatchReverse::invoke (obj, cl, klute); +} + +template +void OopIteratorClosureDispatch::oop_oop_iterate_bounded(oop obj, OopClosureType* cl, MemRegion mr) { + const KlassLUTEntry klute = obj->get_klute(); + OopOopIterateDispatchBounded::invoke(obj, cl, mr, klute); +} + template -typename OopOopIterateDispatchBoundedReturnSize::Table -OopOopIterateDispatchBoundedReturnSize::_table; +size_t OopIteratorClosureDispatch::oop_oop_iterate_size(oop obj, OopClosureType* cl) { + const KlassLUTEntry klute = obj->get_klute(); + return OopOopIterateDispatchReturnSize::invoke(obj, cl, klute); +} template -size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { +size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr) { + const KlassLUTEntry klute = obj->get_klute(); return OopOopIterateDispatchBoundedReturnSize::invoke(obj, cl, mr, klute); } diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 9d82c1b21dfff..5da279311f02c 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -84,9 +84,14 @@ class oopDesc { // objects during a GC) -- requires a valid klass pointer inline void init_mark(); + template + inline KlassLUTEntry get_klute() const; inline KlassLUTEntry get_klute() const; + template + inline Klass* klass() const; inline Klass* klass() const; + inline Klass* klass_or_null() const; inline Klass* klass_or_null_acquire() const; // Get the klass without running any asserts. diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index fe7d510673a38..7030157f113b3 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -97,8 +97,9 @@ void oopDesc::init_mark() { set_mark(prototype_mark()); } +template KlassLUTEntry oopDesc::get_klute() const { - switch (ObjLayout::klass_mode()) { + switch (mode) { case HeaderMode::Compact: return KlassInfoLUT::lookup(mark().narrow_klass()); case HeaderMode::Compressed: @@ -108,8 +109,18 @@ KlassLUTEntry oopDesc::get_klute() const { } } -Klass* oopDesc::klass() const { +KlassLUTEntry oopDesc::get_klute() const { switch (ObjLayout::klass_mode()) { + case HeaderMode::Compact: return get_klute(); + case HeaderMode::Compressed: return get_klute(); + default: + return get_klute(); + } +} + +template +Klass* oopDesc::klass() const { + switch (mode) { case HeaderMode::Compact: return mark().klass(); case HeaderMode::Compressed: @@ -119,6 +130,15 @@ Klass* oopDesc::klass() const { } } +Klass* oopDesc::klass() const { + switch (ObjLayout::klass_mode()) { + case HeaderMode::Compact: return klass(); + case HeaderMode::Compressed: return klass(); + default: + return klass(); + } +} + Klass* oopDesc::klass_or_null() const { switch (ObjLayout::klass_mode()) { case HeaderMode::Compact: @@ -397,32 +417,27 @@ void oopDesc::incr_age() { template void oopDesc::oop_iterate(OopClosureType* cl) { - const KlassLUTEntry klute = get_klute(); - OopIteratorClosureDispatch::oop_oop_iterate(this, cl, klute); + OopIteratorClosureDispatch::oop_oop_iterate(this, cl); } template void oopDesc::oop_iterate(OopClosureType* cl, MemRegion mr) { - const KlassLUTEntry klute = get_klute(); - OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr, klute); + OopIteratorClosureDispatch::oop_oop_iterate_bounded(this, cl, mr); } template size_t oopDesc::oop_iterate_size(OopClosureType* cl) { - const KlassLUTEntry klute = get_klute(); - return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl, klute); + return OopIteratorClosureDispatch::oop_oop_iterate_size(this, cl); } template size_t oopDesc::oop_iterate_size(OopClosureType* cl, MemRegion mr) { - const KlassLUTEntry klute = get_klute(); - return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr, klute); + return OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(this, cl, mr); } template void oopDesc::oop_iterate_backwards(OopClosureType* cl) { - const KlassLUTEntry klute = get_klute(); - OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl, klute); + OopIteratorClosureDispatch::oop_oop_iterate_reverse(this, cl); } bool oopDesc::is_instanceof_or_null(oop obj, Klass* klass) { From 4b12553f0bee1b15693e7030510bd5d49b6c1437 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 18 Apr 2025 19:36:09 +0200 Subject: [PATCH 055/101] use HeaderMode template parameter for klute size calc functions --- src/hotspot/share/memory/iterator.inline.hpp | 36 ++++++------- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 5 +- .../share/oops/klassInfoLUTEntry.inline.hpp | 41 ++++++++------- src/hotspot/share/oops/objLayout.cpp | 11 ++-- src/hotspot/share/oops/objLayout.hpp | 51 +++++++++---------- src/hotspot/share/oops/objLayout.inline.hpp | 26 +++++----- src/hotspot/share/oops/oop.hpp | 6 +-- src/hotspot/share/oops/oop.inline.hpp | 6 +-- 8 files changed, 93 insertions(+), 89 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index dd206747bac9a..b80150b10c261 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -40,6 +40,7 @@ #include "oops/instanceStackChunkKlass.inline.hpp" #include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/objArrayKlass.inline.hpp" +#include "oops/objLayout.inline.hpp" #include "oops/typeArrayKlass.inline.hpp" #include "utilities/debug.hpp" @@ -109,18 +110,18 @@ class DispatchBase { protected: // Return the size of an object; uses as much hard-coded information as possible - template + template static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { size_t s; constexpr Klass::KlassKind kind = KlassType::Kind; assert(kind == klute.kind(), "Bad call"); switch (kind) { case Klass::ObjArrayKlassKind: { - s = klute.oak_calculate_wordsize_given_oop_fast(obj); + s = klute.oak_calculate_wordsize_given_oop_fast(obj); break; } case Klass::TypeArrayKlassKind: { - s = klute.tak_calculate_wordsize_given_oop_fast(obj); + s = klute.tak_calculate_wordsize_given_oop_fast(obj); break; } default: { @@ -129,7 +130,8 @@ class DispatchBase { s = klute.ik_wordsize(); } else { // Rare path: size not statically computable (e.g. for MirrorKlass instances); calculate using regular Klass - s = obj->size(); + const Klass* k = obj->klass(); + s = obj->size_given_klass(k); } } } @@ -335,10 +337,10 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { FunctionType _function [Klass::KLASS_KIND_COUNT]; - template + template static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { KlassType::template oop_oop_iterate (obj, cl, klute); - return calculate_size_for_object_fast(klute, obj); + return calculate_size_for_object_fast(klute, obj); } template @@ -374,15 +376,15 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } else { if (UseCompressedOops) { if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } } else { if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } } } @@ -420,15 +422,15 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { FunctionType _function [Klass::KLASS_KIND_COUNT]; - template + template static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); - return calculate_size_for_object_fast(klute, obj); + return calculate_size_for_object_fast(klute, obj); } template static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); return obj->size(); } @@ -459,15 +461,15 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } else { if (UseCompressedOops) { if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } } else { if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } else { - _function[KlassType::Kind] = &invoke; + _function[KlassType::Kind] = &invoke; } } } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index ac42e1c501b77..f0616dde0728c 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -28,6 +28,7 @@ // Included by oop.hpp and klass.hpp, keep includes short #include "memory/allStatic.hpp" +#include "oops/objLayout.hpp" #include "utilities/globalDefinitions.hpp" class ArrayKlass; @@ -231,11 +232,11 @@ class KlassLUTEntry { inline unsigned ak_header_size() const { return _v.ake.hsz; } // for an oak, calculates word size given header size, element size, and array length - template + template inline size_t oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; // for a tak, calculates word size given header size, element size, and array length - template + template inline size_t tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; // Helper function, prints current limits diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index ba71632730a54..c0e6192fb8929 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -29,6 +29,7 @@ #include "oops/instanceKlass.hpp" #include "oops/klass.hpp" // for KlassKind #include "oops/klassInfoLUTEntry.hpp" +#include "oops/objLayout.inline.hpp" #include "oops/oop.hpp" #include "utilities/debug.hpp" @@ -69,30 +70,35 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { return _v.ike.omb_offset_2; } +template +static void check_header_mode() { + assert(mode != HeaderMode::Uncompressed, "bad call"); + assert(UseCompressedClassPointers, "bad call"); + if (mode == HeaderMode::Compact) { + assert(UseCompactObjectHeaders, "bad call"); + } + assert(MinObjAlignmentInBytes == BytesPerWord, "Bad call"); +} + // calculates word size given header size, element size, and array length -template +template inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { - // The purpose of this function is to hard-code as much as we can via template parameters. + check_header_mode(); + assert(sizeof(OopType) == (UseCompressedOops ? 4 : 8), "bad call"); assert(is_obj_array(), "Bad call"); assert(obj->is_objArray(), "Bad call"); - assert(sizeof(OopType) == 4 || sizeof(OopType) == 8, "Bad oop type"); // Only call for +UCCP and for standard ObjectAlignmentInBytes - assert(UseCompressedClassPointers, "Bad call"); constexpr int obj_alignment = BytesPerWord; - assert(MinObjAlignmentInBytes == obj_alignment, "Bad call"); - constexpr int log2_oopsize = (sizeof(OopType) == 4 ? 2 : 3); // narrowOop or Oop - assert(UseCompressedOops == (log2_oopsize == 2), "Bad call"); - - assert(UseCompactObjectHeaders == compact_headers, "Bad call"); - constexpr unsigned length_field_offset = compact_headers ? 8 : 12; - constexpr unsigned first_element_offset = align_up(length_field_offset + BytesPerInt, sizeof(OopType)); + constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::oop_base_offset_in_bytes(); + constexpr unsigned first_element_offset = (unsigned)ObjLayoutHelpers::array_first_element_offset_in_bytes(); assert(first_element_offset == ak_header_size(), "sanity"); // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); const unsigned array_length = (size_t) (*array_len_addr); + assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); // Calculate size const unsigned size_in_bytes = (array_length << log2_oopsize) + first_element_offset; @@ -100,22 +106,19 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) } // calculates word size given header size, element size, and array length -template +template inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { + check_header_mode(); // The purpose of this function is to hard-code as much as we can via template parameters. assert(is_type_array(), "Bad call"); assert(obj->is_typeArray(), "Bad call"); // Only call for +UCCP and for standard ObjectAlignmentInBytes - assert(UseCompressedClassPointers, "Bad call"); constexpr int obj_alignment = BytesPerWord; - assert(MinObjAlignmentInBytes == obj_alignment, "Bad call"); - const int log2_elemsize = ak_log2_elem_size(); - - assert(UseCompactObjectHeaders == compact_headers, "Bad call"); - constexpr unsigned length_field_offset = compact_headers ? 8 : 12; - const unsigned first_element_offset = ak_header_size(); + constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::oop_base_offset_in_bytes(); + const unsigned first_element_offset = ak_header_size(); // from klute, cannot calculate at build time + assert(first_element_offset == ak_header_size(), "sanity"); // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); diff --git a/src/hotspot/share/oops/objLayout.cpp b/src/hotspot/share/oops/objLayout.cpp index 95c1afd394916..f6109d2c0f113 100644 --- a/src/hotspot/share/oops/objLayout.cpp +++ b/src/hotspot/share/oops/objLayout.cpp @@ -27,7 +27,7 @@ #include "runtime/globals.hpp" #include "utilities/debug.hpp" -HeaderMode::Mode ObjLayout::_mode; +HeaderMode ObjLayout::_mode; int ObjLayout::_oop_base_offset_in_bytes = 0; bool ObjLayout::_oop_has_klass_gap = false; @@ -35,12 +35,15 @@ void ObjLayout::initialize() { assert(!is_initialized(), "ObjLayout initialized twice"); if (UseCompactObjectHeaders) { _mode = HeaderMode::Compact; + _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } else if (UseCompressedClassPointers) { _mode = HeaderMode::Compressed; + _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } else { _mode = HeaderMode::Uncompressed; + _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } - HeaderMode hm(_mode); - _oop_base_offset_in_bytes = hm.base_offset_in_bytes(); - _oop_has_klass_gap = hm.has_klass_gap(); } diff --git a/src/hotspot/share/oops/objLayout.hpp b/src/hotspot/share/oops/objLayout.hpp index 9ef41d59a05b7..6f389ab99160b 100644 --- a/src/hotspot/share/oops/objLayout.hpp +++ b/src/hotspot/share/oops/objLayout.hpp @@ -25,29 +25,13 @@ #ifndef SHARE_OOPS_OBJLAYOUT_HPP #define SHARE_OOPS_OBJLAYOUT_HPP -class HeaderMode { -public: - enum Mode { - // +UseCompactObjectHeaders (implies +UseCompressedClassPointers) - Compact = 0, - // +UseCompressedClassPointers (-UseCompactObjectHeaders) - Compressed, - // -UseCompressedClassPointers (-UseCompactObjectHeaders) - Uncompressed - }; -private: - const Mode _mode; -public: - HeaderMode(Mode mode) : _mode(mode) {} - - inline bool has_klass_gap() const; - - // Size of markword, or markword+klassword; offset of length for arrays - inline int base_offset_in_bytes() const; - - // Size of markword, or markword+klassword; offset of length for arrays - template - inline int array_first_element_offset_in_bytes() const; +enum class HeaderMode { + // +UseCompactObjectHeaders (implies +UseCompressedClassPointers) + Compact = 0, + // +UseCompressedClassPointers (-UseCompactObjectHeaders) + Compressed, + // -UseCompressedClassPointers (-UseCompactObjectHeaders) + Uncompressed }; /* @@ -61,7 +45,7 @@ class HeaderMode { */ class ObjLayout : public AllStatic { - static HeaderMode::Mode _mode; + static HeaderMode _mode; static int _oop_base_offset_in_bytes; static bool _oop_has_klass_gap; @@ -70,10 +54,9 @@ class ObjLayout : public AllStatic { } public: + static void initialize(); - static inline HeaderMode::Mode klass_mode() { - return _mode; - } + static inline HeaderMode klass_mode() { return _mode; } static inline int oop_base_offset_in_bytes() { return _oop_base_offset_in_bytes; } @@ -82,4 +65,18 @@ class ObjLayout : public AllStatic { } }; +// Has to be a separate class from ObjLayout +struct ObjLayoutHelpers { + + template + static constexpr inline bool oop_has_klass_gap(); + + template + static constexpr inline int oop_base_offset_in_bytes(); + + template + static constexpr inline int array_first_element_offset_in_bytes(); + +}; + #endif // SHARE_OOPS_OBJLAYOUT_HPP diff --git a/src/hotspot/share/oops/objLayout.inline.hpp b/src/hotspot/share/oops/objLayout.inline.hpp index 7d0cea947bdc9..432d8dc7766e1 100644 --- a/src/hotspot/share/oops/objLayout.inline.hpp +++ b/src/hotspot/share/oops/objLayout.inline.hpp @@ -27,26 +27,24 @@ #include "oops/objLayout.hpp" -inline bool HeaderMode::has_klass_gap() const { - return _mode == Compressed; +template +constexpr inline bool ObjLayoutHelpers::oop_has_klass_gap() { + return mode == HeaderMode::Compressed; } - -// Size of markword, or markword+klassword; offset of length for arrays -inline int HeaderMode::base_offset_in_bytes() const { - switch (_mode) { - case Uncompressed: return sizeof(markWord) + sizeof(Klass*); - case Compressed: return sizeof(markWord) + sizeof(narrowKlass); - case Compact: return sizeof(markWord); +template +constexpr inline int ObjLayoutHelpers::oop_base_offset_in_bytes() { + switch (mode) { + case HeaderMode::Uncompressed: return sizeof(markWord) + sizeof(Klass*); + case HeaderMode::Compressed: return sizeof(markWord) + sizeof(narrowKlass); + case HeaderMode::Compact: return sizeof(markWord); } - ShouldNotReachHere(); return 0; } -// Size of markword, or markword+klassword; offset of length for arrays -template -inline int HeaderMode::array_first_element_offset_in_bytes() const { - return align_up(base_offset_in_bytes() + BytesPerInt, sizeof(T)); +template +constexpr inline int ObjLayoutHelpers::array_first_element_offset_in_bytes() { + return align_up(oop_base_offset_in_bytes() + BytesPerInt, sizeof(elemtype)); } #endif // SHARE_OOPS_OBJLAYOUT_INLINE_HPP diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 5da279311f02c..27bf15fc850a9 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -84,11 +84,11 @@ class oopDesc { // objects during a GC) -- requires a valid klass pointer inline void init_mark(); - template + template inline KlassLUTEntry get_klute() const; inline KlassLUTEntry get_klute() const; - template + template inline Klass* klass() const; inline Klass* klass() const; @@ -121,7 +121,7 @@ class oopDesc { // Sometimes (for complicated concurrency-related reasons), it is useful // to be able to figure out the size of an object knowing its klass. - inline size_t size_given_klass(Klass* klass); + inline size_t size_given_klass(const Klass* klass); // type test operations (inlined in oop.inline.hpp) inline bool is_instance() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 7030157f113b3..694329ce80cc8 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -97,7 +97,7 @@ void oopDesc::init_mark() { set_mark(prototype_mark()); } -template +template KlassLUTEntry oopDesc::get_klute() const { switch (mode) { case HeaderMode::Compact: @@ -118,7 +118,7 @@ KlassLUTEntry oopDesc::get_klute() const { } } -template +template Klass* oopDesc::klass() const { switch (mode) { case HeaderMode::Compact: @@ -209,7 +209,7 @@ size_t oopDesc::size() { return size_given_klass(klass()); } -size_t oopDesc::size_given_klass(Klass* klass) { +size_t oopDesc::size_given_klass(const Klass* klass) { int lh = klass->layout_helper(); size_t s; From 0b1e9bb7477bbb9f51e222fd468ae6e5956ceae7 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 19 Apr 2025 08:43:12 +0200 Subject: [PATCH 056/101] IK oop iteration functions reshuffling, simplification --- src/hotspot/share/oops/instanceKlass.hpp | 34 +++++-------------- .../share/oops/instanceKlass.inline.hpp | 28 ++------------- 2 files changed, 11 insertions(+), 51 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 1a784a081c4ef..d43b42934de2c 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1002,39 +1002,21 @@ class InstanceKlass: public Klass { // // The InstanceKlass iterators also visits the Object's klass. - // Forward iteration - // Iterate over all oop fields in the oop maps. + // Iterate over single oop map given by count and offset template - inline void oop_oop_iterate_oop_maps(oop obj, OopClosureType* closure); - - // Iterate over all oop fields in a single oop map. - template - static inline void oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure); - - // Iterate over all oop fields in the oop maps. - template - inline void oop_oop_iterate_oop_maps_reverse(oop obj, OopClosureType* closure); - - // Iterate over all oop fields in one oop map. - template - static inline void oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure); - - // Bounded range iteration - // Iterate over all oop fields in the oop maps. + static inline void oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count); template - inline void oop_oop_iterate_oop_maps_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // Iterate over all oop fields in one oop map. + static inline void oop_oop_iterate_single_oop_map_reverse(oop obj, OopClosureType* closure, unsigned offset, unsigned count); template - static inline void oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr); + static inline void oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count); - // Single oop map iteration given by count and offset + // Iterate over multiple oop maps (non-Klute fallback version for classes >2 oop map entries) template - static inline void oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count); + inline void oop_oop_iterate_oop_maps(oop obj, OopClosureType* closure); template - static inline void oop_oop_iterate_single_oop_map_reverse(oop obj, OopClosureType* closure, unsigned offset, unsigned count); + inline void oop_oop_iterate_oop_maps_reverse(oop obj, OopClosureType* closure); template - static inline void oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count); + inline void oop_oop_iterate_oop_maps_bounded(oop obj, OopClosureType* closure, MemRegion mr); static inline ClassLoaderData* cld_from_klut_or_klass(oop obj, KlassLUTEntry klute); diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 1679b071dfd01..6d6b5d9536d00 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -80,28 +80,6 @@ inline void InstanceKlass::release_set_methods_jmethod_ids(jmethodID* jmeths) { Atomic::release_store(&_methods_jmethod_ids, jmeths); } -// The iteration over the oops in objects is a hot path in the GC code. -// By force inlining the following functions, we get similar GC performance -// as the previous macro based implementation. - -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map(OopMapBlock* map, oop obj, OopClosureType* closure) { - assert(map->offset() > 0, "must be"); - oop_oop_iterate_single_oop_map(obj, closure, (unsigned)map->offset(), map->count()); -} - -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_reverse(OopMapBlock* map, oop obj, OopClosureType* closure) { - assert(map->offset() > 0, "must be"); - oop_oop_iterate_single_oop_map_reverse(obj, closure, (unsigned)map->offset(), map->count()); -} - -template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_map_bounded(OopMapBlock* map, oop obj, OopClosureType* closure, MemRegion mr) { - assert(map->offset() > 0, "must be"); - oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, (unsigned)map->offset(), map->count()); -} - template ALWAYSINLINE void InstanceKlass::oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count) { T* p = obj->field_addr(offset); @@ -152,7 +130,7 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps(oop obj, OopClosureTyp OopMapBlock* const end_map = map + nonstatic_oop_map_count(); for (; map < end_map; ++map) { - oop_oop_iterate_oop_map(map, obj, closure); + oop_oop_iterate_single_oop_map(obj, closure, (unsigned)map->offset(), map->count()); } } @@ -163,7 +141,7 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_reverse(oop obj, OopCl while (start_map < map) { --map; - oop_oop_iterate_oop_map_reverse(map, obj, closure); + oop_oop_iterate_single_oop_map_reverse(obj, closure, (unsigned)map->offset(), map->count()); } } @@ -173,7 +151,7 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl OopMapBlock* const end_map = map + nonstatic_oop_map_count(); for (;map < end_map; ++map) { - oop_oop_iterate_oop_map_bounded(map, obj, closure, mr); + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, (unsigned)map->offset(), map->count()); } } From a770cd08d4ed68b86fce0d0158f5115ce1aaebfe Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sun, 20 Apr 2025 07:21:06 +0200 Subject: [PATCH 057/101] iterator.inline T->OopType --- src/hotspot/share/memory/iterator.inline.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index b80150b10c261..b5b6d9a7787a7 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -156,9 +156,9 @@ class OopOopIterateDispatch : public DispatchBase { FunctionType _function [Klass::KLASS_KIND_COUNT]; - template + template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate(obj, cl, klute); + KlassType::template oop_oop_iterate(obj, cl, klute); } template @@ -216,9 +216,9 @@ class OopOopIterateDispatchReverse { FunctionType _function [Klass::KLASS_KIND_COUNT]; - template + template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_reverse (obj, cl, klute); + KlassType::template oop_oop_iterate_reverse (obj, cl, klute); } template @@ -276,9 +276,9 @@ class OopOopIterateDispatchBounded { FunctionType _function [Klass::KLASS_KIND_COUNT]; - template + template static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); } template From 0637011404bb2e022e2160ec695ab637129dd664 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 22 Apr 2025 09:24:43 +0200 Subject: [PATCH 058/101] iteration dispatch: specify template arg OopClosureType explicitly --- src/hotspot/share/memory/iterator.inline.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index b5b6d9a7787a7..7e78eed34937a 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -158,7 +158,7 @@ class OopOopIterateDispatch : public DispatchBase { template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate(obj, cl, klute); + KlassType::template oop_oop_iterate(obj, cl, klute); } template @@ -218,7 +218,7 @@ class OopOopIterateDispatchReverse { template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_reverse (obj, cl, klute); + KlassType::template oop_oop_iterate_reverse (obj, cl, klute); } template @@ -278,7 +278,7 @@ class OopOopIterateDispatchBounded { template static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); } template @@ -339,13 +339,13 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { template static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate (obj, cl, klute); + KlassType::template oop_oop_iterate (obj, cl, klute); return calculate_size_for_object_fast(klute, obj); } template static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate (obj, cl, klute); + KlassType::template oop_oop_iterate (obj, cl, klute); return obj->size(); } @@ -424,13 +424,13 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return calculate_size_for_object_fast(klute, obj); } template static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); return obj->size(); } From 0790a3eff5a2d43cca3ba38acf898f5306f3137d Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 22 Apr 2025 09:26:40 +0200 Subject: [PATCH 059/101] fix whitespaces --- src/hotspot/share/oops/klass.hpp | 4 ++-- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 540a839e0973a..eacc2418bae15 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -92,7 +92,7 @@ class Klass : public Metadata { static inline TYPE* cast_exact( Klass* k); #define DEFINE_EXACT_CAST_FUNCTIONS(TYPE) \ - inline const TYPE* TYPE::cast_exact(const Klass* k) { \ + inline const TYPE* TYPE::cast_exact(const Klass* k) { \ assert(k != nullptr, "klass null"); \ assert(k->kind() == Klass::TYPE ## Kind, \ "Klass @" PTR_FORMAT ": wrong kind %d", p2i(k), k->kind()) ; \ @@ -108,7 +108,7 @@ class Klass : public Metadata { static inline TYPE* narrow_klass_to_klass(narrowKlass nk); #define DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(TYPE) \ - inline const TYPE* TYPE::narrow_klass_to_const_klass(narrowKlass nk) { \ + inline const TYPE* TYPE::narrow_klass_to_const_klass(narrowKlass nk) { \ const Klass* const k = CompressedKlassPointers::decode_not_null(nk); \ return cast_exact(k); \ } \ diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index f0616dde0728c..27bb13a1fe620 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -74,8 +74,8 @@ class outputStream; class KlassLUTEntry { #define ALL_KLASS_KINDS_DO(what) \ - what(InstanceKlass, IK) \ - what(InstanceRefKlass, IRK) \ + what(InstanceKlass, IK) \ + what(InstanceRefKlass, IRK) \ what(InstanceMirrorKlass, IMK) \ what(InstanceClassLoaderKlass, ICLK) \ what(InstanceStackChunkKlass, ISCK) \ From b05064e6639e134c268373ed172d9547d93a3aa1 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 22 Apr 2025 16:51:02 +0200 Subject: [PATCH 060/101] arrayOopDesc, objArrayOopDesc, templatized versions of length/base etc --- src/hotspot/share/oops/arrayOop.hpp | 9 +++++++ .../share/oops/klassInfoLUTEntry.inline.hpp | 4 ++-- src/hotspot/share/oops/objArrayOop.hpp | 7 ++++++ src/hotspot/share/oops/objArrayOop.inline.hpp | 16 +++++++++++-- src/hotspot/share/oops/objLayout.cpp | 6 ++--- src/hotspot/share/oops/objLayout.hpp | 6 +++-- src/hotspot/share/oops/objLayout.inline.hpp | 6 +++-- test/hotspot/gtest/oops/test_arrayOop.cpp | 14 ++++++++++- test/hotspot/gtest/oops/test_objArrayOop.cpp | 24 ++++++++++++++++++- 9 files changed, 79 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/oops/arrayOop.hpp b/src/hotspot/share/oops/arrayOop.hpp index f0c476a248607..06eadbe97df85 100644 --- a/src/hotspot/share/oops/arrayOop.hpp +++ b/src/hotspot/share/oops/arrayOop.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_ARRAYOOP_HPP #define SHARE_OOPS_ARRAYOOP_HPP +#include "oops/objLayout.hpp" #include "oops/oop.hpp" #include "utilities/align.hpp" #include "utilities/globalDefinitions.hpp" @@ -126,6 +127,14 @@ class arrayOopDesc : public oopDesc { *length_addr_impl(mem) = length; } + // Variants of length_offset_in_bytes/length_addr/length that avoid dynamic switching for UseCompressedClassPointers/UseCompactObjectHeaders + template inline + constexpr static int length_offset_in_bytes_nobranches(); + template + inline int* length_addr_nobranches() const; + template + inline int length_nobranches() const; + // Return the maximum length of an array of BasicType. The length can be passed // to typeArrayOop::object_size(scale, length, header_size) without causing an // overflow. We also need to make sure that this will not overflow a size_t on diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index c0e6192fb8929..6acdfd8a8e5bc 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -91,7 +91,7 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Only call for +UCCP and for standard ObjectAlignmentInBytes constexpr int obj_alignment = BytesPerWord; constexpr int log2_oopsize = (sizeof(OopType) == 4 ? 2 : 3); // narrowOop or Oop - constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::oop_base_offset_in_bytes(); + constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::markword_plus_klass_in_bytes(); constexpr unsigned first_element_offset = (unsigned)ObjLayoutHelpers::array_first_element_offset_in_bytes(); assert(first_element_offset == ak_header_size(), "sanity"); @@ -116,7 +116,7 @@ inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Only call for +UCCP and for standard ObjectAlignmentInBytes constexpr int obj_alignment = BytesPerWord; const int log2_elemsize = ak_log2_elem_size(); - constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::oop_base_offset_in_bytes(); + constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::markword_plus_klass_in_bytes(); const unsigned first_element_offset = ak_header_size(); // from klute, cannot calculate at build time assert(first_element_offset == ak_header_size(), "sanity"); diff --git a/src/hotspot/share/oops/objArrayOop.hpp b/src/hotspot/share/oops/objArrayOop.hpp index 20e2953fee9f5..9276c9b688b60 100644 --- a/src/hotspot/share/oops/objArrayOop.hpp +++ b/src/hotspot/share/oops/objArrayOop.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_OBJARRAYOOP_HPP #include "oops/arrayOop.hpp" +#include "oops/objLayout.hpp" #include "utilities/align.hpp" #include @@ -60,6 +61,12 @@ class objArrayOopDesc : public arrayOopDesc { // base is the address following the header. HeapWord* base() const; + // Variants of base_offset_in_bytes/base avoid dynamic switching for UseCompressedOops/UseCompressedClassPointers/UseCompactObjectHeaders + template + inline constexpr static int base_offset_in_bytes_nobranches(); + template + inline HeapWord* base_nobranches() const; + // Accessing oop obj_at(int index) const; diff --git a/src/hotspot/share/oops/objArrayOop.inline.hpp b/src/hotspot/share/oops/objArrayOop.inline.hpp index 21f95b756de07..9b1887fffa944 100644 --- a/src/hotspot/share/oops/objArrayOop.inline.hpp +++ b/src/hotspot/share/oops/objArrayOop.inline.hpp @@ -25,10 +25,10 @@ #ifndef SHARE_OOPS_OBJARRAYOOP_INLINE_HPP #define SHARE_OOPS_OBJARRAYOOP_INLINE_HPP -#include "oops/objArrayOop.hpp" - #include "oops/access.hpp" #include "oops/arrayOop.hpp" +#include "oops/objArrayOop.hpp" +#include "oops/objLayout.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/globals.hpp" @@ -51,4 +51,16 @@ inline void objArrayOopDesc::obj_at_put(int index, oop value) { HeapAccess::oop_store_at(as_oop(), offset, value); } +// Variants of base_offset_in_bytes/base avoid dynamic switching for UseCompressedOops/UseCompressedClassPointers/UseCompactObjectHeaders +template +inline constexpr int objArrayOopDesc::base_offset_in_bytes_nobranches() { + return ObjLayoutHelpers::array_first_element_offset_in_bytes(); +} + +template +inline HeapWord* objArrayOopDesc::base_nobranches() const { + constexpr int offset = base_offset_in_bytes_nobranches(); + return field_addr(offset); +} + #endif // SHARE_OOPS_OBJARRAYOOP_INLINE_HPP diff --git a/src/hotspot/share/oops/objLayout.cpp b/src/hotspot/share/oops/objLayout.cpp index f6109d2c0f113..dc3e6da995a8c 100644 --- a/src/hotspot/share/oops/objLayout.cpp +++ b/src/hotspot/share/oops/objLayout.cpp @@ -35,15 +35,15 @@ void ObjLayout::initialize() { assert(!is_initialized(), "ObjLayout initialized twice"); if (UseCompactObjectHeaders) { _mode = HeaderMode::Compact; - _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_base_offset_in_bytes = ObjLayoutHelpers::markword_plus_klass_in_bytes(); _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } else if (UseCompressedClassPointers) { _mode = HeaderMode::Compressed; - _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_base_offset_in_bytes = ObjLayoutHelpers::markword_plus_klass_in_bytes(); _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } else { _mode = HeaderMode::Uncompressed; - _oop_base_offset_in_bytes = ObjLayoutHelpers::oop_base_offset_in_bytes(); + _oop_base_offset_in_bytes = ObjLayoutHelpers::markword_plus_klass_in_bytes(); _oop_has_klass_gap = ObjLayoutHelpers::oop_has_klass_gap(); } } diff --git a/src/hotspot/share/oops/objLayout.hpp b/src/hotspot/share/oops/objLayout.hpp index 6f389ab99160b..8f01c93e63ae2 100644 --- a/src/hotspot/share/oops/objLayout.hpp +++ b/src/hotspot/share/oops/objLayout.hpp @@ -25,6 +25,8 @@ #ifndef SHARE_OOPS_OBJLAYOUT_HPP #define SHARE_OOPS_OBJLAYOUT_HPP +// Be frugal with includes here to prevent circularities. + enum class HeaderMode { // +UseCompactObjectHeaders (implies +UseCompressedClassPointers) Compact = 0, @@ -43,7 +45,7 @@ enum class HeaderMode { * the Klass* is accessed frequently, especially by GC oop iterators * and stack-trace builders. */ -class ObjLayout : public AllStatic { +class ObjLayout { static HeaderMode _mode; static int _oop_base_offset_in_bytes; @@ -72,7 +74,7 @@ struct ObjLayoutHelpers { static constexpr inline bool oop_has_klass_gap(); template - static constexpr inline int oop_base_offset_in_bytes(); + static constexpr inline int markword_plus_klass_in_bytes(); template static constexpr inline int array_first_element_offset_in_bytes(); diff --git a/src/hotspot/share/oops/objLayout.inline.hpp b/src/hotspot/share/oops/objLayout.inline.hpp index 432d8dc7766e1..55eeed860d178 100644 --- a/src/hotspot/share/oops/objLayout.inline.hpp +++ b/src/hotspot/share/oops/objLayout.inline.hpp @@ -25,6 +25,8 @@ #ifndef SHARE_OOPS_OBJLAYOUT_INLINE_HPP #define SHARE_OOPS_OBJLAYOUT_INLINE_HPP +// Be frugal with includes here to prevent circularities. + #include "oops/objLayout.hpp" template @@ -33,7 +35,7 @@ constexpr inline bool ObjLayoutHelpers::oop_has_klass_gap() { } template -constexpr inline int ObjLayoutHelpers::oop_base_offset_in_bytes() { +constexpr inline int ObjLayoutHelpers::markword_plus_klass_in_bytes() { switch (mode) { case HeaderMode::Uncompressed: return sizeof(markWord) + sizeof(Klass*); case HeaderMode::Compressed: return sizeof(markWord) + sizeof(narrowKlass); @@ -44,7 +46,7 @@ constexpr inline int ObjLayoutHelpers::oop_base_offset_in_bytes() { template constexpr inline int ObjLayoutHelpers::array_first_element_offset_in_bytes() { - return align_up(oop_base_offset_in_bytes() + BytesPerInt, sizeof(elemtype)); + return align_up(markword_plus_klass_in_bytes() + BytesPerInt, sizeof(elemtype)); } #endif // SHARE_OOPS_OBJLAYOUT_INLINE_HPP diff --git a/test/hotspot/gtest/oops/test_arrayOop.cpp b/test/hotspot/gtest/oops/test_arrayOop.cpp index 33163de427c8f..dcf738dbd2e0a 100644 --- a/test/hotspot/gtest/oops/test_arrayOop.cpp +++ b/test/hotspot/gtest/oops/test_arrayOop.cpp @@ -21,7 +21,7 @@ * questions. */ -#include "oops/arrayOop.hpp" +#include "oops/arrayOop.inline.hpp" #include "oops/oop.inline.hpp" #include "unittest.hpp" #include "utilities/globalDefinitions.hpp" @@ -138,3 +138,15 @@ TEST_VM(arrayOopDesc, base_offset) { EXPECT_EQ(arrayOopDesc::base_offset_in_bytes(T_ARRAY), 12); #endif } + +TEST_VM(arrayOopDesc, nobranches_functions) { + int tmp[] alignas(uint64_t) = { + INT_MAX, INT_MAX - 1, INT_MAX - 2, INT_MAX - 3, INT_MAX - 4 + }; + const arrayOopDesc* const oop = (arrayOopDesc*)tmp; + // Note: length_nobranches uses length_addr_nobranches uses length_offset_in_bytes_nobranches + // so this tests all three + EXPECT_EQ(oop->length_nobranches(), INT_MAX - 4); + EXPECT_EQ(oop->length_nobranches(), INT_MAX - 3); + EXPECT_EQ(oop->length_nobranches(), INT_MAX - 2); +} diff --git a/test/hotspot/gtest/oops/test_objArrayOop.cpp b/test/hotspot/gtest/oops/test_objArrayOop.cpp index 22c9b2efc1119..7e28276dbfda5 100644 --- a/test/hotspot/gtest/oops/test_objArrayOop.cpp +++ b/test/hotspot/gtest/oops/test_objArrayOop.cpp @@ -21,7 +21,7 @@ * questions. */ -#include "oops/objArrayOop.hpp" +#include "oops/objArrayOop.inline.hpp" #include "unittest.hpp" #include "utilities/globalDefinitions.hpp" @@ -61,3 +61,25 @@ TEST_VM(objArrayOop, osize) { } } } + +TEST_VM(objArrayOop, nobranches_functions) { + int tmp[] alignas(uint64_t) = { + INT_MAX, INT_MAX - 1, INT_MAX - 2, INT_MAX - 3, INT_MAX - 4, INT_MAX - 5, INT_MAX - 6 + }; + objArrayOopDesc* const o = (objArrayOopDesc*)tmp; + + // Note: length_nobranches uses length_addr_nobranches uses length_offset_in_bytes_nobranches + // so this tests all three + const int* base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 6); + base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 5); + base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 4); + base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 4); + base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 4); + base = (int*)o->base_nobranches(); + EXPECT_EQ(*base, INT_MAX - 3); +} From 1aaa0919981b147be94309ba582d04b79afb0d5c Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 23 Apr 2025 07:53:45 +0200 Subject: [PATCH 061/101] rework ObjArrayKlass partial iteration --- .../share/gc/g1/g1FullGCMarker.inline.hpp | 3 +- .../share/gc/g1/g1ParScanThreadState.cpp | 12 ++-- src/hotspot/share/gc/serial/serialFullGC.cpp | 2 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 6 +- src/hotspot/share/gc/z/zIterator.inline.hpp | 2 +- src/hotspot/share/memory/iterator.hpp | 2 +- src/hotspot/share/memory/iterator.inline.hpp | 61 +++++++++++++++++++ src/hotspot/share/oops/arrayOop.inline.hpp | 48 +++++++++++++++ .../share/oops/klassInfoLUTEntry.inline.hpp | 4 +- src/hotspot/share/oops/objArrayKlass.hpp | 7 ++- .../share/oops/objArrayKlass.inline.hpp | 25 ++++---- src/hotspot/share/oops/objArrayOop.hpp | 4 -- src/hotspot/share/oops/objArrayOop.inline.hpp | 4 +- 13 files changed, 146 insertions(+), 34 deletions(-) create mode 100644 src/hotspot/share/oops/arrayOop.inline.hpp diff --git a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp index 6cbfe2674e8e8..601b1109dac33 100644 --- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp @@ -39,6 +39,7 @@ #include "gc/shared/continuationGCSupport.inline.hpp" #include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/stringdedup/stringDedup.hpp" +#include "memory/iterator.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" @@ -108,7 +109,7 @@ void G1FullGCMarker::follow_array_chunk(objArrayOop array, int index) { push_objarray(array, end_index); } - array->oop_iterate_range(mark_closure(), beg_index, end_index); + OopIteratorClosureDispatch::oop_oop_iterate_range(array, mark_closure(), beg_index, end_index); } inline void G1FullGCMarker::follow_object(oop obj) { diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 6713c62e69358..bdddd5e111d08 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -40,6 +40,7 @@ #include "gc/shared/stringdedup/stringDedup.hpp" #include "gc/shared/taskqueue.inline.hpp" #include "memory/allocation.inline.hpp" +#include "memory/iterator.inline.hpp" #include "oops/access.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" @@ -232,9 +233,10 @@ void G1ParScanThreadState::do_partial_array(PartialArrayState* state, bool stole G1HeapRegionAttr dest_attr = _g1h->region_attr(to_array); G1SkipCardEnqueueSetter x(&_scanner, dest_attr.is_new_survivor()); // Process claimed task. - to_array->oop_iterate_range(&_scanner, - checked_cast(claim._start), - checked_cast(claim._end)); + OopIteratorClosureDispatch::oop_oop_iterate_range(to_array, + &_scanner, + checked_cast(claim._start), + checked_cast(claim._end)); } MAYBE_INLINE_EVACUATION @@ -254,7 +256,9 @@ void G1ParScanThreadState::start_partial_objarray(oop from_obj, // Process the initial chunk. No need to process the type in the // klass, as it will already be handled by processing the built-in // module. - to_array->oop_iterate_range(&_scanner, 0, checked_cast(initial_chunk_size)); + OopIteratorClosureDispatch::oop_oop_iterate_range(to_array, + &_scanner, + 0, checked_cast(initial_chunk_size)); } MAYBE_INLINE_EVACUATION diff --git a/src/hotspot/share/gc/serial/serialFullGC.cpp b/src/hotspot/share/gc/serial/serialFullGC.cpp index 1546870454754..70292765de401 100644 --- a/src/hotspot/share/gc/serial/serialFullGC.cpp +++ b/src/hotspot/share/gc/serial/serialFullGC.cpp @@ -412,7 +412,7 @@ void SerialFullGC::follow_array_chunk(objArrayOop array, int index) { const int stride = MIN2(len - beg_index, (int) ObjArrayMarkingStride); const int end_index = beg_index + stride; - array->oop_iterate_range(&mark_and_push_closure, beg_index, end_index); + OopIteratorClosureDispatch::oop_oop_iterate_range(array, &mark_and_push_closure, beg_index, end_index); if (end_index < len) { SerialFullGC::push_objarray(array, end_index); // Push the continuation. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 2dc0813e51354..2aaa493885bc5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -167,7 +167,7 @@ inline void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, if (len <= (int) ObjArrayMarkingStride*2) { // A few slices only, process directly - array->oop_iterate_range(cl, 0, len); + OopIteratorClosureDispatch::oop_oop_iterate_range(array, cl, 0, len); } else { int bits = log2i_graceful(len); // Compensate for non-power-of-two arrays, cover the array in excess: @@ -216,7 +216,7 @@ inline void ShenandoahMark::do_chunked_array_start(ShenandoahObjToScanQueue* q, // Process the irregular tail, if present int from = last_idx; if (from < len) { - array->oop_iterate_range(cl, from, len); + OopIteratorClosureDispatch::oop_oop_iterate_range(array, cl, from, len); } } } @@ -248,7 +248,7 @@ inline void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, assert (0 < to && to <= len, "to is sane: %d/%d", to, len); #endif - array->oop_iterate_range(cl, from, to); + OopIteratorClosureDispatch::oop_oop_iterate_range(array, cl, from, to); } template diff --git a/src/hotspot/share/gc/z/zIterator.inline.hpp b/src/hotspot/share/gc/z/zIterator.inline.hpp index fb20a424288d6..e4639e1971966 100644 --- a/src/hotspot/share/gc/z/zIterator.inline.hpp +++ b/src/hotspot/share/gc/z/zIterator.inline.hpp @@ -68,7 +68,7 @@ void ZIterator::oop_iterate(oop obj, OopClosureT* cl) { template void ZIterator::oop_iterate_range(objArrayOop obj, OopClosureT* cl, int start, int end) { assert(!is_invisible_object_array(obj), "not safe"); - obj->oop_iterate_range(cl, start, end); + OopIteratorClosureDispatch::oop_oop_iterate_range(obj, cl, start, end); } template diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 03d6510ece64b..6f4107cfa25c5 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -320,7 +320,7 @@ class OopIteratorClosureDispatch { template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr); template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl); template static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr); - + template static void oop_oop_iterate_range (objArrayOop obj, OopClosureType* cl, int start, int end); }; #endif // SHARE_MEMORY_ITERATOR_HPP diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 7e78eed34937a..fd5b9c28f6b1c 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -499,6 +499,59 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { }; +//////////////////////////////////////////////// +// Abridged dispatch table (just one entry) for oop_iterate_range on ObjArrayOop + +template +class OopOopIterateDispatchRange : public DispatchBase { + typedef void (*FunctionType) (objArrayOop obj, OopClosureType* cl, int start, int end); + + struct Table { + + FunctionType _function; + + template + static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { + ObjArrayKlass::oop_oop_iterate_range(obj, cl, start, end); + } + + static void init_and_execute(objArrayOop obj, OopClosureType* cl, int start, int end) { + OopOopIterateDispatchRange::_table.set_resolve_function_and_execute(obj, cl, start, end); + } + + void set_resolve_function_and_execute(objArrayOop obj, OopClosureType* cl, int start, int end) { + set_resolve_function(); + _function(obj, cl, start, end); + } + + void set_resolve_function() { + if (UseCompressedOops) { + if (UseCompactObjectHeaders) { + _function = &invoke; + } else { + _function = &invoke; + } + } else { + if (UseCompactObjectHeaders) { + _function = &invoke; + } else { + _function = &invoke; + } + } + } + Table() { + _function = &init_and_execute; + } + }; + + static Table _table; + +public: + static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { + _table._function(obj, cl, start, end); + } +}; + //////////////////////////////////////////////// // Dispatcher tables @@ -517,6 +570,9 @@ typename OopOopIterateDispatchReturnSize::Table OopOopIterateDis template typename OopOopIterateDispatchBoundedReturnSize::Table OopOopIterateDispatchBoundedReturnSize::_table; +template +typename OopOopIterateDispatchRange::Table OopOopIterateDispatchRange::_table; + //////////////////////////////////////////////// // Dispatcher entriy points @@ -550,4 +606,9 @@ size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(oop obj, OopClos return OopOopIterateDispatchBoundedReturnSize::invoke(obj, cl, mr, klute); } +template +void OopIteratorClosureDispatch::oop_oop_iterate_range(objArrayOop obj, OopClosureType* cl, int start, int end) { + OopOopIterateDispatchRange::invoke(obj, cl, start, end); +} + #endif // SHARE_MEMORY_ITERATOR_INLINE_HPP diff --git a/src/hotspot/share/oops/arrayOop.inline.hpp b/src/hotspot/share/oops/arrayOop.inline.hpp new file mode 100644 index 0000000000000..eb2d87e882017 --- /dev/null +++ b/src/hotspot/share/oops/arrayOop.inline.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_ARRAYOOP_INLINE_HPP +#define SHARE_OOPS_ARRAYOOP_INLINE_HPP + +#include "oops/arrayOop.hpp" +#include "oops/objLayout.inline.hpp" + +template +inline constexpr int arrayOopDesc::length_offset_in_bytes_nobranches() { + return ObjLayoutHelpers::markword_plus_klass_in_bytes(); +} + +template +inline int* arrayOopDesc::length_addr_nobranches() const { + return field_addr(length_offset_in_bytes_nobranches()); +} + +template +inline int arrayOopDesc::length_nobranches() const { + const int len = *length_addr_nobranches(); + assert(len == length(), "Sanity"); + return len; +} + +#endif // SHARE_OOPS_ARRAYOOP_INLINE_HPP diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index 6acdfd8a8e5bc..0aaea18b1886f 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -97,7 +97,7 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); - const unsigned array_length = (size_t) (*array_len_addr); + const unsigned array_length = (unsigned) (*array_len_addr); assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); // Calculate size @@ -122,7 +122,7 @@ inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); - const unsigned array_length = (size_t) (*array_len_addr); + const unsigned array_length = (unsigned) (*array_len_addr); assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); // Calculate size diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 8b71f016535fe..142e3fe61c0d8 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -27,6 +27,7 @@ #include "oops/arrayKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/objLayout.hpp" #include "utilities/macros.hpp" class ClassLoaderData; @@ -130,15 +131,15 @@ class ObjArrayKlass : public ArrayKlass { static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); // Iterate over oop elements within [start, end), and metadata. - template + template static inline void oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end); - public: + private: + // Iterate over all oop elements. template static inline void oop_oop_iterate_elements(objArrayOop a, OopClosureType* closure); - private: // Iterate over all oop elements with indices within mr. template static inline void oop_oop_iterate_elements_bounded(objArrayOop a, OopClosureType* closure, void* low, void* high); diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 3d49cf2df21c4..cd4810c593d8a 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -29,7 +29,7 @@ #include "memory/memRegion.hpp" #include "oops/arrayKlass.hpp" -#include "oops/arrayOop.hpp" +#include "oops/arrayOop.inline.hpp" #include "oops/klass.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" @@ -100,21 +100,20 @@ void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, Kl // Like oop_oop_iterate but only iterates over a specified range and only used // for objArrayOops. -template +template void ObjArrayKlass::oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end) { - T* low = (T*)a->base() + start; - T* high = (T*)a->base() + end; + assert(start <= end, "Sanity"); - oop_oop_iterate_elements_bounded(a, closure, low, high); -} + assert(start >= 0, "Sanity"); + + const int len = a->length_nobranches(); + if (len < end) { + end = len; + } -// Placed here to resolve include cycle between objArrayKlass.inline.hpp and objArrayOop.inline.hpp -template -void objArrayOopDesc::oop_iterate_range(OopClosureType* blk, int start, int end) { - if (UseCompressedOops) { - ((ObjArrayKlass*)klass())->oop_oop_iterate_range(this, blk, start, end); - } else { - ((ObjArrayKlass*)klass())->oop_oop_iterate_range(this, blk, start, end); + T* const b = (T*)a->base_nobranches(); + for (int i = start; i < end; i++) { + Devirtualizer::do_oop(closure, b + i); } } diff --git a/src/hotspot/share/oops/objArrayOop.hpp b/src/hotspot/share/oops/objArrayOop.hpp index 9276c9b688b60..d27c65d432213 100644 --- a/src/hotspot/share/oops/objArrayOop.hpp +++ b/src/hotspot/share/oops/objArrayOop.hpp @@ -88,10 +88,6 @@ class objArrayOopDesc : public arrayOopDesc { Klass* element_klass(); -public: - // special iterators for index ranges, returns size of object - template - void oop_iterate_range(OopClosureType* blk, int start, int end); }; // See similar requirement for oopDesc. diff --git a/src/hotspot/share/oops/objArrayOop.inline.hpp b/src/hotspot/share/oops/objArrayOop.inline.hpp index 9b1887fffa944..9cdec41cfa6f3 100644 --- a/src/hotspot/share/oops/objArrayOop.inline.hpp +++ b/src/hotspot/share/oops/objArrayOop.inline.hpp @@ -60,7 +60,9 @@ inline constexpr int objArrayOopDesc::base_offset_in_bytes_nobranches() { template inline HeapWord* objArrayOopDesc::base_nobranches() const { constexpr int offset = base_offset_in_bytes_nobranches(); - return field_addr(offset); + HeapWord* const rc = field_addr(offset); + assert(rc == base(), "Sanity"); + return rc; } #endif // SHARE_OOPS_OBJARRAYOOP_INLINE_HPP From d74b219b424f70584cf4040768f93faf18a9b574 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 23 Apr 2025 15:15:43 +0200 Subject: [PATCH 062/101] Fix crash for uncompressed class pointers --- src/hotspot/share/memory/iterator.inline.hpp | 20 +++++++++---------- src/hotspot/share/oops/arrayOop.inline.hpp | 1 - src/hotspot/share/oops/klassInfoLUTEntry.cpp | 2 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 6 +++--- .../share/oops/klassInfoLUTEntry.inline.hpp | 5 ++--- .../share/oops/objArrayKlass.inline.hpp | 6 ++++-- src/hotspot/share/oops/objArrayOop.inline.hpp | 1 - test/hotspot/gtest/oops/test_arrayOop.cpp | 3 +++ 8 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index fd5b9c28f6b1c..715218bb6a033 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -526,17 +526,17 @@ class OopOopIterateDispatchRange : public DispatchBase { void set_resolve_function() { if (UseCompressedOops) { - if (UseCompactObjectHeaders) { - _function = &invoke; - } else { - _function = &invoke; - } + switch (ObjLayout::klass_mode()) { + case HeaderMode::Compact: _function = &invoke; break; + case HeaderMode::Compressed: _function = &invoke; break; + case HeaderMode::Uncompressed: _function = &invoke; break; + }; } else { - if (UseCompactObjectHeaders) { - _function = &invoke; - } else { - _function = &invoke; - } + switch (ObjLayout::klass_mode()) { + case HeaderMode::Compact: _function = &invoke; break; + case HeaderMode::Compressed: _function = &invoke; break; + case HeaderMode::Uncompressed: _function = &invoke; break; + }; } } Table() { diff --git a/src/hotspot/share/oops/arrayOop.inline.hpp b/src/hotspot/share/oops/arrayOop.inline.hpp index eb2d87e882017..3a4251a4ad86c 100644 --- a/src/hotspot/share/oops/arrayOop.inline.hpp +++ b/src/hotspot/share/oops/arrayOop.inline.hpp @@ -41,7 +41,6 @@ inline int* arrayOopDesc::length_addr_nobranches() const { template inline int arrayOopDesc::length_nobranches() const { const int len = *length_addr_nobranches(); - assert(len == length(), "Sanity"); return len; } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 3c082a3bd1c6a..8b4b99ce284c3 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -227,7 +227,7 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { // compare our (truncated) lh with the real one const LayoutHelperHelper lhu = { (unsigned) real_lh }; assert(lhu.bytes.lh_esz == ak_log2_elem_size() && - lhu.bytes.lh_hsz == ak_header_size() && + lhu.bytes.lh_hsz == ak_first_element_offset_in_bytes() && ( (lhu.bytes.lh_tag == 0xC0 && real_kind == Klass::TypeArrayKlassKind) || (lhu.bytes.lh_tag == 0x80 && real_kind == Klass::ObjArrayKlassKind) ), "layouthelper mismatch (layouthelper: 0x%x, klute: 0x%x)", real_lh, _v.raw); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 27bb13a1fe620..7fd3424cc2232 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -225,11 +225,11 @@ class KlassLUTEntry { // Following methods only if AK: - // returns log2 element size + // returns log2 element size in bytes inline unsigned ak_log2_elem_size() const { return _v.ake.l2esz; } - // returns distance to first element - inline unsigned ak_header_size() const { return _v.ake.hsz; } + // returns offset of first array element, in bytes + inline unsigned ak_first_element_offset_in_bytes() const { return _v.ake.hsz; } // for an oak, calculates word size given header size, element size, and array length template diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index 0aaea18b1886f..2f45e193c436c 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -93,7 +93,7 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) constexpr int log2_oopsize = (sizeof(OopType) == 4 ? 2 : 3); // narrowOop or Oop constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::markword_plus_klass_in_bytes(); constexpr unsigned first_element_offset = (unsigned)ObjLayoutHelpers::array_first_element_offset_in_bytes(); - assert(first_element_offset == ak_header_size(), "sanity"); + assert(first_element_offset == ak_first_element_offset_in_bytes(), "sanity"); // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); @@ -117,8 +117,7 @@ inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) constexpr int obj_alignment = BytesPerWord; const int log2_elemsize = ak_log2_elem_size(); constexpr unsigned length_field_offset = (unsigned)ObjLayoutHelpers::markword_plus_klass_in_bytes(); - const unsigned first_element_offset = ak_header_size(); // from klute, cannot calculate at build time - assert(first_element_offset == ak_header_size(), "sanity"); + const unsigned first_element_offset = ak_first_element_offset_in_bytes(); // from klute, cannot calculate at build time // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index cd4810c593d8a..01d3c66fdac3d 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -103,15 +103,17 @@ void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, Kl template void ObjArrayKlass::oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end) { assert(start <= end, "Sanity"); - assert(start >= 0, "Sanity"); const int len = a->length_nobranches(); + assert(a->length() == len, "Sanity (%d vs %d)", len, a->length()); + T* const b = (T*)a->base_nobranches(); + assert(((T*)a->base()) == b, "Sanity"); + if (len < end) { end = len; } - T* const b = (T*)a->base_nobranches(); for (int i = start; i < end; i++) { Devirtualizer::do_oop(closure, b + i); } diff --git a/src/hotspot/share/oops/objArrayOop.inline.hpp b/src/hotspot/share/oops/objArrayOop.inline.hpp index 9cdec41cfa6f3..bc0bff6daba49 100644 --- a/src/hotspot/share/oops/objArrayOop.inline.hpp +++ b/src/hotspot/share/oops/objArrayOop.inline.hpp @@ -61,7 +61,6 @@ template inline HeapWord* objArrayOopDesc::base_nobranches() const { constexpr int offset = base_offset_in_bytes_nobranches(); HeapWord* const rc = field_addr(offset); - assert(rc == base(), "Sanity"); return rc; } diff --git a/test/hotspot/gtest/oops/test_arrayOop.cpp b/test/hotspot/gtest/oops/test_arrayOop.cpp index dcf738dbd2e0a..a61b93d30e89f 100644 --- a/test/hotspot/gtest/oops/test_arrayOop.cpp +++ b/test/hotspot/gtest/oops/test_arrayOop.cpp @@ -146,6 +146,9 @@ TEST_VM(arrayOopDesc, nobranches_functions) { const arrayOopDesc* const oop = (arrayOopDesc*)tmp; // Note: length_nobranches uses length_addr_nobranches uses length_offset_in_bytes_nobranches // so this tests all three + EXPECT_EQ(oop->length_offset_in_bytes_nobranches(), 16); + EXPECT_EQ(oop->length_offset_in_bytes_nobranches(), 12); + EXPECT_EQ(oop->length_offset_in_bytes_nobranches(), 8); EXPECT_EQ(oop->length_nobranches(), INT_MAX - 4); EXPECT_EQ(oop->length_nobranches(), INT_MAX - 3); EXPECT_EQ(oop->length_nobranches(), INT_MAX - 2); From ad1bf81ed5baceeed93a1e64514fa558e4a0be55 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 23 Apr 2025 15:44:59 +0200 Subject: [PATCH 063/101] fixes --- src/hotspot/share/cds/archiveBuilder.cpp | 4 ++- src/hotspot/share/oops/compressedKlass.cpp | 7 ++-- .../share/oops/instanceClassLoaderKlass.hpp | 4 --- .../oops/instanceClassLoaderKlass.inline.hpp | 4 --- src/hotspot/share/oops/instanceKlass.hpp | 3 -- .../share/oops/instanceKlass.inline.hpp | 4 --- .../share/oops/instanceMirrorKlass.hpp | 6 ---- .../share/oops/instanceMirrorKlass.inline.hpp | 4 --- src/hotspot/share/oops/instanceRefKlass.hpp | 4 --- .../share/oops/instanceRefKlass.inline.hpp | 6 +--- .../share/oops/instanceStackChunkKlass.hpp | 7 +--- .../oops/instanceStackChunkKlass.inline.hpp | 19 +++++------ src/hotspot/share/oops/klass.hpp | 33 ------------------- src/hotspot/share/oops/objArrayKlass.hpp | 4 --- .../share/oops/objArrayKlass.inline.hpp | 4 --- src/hotspot/share/oops/typeArrayKlass.hpp | 4 --- .../share/oops/typeArrayKlass.inline.hpp | 4 --- .../share/utilities/devirtualizer.inline.hpp | 1 - test/hotspot/gtest/oops/test_arrayOop.cpp | 2 +- test/hotspot/gtest/oops/test_objArrayOop.cpp | 2 +- 20 files changed, 20 insertions(+), 106 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 2e1f1ef021c71..6d675f4308ea8 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -779,7 +779,9 @@ void ArchiveBuilder::make_klasses_shareable() { if (k->is_instance_klass()) { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } - // every archived Klass shall carry a valid KLUTE. + // Every archived Klass must carry a valid klute. That is because every archived Klass + // would have been created via the usual dynamic class loading or - generation, which should + // have registered the Klass with klut. DEBUG_ONLY(k->klute().verify_against_klass(k);) } diff --git a/src/hotspot/share/oops/compressedKlass.cpp b/src/hotspot/share/oops/compressedKlass.cpp index 0bbaef1f1aa58..85e70ee831834 100644 --- a/src/hotspot/share/oops/compressedKlass.cpp +++ b/src/hotspot/share/oops/compressedKlass.cpp @@ -217,13 +217,14 @@ void CompressedKlassPointers::initialize(address addr, size_t len) { // Calculate Base and Shift: if (UseCompactObjectHeaders) { - - // TODO: can this now merged with !COH? + // We use a large shift in order to shrink the narrowKlass down to 22bit while still retaining + // the ability to address a large (4GB) memory range. This also yields narrowKlass values that + // are ID-like (dense, with almost every value in the 22-bit range addressing another Klass), + // which is perfect for use as lookup index into the Klass info lookup table (KLUT). _base = addr; _shift = max_shift(); } else { - // Traditional (non-compact) header mode const uintptr_t unscaled_max = nth_bit(narrow_klass_pointer_bits()); const uintptr_t zerobased_max = nth_bit(narrow_klass_pointer_bits() + max_shift()); diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index 59787890bf4e1..0bca9eefb5b04 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -72,10 +72,6 @@ class InstanceClassLoaderKlass: public InstanceKlass { // Iterate over the oop fields and metadata. template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); - - DECLARE_EXACT_CAST_FUNCTIONS(InstanceClassLoaderKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceClassLoaderKlass) - }; #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_HPP diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index 6edcfa8d7832a..8918c2f42aa62 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -77,8 +77,4 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosur InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); oop_oop_iterate_metadata_bounded(obj, closure, mr); } - -DEFINE_EXACT_CAST_FUNCTIONS(InstanceClassLoaderKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceClassLoaderKlass) - #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index d43b42934de2c..b2fd6f7264584 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -909,9 +909,6 @@ class InstanceKlass: public Klass { return static_cast(k); } - DECLARE_EXACT_CAST_FUNCTIONS(InstanceKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceKlass) - virtual InstanceKlass* java_super() const { return (super() == nullptr) ? nullptr : cast(super()); } diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 6d6b5d9536d00..85f4b913d8445 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -236,8 +236,4 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType ik->oop_oop_iterate_oop_maps_bounded(obj, closure, mr); } } - -DEFINE_EXACT_CAST_FUNCTIONS(InstanceKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceKlass) - #endif // SHARE_OOPS_INSTANCEKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index 1f5afac9907c7..a4b3757f09fad 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -126,12 +126,6 @@ class InstanceMirrorKlass: public InstanceKlass { static inline void oop_oop_iterate_metadata(oop obj, OopClosureType* closure); template static inline void oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - public: - - DECLARE_EXACT_CAST_FUNCTIONS(InstanceMirrorKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceMirrorKlass) - }; #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_HPP diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index f3ad1ee484de8..b8c780ce44d6f 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -139,8 +139,4 @@ void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closu oop_oop_iterate_metadata_bounded(obj, closure, mr); oop_oop_iterate_statics_bounded(obj, closure, mr); } - -DEFINE_EXACT_CAST_FUNCTIONS(InstanceMirrorKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceMirrorKlass) - #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index 362d67a3c8dfd..bc8c7109642ba 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -128,10 +128,6 @@ class InstanceRefKlass: public InstanceKlass { public: // Verification void oop_verify_on(oop obj, outputStream* st); - - DECLARE_EXACT_CAST_FUNCTIONS(InstanceRefKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceRefKlass) - }; #endif // SHARE_OOPS_INSTANCEREFKLASS_HPP diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index 2b3efdb3dad46..81f80d7479c2f 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -187,9 +187,5 @@ void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) { stream.print_contents_cr(discovered_addr); } } -#endif - -DEFINE_EXACT_CAST_FUNCTIONS(InstanceRefKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceRefKlass) - +#endif // ASSERT #endif // SHARE_OOPS_INSTANCEREFKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index c6587dcdc4b86..ad9774cb5f81c 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -164,6 +164,7 @@ class InstanceStackChunkKlass: public InstanceKlass { private: + // non-static (external iteration function just redirect to these) template inline void oop_oop_iterate(oop obj, OopClosureType* closure); template @@ -192,12 +193,6 @@ class InstanceStackChunkKlass: public InstanceKlass { void do_methods(stackChunkOop chunk, OopIterateClosure* cl); void oop_oop_iterate_stack_slow(stackChunkOop chunk, OopIterateClosure* closure, MemRegion mr); - - public: - - DECLARE_EXACT_CAST_FUNCTIONS(InstanceStackChunkKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceStackChunkKlass) - }; #endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 2e43d5ed90422..8e37ec34179f2 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -90,22 +90,25 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c oop_oop_iterate_lockstack(chunk, closure, mr); } +static InstanceStackChunkKlass* get_isck(oop obj) { + Klass* const k = obj->klass(); + assert(k->is_stack_chunk_instance_klass(), "must be"); + return (InstanceStackChunkKlass*)k; +} + template void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); - k->oop_oop_iterate(obj, closure); + get_isck(obj)->oop_oop_iterate(obj, closure); } template void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); - k->oop_oop_iterate_reverse(obj, closure); + get_isck(obj)->oop_oop_iterate_reverse(obj, closure); } template void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - InstanceStackChunkKlass* const k = InstanceStackChunkKlass::cast_exact(obj->klass()); - k->oop_oop_iterate_bounded(obj, closure, mr); + get_isck(obj)->oop_oop_iterate_bounded(obj, closure, mr); } template @@ -181,8 +184,4 @@ void InstanceStackChunkKlass::oop_oop_iterate_stack_with_bitmap(stackChunkOop ch chunk->bitmap().iterate(&bitmap_closure, chunk->bit_index_for((T*)start), chunk->bit_index_for((T*)end)); } } - -DEFINE_EXACT_CAST_FUNCTIONS(InstanceStackChunkKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(InstanceStackChunkKlass) - #endif // SHARE_OOPS_INSTANCESTACKCHUNKKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index eacc2418bae15..882c0a9a08910 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -82,41 +82,8 @@ class Klass : public Metadata { #undef WHAT UnknownKlassKind }; - static const uint KLASS_KIND_COUNT = ObjArrayKlassKind + 1; - // TODO simplify cleanup - -#define DECLARE_EXACT_CAST_FUNCTIONS(TYPE) \ - static inline const TYPE* cast_exact(const Klass* k); \ - static inline TYPE* cast_exact( Klass* k); - -#define DEFINE_EXACT_CAST_FUNCTIONS(TYPE) \ - inline const TYPE* TYPE::cast_exact(const Klass* k) { \ - assert(k != nullptr, "klass null"); \ - assert(k->kind() == Klass::TYPE ## Kind, \ - "Klass @" PTR_FORMAT ": wrong kind %d", p2i(k), k->kind()) ; \ - return static_cast(k); \ - } \ - inline TYPE* TYPE::cast_exact(Klass* k) { \ - const TYPE* const ck = TYPE::cast_exact((const Klass*)k); \ - return const_cast(ck); \ - } - -#define DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(TYPE) \ - static inline const TYPE* narrow_klass_to_const_klass(narrowKlass nk); \ - static inline TYPE* narrow_klass_to_klass(narrowKlass nk); - -#define DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(TYPE) \ - inline const TYPE* TYPE::narrow_klass_to_const_klass(narrowKlass nk) { \ - const Klass* const k = CompressedKlassPointers::decode_not_null(nk); \ - return cast_exact(k); \ - } \ - inline TYPE* TYPE::narrow_klass_to_klass(narrowKlass nk) { \ - Klass* const k = CompressedKlassPointers::decode_not_null(nk); \ - return cast_exact(k); \ - } \ - protected: // If you add a new field that points to any metaspace object, you diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 142e3fe61c0d8..970d5aa8f2531 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -163,10 +163,6 @@ class ObjArrayKlass : public ArrayKlass { void verify_on(outputStream* st); void oop_verify_on(oop obj, outputStream* st); - - DECLARE_EXACT_CAST_FUNCTIONS(ObjArrayKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(ObjArrayKlass) - }; #endif // SHARE_OOPS_OBJARRAYKLASS_HPP diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 01d3c66fdac3d..3981845bef8d1 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -118,8 +118,4 @@ void ObjArrayKlass::oop_oop_iterate_range(objArrayOop a, OopClosureType* closure Devirtualizer::do_oop(closure, b + i); } } - -DEFINE_EXACT_CAST_FUNCTIONS(ObjArrayKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(ObjArrayKlass) - #endif // SHARE_OOPS_OBJARRAYKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index 5b3815cd3a747..9de494a7f2766 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -125,10 +125,6 @@ class TypeArrayKlass : public ArrayKlass { ModuleEntry* module() const; PackageEntry* package() const; - - DECLARE_EXACT_CAST_FUNCTIONS(TypeArrayKlass) - DECLARE_NARROW_KLASS_UTILITY_FUNCTIONS(TypeArrayKlass) - }; #endif // SHARE_OOPS_TYPEARRAYKLASS_HPP diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index b14b230dc90a5..20b0d43e62597 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -54,8 +54,4 @@ template void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); } - -DEFINE_EXACT_CAST_FUNCTIONS(TypeArrayKlass) -DEFINE_NARROW_KLASS_UTILITY_FUNCTIONS(TypeArrayKlass) - #endif // SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP diff --git a/src/hotspot/share/utilities/devirtualizer.inline.hpp b/src/hotspot/share/utilities/devirtualizer.inline.hpp index a1dbdc01cdcc4..0cae27f87e95b 100644 --- a/src/hotspot/share/utilities/devirtualizer.inline.hpp +++ b/src/hotspot/share/utilities/devirtualizer.inline.hpp @@ -28,7 +28,6 @@ #include "utilities/devirtualizer.hpp" #include "classfile/classLoaderData.hpp" -#include "oops/compressedKlass.inline.hpp" #include "oops/access.inline.hpp" #include "utilities/debug.hpp" diff --git a/test/hotspot/gtest/oops/test_arrayOop.cpp b/test/hotspot/gtest/oops/test_arrayOop.cpp index a61b93d30e89f..d9b2f949ebdf4 100644 --- a/test/hotspot/gtest/oops/test_arrayOop.cpp +++ b/test/hotspot/gtest/oops/test_arrayOop.cpp @@ -140,7 +140,7 @@ TEST_VM(arrayOopDesc, base_offset) { } TEST_VM(arrayOopDesc, nobranches_functions) { - int tmp[] alignas(uint64_t) = { + int tmp[] = { INT_MAX, INT_MAX - 1, INT_MAX - 2, INT_MAX - 3, INT_MAX - 4 }; const arrayOopDesc* const oop = (arrayOopDesc*)tmp; diff --git a/test/hotspot/gtest/oops/test_objArrayOop.cpp b/test/hotspot/gtest/oops/test_objArrayOop.cpp index 7e28276dbfda5..07d526f177245 100644 --- a/test/hotspot/gtest/oops/test_objArrayOop.cpp +++ b/test/hotspot/gtest/oops/test_objArrayOop.cpp @@ -63,7 +63,7 @@ TEST_VM(objArrayOop, osize) { } TEST_VM(objArrayOop, nobranches_functions) { - int tmp[] alignas(uint64_t) = { + int tmp[] = { INT_MAX, INT_MAX - 1, INT_MAX - 2, INT_MAX - 3, INT_MAX - 4, INT_MAX - 5, INT_MAX - 6 }; objArrayOopDesc* const o = (objArrayOopDesc*)tmp; From 12b9125031ee4a960fb136e32c940ab02032b307 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 23 Apr 2025 16:32:53 +0200 Subject: [PATCH 064/101] diff reduction --- .../share/oops/instanceClassLoaderKlass.hpp | 9 ++--- .../oops/instanceClassLoaderKlass.inline.hpp | 35 +++++++------------ src/hotspot/share/oops/instanceKlass.hpp | 11 ++++-- .../share/oops/instanceKlass.inline.hpp | 1 + 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index 0bca9eefb5b04..eb6e1938fa65b 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -46,11 +46,6 @@ class InstanceClassLoaderKlass: public InstanceKlass { private: InstanceClassLoaderKlass(const ClassFileParser& parser) : InstanceKlass(parser, Kind) {} - template - static inline void oop_oop_iterate_metadata(oop obj, OopClosureType* closure); - template - static inline void oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr); - public: InstanceClassLoaderKlass(); @@ -59,12 +54,12 @@ class InstanceClassLoaderKlass: public InstanceKlass { // The InstanceClassLoaderKlass iterators also visit the CLD pointer // Forward iteration - // Iterate over the oop fields and metadata + // Iterate over the oop fields and metadata. template static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration - // Iterate over the oop fields and metadata + // Iterate over the oop fields and metadata. template static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index 8918c2f42aa62..b87df20ef9601 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -35,7 +35,9 @@ #include "utilities/macros.hpp" template -inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata(oop obj, OopClosureType* closure) { +inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate(obj, closure, klute); + if (Devirtualizer::do_metadata(closure)) { ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); // cld can be null if we have a non-registered class loader. @@ -46,7 +48,16 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata(oop obj, OopClosu } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr) { +inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); + assert(!Devirtualizer::do_metadata(closure), + "Code to handle metadata is not implemented"); +} + +template +inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); + if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { ClassLoaderData* cld = java_lang_ClassLoader::loader_data(obj); @@ -57,24 +68,4 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_metadata_bounded(oop obj, } } } - -#define NO_METADATA_ITERATION assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceKlass::oop_oop_iterate(obj, closure, klute); - oop_oop_iterate_metadata(obj, closure); -} - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); - NO_METADATA_ITERATION -} - -template -inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); - oop_oop_iterate_metadata_bounded(obj, closure, mr); -} #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index b2fd6f7264584..e8d6720eaaa2c 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -338,6 +338,7 @@ class InstanceKlass: public Klass { void set_nonstatic_field_size(int size) { _nonstatic_field_size = size; } int static_field_size() const { return _static_field_size; } + void set_static_field_size(int size) { _static_field_size = size; } int static_oop_field_count() const { return (int)_static_oop_field_count; } void set_static_oop_field_count(u2 size) { _static_oop_field_count = size; } @@ -999,7 +1000,7 @@ class InstanceKlass: public Klass { // // The InstanceKlass iterators also visits the Object's klass. - // Iterate over single oop map given by count and offset + // Iterate over single oop map entry given by count and offset template static inline void oop_oop_iterate_single_oop_map(oop obj, OopClosureType* closure, unsigned offset, unsigned count); template @@ -1007,7 +1008,7 @@ class InstanceKlass: public Klass { template static inline void oop_oop_iterate_single_oop_map_bounded(oop obj, OopClosureType* closure, MemRegion mr, unsigned offset, unsigned count); - // Iterate over multiple oop maps (non-Klute fallback version for classes >2 oop map entries) + // Iterate over all fields in all oop map entries (fallback version for classes with more than two oop map entries) template inline void oop_oop_iterate_oop_maps(oop obj, OopClosureType* closure); template @@ -1019,12 +1020,18 @@ class InstanceKlass: public Klass { public: + // Forward iteration + // Iterate over all oop fields and metadata. template static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + // Reverse iteration + // Iterate over all oop fields and metadata. template static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + // Bounded range iteration + // Iterate over all oop fields and metadata. template static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 85f4b913d8445..4925da3074523 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -236,4 +236,5 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType ik->oop_oop_iterate_oop_maps_bounded(obj, closure, mr); } } + #endif // SHARE_OOPS_INSTANCEKLASS_INLINE_HPP From a98b625cb03bafca5f838f43e15209e5a98c5e86 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 23 Apr 2025 16:37:29 +0200 Subject: [PATCH 065/101] make PrintKLUTStatistics diagnostic --- src/hotspot/share/oops/klass.inline.hpp | 1 - src/hotspot/share/runtime/globals.hpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/klass.inline.hpp b/src/hotspot/share/oops/klass.inline.hpp index e7282edc92574..4f9401f1709f0 100644 --- a/src/hotspot/share/oops/klass.inline.hpp +++ b/src/hotspot/share/oops/klass.inline.hpp @@ -25,7 +25,6 @@ #ifndef SHARE_OOPS_KLASS_INLINE_HPP #define SHARE_OOPS_KLASS_INLINE_HPP - #include "oops/klass.hpp" #include "classfile/classLoaderData.inline.hpp" diff --git a/src/hotspot/share/runtime/globals.hpp b/src/hotspot/share/runtime/globals.hpp index cd8b652e1bf72..ab6368c7c11f9 100644 --- a/src/hotspot/share/runtime/globals.hpp +++ b/src/hotspot/share/runtime/globals.hpp @@ -1399,7 +1399,7 @@ const int ObjectAlignmentInBytes = 8; "Force the class space to be allocated at this address or " \ "fails VM initialization (requires -Xshare=off.") \ \ - product(bool, PrintKLUTStatistics, false, \ + product(bool, PrintKLUTStatistics, false, DIAGNOSTIC, \ "Print KLUT statistics at exit") \ \ develop(bool, RandomizeClassSpaceLocation, true, \ From 57dc839e46b5241e773c6876a2166827afb212f4 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 06:59:32 +0200 Subject: [PATCH 066/101] delete unnecessary include --- src/hotspot/share/utilities/devirtualizer.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/share/utilities/devirtualizer.hpp b/src/hotspot/share/utilities/devirtualizer.hpp index 40b0c978f3f79..b4d444dc5a85a 100644 --- a/src/hotspot/share/utilities/devirtualizer.hpp +++ b/src/hotspot/share/utilities/devirtualizer.hpp @@ -25,7 +25,6 @@ #ifndef SHARE_UTILITIES_DEVIRTUALIZER_HPP #define SHARE_UTILITIES_DEVIRTUALIZER_HPP -#include "oops/klassInfoLUTEntry.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/bitMap.hpp" From 3dac8c9ea66aa18cb52adb47e6e469b1626e8216 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 07:43:45 +0200 Subject: [PATCH 067/101] reduce diffs in ISCK IMK --- .../share/oops/instanceMirrorKlass.hpp | 24 +++-- .../share/oops/instanceMirrorKlass.inline.hpp | 90 +++++++++---------- .../oops/instanceStackChunkKlass.inline.hpp | 15 ++-- src/hotspot/share/oops/klass.hpp | 8 ++ 4 files changed, 67 insertions(+), 70 deletions(-) diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index a4b3757f09fad..6a063ebdbbcc5 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -26,8 +26,8 @@ #define SHARE_OOPS_INSTANCEMIRRORKLASS_HPP #include "classfile/vmClasses.hpp" -#include "oops/instanceKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/instanceKlass.hpp" #include "runtime/handles.hpp" #include "utilities/macros.hpp" @@ -46,7 +46,7 @@ class InstanceMirrorKlass: public InstanceKlass { friend class InstanceKlass; public: - static constexpr KlassKind Kind = InstanceMirrorKlassKind; + static const KlassKind Kind = InstanceMirrorKlassKind; private: static int _offset_of_static_fields; @@ -98,34 +98,30 @@ class InstanceMirrorKlass: public InstanceKlass { // // The InstanceMirrorKlass iterators also visit the hidden Klass pointer. + // Iterate over the static fields. + template + inline void oop_oop_iterate_statics(oop obj, OopClosureType* closure); + // Forward iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: // Iterate over the static fields. template - static inline void oop_oop_iterate_statics(oop obj, OopClosureType* closure); - template - static inline void oop_oop_iterate_statics_bounded(oop obj, OopClosureType* closure, MemRegion mr); - - // Iterate over the metadata - template - static inline void oop_oop_iterate_metadata(oop obj, OopClosureType* closure); - template - static inline void oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr); + inline void oop_oop_iterate_statics_bounded(oop obj, OopClosureType* closure, MemRegion mr); }; #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_HPP diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index b8c780ce44d6f..c663e9ed3a42b 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -47,32 +47,9 @@ void InstanceMirrorKlass::oop_oop_iterate_statics(oop obj, OopClosureType* closu } template -void InstanceMirrorKlass::oop_oop_iterate_statics_bounded(oop obj, - OopClosureType* closure, - MemRegion mr) { - T* p = (T*)start_of_static_fields(obj); - T* end = p + java_lang_Class::static_oop_field_count(obj); - - T* const l = (T*)mr.start(); - T* const h = (T*)mr.end(); - assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && - mask_bits((intptr_t)h, sizeof(T)-1) == 0, - "bounded region must be properly aligned"); - - if (p < l) { - p = l; - } - if (end > h) { - end = h; - } - - for (;p < end; ++p) { - Devirtualizer::do_oop(closure, p); - } -} +void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate(obj, closure, klute); -template -void InstanceMirrorKlass::oop_oop_iterate_metadata(oop obj, OopClosureType* closure) { if (Devirtualizer::do_metadata(closure)) { Klass* klass = java_lang_Class::as_Klass(obj); // We'll get null for primitive mirrors. @@ -102,41 +79,60 @@ void InstanceMirrorKlass::oop_oop_iterate_metadata(oop obj, OopClosureType* clos // assert(java_lang_Class::is_primitive(obj), "Sanity check"); } } -} -template -void InstanceMirrorKlass::oop_oop_iterate_metadata_bounded(oop obj, OopClosureType* closure, MemRegion mr) { - if (Devirtualizer::do_metadata(closure)) { - if (mr.contains(obj)) { - Klass* klass = java_lang_Class::as_Klass(obj); - // We'll get null for primitive mirrors. - if (klass != nullptr) { - Devirtualizer::do_klass(closure, klass); - } - } - } + InstanceMirrorKlass* const ik = obj->klass()->as_InstanceMirrorKlass(); + ik->oop_oop_iterate_statics(obj, closure); } template -void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceKlass::oop_oop_iterate(obj, closure, klute); - oop_oop_iterate_metadata(obj, closure); - oop_oop_iterate_statics(obj, closure); +void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { + InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); + InstanceMirrorKlass* const ik = obj->klass()->as_InstanceMirrorKlass(); + ik->oop_oop_iterate_statics(obj, closure); } template -void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); +void InstanceMirrorKlass::oop_oop_iterate_statics_bounded(oop obj, + OopClosureType* closure, + MemRegion mr) { + T* p = (T*)start_of_static_fields(obj); + T* end = p + java_lang_Class::static_oop_field_count(obj); - InstanceMirrorKlass::oop_oop_iterate_statics(obj, closure); -} + T* const l = (T*)mr.start(); + T* const h = (T*)mr.end(); + assert(mask_bits((intptr_t)l, sizeof(T)-1) == 0 && + mask_bits((intptr_t)h, sizeof(T)-1) == 0, + "bounded region must be properly aligned"); + + if (p < l) { + p = l; + } + if (end > h) { + end = h; + } + for (;p < end; ++p) { + Devirtualizer::do_oop(closure, p); + } +} template void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); - oop_oop_iterate_metadata_bounded(obj, closure, mr); - oop_oop_iterate_statics_bounded(obj, closure, mr); + + if (Devirtualizer::do_metadata(closure)) { + if (mr.contains(obj)) { + Klass* klass = java_lang_Class::as_Klass(obj); + // We'll get null for primitive mirrors. + if (klass != nullptr) { + Devirtualizer::do_klass(closure, klass); + } + } + } + + InstanceMirrorKlass* const ik = obj->klass()->as_InstanceMirrorKlass(); + ik->oop_oop_iterate_statics_bounded(obj, closure, mr); } + #endif // SHARE_OOPS_INSTANCEMIRRORKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 8e37ec34179f2..3ce9e37093cab 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -90,25 +90,22 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c oop_oop_iterate_lockstack(chunk, closure, mr); } -static InstanceStackChunkKlass* get_isck(oop obj) { - Klass* const k = obj->klass(); - assert(k->is_stack_chunk_instance_klass(), "must be"); - return (InstanceStackChunkKlass*)k; -} - template void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - get_isck(obj)->oop_oop_iterate(obj, closure); + InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); + ik->oop_oop_iterate(obj, closure); } template void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - get_isck(obj)->oop_oop_iterate_reverse(obj, closure); + InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); + ik->oop_oop_iterate_reverse(obj, closure); } template void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - get_isck(obj)->oop_oop_iterate_bounded(obj, closure, mr); + InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); + ik->oop_oop_iterate_bounded(obj, closure, mr); } template diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 882c0a9a08910..4bb2f372c1b6d 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -84,6 +84,14 @@ class Klass : public Metadata { }; static const uint KLASS_KIND_COUNT = ObjArrayKlassKind + 1; + // Define a set of handy cast functions (e.g. "as_InstanceStackChunkKlass") +#define WHAT(name, shortname) \ + name* as_##name() { \ + assert(_kind == name ## Kind, "not a " #name ); \ + return (name*) this; } + KLASS_ALL_KINDS_DO(WHAT) +#undef WHAT + protected: // If you add a new field that points to any metaspace object, you From 72bda7cff7c989ad2dc1921701b7320f6e37ed3a Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 07:45:23 +0200 Subject: [PATCH 068/101] constexpr --- src/hotspot/share/oops/instanceMirrorKlass.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index 6a063ebdbbcc5..2ec4c5faa61bb 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -46,7 +46,7 @@ class InstanceMirrorKlass: public InstanceKlass { friend class InstanceKlass; public: - static const KlassKind Kind = InstanceMirrorKlassKind; + static constexpr KlassKind Kind = InstanceMirrorKlassKind; private: static int _offset_of_static_fields; From e1dc0e5db416a0f71747b68de0e37af7e804e04d Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 11:01:54 +0200 Subject: [PATCH 069/101] AOTLinkedClassBulkLoader: register cld --- src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp | 2 ++ src/hotspot/share/oops/instanceKlass.inline.hpp | 12 ++++++++---- src/hotspot/share/oops/klassInfoLUT.cpp | 10 ++++++---- src/hotspot/share/oops/klassInfoLUT.hpp | 4 ++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp index 31d95024e3bfd..a96ea08c6006e 100644 --- a/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp +++ b/src/hotspot/share/cds/aotLinkedClassBulkLoader.cpp @@ -36,6 +36,7 @@ #include "memory/resourceArea.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "runtime/handles.inline.hpp" #include "runtime/java.hpp" @@ -176,6 +177,7 @@ void AOTLinkedClassBulkLoader::load_classes_impl(AOTLinkedClassCategory class_ca } ClassLoaderData* loader_data = ClassLoaderData::class_loader_data(loader()); + KlassInfoLUT::register_cld_if_needed(loader_data); for (int i = 0; i < classes->length(); i++) { InstanceKlass* ik = classes->at(i); diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 4925da3074523..c8060c0b6face 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -158,11 +158,15 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, KlassLUTEntry klute) { const unsigned perma_cld_index = klute.loader_index(); ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); - if (cld == nullptr) { - // Rare path - cld = obj->klass()->class_loader_data(); + if (cld != nullptr) { +#ifdef ASSERT + const ClassLoaderData* const cld_real = obj->klass()->class_loader_data(); + assert(cld == cld_real, "CLD mismatch (" PTR_FORMAT " vs " PTR_FORMAT ")", p2i(cld_real), p2i(cld)); +#endif + return cld; } - return cld; + // Rare path + return obj->klass()->class_loader_data(); } // Iterate over all oop fields and metadata. diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index e4e29dc83b935..4d10c4a24c649 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -196,10 +196,12 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); - // In the CDS case, we expect the original class to have been registered during dumptime; so - // it should carry a valid klute. We just copy that value into the lookup table. Note that - // we cannot calculate the klute from the Klass here even if we wanted, since the Klass may not - // yet carry a CLD (CDS calls Klass functions on Klass structures that are not yet fully initialized). + ClassLoaderData* const cld = k->class_loader_data(); + if (cld != nullptr) { + // CDS may call functions on Klass without Klass being fully initialized (CLD null) + // We retain the original loader here + register_cld_if_needed(cld); + } const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index cd55e89896e30..2dbf35f405dc1 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -100,8 +100,6 @@ class KlassInfoLUT : public AllStatic { static bool use_lookup_table() { return _entries != nullptr; } - static void register_cld_if_needed(ClassLoaderData* cld); - public: static void initialize(); @@ -114,6 +112,8 @@ class KlassInfoLUT : public AllStatic { static void print_statistics(outputStream* out); + static void register_cld_if_needed(ClassLoaderData* cld); + }; #endif // SHARE_OOPS_KLASSINFOLUT_HPP From 56654c08b4274d4b4cf895fcc4b0413ade8c8afc Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 11:03:52 +0200 Subject: [PATCH 070/101] reduce diff in IRK --- src/hotspot/share/oops/instanceRefKlass.hpp | 28 +++++++++---------- .../share/oops/instanceRefKlass.inline.hpp | 17 ++++++----- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index bc8c7109642ba..984064cc6073a 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -25,8 +25,8 @@ #ifndef SHARE_OOPS_INSTANCEREFKLASS_HPP #define SHARE_OOPS_INSTANCEREFKLASS_HPP -#include "oops/instanceKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/instanceKlass.hpp" #include "utilities/macros.hpp" class ClassFileParser; @@ -67,58 +67,58 @@ class InstanceRefKlass: public InstanceKlass { // Forward iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Reverse iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); // Bounded range iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); private: // Reference processing part of the iterators. template - static inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains); + inline static void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains); // Only perform reference processing if the referent object is within mr. template - static inline void oop_oop_iterate_ref_processing_bounded(oop obj, OopClosureType* closure, MemRegion mr); + inline static void oop_oop_iterate_ref_processing_bounded(oop obj, OopClosureType* closure, MemRegion mr); // Reference processing template - static inline void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure); + inline static void oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure); // Building blocks for specialized handling. template - static inline void do_referent(oop obj, OopClosureType* closure, Contains& contains); + static void do_referent(oop obj, OopClosureType* closure, Contains& contains); template - static inline void do_discovered(oop obj, OopClosureType* closure, Contains& contains); + static void do_discovered(oop obj, OopClosureType* closure, Contains& contains); template - static inline bool try_discover(oop obj, ReferenceType type, OopClosureType* closure); + static bool try_discover(oop obj, ReferenceType type, OopClosureType* closure); // Do discovery while handling InstanceRefKlasses. Reference discovery // is only done if the closure provides a ReferenceProcessor. template - static inline void oop_oop_iterate_discovery(oop obj, ReferenceType type, OopClosureType* closure, Contains& contains); + static void oop_oop_iterate_discovery(oop obj, OopClosureType* closure, Contains& contains); // Apply the closure to all fields. No reference discovery is done. template - static inline void oop_oop_iterate_fields(oop obj, OopClosureType* closure, Contains& contains); + static void oop_oop_iterate_fields(oop obj, OopClosureType* closure, Contains& contains); // Apply the closure to all fields, except the referent field. No reference discovery is done. template - static inline void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains); + static void oop_oop_iterate_fields_except_referent(oop obj, OopClosureType* closure, Contains& contains); template - static inline void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; + static void trace_reference_gc(const char *s, oop obj) NOT_DEBUG_RETURN; public: // Update non-static oop maps so 'referent', 'nextPending' and diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index 81f80d7479c2f..afcc2855c8e4d 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -79,7 +79,8 @@ bool InstanceRefKlass::try_discover(oop obj, ReferenceType type, OopClosureType* } template -void InstanceRefKlass::oop_oop_iterate_discovery(oop obj, ReferenceType type, OopClosureType* closure, Contains& contains) { +void InstanceRefKlass::oop_oop_iterate_discovery(oop obj, OopClosureType* closure, Contains& contains) { + const ReferenceType type = InstanceKlass::cast(obj->klass())->reference_type(); // Try to discover reference and return if it succeeds. if (try_discover(obj, type, closure)) { return; @@ -106,12 +107,10 @@ void InstanceRefKlass::oop_oop_iterate_fields_except_referent(oop obj, OopClosur template void InstanceRefKlass::oop_oop_iterate_ref_processing(oop obj, OopClosureType* closure, Contains& contains) { switch (closure->reference_iteration_mode()) { - case OopIterateClosure::DO_DISCOVERY: { - const ReferenceType reftype = InstanceKlass::cast(obj->klass())->reference_type(); + case OopIterateClosure::DO_DISCOVERY: trace_reference_gc("do_discovery", obj); - oop_oop_iterate_discovery(obj, reftype, closure, contains); - } - break; + oop_oop_iterate_discovery(obj, closure, contains); + break; case OopIterateClosure::DO_FIELDS: trace_reference_gc("do_fields", obj); oop_oop_iterate_fields(obj, closure, contains); @@ -152,18 +151,21 @@ void InstanceRefKlass::oop_oop_iterate_ref_processing_bounded(oop obj, OopClosur template void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); + oop_oop_iterate_ref_processing(obj, closure); } template void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); + oop_oop_iterate_ref_processing(obj, closure); } template void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); + oop_oop_iterate_ref_processing_bounded(obj, closure, mr); } @@ -187,5 +189,6 @@ void InstanceRefKlass::trace_reference_gc(const char *s, oop obj) { stream.print_contents_cr(discovered_addr); } } -#endif // ASSERT +#endif + #endif // SHARE_OOPS_INSTANCEREFKLASS_INLINE_HPP From be5a1d8365bb2ae06acf3cd425c0faf78b39ef03 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 11:16:54 +0200 Subject: [PATCH 071/101] small fix to Klass::print_on --- src/hotspot/share/oops/klass.cpp | 7 ++++--- src/hotspot/share/oops/typeArrayKlass.inline.hpp | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 8cb9559a9c5f1..184accd84ea62 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -1016,9 +1016,10 @@ void Klass::print_on(outputStream* st) const { st->print("Klass: %s", internal_name()); print_address_on(st); st->cr(); - st->print(" - kind: %d", kind()); st->cr(); - st->print(" - layouthelper raw 0x%x", layout_helper()); st->cr(); - st->print(" - name: "); name()->print_value_on(st); st->cr(); + st->print_cr(" - kind: %d", kind()); + st->print_cr(" - name: "); name()->print_value_on(st); + st->print_cr(" - klute " UINT32_FORMAT_X_0, klute().value()); + st->print_cr(" - layouthelper " INT32_FORMAT_X_0, layout_helper()); } #define BULLET " - " diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp index 20b0d43e62597..3dd090d6f43a9 100644 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.inline.hpp @@ -54,4 +54,5 @@ template void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { oop_oop_iterate_impl(obj, closure); } + #endif // SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP From 6ed0464d71b07fc93fade1d4bf502c64e632bc59 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 12:18:46 +0200 Subject: [PATCH 072/101] Make CLD handling for late-registered CDS/AOT classes more robust --- .../classfile/systemDictionaryShared.cpp | 2 + src/hotspot/share/oops/klass.cpp | 1 + src/hotspot/share/oops/klassInfoLUT.cpp | 47 ++++++++++++++----- src/hotspot/share/oops/klassInfoLUT.hpp | 5 +- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp index 521481889528b..9563bd8bd17cf 100644 --- a/src/hotspot/share/classfile/systemDictionaryShared.cpp +++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp @@ -65,6 +65,7 @@ #include "oops/compressedKlass.inline.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" @@ -170,6 +171,7 @@ InstanceKlass* SystemDictionaryShared::acquire_class_for_current_thread( // No other thread has acquired this yet, so give it to *this thread* ik->set_class_loader_data(loader_data); + KlassInfoLUT::shared_klass_cld_changed(ik); } // No longer holding SharedDictionary_lock diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 184accd84ea62..41e01d2ca2d44 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -870,6 +870,7 @@ void Klass::restore_unshareable_info(ClassLoaderData* loader_data, Handle protec // modify the CLD list outside a safepoint. if (class_loader_data() == nullptr) { set_class_loader_data(loader_data); + KlassInfoLUT::shared_klass_cld_changed(this); // Add to class loader list first before creating the mirror // (same order as class file parsing) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 4d10c4a24c649..8b5d23853545b 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -187,27 +187,52 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { } #if INCLUDE_CDS -// We only tolerate this for CDS. I hope to find a better solution that allows me to -// safely register all Klass structures *before* anyone calls any methods on them. CDS -// makes that very difficult. -// The problem with late_register_class is that it imposes an overhead on every lookup, since -// a lookup table entry may potentially be still invalid. +// We only tolerate this for CDS. +// CDS calls methods on oops for Klass instances that are just mapped into the address space. +// (e.g. oop->size()). These operations use the klute from the KLUT to avoid dereferencing +// Klass itself. However, for CDS the Klass instance may not yet be registered with KLUT, so +// their table entry is missing. +// We currently don't have a good way to iterate all Klass structures that are loaded +// before CDS calls any operations on oops belonging to that Klass. So we self-correct the +// table on the fly. This, unfortunately, adds a branch into the very hot oop iteration path, +// albeit one that should be taken rarely and will hopefully be mitigated by branch prediction. KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); - ClassLoaderData* const cld = k->class_loader_data(); - if (cld != nullptr) { - // CDS may call functions on Klass without Klass being fully initialized (CLD null) - // We retain the original loader here - register_cld_if_needed(cld); - } + // Here we rely on the Klass itself carrying a valid klute that would have been pre-calculated + // during CDS dump time. We just copy that entry. const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); + ClassLoaderData* const cld = k->class_loader_data(); + if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS + register_cld_if_needed(cld); + klute.verify_against_klass(k); + } + // Note: if Klass->class_loader_data is still nullptr, it would be initialized shortly via + // Klass->set_class_loader_data(). There, we will correct the Klass klute if necessary to + // reflect the new CLD. log_klass_registration(k, nk, true, klute.value(), "late-registered"); return klute; } + +void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { + // Called when a shared class gets its ClassLoaderData restored after being loaded. + // The table entry (klute) may already exist, since CDS may have called an operation + // on an oop of that Klass and hence triggered "KlassInfoLUT::late_register_klass". + // But the CLD bits in the klute may need correction. + const KlassLUTEntry klute = k->klute(); + ClassLoaderData* cld = k->class_loader_data(); + assert(cld != nullptr, "must be"); + register_cld_if_needed(cld); + const int cld_index = index_for_cld(cld); + if (klute.loader_index() != cld_index) { + // for simplicity, just recalculate the klute and update the table. + log_debug(klut)("Re-registering Klass after CLD change"); + k->register_with_klut(); + } +} #endif // INCLUDE_CDS // Counters and incrementors diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 2dbf35f405dc1..387611a1de5d5 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -111,9 +111,10 @@ class KlassInfoLUT : public AllStatic { static inline ClassLoaderData* lookup_cld(int index); static void print_statistics(outputStream* out); - static void register_cld_if_needed(ClassLoaderData* cld); - +#if INCLUDE_CDS + static void shared_klass_cld_changed(Klass* k); +#endif }; #endif // SHARE_OOPS_KLASSINFOLUT_HPP From e13cf0cbc4baa619a00f44a68d45b836df7a3033 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 15:29:26 +0200 Subject: [PATCH 073/101] comments --- src/hotspot/share/memory/iterator.hpp | 1 - src/hotspot/share/memory/iterator.inline.hpp | 3 +- src/hotspot/share/oops/klassInfoLUT.cpp | 38 ++++++++++--------- .../share/oops/klassInfoLUT.inline.hpp | 2 - 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 6f4107cfa25c5..45aca68cb6e0f 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -28,7 +28,6 @@ #include "memory/allocation.hpp" #include "memory/memRegion.hpp" #include "oops/oopsHierarchy.hpp" -#include "oops/klassInfoLUT.hpp" class CodeBlob; class nmethod; diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 715218bb6a033..7e3f5f9969110 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -70,7 +70,6 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { m->record_gc_epoch(); } -// TODO update comment // Dispatch table implementation for *Klass::oop_oop_iterate // // It allows for a single call to do a multi-dispatch to an optimized version @@ -574,7 +573,7 @@ template typename OopOopIterateDispatchRange::Table OopOopIterateDispatchRange::_table; //////////////////////////////////////////////// -// Dispatcher entriy points +// Dispatcher entry points template void OopIteratorClosureDispatch::oop_oop_iterate (oop obj, OopClosureType* cl) { diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 8b5d23853545b..3f79be88dbbb0 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -187,41 +187,43 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { } #if INCLUDE_CDS -// We only tolerate this for CDS. -// CDS calls methods on oops for Klass instances that are just mapped into the address space. -// (e.g. oop->size()). These operations use the klute from the KLUT to avoid dereferencing -// Klass itself. However, for CDS the Klass instance may not yet be registered with KLUT, so -// their table entry is missing. -// We currently don't have a good way to iterate all Klass structures that are loaded -// before CDS calls any operations on oops belonging to that Klass. So we self-correct the -// table on the fly. This, unfortunately, adds a branch into the very hot oop iteration path, -// albeit one that should be taken rarely and will hopefully be mitigated by branch prediction. +// We only tolerate this for CDS: +// We currently have no simple way to iterate all Klass structures in a CDS/AOT archive +// before the JVM starts calling methods on oops that refer to these classes. This is because +// these Klasses don't go through normal construction but are mapped into the address space +// when the CDS archive is mapped. +// So it can happen, early during CDS initialization, when CDS revives archived heap objects, +// that the entry in the KLUT table for this Klass is still uninitialized. If that happens, +// this function is called where we add the table entry on the fly. +// Unfortunately, this adds a branch into the very hot oop iteration path, albeit one that +// would hopefully be mitigated by branch prediction since this should be exceedingly rare. KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); - // Here we rely on the Klass itself carrying a valid klute that would have been pre-calculated - // during CDS dump time. We just copy that entry. + // Here we rely on the Klass itself carrying a valid klute already. No need to calculate it. + // That klute would have been pre-calculated during CDS dump time when the to-be-dumped Klass + // was dynamically constructed. + // We just copy that entry into the table slot. const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); _entries[nk] = klute.value(); ClassLoaderData* const cld = k->class_loader_data(); if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS register_cld_if_needed(cld); - klute.verify_against_klass(k); + DEBUG_ONLY(klute.verify_against_klass(k);) + } else { + // Note: cld may still be nullptr; in that case it will be initialized by CDS before the Klass + // is used. At that point we may correct the klute entry to account for the new CDS. } - // Note: if Klass->class_loader_data is still nullptr, it would be initialized shortly via - // Klass->set_class_loader_data(). There, we will correct the Klass klute if necessary to - // reflect the new CLD. log_klass_registration(k, nk, true, klute.value(), "late-registered"); return klute; } void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { // Called when a shared class gets its ClassLoaderData restored after being loaded. - // The table entry (klute) may already exist, since CDS may have called an operation - // on an oop of that Klass and hence triggered "KlassInfoLUT::late_register_klass". - // But the CLD bits in the klute may need correction. + // The function makes sure that the CLD bits in the Klass' klute match the new + // ClassLoaderData. const KlassLUTEntry klute = k->klute(); ClassLoaderData* cld = k->class_loader_data(); assert(cld != nullptr, "must be"); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 897fe7b7a9595..3885c3add1892 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -46,8 +46,6 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { if (!e.is_valid()) { // This branch only exists because it is so very difficult to iterate CDS classes after loading // CDS archives. See discussion surrounding 8353225. Hopefully we can remove this in the future. - // And equally hopefully branch prediction takes the sting out of this branch in real oop iteration. - // In order to measure without this branch, build without CDS. return late_register_klass(nk); } #else From 5119745c17be29b7010723dbbb296f5b44139c1b Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 15:40:39 +0200 Subject: [PATCH 074/101] Comments --- src/hotspot/share/oops/objArrayKlass.inline.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 3981845bef8d1..2d66e5bbca746 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -86,7 +86,7 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me objArrayOop a = objArrayOop(obj); if (Devirtualizer::do_metadata(closure)) { - Devirtualizer::do_klass(closure, a->klass()); // Todo: why not "this" ?? + Devirtualizer::do_klass(closure, a->klass()); } oop_oop_iterate_elements_bounded(a, closure, mr.start(), mr.end()); From 1152f58c19f25e616814c51263d9f5fcb99d922d Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 24 Apr 2025 16:25:20 +0200 Subject: [PATCH 075/101] comment --- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 101 ++++++++++++++----- 1 file changed, 78 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 7fd3424cc2232..ea8b80e94d45b 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -38,38 +38,93 @@ class oopDesc; class OopMapBlock; class outputStream; -// msb lsb +// A Klass Info Lookup Table Entry (klute) is a 32-bit value carrying, in a very condensed form, some of the most +// important information about a Klass. // -// invalid_entry (debug zap): 1111 1111 1111 1111 1111 1111 1111 1111 (relies on ClassKind == 0b111 == 7 being invalid) +// It carries the following information: +// - The Klass kind +// - The ClassLoaderData association (if the Klass belongs to one of the three permanent CLDs - boot, system, app) // -// All valid entries: KKKL L... .... .... .... .... .... .... +// - For InstanceKlass klasses, it may carry more information iff the object satisfies the following conditions: +// - its size, in words, is less than 64 heap words (512 bytes) +// - it has less than three oop map entries, and these oop map entries are within certain limits for position and count +// - in that case, the klute carries the object size information and information for both entries. // -// InstanceKlass: KKKL LSSS SSSO OOOO CCCC CCOO OOCC CCCC -// 2 2222 2222 2211 1111 1111 -// InstanceKlass, has_no_addinfo: KKKL L000 0000 0000 0000 0000 0000 0000 (all IK specific bits 0) (note: means that "0" is a valid IK entry with no add. info) -// InstanceKlass, has no oopmap entries: KKKL LSSS SSS. .... .... .... 0000 0000 (omb count bits are 0) (only valid if !has_no_addinfo) +// - For ArrayKlass klasses, it carries parts of the layout helper needed to calculate the object size. // -// ArrayKlass: KKKL L--- ---- ---- ---- ---- -eeh hhhh +// ----------------- Common bits --------------------------------- // +// These bits are always populated. // -// IK specific bits: -// C1 : Count of first OMB (6 bits) -// O1 : Offset, in number-of-oops, of first OMB (4 bits) -// C2 : Count of second OMB (6 bits) -// O2 : Offset, in number-of-oops, of second OMB (5 bits) -// S : Object instance size in words (6 bits) +// Bit 31 27 23 19 15 11 7 3 0 +// K K K L L - - - - - - - - - - - - - - - - - - - - - - - - - - - +// \ / \ / +// ----- - +// kind loader // -// AK specific bits: -// h : header size (5 bits) -// e : log2 element size (2 bits) +// K (3 bits): The Klass kind. Note that 0b111 = 7 is not a valid KlassKind, and therefore used (see below) to designate an invalid klute). +// L (2 bits): Whether the Klass is associated with one of the three permanent CLDs: +// '0' unknown CLD, '1' boot loader CLD, '2' system loader CLD, '3' platform loader CLD // -// Common bits: -// L : Loader (2 bits) -// (0 unknown, 1 boot loader, -// 2 system loader, 3 platform loader) -// K : KlassKind (3 bits) // -// - : unused +// ----------------- InstanceKlass encoding ---------------------- +// +// Bit 31 27 23 19 15 11 7 3 0 +// K K K L L S S S S S O2 O2 O2 O2 O2 C2 C2 C2 C2 C2 C2 C2 O1 O1 O1 O1 C1 C1 C1 C1 C1 C1 +// \ / \ / \ / +// ----------- --------------------------------- -------------------------- +// obj size offset, count for oop map 2 offset, count for oop map 1 +// +// C1 (6 bits): Count of first Oop Map Entry +// O1 (4 bits): Offset, in number-of-(oop|narrowOop), of second Oop Map Entry +// C2 (6 bits): Count of first Oop Map Entry +// O2 (5 bits): Offset, in number-of-(oop|narrowOop), of second Oop Map Entry +// S (5 bits): Object instance size in heap words +// +// If the InstanceKlass cannot be represented by this scheme (instance size too large, too many or too large oop map entries), then +// the IK-specific bits are all zero'd out (this is rare): +// +// Bit 31 27 23 19 15 11 7 3 0 +// K K K L L 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +// +// +// ----------------- ArrayKlass encoding ------------------------- +// +// Bit 31 27 23 19 15 11 7 3 0 +// K K K L L 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 E E H H H H H +// \ / \ / \ / +// -------------------------------------------------------- - ----------- +// (unused) elem header +// size size +// +// H (5 bits): header size, in bytes (same as layouthelper header size) +// E (2 bits): log2 elem size, in bytes +// +// +// ----------------- Invalid Klute encoding ------------------------- +// +// A klute that has all bits set (1) is invalid: +// +// Bit 31 27 23 19 15 11 7 3 0 +// 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 +// +// It is the KLUT table entry initialization value (KLUT is zapped with '-1' on startup). +// +// The "invalid" has two different uses: +// - if CDS is disabled at build time, it simply designates an invalid entry that should never be encountered +// at runtime. In other words, when doing a lookup with a narrowKlass value into the KLUT table, one should +// always find a valid klute, since a narrowKlass value can only result from a Klass that was loaded, and as +// part of Klass creation, the klute table entry is created. +// +// - if CDS is enabled at build time: unfortunately, CDS maps in archived Klass structures into memory and these +// Klass structures never go through a normal loading process; they just appear and then they are just there. +// These may be accessed via narrowKlass values that are the result of a precalculation during CDS archive dump +// time. +// in that case, an "invalid_entry" can mean a Klass that was loaded from CDS archive and for that no table +// entry in the KLUT exists yet. If we encounter such an entry, we generate it on the fly (see KlassInfoLUT::late_register_klass()). +// +// Implementation note: the value -1 (all bits 1) relies on the fact that a KlassKind of 7 (0b111) is invalid. We +// don't use zero as "invalid entry" since zero would encode a valid Klass. class KlassLUTEntry { From 07f9bc1b3c0fe6fa4128c884411fb01a15866a32 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 25 Apr 2025 10:01:33 +0200 Subject: [PATCH 076/101] iterator simplifications --- src/hotspot/share/memory/iterator.inline.hpp | 295 +++++++------------ 1 file changed, 104 insertions(+), 191 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 7e3f5f9969110..8b8d52491d59b 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -113,6 +113,7 @@ class DispatchBase { static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { size_t s; constexpr Klass::KlassKind kind = KlassType::Kind; + assert(kind == obj->klass()->kind(), "Bad call"); assert(kind == klute.kind(), "Bad call"); switch (kind) { case Klass::ObjArrayKlassKind: { @@ -142,17 +143,17 @@ class DispatchBase { static inline bool should_use_slowpath_getsize() { return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; } + }; //////////////////////////////////////////////// -// Normal +// Normal forward iteration, returns void template class OopOopIterateDispatch : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; template @@ -162,42 +163,28 @@ class OopOopIterateDispatch : public DispatchBase { template static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatch::_table.set_resolve_function_and_execute (obj, cl, klute); - } - - template - void set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - set_resolve_function(); - _function[KlassType::Kind] (obj, cl, klute); + _table.resolve(); + _table._function[KlassType::Kind](obj, cl, klute); } template - void set_init_function() { - _function[KlassType::Kind] = &init_and_execute; - } - - template - void set_resolve_function() { + void resolve() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; } Table() { - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); +#define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; + ALL_KLASS_KINDS_DO(WHAT) +#undef WHAT } - }; static Table _table; public: + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); _table._function[slot](obj, cl, klute); @@ -205,135 +192,103 @@ class OopOopIterateDispatch : public DispatchBase { }; //////////////////////////////////////////////// -// Reverse +// Reverse iteration, returns void template -class OopOopIterateDispatchReverse { +class OopOopIterateDispatchReverse : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_reverse (obj, cl, klute); - } - - template - static void init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - OopOopIterateDispatchReverse::_table.set_resolve_function_and_execute(obj, cl, klute); - } - - template - void set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - set_resolve_function(); - _function[KlassType::Kind](obj, cl, klute); + KlassType::template oop_oop_iterate_reverse(obj, cl, klute); } template - void set_init_function() { - _function[KlassType::Kind] = &init_and_execute; + static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + _table.resolve(); + _table._function[KlassType::Kind](obj, cl, klute); } template - void set_resolve_function() { + void resolve() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; } Table() { - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); +#define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; + ALL_KLASS_KINDS_DO(WHAT) +#undef WHAT } - }; static Table _table; public: + static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); - _table._function[slot] (obj, cl, klute); + _table._function[slot](obj, cl, klute); } }; //////////////////////////////////////////////// -// Bounded +// Bounded iteration, returns void template -class OopOopIterateDispatchBounded { +class OopOopIterateDispatchBounded : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; template static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); } template static void init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - OopOopIterateDispatchBounded::_table.set_resolve_function_and_execute (obj, cl, mr, klute); + _table.resolve(); + _table._function[KlassType::Kind](obj, cl, mr, klute); } template - void set_resolve_function_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - set_resolve_function(); - _function[KlassType::Kind] (obj, cl, mr, klute); - } - - template - void set_init_function() { - _function[KlassType::Kind] = &init_and_execute; - } - - template - void set_resolve_function() { + void resolve() { _function[KlassType::Kind] = UseCompressedOops ? &invoke : &invoke; } - Table(){ - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); + Table() { +#define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; + ALL_KLASS_KINDS_DO(WHAT) +#undef WHAT } - }; static Table _table; public: + static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { const int slot = klute.kind(); - _table._function[slot] (obj, cl, mr, klute); + _table._function[slot](obj, cl, mr, klute); } }; //////////////////////////////////////////////// -// Normal, returns size +// Normal forward iteration, returns size template class OopOopIterateDispatchReturnSize : public DispatchBase { - typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; template @@ -349,61 +304,41 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - static size_t init_and_execute (oop obj, OopClosureType* cl, KlassLUTEntry klute) { - return OopOopIterateDispatchReturnSize::_table.set_resolve_function_and_execute (obj, cl, klute); + static size_t init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + _table.resolve(); + return _table._function[KlassType::Kind](obj, cl, klute); } template - void set_init_function() { - _function[KlassType::Kind] = &init_and_execute; - } - - template - size_t set_resolve_function_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - set_resolve_function(); - return _function[KlassType::Kind] (obj, cl, klute); - } - - template - void set_resolve_function() { - if (should_use_slowpath_getsize()) { - if (UseCompressedOops) { - _function[KlassType::Kind] = &invoke_slow; - } else { - _function[KlassType::Kind] = &invoke_slow; - } + void resolve() { + if (should_use_slowpath_getsize()) { // non-standard obj alignment, or uncompressed klass pointers + _function[KlassType::Kind] = UseCompressedOops ? + &invoke_slow : + &invoke_slow; } else { - if (UseCompressedOops) { - if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; - } else { - _function[KlassType::Kind] = &invoke; - } + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; } else { - if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; - } else { - _function[KlassType::Kind] = &invoke; - } + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; } } } Table() { - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); +#define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; + ALL_KLASS_KINDS_DO(WHAT) +#undef WHAT } - }; static Table _table; public: + static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { const int slot = klute.kind(); return _table._function[slot](obj, cl, klute); @@ -411,14 +346,13 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { }; //////////////////////////////////////////////// -// Bounded, returns size +// Bounded forward iteration, returns size template class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; template @@ -429,61 +363,40 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); + KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return obj->size(); } template - static size_t init_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - return OopOopIterateDispatchBoundedReturnSize::_table.set_resolve_function_and_execute (obj, cl, mr, klute); - } - - template - void set_init_function() { - _function[KlassType::Kind] = &init_and_execute; + static size_t init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + _table.resolve(); + return _table._function[KlassType::Kind](obj, cl, mr, klute); } template - size_t set_resolve_function_and_execute (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - set_resolve_function(); - return _function[KlassType::Kind] (obj, cl, mr, klute); - } - - template - void set_resolve_function() { - if (should_use_slowpath_getsize()) { - if (UseCompressedOops) { - _function[KlassType::Kind] = &invoke_slow; - } else { - _function[KlassType::Kind] = &invoke_slow; - } + void resolve() { + if (should_use_slowpath_getsize()) { // non-standard obj alignment, or uncompressed klass pointers + _function[KlassType::Kind] = UseCompressedOops ? + &invoke_slow : + &invoke_slow; } else { - if (UseCompressedOops) { - if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; - } else { - _function[KlassType::Kind] = &invoke; - } + if (UseCompactObjectHeaders) { + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; } else { - if (UseCompactObjectHeaders) { - _function[KlassType::Kind] = &invoke; - } else { - _function[KlassType::Kind] = &invoke; - } + _function[KlassType::Kind] = UseCompressedOops ? + &invoke : + &invoke; } } } Table() { - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); - set_init_function(); +#define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; + ALL_KLASS_KINDS_DO(WHAT) +#undef WHAT } - }; static Table _table; @@ -494,50 +407,49 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { const int slot = klute.kind(); return _table._function[slot](obj, cl, mr, klute); } - }; - //////////////////////////////////////////////// -// Abridged dispatch table (just one entry) for oop_iterate_range on ObjArrayOop +// Limited forward iteration, returns void +// This is a special solution purely to call ObjArrayKlass::oop_oop_iterate_range() (which only +// exists there). It is used for partial scanning of object arrays. ObjArrayKlass::oop_oop_iterate_range() +// is templatized by HeaderMode, which makes it possible to emit coding that treats the form of the +// header (therefore position of length field, position of first array element) as constexpr. template class OopOopIterateDispatchRange : public DispatchBase { - typedef void (*FunctionType) (objArrayOop obj, OopClosureType* cl, int start, int end); + typedef void (*FunctionType) (oop obj, OopClosureType* cl, int start, int end); struct Table { - - FunctionType _function; + FunctionType _function; // only for ObjArrayKlass template - static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { + static void invoke(oop obj, OopClosureType* cl, int start, int end) { ObjArrayKlass::oop_oop_iterate_range(obj, cl, start, end); } - static void init_and_execute(objArrayOop obj, OopClosureType* cl, int start, int end) { - OopOopIterateDispatchRange::_table.set_resolve_function_and_execute(obj, cl, start, end); + static void init_and_execute(oop obj, OopClosureType* cl, int start, int end) { + _table.resolve(); + _table._function(obj, cl, start, end); } - void set_resolve_function_and_execute(objArrayOop obj, OopClosureType* cl, int start, int end) { - set_resolve_function(); - _function(obj, cl, start, end); + void resolve() { + switch (ObjLayout::klass_mode()) { + case HeaderMode::Compact: + _function = UseCompressedOops ? &invoke : + &invoke; + break; + case HeaderMode::Compressed: + _function = UseCompressedOops ? &invoke : + &invoke; + break; + case HeaderMode::Uncompressed: + _function = UseCompressedOops ? &invoke : + &invoke; + break; + }; } - void set_resolve_function() { - if (UseCompressedOops) { - switch (ObjLayout::klass_mode()) { - case HeaderMode::Compact: _function = &invoke; break; - case HeaderMode::Compressed: _function = &invoke; break; - case HeaderMode::Uncompressed: _function = &invoke; break; - }; - } else { - switch (ObjLayout::klass_mode()) { - case HeaderMode::Compact: _function = &invoke; break; - case HeaderMode::Compressed: _function = &invoke; break; - case HeaderMode::Uncompressed: _function = &invoke; break; - }; - } - } Table() { _function = &init_and_execute; } @@ -546,7 +458,8 @@ class OopOopIterateDispatchRange : public DispatchBase { static Table _table; public: - static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { + + static void invoke(oop obj, OopClosureType* cl, int start, int end) { _table._function(obj, cl, start, end); } }; @@ -573,10 +486,10 @@ template typename OopOopIterateDispatchRange::Table OopOopIterateDispatchRange::_table; //////////////////////////////////////////////// -// Dispatcher entry points +// Dispatcher external entry points template -void OopIteratorClosureDispatch::oop_oop_iterate (oop obj, OopClosureType* cl) { +void OopIteratorClosureDispatch::oop_oop_iterate(oop obj, OopClosureType* cl) { const KlassLUTEntry klute = obj->get_klute(); OopOopIterateDispatch::invoke(obj, cl, klute); } From 7509ebff9b378bcfe0a86ffb5dacba2e45a18a55 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 26 Apr 2025 08:07:22 +0200 Subject: [PATCH 077/101] fix make error --- src/hotspot/share/memory/iterator.hpp | 12 ++-- src/hotspot/share/memory/iterator.inline.hpp | 10 ++-- src/hotspot/share/oops/typeArrayKlass.cpp | 2 +- src/hotspot/share/oops/typeArrayKlass.hpp | 20 +++---- .../share/oops/typeArrayKlass.inline.hpp | 58 ------------------- 5 files changed, 20 insertions(+), 82 deletions(-) delete mode 100644 src/hotspot/share/oops/typeArrayKlass.inline.hpp diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp index 45aca68cb6e0f..feb3f3e9dedf4 100644 --- a/src/hotspot/share/memory/iterator.hpp +++ b/src/hotspot/share/memory/iterator.hpp @@ -314,12 +314,12 @@ class CompareClosure : public Closure { class OopIteratorClosureDispatch { public: - template static void oop_oop_iterate (oop obj, OopClosureType* cl); - template static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl); - template static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr); - template static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl); - template static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr); - template static void oop_oop_iterate_range (objArrayOop obj, OopClosureType* cl, int start, int end); + template inline static void oop_oop_iterate (oop obj, OopClosureType* cl); + template inline static void oop_oop_iterate_reverse (oop obj, OopClosureType* cl); + template inline static void oop_oop_iterate_bounded (oop obj, OopClosureType* cl, MemRegion mr); + template inline static size_t oop_oop_iterate_size (oop obj, OopClosureType* cl); + template inline static size_t oop_oop_iterate_bounded_size (oop obj, OopClosureType* cl, MemRegion mr); + template inline static void oop_oop_iterate_range (objArrayOop obj, OopClosureType* cl, int start, int end); }; #endif // SHARE_MEMORY_ITERATOR_HPP diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 8b8d52491d59b..16fd1f06e1654 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -41,7 +41,7 @@ #include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/objLayout.inline.hpp" -#include "oops/typeArrayKlass.inline.hpp" +#include "oops/typeArrayKlass.hpp" #include "utilities/debug.hpp" // Defaults to strong claiming. @@ -418,17 +418,17 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template class OopOopIterateDispatchRange : public DispatchBase { - typedef void (*FunctionType) (oop obj, OopClosureType* cl, int start, int end); + typedef void (*FunctionType) (objArrayOop obj, OopClosureType* cl, int start, int end); struct Table { FunctionType _function; // only for ObjArrayKlass template - static void invoke(oop obj, OopClosureType* cl, int start, int end) { + static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { ObjArrayKlass::oop_oop_iterate_range(obj, cl, start, end); } - static void init_and_execute(oop obj, OopClosureType* cl, int start, int end) { + static void init_and_execute(objArrayOop obj, OopClosureType* cl, int start, int end) { _table.resolve(); _table._function(obj, cl, start, end); } @@ -459,7 +459,7 @@ class OopOopIterateDispatchRange : public DispatchBase { public: - static void invoke(oop obj, OopClosureType* cl, int start, int end) { + static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { _table._function(obj, cl, start, end); } }; diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index 8609f7a022684..aed7e84422b20 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -36,7 +36,7 @@ #include "oops/klass.inline.hpp" #include "oops/objArrayKlass.hpp" #include "oops/oop.inline.hpp" -#include "oops/typeArrayKlass.inline.hpp" +#include "oops/typeArrayKlass.hpp" #include "oops/typeArrayOop.inline.hpp" #include "runtime/handles.inline.hpp" #include "utilities/macros.hpp" diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index 9de494a7f2766..069496e75b668 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -76,23 +76,19 @@ class TypeArrayKlass : public ArrayKlass { // Copying void copy_array(arrayOop s, int src_pos, arrayOop d, int dst_pos, int length, TRAPS); - // Oop iterators. Since there are no oops in TypeArrayKlasses, - // these functions only return the size of the object. - - private: - // The implementation used by all oop_oop_iterate functions in TypeArrayKlasses. - static inline void oop_oop_iterate_impl(oop obj, OopIterateClosure* closure); - public: - template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + // Oop iterators are dummy methods for TypeArrayKlass: + // - there are no oops to iterate + // - there are no metadata to iterate either since TypeArrayKlass is guaranteed + // to be loaded by the boot class loader. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); - + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) {} + template + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) {} template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) {} static TypeArrayKlass* cast(Klass* k) { return const_cast(cast(const_cast(k))); diff --git a/src/hotspot/share/oops/typeArrayKlass.inline.hpp b/src/hotspot/share/oops/typeArrayKlass.inline.hpp deleted file mode 100644 index 3dd090d6f43a9..0000000000000 --- a/src/hotspot/share/oops/typeArrayKlass.inline.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP -#define SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP - -#include "oops/typeArrayKlass.hpp" - -#include "oops/arrayKlass.hpp" -#include "oops/klass.hpp" -#include "oops/oop.inline.hpp" -#include "oops/typeArrayOop.hpp" - -class OopIterateClosure; - -inline void TypeArrayKlass::oop_oop_iterate_impl(oop obj, OopIterateClosure* closure) { - assert(obj->is_typeArray(),"must be a type array"); - // Performance tweak: We skip processing the klass pointer since all - // TypeArrayKlasses are guaranteed processed via the null class loader. -} - -template -inline void TypeArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - oop_oop_iterate_impl(obj, closure); -} - -template -inline void TypeArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { - oop_oop_iterate_impl(obj, closure); -} - -template -void TypeArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { - oop_oop_iterate_impl(obj, closure); -} - -#endif // SHARE_OOPS_TYPEARRAYKLASS_INLINE_HPP From c131cc1aeae8c3da83198ae2f7fc2b3eb8bb8474 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 26 Apr 2025 08:14:49 +0200 Subject: [PATCH 078/101] change tak/oak length fast functions to use the correct oopDesc ttype --- src/hotspot/share/memory/iterator.inline.hpp | 4 ++-- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 6 ++++-- src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 16fd1f06e1654..85498d9d11287 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -117,11 +117,11 @@ class DispatchBase { assert(kind == klute.kind(), "Bad call"); switch (kind) { case Klass::ObjArrayKlassKind: { - s = klute.oak_calculate_wordsize_given_oop_fast(obj); + s = klute.oak_calculate_wordsize_given_oop_fast((objArrayOop)obj); break; } case Klass::TypeArrayKlassKind: { - s = klute.tak_calculate_wordsize_given_oop_fast(obj); + s = klute.tak_calculate_wordsize_given_oop_fast((typeArrayOop)obj); break; } default: { diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index ea8b80e94d45b..8b8a9152362b1 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -35,8 +35,10 @@ class ArrayKlass; class InstanceKlass; class Klass; class oopDesc; +class objArrayOopDesc; class OopMapBlock; class outputStream; +class typeArrayOopDesc; // A Klass Info Lookup Table Entry (klute) is a 32-bit value carrying, in a very condensed form, some of the most // important information about a Klass. @@ -288,11 +290,11 @@ class KlassLUTEntry { // for an oak, calculates word size given header size, element size, and array length template - inline size_t oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; + inline size_t oak_calculate_wordsize_given_oop_fast(objArrayOopDesc* obj) const; // for a tak, calculates word size given header size, element size, and array length template - inline size_t tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const; + inline size_t tak_calculate_wordsize_given_oop_fast(typeArrayOopDesc* obj) const; // Helper function, prints current limits static void print_limits(outputStream* st); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index 2f45e193c436c..7290912fec149 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -82,7 +82,7 @@ static void check_header_mode() { // calculates word size given header size, element size, and array length template -inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { +inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(objArrayOopDesc* obj) const { check_header_mode(); assert(sizeof(OopType) == (UseCompressedOops ? 4 : 8), "bad call"); assert(is_obj_array(), "Bad call"); @@ -98,7 +98,7 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); const unsigned array_length = (unsigned) (*array_len_addr); - assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); + assert(array_length == (unsigned)obj->length(), "sanity"); // Calculate size const unsigned size_in_bytes = (array_length << log2_oopsize) + first_element_offset; @@ -107,7 +107,7 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(oopDesc* obj) // calculates word size given header size, element size, and array length template -inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) const { +inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(typeArrayOopDesc* obj) const { check_header_mode(); // The purpose of this function is to hard-code as much as we can via template parameters. assert(is_type_array(), "Bad call"); @@ -122,7 +122,7 @@ inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(oopDesc* obj) // Load length from object const unsigned* const array_len_addr = (unsigned*)(obj->field_addr(length_field_offset)); const unsigned array_length = (unsigned) (*array_len_addr); - assert(array_length == (unsigned)((typeArrayOop)obj)->length(), "sanity"); + assert(array_length == (unsigned)obj->length(), "sanity"); // Calculate size const unsigned size_in_bytes = (array_length << log2_elemsize) + first_element_offset; From c9a75b692b2e50deb0d1b49b30d062952059dc29 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 26 Apr 2025 08:38:18 +0200 Subject: [PATCH 079/101] move checks up to calculate_size_for_object_fast --- src/hotspot/share/memory/iterator.inline.hpp | 18 +++++++++++++++--- .../share/oops/klassInfoLUTEntry.inline.hpp | 12 ------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 85498d9d11287..f12685bb5e826 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -108,9 +108,20 @@ inline void ClaimMetadataVisitingOopIterateClosure::do_method(Method* m) { class DispatchBase { protected: - // Return the size of an object; uses as much hard-coded information as possible + // Return the size of an object; by templatizing over HeaderMode and OopType, we + // can move most of the decisions (compressed/compact mode etc) from run time to + // build time. + // This variant does not work for uncompressed klass pointers or for non-standard + // values of ObjAlignmentInBytes. template static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { + assert(mode != HeaderMode::Uncompressed && UseCompressedClassPointers, + "Not for uncompressed class pointer mode"); + assert((UseCompactObjectHeaders == true) == (mode == HeaderMode::Compact), "HeaderMode mismatch"); + assert(sizeof(OopType) == 4 || sizeof(OopType) == 8, "odd OopType"); + assert((UseCompressedOops == true) == (sizeof(OopType) == 4), "OopType mismatch"); + assert(MinObjAlignmentInBytes == BytesPerWord, "Bad call"); + size_t s; constexpr Klass::KlassKind kind = KlassType::Kind; assert(kind == obj->klass()->kind(), "Bad call"); @@ -140,10 +151,10 @@ class DispatchBase { return s; } + // Returns true if calculate_size_for_object_fast cannot be used static inline bool should_use_slowpath_getsize() { return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; } - }; //////////////////////////////////////////////// @@ -386,7 +397,8 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { &invoke; } else { _function[KlassType::Kind] = UseCompressedOops ? - &invoke : + //&invoke : + &invoke : &invoke; } } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index 7290912fec149..e22cf12dd8348 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -70,20 +70,9 @@ inline unsigned KlassLUTEntry::ik_omb_offset_2() const { return _v.ike.omb_offset_2; } -template -static void check_header_mode() { - assert(mode != HeaderMode::Uncompressed, "bad call"); - assert(UseCompressedClassPointers, "bad call"); - if (mode == HeaderMode::Compact) { - assert(UseCompactObjectHeaders, "bad call"); - } - assert(MinObjAlignmentInBytes == BytesPerWord, "Bad call"); -} - // calculates word size given header size, element size, and array length template inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(objArrayOopDesc* obj) const { - check_header_mode(); assert(sizeof(OopType) == (UseCompressedOops ? 4 : 8), "bad call"); assert(is_obj_array(), "Bad call"); assert(obj->is_objArray(), "Bad call"); @@ -108,7 +97,6 @@ inline size_t KlassLUTEntry::oak_calculate_wordsize_given_oop_fast(objArrayOopDe // calculates word size given header size, element size, and array length template inline size_t KlassLUTEntry::tak_calculate_wordsize_given_oop_fast(typeArrayOopDesc* obj) const { - check_header_mode(); // The purpose of this function is to hard-code as much as we can via template parameters. assert(is_type_array(), "Bad call"); assert(obj->is_typeArray(), "Bad call"); From 2fc445149e2563c18003bd40cc56d8333b22e30e Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 26 Apr 2025 17:30:30 +0200 Subject: [PATCH 080/101] move KlassKind out of Klass --- src/hotspot/share/memory/iterator.inline.hpp | 27 ++++++++++--------- src/hotspot/share/oops/arrayKlass.hpp | 1 + .../share/oops/instanceClassLoaderKlass.hpp | 1 + src/hotspot/share/oops/instanceKlass.hpp | 1 + .../share/oops/instanceMirrorKlass.hpp | 3 ++- src/hotspot/share/oops/instanceRefKlass.hpp | 1 + .../share/oops/instanceStackChunkKlass.hpp | 1 + src/hotspot/share/oops/klass.hpp | 20 ++------------ src/hotspot/share/oops/klassInfoLUT.cpp | 27 ++++++++++--------- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 11 +++----- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 23 ++-------------- .../share/oops/klassInfoLUTEntry.inline.hpp | 2 +- src/hotspot/share/oops/objArrayKlass.hpp | 1 + src/hotspot/share/oops/typeArrayKlass.hpp | 3 ++- 14 files changed, 47 insertions(+), 75 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index f12685bb5e826..898d88fe53285 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -33,6 +33,7 @@ #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/klass.hpp" +#include "oops/klassKind.hpp" #include "oops/instanceKlass.inline.hpp" #include "oops/instanceMirrorKlass.inline.hpp" #include "oops/instanceClassLoaderKlass.inline.hpp" @@ -123,15 +124,15 @@ class DispatchBase { assert(MinObjAlignmentInBytes == BytesPerWord, "Bad call"); size_t s; - constexpr Klass::KlassKind kind = KlassType::Kind; + constexpr KlassKind kind = KlassType::Kind; assert(kind == obj->klass()->kind(), "Bad call"); assert(kind == klute.kind(), "Bad call"); switch (kind) { - case Klass::ObjArrayKlassKind: { + case ObjArrayKlassKind: { s = klute.oak_calculate_wordsize_given_oop_fast((objArrayOop)obj); break; } - case Klass::TypeArrayKlassKind: { + case TypeArrayKlassKind: { s = klute.tak_calculate_wordsize_given_oop_fast((typeArrayOop)obj); break; } @@ -165,7 +166,7 @@ class OopOopIterateDispatch : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; + FunctionType _function [KLASS_KIND_COUNT]; template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { @@ -187,7 +188,7 @@ class OopOopIterateDispatch : public DispatchBase { Table() { #define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; - ALL_KLASS_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT } }; @@ -210,7 +211,7 @@ class OopOopIterateDispatchReverse : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; + FunctionType _function [KLASS_KIND_COUNT]; template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { @@ -232,7 +233,7 @@ class OopOopIterateDispatchReverse : public DispatchBase { Table() { #define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; - ALL_KLASS_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT } }; @@ -255,7 +256,7 @@ class OopOopIterateDispatchBounded : public DispatchBase { typedef void (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; + FunctionType _function [KLASS_KIND_COUNT]; template static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { @@ -277,7 +278,7 @@ class OopOopIterateDispatchBounded : public DispatchBase { Table() { #define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; - ALL_KLASS_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT } }; @@ -300,7 +301,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; + FunctionType _function [KLASS_KIND_COUNT]; template static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { @@ -341,7 +342,7 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { Table() { #define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; - ALL_KLASS_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT } }; @@ -364,7 +365,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); struct Table { - FunctionType _function [Klass::KLASS_KIND_COUNT]; + FunctionType _function[KlassKindCount]; template static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { @@ -406,7 +407,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { Table() { #define WHAT(name, ignored) _function[name::Kind] = &init_and_execute; - ALL_KLASS_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT } }; diff --git a/src/hotspot/share/oops/arrayKlass.hpp b/src/hotspot/share/oops/arrayKlass.hpp index 5bfe46573d3f5..da06dbe464cd6 100644 --- a/src/hotspot/share/oops/arrayKlass.hpp +++ b/src/hotspot/share/oops/arrayKlass.hpp @@ -25,6 +25,7 @@ #ifndef SHARE_OOPS_ARRAYKLASS_HPP #define SHARE_OOPS_ARRAYKLASS_HPP +#include "oops/klassKind.hpp" #include "oops/klass.hpp" class fieldDescriptor; diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index eb6e1938fa65b..ad82229980c07 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -26,6 +26,7 @@ #define SHARE_OOPS_INSTANCECLASSLOADERKLASS_HPP #include "oops/instanceKlass.hpp" +#include "oops/klassKind.hpp" #include "oops/klassInfoLUTEntry.hpp" #include "utilities/macros.hpp" diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index e8d6720eaaa2c..161587dea0ca3 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -33,6 +33,7 @@ #include "oops/instanceKlassFlags.hpp" #include "oops/instanceOop.hpp" #include "oops/klassInfoLUT.hpp" +#include "oops/klassKind.hpp" #include "runtime/handles.hpp" #include "runtime/javaThread.hpp" #include "utilities/accessFlags.hpp" diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index 2ec4c5faa61bb..ae96f8d02f0e9 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -26,8 +26,9 @@ #define SHARE_OOPS_INSTANCEMIRRORKLASS_HPP #include "classfile/vmClasses.hpp" -#include "oops/klassInfoLUTEntry.hpp" #include "oops/instanceKlass.hpp" +#include "oops/klassInfoLUTEntry.hpp" +#include "oops/klassKind.hpp" #include "runtime/handles.hpp" #include "utilities/macros.hpp" diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index 984064cc6073a..52dd142b5dbde 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -27,6 +27,7 @@ #include "oops/klassInfoLUTEntry.hpp" #include "oops/instanceKlass.hpp" +#include "oops/klassKind.hpp" #include "utilities/macros.hpp" class ClassFileParser; diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index ad9774cb5f81c..3952b191a07f7 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -27,6 +27,7 @@ #include "oops/instanceKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/klassKind.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/macros.hpp" #include "utilities/ostream.hpp" diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 4bb2f372c1b6d..51e03a6a8343d 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -27,6 +27,7 @@ #include "oops/klassFlags.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/klassKind.hpp" #include "oops/markWord.hpp" #include "oops/metadata.hpp" #include "oops/oop.hpp" @@ -67,29 +68,12 @@ class Klass : public Metadata { friend class JVMCIVMStructs; public: -#define KLASS_ALL_KINDS_DO(what) \ - what(InstanceKlass, IK) \ - what(InstanceRefKlass, IRK) \ - what(InstanceMirrorKlass, IMK) \ - what(InstanceClassLoaderKlass, ICLK) \ - what(InstanceStackChunkKlass, ISCK) \ - what(TypeArrayKlass, TAK) \ - what(ObjArrayKlass, OAK) - - enum KlassKind : u2 { -#define WHAT(name, shortname) name ## Kind, - KLASS_ALL_KINDS_DO(WHAT) -#undef WHAT - UnknownKlassKind - }; - static const uint KLASS_KIND_COUNT = ObjArrayKlassKind + 1; - // Define a set of handy cast functions (e.g. "as_InstanceStackChunkKlass") #define WHAT(name, shortname) \ name* as_##name() { \ assert(_kind == name ## Kind, "not a " #name ); \ return (name*) this; } - KLASS_ALL_KINDS_DO(WHAT) + KLASSKIND_ALL_KINDS_DO(WHAT) #undef WHAT protected: diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 3f79be88dbbb0..3cd923748dfc1 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -30,6 +30,7 @@ #include "oops/klass.hpp" #include "oops/klassInfoLUT.inline.hpp" #include "oops/klassInfoLUTEntry.inline.hpp" +#include "oops/klassKind.hpp" #include "runtime/atomic.hpp" #include "utilities/debug.hpp" #include "utilities/ostream.hpp" @@ -170,13 +171,13 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { // update register stats switch (k->kind()) { - case Klass::InstanceKlassKind: inc_registered_IK(); break; - case Klass::InstanceRefKlassKind: inc_registered_IRK(); break; - case Klass::InstanceMirrorKlassKind: inc_registered_IMK(); break; - case Klass::InstanceClassLoaderKlassKind: inc_registered_ICLK(); break; - case Klass::InstanceStackChunkKlassKind: inc_registered_ISCK(); break; - case Klass::TypeArrayKlassKind: inc_registered_TAK(); break; - case Klass::ObjArrayKlassKind: inc_registered_OAK(); break; + case InstanceKlassKind: inc_registered_IK(); break; + case InstanceRefKlassKind: inc_registered_IRK(); break; + case InstanceMirrorKlassKind: inc_registered_IMK(); break; + case InstanceClassLoaderKlassKind: inc_registered_ICLK(); break; + case InstanceStackChunkKlassKind: inc_registered_ISCK(); break; + case TypeArrayKlassKind: inc_registered_TAK(); break; + case ObjArrayKlassKind: inc_registered_OAK(); break; default: ShouldNotReachHere(); }; if (k->is_abstract() || k->is_interface()) { @@ -268,7 +269,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { st->print_cr(" Registered classes, total: " UINT64_FORMAT, registered_all); #define XX(name, shortname) PRINT_WITH_PERCENTAGE("Registered, " #shortname, counter_registered_##shortname, registered_all); - ALL_KLASS_KINDS_DO(XX) + KLASSKIND_ALL_KINDS_DO(XX) #undef XX const uint64_t registered_AK = counter_registered_OAK - counter_registered_TAK; @@ -284,7 +285,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { st->print_cr(" Hits, total: " UINT64_FORMAT, hits); #define XX(name, shortname) PRINT_WITH_PERCENTAGE("Hits, " #shortname, counter_hits_##shortname, hits); - ALL_KLASS_KINDS_DO(XX) + KLASSKIND_ALL_KINDS_DO(XX) #undef XX const uint64_t hits_ak = counter_hits_OAK + counter_hits_TAK; @@ -339,15 +340,15 @@ void KlassInfoLUT::print_statistics(outputStream* st) { #ifdef KLUT_ENABLE_EXPENSIVE_STATS void KlassInfoLUT::update_hit_stats(KlassLUTEntry klute) { switch (klute.kind()) { -#define XX(name, shortname) case Klass::name ## Kind: inc_hits_ ## shortname(); break; - ALL_KLASS_KINDS_DO(XX) +#define XX(name, shortname) case name ## Kind: inc_hits_ ## shortname(); break; + KLASSKIND_ALL_KINDS_DO(XX) #undef XX default: ShouldNotReachHere(); }; if (klute.is_instance() && !klute.ik_carries_infos()) { switch (klute.kind()) { - case Klass::InstanceClassLoaderKlassKind: inc_noinfo_ICLK(); break; - case Klass::InstanceMirrorKlassKind: inc_noinfo_IMK(); break; + case InstanceClassLoaderKlassKind: inc_noinfo_ICLK(); break; + case InstanceMirrorKlassKind: inc_noinfo_IMK(); break; default: inc_noinfo_IK_other(); break; } } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 8b4b99ce284c3..efb03e12bbd81 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -29,6 +29,7 @@ #include "oops/klass.inline.hpp" #include "oops/klassInfoLUT.hpp" #include "oops/klassInfoLUTEntry.inline.hpp" +#include "oops/klassKind.hpp" #include "utilities/debug.hpp" // See klass.hpp @@ -195,11 +196,7 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { STATIC_ASSERT(sizeof(U) == sizeof(uint32_t)); // Kind must fit into 3 bits - STATIC_ASSERT(Klass::KLASS_KIND_COUNT < nth_bit(bits_kind)); - -#define XX(name, ignored) STATIC_ASSERT((int)name ## Kind == (int)Klass::name ## Kind); - ALL_KLASS_KINDS_DO(XX) -#undef XX + STATIC_ASSERT(KlassKindCount < nth_bit(bits_kind)); assert(is_valid(), "klute should be valid (%x)", _v.raw); @@ -228,8 +225,8 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { const LayoutHelperHelper lhu = { (unsigned) real_lh }; assert(lhu.bytes.lh_esz == ak_log2_elem_size() && lhu.bytes.lh_hsz == ak_first_element_offset_in_bytes() && - ( (lhu.bytes.lh_tag == 0xC0 && real_kind == Klass::TypeArrayKlassKind) || - (lhu.bytes.lh_tag == 0x80 && real_kind == Klass::ObjArrayKlassKind) ), + ( (lhu.bytes.lh_tag == 0xC0 && real_kind == TypeArrayKlassKind) || + (lhu.bytes.lh_tag == 0x80 && real_kind == ObjArrayKlassKind) ), "layouthelper mismatch (layouthelper: 0x%x, klute: 0x%x)", real_lh, _v.raw); } else { diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 8b8a9152362b1..6ef8bd629e0e6 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -28,6 +28,7 @@ // Included by oop.hpp and klass.hpp, keep includes short #include "memory/allStatic.hpp" +#include "oops/klassKind.hpp" #include "oops/objLayout.hpp" #include "utilities/globalDefinitions.hpp" @@ -130,26 +131,6 @@ class typeArrayOopDesc; class KlassLUTEntry { -#define ALL_KLASS_KINDS_DO(what) \ - what(InstanceKlass, IK) \ - what(InstanceRefKlass, IRK) \ - what(InstanceMirrorKlass, IMK) \ - what(InstanceClassLoaderKlass, ICLK) \ - what(InstanceStackChunkKlass, ISCK) \ - what(TypeArrayKlass, TAK) \ - what(ObjArrayKlass, OAK) \ - - // Todo: move KlassKind out of Klass - // Don't want to include it here for all the baggage it brings - enum LocalKlassKind { -#define XX(name, ignored) name ## Kind, - ALL_KLASS_KINDS_DO(XX) -#undef XX - UnknownKlassKind, - LastKlassKind = ObjArrayKlassKind, - FirstArrayKlassKind = TypeArrayKlassKind - }; - static constexpr int bits_total = 32; // All valid entries: KKKB ---- ---- ---- ---- ---- ---- ---- @@ -251,7 +232,7 @@ class KlassLUTEntry { // returns loader index (0 for unknown) inline int loader_index() const { return _v.common.loader; } - bool is_array() const { return _v.common.kind >= FirstArrayKlassKind; } + bool is_array() const { return _v.common.kind >= TypeArrayKlassKind; } bool is_instance() const { return !is_array(); } bool is_obj_array() const { return _v.common.kind == ObjArrayKlassKind; } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp index e22cf12dd8348..387f1260a6509 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.inline.hpp @@ -27,9 +27,9 @@ #define SHARE_OOPS_KLASSINFOLUTENTRY_INLINE_HPP #include "oops/instanceKlass.hpp" -#include "oops/klass.hpp" // for KlassKind #include "oops/klassInfoLUTEntry.hpp" #include "oops/objLayout.inline.hpp" +#include "oops/arrayOop.hpp" #include "oops/oop.hpp" #include "utilities/debug.hpp" diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index 970d5aa8f2531..dfaaabd21349d 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -27,6 +27,7 @@ #include "oops/arrayKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/klassKind.hpp" #include "oops/objLayout.hpp" #include "utilities/macros.hpp" diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index 069496e75b668..d312b9a46fd76 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -27,6 +27,7 @@ #include "oops/arrayKlass.hpp" #include "oops/klassInfoLUTEntry.hpp" +#include "oops/klassKind.hpp" class ClassLoaderData; @@ -78,7 +79,7 @@ class TypeArrayKlass : public ArrayKlass { public: - // Oop iterators are dummy methods for TypeArrayKlass: + // Oop iterators are dummy methods for TypeArrayKlass since: // - there are no oops to iterate // - there are no metadata to iterate either since TypeArrayKlass is guaranteed // to be loaded by the boot class loader. From 8a96277fe706ea71d495a3978fa61ed49e1dd598 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 28 Apr 2025 07:24:27 +0200 Subject: [PATCH 081/101] Remove Klass printing function changes --- src/hotspot/share/oops/instanceKlass.cpp | 16 +++------------- src/hotspot/share/oops/klass.cpp | 6 +----- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index cbd5a73ce8430..8c0b10986c389 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -3643,23 +3643,13 @@ const char* InstanceKlass::init_state_name() const { void InstanceKlass::print_on(outputStream* st) const { assert(is_klass(), "must be klass"); Klass::print_on(st); - if (UseCompressedClassPointers) { - if (CompressedKlassPointers::is_encodable(this)) { - st->print(BULLET"narrow Klass: %x", CompressedKlassPointers::encode(const_cast(this))); - } else { - st->print(BULLET"narrow Klass: (not in class space)"); - } - } + st->print(BULLET"instance size: %d", size_helper()); st->cr(); st->print(BULLET"klass size: %d", size()); st->cr(); - st->print(BULLET"klass header size: %d", header_size()); st->cr(); - st->print(BULLET"vtable size: %d", itable_length()); st->cr(); - st->print(BULLET"itable size: %d", vtable_length()); st->cr(); - st->print(BULLET"nonstatic_oopmap size: %d", nonstatic_oop_map_size()); st->cr(); - st->print(BULLET"access: "); access_flags().print_on(st); st->cr(); st->print(BULLET"flags: "); _misc_flags.print_on(st); st->cr(); st->print(BULLET"state: "); st->print_cr("%s", init_state_name()); + st->print(BULLET"name: "); name()->print_value_on(st); st->cr(); st->print(BULLET"super: "); Metadata::print_value_on_maybe_null(st, super()); st->cr(); st->print(BULLET"sub: "); Klass* sub = subklass(); @@ -3783,7 +3773,7 @@ void InstanceKlass::print_on(outputStream* st) const { OopMapBlock* map = start_of_nonstatic_oop_maps(); OopMapBlock* end_map = map + nonstatic_oop_map_count(); while (map < end_map) { - st->print_cr("[@%d-@%d) (%d oops) ", map->offset(), map->offset() + heapOopSize * map->count(), map->count()); + st->print("%d-%d ", map->offset(), map->offset() + heapOopSize*(map->count() - 1)); map++; } st->cr(); diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 41e01d2ca2d44..a570cb6ebbd97 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -1014,13 +1014,9 @@ jint Klass::jvmti_class_status() const { void Klass::print_on(outputStream* st) const { ResourceMark rm; // print title - st->print("Klass: %s", internal_name()); + st->print("%s", internal_name()); print_address_on(st); st->cr(); - st->print_cr(" - kind: %d", kind()); - st->print_cr(" - name: "); name()->print_value_on(st); - st->print_cr(" - klute " UINT32_FORMAT_X_0, klute().value()); - st->print_cr(" - layouthelper " INT32_FORMAT_X_0, layout_helper()); } #define BULLET " - " From 2006f8512721296904722db9999c376e6aedd441 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 28 Apr 2025 08:30:08 +0200 Subject: [PATCH 082/101] Add missing header --- src/hotspot/share/oops/klassKind.hpp | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/hotspot/share/oops/klassKind.hpp diff --git a/src/hotspot/share/oops/klassKind.hpp b/src/hotspot/share/oops/klassKind.hpp new file mode 100644 index 0000000000000..d5a52dddf14c7 --- /dev/null +++ b/src/hotspot/share/oops/klassKind.hpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_OOPS_KLASSKIND_HPP +#define SHARE_OOPS_KLASSKIND_HPP + +// No unnecessary includes! + +#define KLASSKIND_ALL_KINDS_DO(what) \ + what(InstanceKlass, IK) \ + what(InstanceRefKlass, IRK) \ + what(InstanceMirrorKlass, IMK) \ + what(InstanceClassLoaderKlass, ICLK) \ + what(InstanceStackChunkKlass, ISCK) \ + what(TypeArrayKlass, TAK) \ + what(ObjArrayKlass, OAK) + + enum KlassKind : u2 { +#define WHAT(name, shortname) name ## Kind, + KLASSKIND_ALL_KINDS_DO(WHAT) +#undef WHAT + KlassKindCount, + UnknownKlassKind + }; + static const uint KLASS_KIND_COUNT = KlassKindCount; + +#endif // SHARE_OOPS_KLASSKIND_HPP From 8df0c7452454e91c63195e7f34319dbf52c83be3 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 28 Apr 2025 08:30:58 +0200 Subject: [PATCH 083/101] Code grooming, diff reduction --- src/hotspot/share/oops/compressedKlass.cpp | 1 + .../oops/instanceClassLoaderKlass.inline.hpp | 1 + src/hotspot/share/oops/klass.hpp | 2 +- src/hotspot/share/oops/klassInfoLUT.cpp | 42 ++++++------- src/hotspot/share/oops/klassInfoLUT.hpp | 61 +++++++++++++++---- .../share/oops/klassInfoLUT.inline.hpp | 11 ++-- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 6 +- 7 files changed, 80 insertions(+), 44 deletions(-) diff --git a/src/hotspot/share/oops/compressedKlass.cpp b/src/hotspot/share/oops/compressedKlass.cpp index 85e70ee831834..c97cd670854f7 100644 --- a/src/hotspot/share/oops/compressedKlass.cpp +++ b/src/hotspot/share/oops/compressedKlass.cpp @@ -225,6 +225,7 @@ void CompressedKlassPointers::initialize(address addr, size_t len) { _shift = max_shift(); } else { + // Traditional (non-compact) header mode const uintptr_t unscaled_max = nth_bit(narrow_klass_pointer_bits()); const uintptr_t zerobased_max = nth_bit(narrow_klass_pointer_bits() + max_shift()); diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index b87df20ef9601..625a3a8277724 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -68,4 +68,5 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosur } } } + #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_INLINE_HPP diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 51e03a6a8343d..84afb53b21fe7 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -68,7 +68,7 @@ class Klass : public Metadata { friend class JVMCIVMStructs; public: - // Define a set of handy cast functions (e.g. "as_InstanceStackChunkKlass") + // Define a set of handy cast functions (e.g. "as_InstanceStackChunkKlass()") #define WHAT(name, shortname) \ name* as_##name() { \ assert(_kind == name ## Kind, "not a " #name ); \ diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 3cd923748dfc1..6951380061f11 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -36,8 +36,8 @@ #include "utilities/ostream.hpp" ClassLoaderData* KlassInfoLUT::_common_loaders[4] = { nullptr }; -uint32_t* KlassInfoLUT::_entries = nullptr; -unsigned KlassInfoLUT::_num_entries = -1; +uint32_t* KlassInfoLUT::_table = nullptr; +unsigned KlassInfoLUT::_max_entries = -1; void KlassInfoLUT::initialize() { @@ -58,26 +58,26 @@ void KlassInfoLUT::initialize() { const size_t large_page_size = os::large_page_size(); if (large_page_size < 16 * M) { // not for freakishly large pages table_size_in_bytes = align_up(table_size_in_bytes, large_page_size); - _entries = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); - if (_entries != nullptr) { + _table = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); + if (_table != nullptr) { uses_large_pages = true; - _num_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); + _max_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); } } } - if (_entries == nullptr) { + if (_table == nullptr) { table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); - _entries = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); - os::commit_memory_or_exit((char*)_entries, table_size_in_bytes, false, "KLUT"); - _num_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); + _table = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); + os::commit_memory_or_exit((char*)_table, table_size_in_bytes, false, "KLUT"); + _max_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); } log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, - _num_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_entries, table_size_in_bytes)); + _max_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_table, table_size_in_bytes)); // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. - memset(_entries, 0xff, _num_entries * sizeof(uint32_t)); - assert(_entries[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff + memset(_table, 0xff, _max_entries * sizeof(uint32_t)); + assert(_table[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff } } @@ -157,7 +157,7 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { // Calculate klute from Klass properties and update the table value. klute = KlassLUTEntry::build_from_klass(k); if (add_to_table) { - _entries[nk] = klute.value(); + _table[nk] = klute.value(); } log_klass_registration(k, nk, add_to_table, klute, "registered"); @@ -171,13 +171,9 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { // update register stats switch (k->kind()) { - case InstanceKlassKind: inc_registered_IK(); break; - case InstanceRefKlassKind: inc_registered_IRK(); break; - case InstanceMirrorKlassKind: inc_registered_IMK(); break; - case InstanceClassLoaderKlassKind: inc_registered_ICLK(); break; - case InstanceStackChunkKlassKind: inc_registered_ISCK(); break; - case TypeArrayKlassKind: inc_registered_TAK(); break; - case ObjArrayKlassKind: inc_registered_OAK(); break; +#define WHAT(name, shorthand) case name ## Kind : inc_registered_ ## shorthand(); break; + KLASSKIND_ALL_KINDS_DO(WHAT) +#undef WHAT default: ShouldNotReachHere(); }; if (k->is_abstract() || k->is_interface()) { @@ -208,7 +204,7 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { // We just copy that entry into the table slot. const KlassLUTEntry klute = k->klute(); assert(klute.is_valid(), "Must be a valid klute"); - _entries[nk] = klute.value(); + _table[nk] = klute.value(); ClassLoaderData* const cld = k->class_loader_data(); if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS register_cld_if_needed(cld); @@ -255,7 +251,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { st->print_cr("KLUT statistics:"); if (use_lookup_table()) { - st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _num_entries, _num_entries * sizeof(uint32_t)); + st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(uint32_t)); } const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + @@ -314,7 +310,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) constexpr int chacheline_size = 64; constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); - const int num_cachelines = num_entries() / slots_per_cacheline; + const int num_cachelines = max_entries() / slots_per_cacheline; int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; for (int i = 0; i < num_cachelines; i++) { int n = 0; diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 387611a1de5d5..cfd9ce6a0cda4 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -39,21 +39,57 @@ class ClassLoaderData; //#define KLUT_ENABLE_EXPENSIVE_LOG #endif +// The Klass Info Lookup Table (KLUT) is a table of 32-bit values. Each value represents +// a Klass. It contains some important information about its class in a very condensed +// form. For details about the KLUT entry encoding, please see comments in +// klassInfoLUTEntry.hpp. +// +// The purpose of this table is to make it (mostly) unnecessary to dereference Klass to +// get Klass meta information; instead, the KLUT entry is read, which means instead of +// reading from several memory locations spread out over different cache lines, we read +// just one datum from a very condensed data store. The result is fewer memory traffic +// and better spatial locality. +// +// The KLUT is only allocated when compact object headers are used. With Compact Object Headers, +// we do have very "tight" (condensed) narrowKlass value space that is perfect for use as +// indexes into the KLUT lookup table. +// Without Compact Headers, we still calculate KLUT entries, but store then in and retrieve +// them from the Klass directly. This gives a (more modest) performance benefit for non-COH +// scenarios as well as serves to unify oop iteration code. +// +// KLUT entry life cycle: +// +// When a Klass is dynamically loaded, the KLUT entry is calculated, entered into the table +// (table[narrowKlass] = klute) and also stored in the Klass itself. See KlassInfoLUT::register_klass(). +// +// The KLUT entry is never removed from the table. When a class is unloaded, the entry gets +// stale; that is fine, since the narrowKlass value that could be used to look up the KLUT +// entry is also stale - referring to an unloaded Klass via its narrowKlass value is an error. +// A new future Klass that would be created at the same position in the Class Space will +// get the same narrowKlass and overwrite, as part of its creation, the old stale table slot +// with a newly generated KLUT entry. +// +// It gets a bit more complicated with CDS. CDS maps Klass instances into the process memory +// without going through the process of initialization. It also refers to that Klass via +// a narrowKlass value that is the result of a precalculation during dump time (e.g. by +// accessing obj->klass()->kind() on object that are re-animated from the CDS archive). +// Since there is no secure way to initialize the KLUT entry for these classes, we (for the +// moment) allow a form of "self-healing": if the KLUT entry for a class is requested but +// not yet added, we add it to the KLUT table on the fly. See KlassInfoLUT::late_register_klass(). + class KlassInfoLUT : public AllStatic { static ClassLoaderData* _common_loaders[4]; // See "loader" bits in Klute - static uint32_t* _entries; - - static unsigned _num_entries; - static inline unsigned num_entries() { return _num_entries; } + static unsigned _max_entries; + static uint32_t* _table; + static inline unsigned max_entries() { return _max_entries; } static inline uint32_t at(unsigned index); - static void allocate_lookup_table(); - // Statistics about the distribution of Klasses registered. These - // are not expensive (counter hit per class) + // Klass registration statistics. These are not expensive and therefore + // we carry them always. #define REGISTER_STATS_DO(f) \ f(registered_IK) \ f(registered_IRK) \ @@ -68,7 +104,7 @@ class KlassInfoLUT : public AllStatic { #undef XX // Statistics about the hit rate of the lookup table. These are - // very expensive. + // expensive and only enabled when needed. #ifdef KLUT_ENABLE_EXPENSIVE_STATS #define HIT_STATS_DO(f) \ f(hits_IK) \ @@ -98,7 +134,7 @@ class KlassInfoLUT : public AllStatic { static KlassLUTEntry late_register_klass(narrowKlass nk); #endif - static bool use_lookup_table() { return _entries != nullptr; } + static bool use_lookup_table() { return _table != nullptr; } public: @@ -107,14 +143,15 @@ class KlassInfoLUT : public AllStatic { static KlassLUTEntry register_klass(const Klass* k); static inline KlassLUTEntry lookup(narrowKlass k); + // ClassLoaderData handling + static void register_cld_if_needed(ClassLoaderData* cld); static int index_for_cld(const ClassLoaderData* cld); static inline ClassLoaderData* lookup_cld(int index); - - static void print_statistics(outputStream* out); - static void register_cld_if_needed(ClassLoaderData* cld); #if INCLUDE_CDS static void shared_klass_cld_changed(Klass* k); #endif + + static void print_statistics(outputStream* out); }; #endif // SHARE_OOPS_KLASSINFOLUT_HPP diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 3885c3add1892..b189ae7c26ec5 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -33,9 +33,9 @@ #include "utilities/debug.hpp" ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { - assert(_entries != nullptr, "LUT table does not exist"); - assert(index < num_entries(), "oob (%x vs %x)", index, num_entries()); - return _entries[index]; + assert(_table != nullptr, "LUT table does not exist"); + assert(index < max_entries(), "oob (%x vs %x)", index, max_entries()); + return _table[index]; } ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { @@ -44,8 +44,9 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { KlassLUTEntry e(v); #if INCLUDE_CDS if (!e.is_valid()) { - // This branch only exists because it is so very difficult to iterate CDS classes after loading - // CDS archives. See discussion surrounding 8353225. Hopefully we can remove this in the future. + // This branch, and the late_register_klass mechanic, only exists because it is + // so difficult to iterate CDS classes after loading CDS archives. See discussion + // surrounding 8353225. Hopefully we can remove this in the future. return late_register_klass(nk); } #else diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index efb03e12bbd81..0db5db0a8923c 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -37,10 +37,10 @@ union LayoutHelperHelper { unsigned raw; struct { // lsb - unsigned lh_esz : 8; // element size - unsigned lh_ebt : 8; // element BasicType (currently unused) + unsigned lh_esz : 8; // log2 element size + unsigned lh_ebt : 8; // element BasicType unsigned lh_hsz : 8; // header size (offset to first element) - unsigned lh_tag : 8; // 0x80 or 0xc0 + unsigned lh_tag : 8; // 0x80 (IK) or 0xc0 (AK) // msb } bytes; }; From 802eee1f7be884de9281d0cda8feb169c3c153ca Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 29 Apr 2025 14:50:54 +0200 Subject: [PATCH 084/101] fix bug in objArray oop iteration --- src/hotspot/share/memory/iterator.inline.hpp | 35 +++++++++++++++---- .../share/oops/objArrayKlass.inline.hpp | 1 - 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index 898d88fe53285..ba7589e3b33a8 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -116,12 +116,12 @@ class DispatchBase { // values of ObjAlignmentInBytes. template static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { + check_oopType(); + check_headerMode(); + assert(MinObjAlignmentInBytes == BytesPerWord, + "Don't call for non-standard ObjAlignmentInBytes"); assert(mode != HeaderMode::Uncompressed && UseCompressedClassPointers, - "Not for uncompressed class pointer mode"); - assert((UseCompactObjectHeaders == true) == (mode == HeaderMode::Compact), "HeaderMode mismatch"); - assert(sizeof(OopType) == 4 || sizeof(OopType) == 8, "odd OopType"); - assert((UseCompressedOops == true) == (sizeof(OopType) == 4), "OopType mismatch"); - assert(MinObjAlignmentInBytes == BytesPerWord, "Bad call"); + "Don't call for uncompressed Klasspointers"); size_t s; constexpr KlassKind kind = KlassType::Kind; @@ -156,6 +156,18 @@ class DispatchBase { static inline bool should_use_slowpath_getsize() { return !UseCompressedClassPointers || ObjectAlignmentInBytes != BytesPerWord; } + + template + static inline void check_headerMode() { + assert(ObjLayout::klass_mode() == mode, "Header mode mismatch"); + } + + template + static inline void check_oopType() { + assert(sizeof(OopType) == 8 || sizeof(OopType) == 4, "strange OopType"); + assert( (sizeof(OopType) == 4) == (UseCompressedOops == true), "OopType mismatch"); + } + }; //////////////////////////////////////////////// @@ -170,6 +182,7 @@ class OopOopIterateDispatch : public DispatchBase { template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + check_oopType(); KlassType::template oop_oop_iterate(obj, cl, klute); } @@ -215,6 +228,7 @@ class OopOopIterateDispatchReverse : public DispatchBase { template static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + check_oopType(); KlassType::template oop_oop_iterate_reverse(obj, cl, klute); } @@ -260,6 +274,7 @@ class OopOopIterateDispatchBounded : public DispatchBase { template static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + check_oopType(); KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); } @@ -305,12 +320,15 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { template static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + check_headerMode(); + check_oopType(); KlassType::template oop_oop_iterate (obj, cl, klute); return calculate_size_for_object_fast(klute, obj); } template static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + check_oopType(); KlassType::template oop_oop_iterate (obj, cl, klute); return obj->size(); } @@ -369,12 +387,15 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { template static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + check_headerMode(); + check_oopType(); KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return calculate_size_for_object_fast(klute, obj); } template static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + check_oopType(); KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return obj->size(); } @@ -399,7 +420,7 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } else { _function[KlassType::Kind] = UseCompressedOops ? //&invoke : - &invoke : + &invoke : &invoke; } } @@ -438,6 +459,8 @@ class OopOopIterateDispatchRange : public DispatchBase { template static void invoke(objArrayOop obj, OopClosureType* cl, int start, int end) { + check_headerMode(); + check_oopType(); ObjArrayKlass::oop_oop_iterate_range(obj, cl, start, end); } diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 2d66e5bbca746..1fdac791ec91e 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -104,7 +104,6 @@ template void ObjArrayKlass::oop_oop_iterate_range(objArrayOop a, OopClosureType* closure, int start, int end) { assert(start <= end, "Sanity"); assert(start >= 0, "Sanity"); - const int len = a->length_nobranches(); assert(a->length() == len, "Sanity (%d vs %d)", len, a->length()); T* const b = (T*)a->base_nobranches(); From 083dfc935052ef49c3e6bf9cfd1a9f90b5469df1 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 30 Apr 2025 16:18:45 +0200 Subject: [PATCH 085/101] AK KLUTE: 8bit per header size and elem size --- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 6ef8bd629e0e6..ebc51b9c5255e 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -73,10 +73,10 @@ class typeArrayOopDesc; // ----------------- InstanceKlass encoding ---------------------- // // Bit 31 27 23 19 15 11 7 3 0 -// K K K L L S S S S S O2 O2 O2 O2 O2 C2 C2 C2 C2 C2 C2 C2 O1 O1 O1 O1 C1 C1 C1 C1 C1 C1 -// \ / \ / \ / -// ----------- --------------------------------- -------------------------- -// obj size offset, count for oop map 2 offset, count for oop map 1 +// K K K L L S S S S S S O2 O2 O2 O2 O2 C2 C2 C2 C2 C2 C2 O1 O1 O1 O1 C1 C1 C1 C1 C1 C1 +// \ / \ / \ / +// ------------- ------------------------------ -------------------------- +// obj size offset, count for oop map 2 offset, count for oop map 1 // // C1 (6 bits): Count of first Oop Map Entry // O1 (4 bits): Offset, in number-of-(oop|narrowOop), of second Oop Map Entry @@ -94,11 +94,11 @@ class typeArrayOopDesc; // ----------------- ArrayKlass encoding ------------------------- // // Bit 31 27 23 19 15 11 7 3 0 -// K K K L L 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 E E H H H H H -// \ / \ / \ / -// -------------------------------------------------------- - ----------- -// (unused) elem header -// size size +// K K K L L 0 0 0 0 0 0 0 0 0 0 0 E E E E E E E E H H H H H H H H +// \ / \ / \ / +// ----------------------------- -------------------- -------------------- +// unused log2 elem size header size +// // // H (5 bits): header size, in bytes (same as layouthelper header size) // E (2 bits): log2 elem size, in bytes @@ -167,8 +167,8 @@ class KlassLUTEntry { }; // Bits only valid for ArrayKlass - static constexpr int bits_ak_l2esz = 2; - static constexpr int bits_ak_hsz = 5; + static constexpr int bits_ak_l2esz = 8; + static constexpr int bits_ak_hsz = 8; struct AKE { // lsb unsigned hsz : bits_ak_hsz; // header size (offset to first element) in bytes From a42a022e0bfbe5ac2d5adcd2185c4575cf4297ad Mon Sep 17 00:00:00 2001 From: tstuefe Date: Fri, 2 May 2025 16:45:59 +0200 Subject: [PATCH 086/101] klute const - make KlassLUTEntry dumber and fix instruction bloat --- src/hotspot/share/cds/archiveBuilder.cpp | 2 +- .../gc/parallel/psPromotionManager.inline.hpp | 4 +- src/hotspot/share/memory/iterator.inline.hpp | 79 ++++++++++--------- .../share/oops/instanceClassLoaderKlass.hpp | 6 +- .../oops/instanceClassLoaderKlass.inline.hpp | 6 +- src/hotspot/share/oops/instanceKlass.hpp | 8 +- .../share/oops/instanceKlass.inline.hpp | 46 +++++------ .../share/oops/instanceMirrorKlass.hpp | 6 +- .../share/oops/instanceMirrorKlass.inline.hpp | 6 +- src/hotspot/share/oops/instanceRefKlass.hpp | 6 +- .../share/oops/instanceRefKlass.inline.hpp | 6 +- .../share/oops/instanceStackChunkKlass.hpp | 6 +- .../oops/instanceStackChunkKlass.inline.hpp | 6 +- src/hotspot/share/oops/klass.hpp | 4 +- src/hotspot/share/oops/klassInfoLUT.cpp | 65 +++++++-------- src/hotspot/share/oops/klassInfoLUT.hpp | 14 ++-- .../share/oops/klassInfoLUT.inline.hpp | 11 ++- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 10 +-- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 27 +++---- src/hotspot/share/oops/objArrayKlass.hpp | 6 +- .../share/oops/objArrayKlass.inline.hpp | 6 +- src/hotspot/share/oops/oop.hpp | 4 +- src/hotspot/share/oops/oop.inline.hpp | 8 +- src/hotspot/share/oops/typeArrayKlass.hpp | 6 +- 24 files changed, 172 insertions(+), 176 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 6d675f4308ea8..b5a2c7d78cc99 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -782,7 +782,7 @@ void ArchiveBuilder::make_klasses_shareable() { // Every archived Klass must carry a valid klute. That is because every archived Klass // would have been created via the usual dynamic class loading or - generation, which should // have registered the Klass with klut. - DEBUG_ONLY(k->klute().verify_against_klass(k);) + DEBUG_ONLY(KlassLUTEntry(k->klute()).verify_against_klass(k);) } for (int i = 0; i < klasses()->length(); i++) { diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp index 65190ce9c8ff0..0ffc33f260f10 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp @@ -115,13 +115,13 @@ class PSPushContentsClosure: public BasicOopIterateClosure { // order of these function calls. // template <> -inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, KlassLUTEntry klute) { +inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, klute_raw_t klute) { oop_oop_iterate_ref_processing(obj, closure); InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); } template <> -inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, KlassLUTEntry klute) { +inline void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, PSPushContentsClosure* closure, klute_raw_t klute) { oop_oop_iterate_ref_processing(obj, closure); InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); } diff --git a/src/hotspot/share/memory/iterator.inline.hpp b/src/hotspot/share/memory/iterator.inline.hpp index ba7589e3b33a8..fd988a0fffcd6 100644 --- a/src/hotspot/share/memory/iterator.inline.hpp +++ b/src/hotspot/share/memory/iterator.inline.hpp @@ -115,7 +115,7 @@ class DispatchBase { // This variant does not work for uncompressed klass pointers or for non-standard // values of ObjAlignmentInBytes. template - static inline size_t calculate_size_for_object_fast(KlassLUTEntry klute, oop obj) { + static inline size_t calculate_size_for_object_fast(klute_raw_t klute, oop obj) { check_oopType(); check_headerMode(); assert(MinObjAlignmentInBytes == BytesPerWord, @@ -124,22 +124,23 @@ class DispatchBase { "Don't call for uncompressed Klasspointers"); size_t s; + const KlassLUTEntry klutehelper(klute); constexpr KlassKind kind = KlassType::Kind; assert(kind == obj->klass()->kind(), "Bad call"); - assert(kind == klute.kind(), "Bad call"); + assert(kind == klutehelper.kind(), "Bad call"); switch (kind) { case ObjArrayKlassKind: { - s = klute.oak_calculate_wordsize_given_oop_fast((objArrayOop)obj); + s = klutehelper.oak_calculate_wordsize_given_oop_fast((objArrayOop)obj); break; } case TypeArrayKlassKind: { - s = klute.tak_calculate_wordsize_given_oop_fast((typeArrayOop)obj); + s = klutehelper.tak_calculate_wordsize_given_oop_fast((typeArrayOop)obj); break; } default: { // all InstanceKlass variants - if (klute.ik_carries_infos()) { - s = klute.ik_wordsize(); + if (klutehelper.ik_carries_infos()) { + s = klutehelper.ik_wordsize(); } else { // Rare path: size not statically computable (e.g. for MirrorKlass instances); calculate using regular Klass const Klass* k = obj->klass(); @@ -148,7 +149,7 @@ class DispatchBase { } } assert(s == obj->size(), "Unexpected size (klute %X, %zu vs %zu)", - klute.value(), s, obj->size()); + klute, s, obj->size()); return s; } @@ -175,19 +176,19 @@ class DispatchBase { template class OopOopIterateDispatch : public DispatchBase { - typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + typedef void (*FunctionType) (oop obj, OopClosureType* cl, klute_raw_t klute); struct Table { FunctionType _function [KLASS_KIND_COUNT]; template - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static void invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { check_oopType(); KlassType::template oop_oop_iterate(obj, cl, klute); } template - static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static void init_and_execute(oop obj, OopClosureType* cl, klute_raw_t klute) { _table.resolve(); _table._function[KlassType::Kind](obj, cl, klute); } @@ -210,8 +211,8 @@ class OopOopIterateDispatch : public DispatchBase { public: - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - const int slot = klute.kind(); + static void invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { + const int slot = KlassLUTEntry(klute).kind(); _table._function[slot](obj, cl, klute); } }; @@ -221,19 +222,19 @@ class OopOopIterateDispatch : public DispatchBase { template class OopOopIterateDispatchReverse : public DispatchBase { - typedef void (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + typedef void (*FunctionType) (oop obj, OopClosureType* cl, klute_raw_t klute); struct Table { FunctionType _function [KLASS_KIND_COUNT]; template - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static void invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { check_oopType(); KlassType::template oop_oop_iterate_reverse(obj, cl, klute); } template - static void init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static void init_and_execute(oop obj, OopClosureType* cl, klute_raw_t klute) { _table.resolve(); _table._function[KlassType::Kind](obj, cl, klute); } @@ -256,8 +257,8 @@ class OopOopIterateDispatchReverse : public DispatchBase { public: - static void invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - const int slot = klute.kind(); + static void invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { + const int slot = KlassLUTEntry(klute).kind(); _table._function[slot](obj, cl, klute); } }; @@ -267,19 +268,19 @@ class OopOopIterateDispatchReverse : public DispatchBase { template class OopOopIterateDispatchBounded : public DispatchBase { - typedef void (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + typedef void (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute); struct Table { FunctionType _function [KLASS_KIND_COUNT]; template - static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static void invoke(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { check_oopType(); KlassType::template oop_oop_iterate_bounded(obj, cl, mr, klute); } template - static void init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static void init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { _table.resolve(); _table._function[KlassType::Kind](obj, cl, mr, klute); } @@ -302,8 +303,8 @@ class OopOopIterateDispatchBounded : public DispatchBase { public: - static void invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - const int slot = klute.kind(); + static void invoke(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { + const int slot = KlassLUTEntry(klute).kind(); _table._function[slot](obj, cl, mr, klute); } }; @@ -313,13 +314,13 @@ class OopOopIterateDispatchBounded : public DispatchBase { template class OopOopIterateDispatchReturnSize : public DispatchBase { - typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, KlassLUTEntry klute); + typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, klute_raw_t klute); struct Table { FunctionType _function [KLASS_KIND_COUNT]; template - static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static size_t invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { check_headerMode(); check_oopType(); KlassType::template oop_oop_iterate (obj, cl, klute); @@ -327,14 +328,14 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { } template - static size_t invoke_slow(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static size_t invoke_slow(oop obj, OopClosureType* cl, klute_raw_t klute) { check_oopType(); KlassType::template oop_oop_iterate (obj, cl, klute); return obj->size(); } template - static size_t init_and_execute(oop obj, OopClosureType* cl, KlassLUTEntry klute) { + static size_t init_and_execute(oop obj, OopClosureType* cl, klute_raw_t klute) { _table.resolve(); return _table._function[KlassType::Kind](obj, cl, klute); } @@ -369,8 +370,8 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { public: - static size_t invoke(oop obj, OopClosureType* cl, KlassLUTEntry klute) { - const int slot = klute.kind(); + static size_t invoke(oop obj, OopClosureType* cl, klute_raw_t klute) { + const int slot = KlassLUTEntry(klute).kind(); return _table._function[slot](obj, cl, klute); } }; @@ -380,13 +381,13 @@ class OopOopIterateDispatchReturnSize : public DispatchBase { template class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { - typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute); + typedef size_t (*FunctionType) (oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute); struct Table { FunctionType _function[KlassKindCount]; template - static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { check_headerMode(); check_oopType(); KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); @@ -394,14 +395,14 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { } template - static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static size_t invoke_slow(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { check_oopType(); KlassType::template oop_oop_iterate_bounded (obj, cl, mr, klute); return obj->size(); } template - static size_t init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { + static size_t init_and_execute(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { _table.resolve(); return _table._function[KlassType::Kind](obj, cl, mr, klute); } @@ -437,8 +438,8 @@ class OopOopIterateDispatchBoundedReturnSize : public DispatchBase { public: - static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, KlassLUTEntry klute) { - const int slot = klute.kind(); + static size_t invoke(oop obj, OopClosureType* cl, MemRegion mr, klute_raw_t klute) { + const int slot = KlassLUTEntry(klute).kind(); return _table._function[slot](obj, cl, mr, klute); } }; @@ -526,31 +527,31 @@ typename OopOopIterateDispatchRange::Table OopOopIterateDispatch template void OopIteratorClosureDispatch::oop_oop_iterate(oop obj, OopClosureType* cl) { - const KlassLUTEntry klute = obj->get_klute(); + const klute_raw_t klute = obj->get_klute(); OopOopIterateDispatch::invoke(obj, cl, klute); } template void OopIteratorClosureDispatch::oop_oop_iterate_reverse(oop obj, OopClosureType* cl) { - const KlassLUTEntry klute = obj->get_klute(); + const klute_raw_t klute = obj->get_klute(); OopOopIterateDispatchReverse::invoke (obj, cl, klute); } template void OopIteratorClosureDispatch::oop_oop_iterate_bounded(oop obj, OopClosureType* cl, MemRegion mr) { - const KlassLUTEntry klute = obj->get_klute(); + const klute_raw_t klute = obj->get_klute(); OopOopIterateDispatchBounded::invoke(obj, cl, mr, klute); } template size_t OopIteratorClosureDispatch::oop_oop_iterate_size(oop obj, OopClosureType* cl) { - const KlassLUTEntry klute = obj->get_klute(); + const klute_raw_t klute = obj->get_klute(); return OopOopIterateDispatchReturnSize::invoke(obj, cl, klute); } template size_t OopIteratorClosureDispatch::oop_oop_iterate_bounded_size(oop obj, OopClosureType* cl, MemRegion mr) { - const KlassLUTEntry klute = obj->get_klute(); + const klute_raw_t klute = obj->get_klute(); return OopOopIterateDispatchBoundedReturnSize::invoke(obj, cl, mr, klute); } diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp index ad82229980c07..9d6e21618ac0e 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.hpp @@ -57,17 +57,17 @@ class InstanceClassLoaderKlass: public InstanceKlass { // Forward iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Reverse iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); }; #endif // SHARE_OOPS_INSTANCECLASSLOADERKLASS_HPP diff --git a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp index 625a3a8277724..a41b9144cb07b 100644 --- a/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceClassLoaderKlass.inline.hpp @@ -35,7 +35,7 @@ #include "utilities/macros.hpp" template -inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); if (Devirtualizer::do_metadata(closure)) { @@ -48,14 +48,14 @@ inline void InstanceClassLoaderKlass::oop_oop_iterate(oop obj, OopClosureType* c } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +inline void InstanceClassLoaderKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); } template -inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +inline void InstanceClassLoaderKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); if (Devirtualizer::do_metadata(closure)) { diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 161587dea0ca3..230ba1bb69ef5 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1017,24 +1017,24 @@ class InstanceKlass: public Klass { template inline void oop_oop_iterate_oop_maps_bounded(oop obj, OopClosureType* closure, MemRegion mr); - static inline ClassLoaderData* cld_from_klut_or_klass(oop obj, KlassLUTEntry klute); + static inline ClassLoaderData* cld_from_klut_or_klass(oop obj, klute_raw_t klute); public: // Forward iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Reverse iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Bounded range iteration // Iterate over all oop fields and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); u2 idnum_allocated_count() const { return _idnum_allocated_count; } diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index c8060c0b6face..eaa692ba1fab0 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -155,8 +155,8 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl } } -ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, KlassLUTEntry klute) { - const unsigned perma_cld_index = klute.loader_index(); +ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, klute_raw_t klute) { + const unsigned perma_cld_index = KlassLUTEntry(klute).loader_index(); ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); if (cld != nullptr) { #ifdef ASSERT @@ -171,18 +171,18 @@ ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, Kla // Iterate over all oop fields and metadata. template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { if (Devirtualizer::do_metadata(closure)) { ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); Devirtualizer::do_cld(closure, cld); } - - if (klute.ik_carries_infos()) { + const KlassLUTEntry klutehelper(klute); + if (klutehelper.ik_carries_infos()) { // klute may encode 0, 1 or 2 oop maps. Iterate those. - if (klute.ik_omb_count_1() > 0) { - oop_oop_iterate_single_oop_map(obj, closure, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); - if (klute.ik_omb_count_2() > 0) { - oop_oop_iterate_single_oop_map(obj, closure, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + if (klutehelper.ik_omb_count_1() > 0) { + oop_oop_iterate_single_oop_map(obj, closure, klutehelper.ik_omb_offset_1() * sizeof(T), klutehelper.ik_omb_count_1()); + if (klutehelper.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map(obj, closure, klutehelper.ik_omb_offset_2() * sizeof(T), klutehelper.ik_omb_count_2()); } } } else { @@ -195,17 +195,17 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closur // Iterate over all oop fields in the oop maps (no metadata traversal) template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { assert(!Devirtualizer::do_metadata(closure), "Code to handle metadata is not implemented"); - - if (klute.ik_carries_infos()) { + const KlassLUTEntry klutehelper(klute); + if (klutehelper.ik_carries_infos()) { // klute may encode 0, 1 or 2 oop maps. Iterate those (reversely). - if (klute.ik_omb_count_1() > 0) { - if (klute.ik_omb_count_2() > 0) { - oop_oop_iterate_single_oop_map_reverse(obj, closure, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + if (klutehelper.ik_omb_count_1() > 0) { + if (klutehelper.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map_reverse(obj, closure, klutehelper.ik_omb_offset_2() * sizeof(T), klutehelper.ik_omb_count_2()); } - oop_oop_iterate_single_oop_map_reverse(obj, closure, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); + oop_oop_iterate_single_oop_map_reverse(obj, closure, klutehelper.ik_omb_offset_1() * sizeof(T), klutehelper.ik_omb_count_1()); } } else { // Rare path @@ -217,20 +217,20 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_reverse(oop obj, OopClosureType // Iterate over all oop fields and metadata. template -ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); Devirtualizer::do_cld(closure, cld); } } - - if (klute.ik_carries_infos()) { + const KlassLUTEntry klutehelper(klute); + if (klutehelper.ik_carries_infos()) { // klute may encode 0, 1 or 2 oop maps. Iterate those (bounded). - if (klute.ik_omb_count_1() > 0) { - oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klute.ik_omb_offset_1() * sizeof(T), klute.ik_omb_count_1()); - if (klute.ik_omb_count_2() > 0) { - oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klute.ik_omb_offset_2() * sizeof(T), klute.ik_omb_count_2()); + if (klutehelper.ik_omb_count_1() > 0) { + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klutehelper.ik_omb_offset_1() * sizeof(T), klutehelper.ik_omb_count_1()); + if (klutehelper.ik_omb_count_2() > 0) { + oop_oop_iterate_single_oop_map_bounded(obj, closure, mr, klutehelper.ik_omb_offset_2() * sizeof(T), klutehelper.ik_omb_count_2()); } } } else { diff --git a/src/hotspot/share/oops/instanceMirrorKlass.hpp b/src/hotspot/share/oops/instanceMirrorKlass.hpp index ae96f8d02f0e9..9de9952151270 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.hpp @@ -106,17 +106,17 @@ class InstanceMirrorKlass: public InstanceKlass { // Forward iteration // Iterate over the oop fields and metadata. template - inline static void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Reverse iteration // Iterate over the oop fields and metadata. template - inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); private: diff --git a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp index c663e9ed3a42b..bb0a8cc9ec481 100644 --- a/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceMirrorKlass.inline.hpp @@ -47,7 +47,7 @@ void InstanceMirrorKlass::oop_oop_iterate_statics(oop obj, OopClosureType* closu } template -void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); if (Devirtualizer::do_metadata(closure)) { @@ -85,7 +85,7 @@ void InstanceMirrorKlass::oop_oop_iterate(oop obj, OopClosureType* closure, Klas } template -void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceMirrorKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); InstanceMirrorKlass* const ik = obj->klass()->as_InstanceMirrorKlass(); @@ -118,7 +118,7 @@ void InstanceMirrorKlass::oop_oop_iterate_statics_bounded(oop obj, } template -void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +void InstanceMirrorKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); if (Devirtualizer::do_metadata(closure)) { diff --git a/src/hotspot/share/oops/instanceRefKlass.hpp b/src/hotspot/share/oops/instanceRefKlass.hpp index 52dd142b5dbde..ff19d68decb2b 100644 --- a/src/hotspot/share/oops/instanceRefKlass.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.hpp @@ -68,17 +68,17 @@ class InstanceRefKlass: public InstanceKlass { // Forward iteration // Iterate over all oop fields and metadata. template - inline static void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Reverse iteration // Iterate over all oop fields and metadata. template - inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + inline static void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Bounded range iteration // Iterate over all oop fields and metadata. template - inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + inline static void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); private: diff --git a/src/hotspot/share/oops/instanceRefKlass.inline.hpp b/src/hotspot/share/oops/instanceRefKlass.inline.hpp index afcc2855c8e4d..0e2cac9aa2c2f 100644 --- a/src/hotspot/share/oops/instanceRefKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceRefKlass.inline.hpp @@ -149,21 +149,21 @@ void InstanceRefKlass::oop_oop_iterate_ref_processing_bounded(oop obj, OopClosur } template -void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceRefKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate(obj, closure, klute); oop_oop_iterate_ref_processing(obj, closure); } template -void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceRefKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_reverse(obj, closure, klute); oop_oop_iterate_ref_processing(obj, closure); } template -void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +void InstanceRefKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { InstanceKlass::oop_oop_iterate_bounded(obj, closure, mr, klute); oop_oop_iterate_ref_processing_bounded(obj, closure, mr); diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.hpp index 3952b191a07f7..2bb71a0ad09dd 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.hpp @@ -151,17 +151,17 @@ class InstanceStackChunkKlass: public InstanceKlass { // Forward iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Reverse iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Bounded range iteration // Iterate over the oop fields and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); private: diff --git a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp index 3ce9e37093cab..a5ce3a43da79d 100644 --- a/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp @@ -91,19 +91,19 @@ void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* c } template -void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceStackChunkKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); ik->oop_oop_iterate(obj, closure); } template -void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void InstanceStackChunkKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); ik->oop_oop_iterate_reverse(obj, closure); } template -void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +void InstanceStackChunkKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { InstanceStackChunkKlass* ik = obj->klass()->as_InstanceStackChunkKlass(); ik->oop_oop_iterate_bounded(obj, closure, mr); } diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 84afb53b21fe7..f4d21aefe6f1f 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -114,7 +114,7 @@ class Klass : public Metadata { jint _layout_helper; // KLUT entry - KlassLUTEntry _klute; + klute_raw_t _klute; // Klass kind used to resolve the runtime type of the instance. // - Used to implement devirtualized oop closure dispatching. @@ -218,7 +218,7 @@ class Klass : public Metadata { enum class PrivateLookupMode { find, skip }; // Klute handling - KlassLUTEntry klute() const { return _klute; } + klute_raw_t klute() const { return _klute; } void register_with_klut(); virtual bool is_klass() const { return true; } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 6951380061f11..f34fdd0c4469c 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -36,7 +36,7 @@ #include "utilities/ostream.hpp" ClassLoaderData* KlassInfoLUT::_common_loaders[4] = { nullptr }; -uint32_t* KlassInfoLUT::_table = nullptr; +klute_raw_t* KlassInfoLUT::_table = nullptr; unsigned KlassInfoLUT::_max_entries = -1; void KlassInfoLUT::initialize() { @@ -52,31 +52,31 @@ void KlassInfoLUT::initialize() { assert(CompressedKlassPointers::shift() == 10, "must be (for density)"); const narrowKlass highest_nk = CompressedKlassPointers::highest_valid_narrow_klass_id(); - size_t table_size_in_bytes = sizeof(uint32_t) * highest_nk; + size_t table_size_in_bytes = sizeof(klute_raw_t) * highest_nk; bool uses_large_pages = false; if (UseLargePages) { const size_t large_page_size = os::large_page_size(); if (large_page_size < 16 * M) { // not for freakishly large pages table_size_in_bytes = align_up(table_size_in_bytes, large_page_size); - _table = (uint32_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); + _table = (klute_raw_t*) os::reserve_memory_special(table_size_in_bytes, large_page_size, large_page_size, nullptr, false); if (_table != nullptr) { uses_large_pages = true; - _max_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); + _max_entries = (unsigned)(table_size_in_bytes / sizeof(klute_raw_t)); } } } if (_table == nullptr) { table_size_in_bytes = align_up(table_size_in_bytes, os::vm_page_size()); - _table = (uint32_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); + _table = (klute_raw_t*)os::reserve_memory(table_size_in_bytes, false, mtKLUT); os::commit_memory_or_exit((char*)_table, table_size_in_bytes, false, "KLUT"); - _max_entries = (unsigned)(table_size_in_bytes / sizeof(uint32_t)); + _max_entries = (unsigned)(table_size_in_bytes / sizeof(klute_raw_t)); } log_info(klut)("Lookup table initialized (%u entries, using %s pages): " RANGEFMT, _max_entries, (uses_large_pages ? "large" : "normal"), RANGEFMTARGS(_table, table_size_in_bytes)); // We need to zap the whole LUT if CDS is enabled or dumping, since we may need to late-register classes. - memset(_table, 0xff, _max_entries * sizeof(uint32_t)); + memset(_table, 0xff, _max_entries * sizeof(klute_raw_t)); assert(_table[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff } @@ -126,18 +126,19 @@ int KlassInfoLUT::index_for_cld(const ClassLoaderData* cld) { } static void log_klass_registration(const Klass* k, narrowKlass nk, bool added_to_table, - KlassLUTEntry klute, const char* message) { + klute_raw_t klute, const char* message) { char tmp[1024]; + const KlassLUTEntry klutehelper(klute); log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u(%c), klute: " INT32_FORMAT_X_0 ": %s %s%s", - p2i(k), common_loader_names[klute.loader_index()], nk, + p2i(k), common_loader_names[klutehelper.loader_index()], nk, (added_to_table ? '+' : '-'), - klute.value(), + klute, message, (k->is_shared() ? "(shared) " : ""), k->name()->as_C_string(tmp, sizeof(tmp))); } -KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { +klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // First register the CLD in case we did not already do that ClassLoaderData* const cld = k->class_loader_data(); @@ -152,20 +153,18 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { const bool add_to_table = use_lookup_table() ? CompressedKlassPointers::is_encodable(k) : false; const narrowKlass nk = add_to_table ? CompressedKlassPointers::encode(const_cast(k)) : 0; - KlassLUTEntry klute = k->klute(); - // Calculate klute from Klass properties and update the table value. - klute = KlassLUTEntry::build_from_klass(k); + const klute_raw_t klute = KlassLUTEntry::build_from_klass(k); if (add_to_table) { - _table[nk] = klute.value(); + _table[nk] = klute; } log_klass_registration(k, nk, add_to_table, klute, "registered"); #ifdef ASSERT - klute.verify_against_klass(k); + KlassLUTEntry(klute).verify_against_klass(k); if (add_to_table) { KlassLUTEntry e2(at(nk)); - assert(e2 == klute, "sanity"); + assert(e2.value() == klute, "sanity"); } #endif // ASSERT @@ -194,7 +193,7 @@ KlassLUTEntry KlassInfoLUT::register_klass(const Klass* k) { // this function is called where we add the table entry on the fly. // Unfortunately, this adds a branch into the very hot oop iteration path, albeit one that // would hopefully be mitigated by branch prediction since this should be exceedingly rare. -KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { +klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); @@ -202,18 +201,19 @@ KlassLUTEntry KlassInfoLUT::late_register_klass(narrowKlass nk) { // That klute would have been pre-calculated during CDS dump time when the to-be-dumped Klass // was dynamically constructed. // We just copy that entry into the table slot. - const KlassLUTEntry klute = k->klute(); - assert(klute.is_valid(), "Must be a valid klute"); - _table[nk] = klute.value(); + const klute_raw_t klute = k->klute(); + const KlassLUTEntry klutehelper(klute); + assert(klutehelper.is_valid(), "Must be a valid klute"); + _table[nk] = klutehelper.value(); ClassLoaderData* const cld = k->class_loader_data(); if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS register_cld_if_needed(cld); - DEBUG_ONLY(klute.verify_against_klass(k);) + DEBUG_ONLY(klutehelper.verify_against_klass(k);) } else { // Note: cld may still be nullptr; in that case it will be initialized by CDS before the Klass // is used. At that point we may correct the klute entry to account for the new CDS. } - log_klass_registration(k, nk, true, klute.value(), "late-registered"); + log_klass_registration(k, nk, true, klute, "late-registered"); return klute; } @@ -221,12 +221,12 @@ void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { // Called when a shared class gets its ClassLoaderData restored after being loaded. // The function makes sure that the CLD bits in the Klass' klute match the new // ClassLoaderData. - const KlassLUTEntry klute = k->klute(); + const klute_raw_t klute = k->klute(); ClassLoaderData* cld = k->class_loader_data(); assert(cld != nullptr, "must be"); register_cld_if_needed(cld); const int cld_index = index_for_cld(cld); - if (klute.loader_index() != cld_index) { + if (KlassLUTEntry(klute).loader_index() != cld_index) { // for simplicity, just recalculate the klute and update the table. log_debug(klut)("Re-registering Klass after CLD change"); k->register_with_klut(); @@ -251,7 +251,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { st->print_cr("KLUT statistics:"); if (use_lookup_table()) { - st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(uint32_t)); + st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(klute_raw_t)); } const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + @@ -334,21 +334,22 @@ void KlassInfoLUT::print_statistics(outputStream* st) { } #ifdef KLUT_ENABLE_EXPENSIVE_STATS -void KlassInfoLUT::update_hit_stats(KlassLUTEntry klute) { - switch (klute.kind()) { +void KlassInfoLUT::update_hit_stats(klute_raw_t klute) { + const KlassLUTEntry klutehelper(klute); + switch (klutehelper.kind()) { #define XX(name, shortname) case name ## Kind: inc_hits_ ## shortname(); break; KLASSKIND_ALL_KINDS_DO(XX) #undef XX default: ShouldNotReachHere(); }; - if (klute.is_instance() && !klute.ik_carries_infos()) { - switch (klute.kind()) { + if (klutehelper.is_instance() && !klutehelper.ik_carries_infos()) { + switch (klutehelper.kind()) { case InstanceClassLoaderKlassKind: inc_noinfo_ICLK(); break; case InstanceMirrorKlassKind: inc_noinfo_IMK(); break; default: inc_noinfo_IK_other(); break; } } - switch (klute.loader_index()) { + switch (klutehelper.loader_index()) { case 1: inc_hits_bootloader(); break; case 2: inc_hits_sysloader(); break; case 3: inc_hits_platformloader(); break; @@ -357,7 +358,7 @@ void KlassInfoLUT::update_hit_stats(KlassLUTEntry klute) { #endif // KLUT_ENABLE_EXPENSIVE_STATS #ifdef KLUT_ENABLE_EXPENSIVE_LOG -void KlassInfoLUT::log_hit(KlassLUTEntry klute) { +void KlassInfoLUT::log_hit(klute_raw_t klute) { //log_debug(klut)("retrieval: klute: name: %s kind: %d", k->name()->as_C_string(), k->kind()); } #endif diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index cfd9ce6a0cda4..9c8b154a5a564 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -83,9 +83,9 @@ class KlassInfoLUT : public AllStatic { static unsigned _max_entries; - static uint32_t* _table; + static klute_raw_t* _table; static inline unsigned max_entries() { return _max_entries; } - static inline uint32_t at(unsigned index); + static inline klute_raw_t at(unsigned index); static void allocate_lookup_table(); // Klass registration statistics. These are not expensive and therefore @@ -123,15 +123,15 @@ class KlassInfoLUT : public AllStatic { #define XX(xx) static void inc_##xx(); HIT_STATS_DO(XX) #undef XX - static void update_hit_stats(KlassLUTEntry klute); + static void update_hit_stats(klute_raw_t klute); #endif // KLUT_ENABLE_EXPENSIVE_STATS #ifdef KLUT_ENABLE_EXPENSIVE_LOG - static void log_hit(KlassLUTEntry klute); + static void log_hit(klute_raw_t klute); #endif #if INCLUDE_CDS - static KlassLUTEntry late_register_klass(narrowKlass nk); + static klute_raw_t late_register_klass(narrowKlass nk); #endif static bool use_lookup_table() { return _table != nullptr; } @@ -140,8 +140,8 @@ class KlassInfoLUT : public AllStatic { static void initialize(); - static KlassLUTEntry register_klass(const Klass* k); - static inline KlassLUTEntry lookup(narrowKlass k); + static klute_raw_t register_klass(const Klass* k); + static inline klute_raw_t lookup(narrowKlass k); // ClassLoaderData handling static void register_cld_if_needed(ClassLoaderData* cld); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index b189ae7c26ec5..36ea060876bf2 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -38,12 +38,11 @@ ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { return _table[index]; } -ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { +ALWAYSINLINE klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); - const uint32_t v = at(nk); - KlassLUTEntry e(v); + const klute_raw_t klute = at(nk); #if INCLUDE_CDS - if (!e.is_valid()) { + if (klute == KlassLUTEntry::invalid_entry) { // This branch, and the late_register_klass mechanic, only exists because it is // so difficult to iterate CDS classes after loading CDS archives. See discussion // surrounding 8353225. Hopefully we can remove this in the future. @@ -54,14 +53,14 @@ ALWAYSINLINE KlassLUTEntry KlassInfoLUT::lookup(narrowKlass nk) { #endif #ifdef KLUT_ENABLE_EXPENSIVE_STATS - update_hit_stats(e); + update_hit_stats(klute); #endif #ifdef KLUT_ENABLE_EXPENSIVE_LOG log_hit(e); #endif - return e; + return klute; } ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(int index) { diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 0db5db0a8923c..bb2a6cbbc14f0 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -57,7 +57,7 @@ static void read_and_check_omb_values(const OopMapBlock* omb, unsigned& omb_offs omb_count = count; } -uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_encodable_reason) { +klute_raw_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_encodable_reason) { assert(ik->is_instance_klass(), "sanity"); @@ -140,7 +140,7 @@ uint32_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_ } -uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { +klute_raw_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { assert(ak->is_array_klass(), "sanity"); @@ -164,9 +164,9 @@ uint32_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { } -KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { +klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { - uint32_t value = invalid_entry; + klute_raw_t value = invalid_entry; if (k->is_array_klass()) { value = build_from_ak(ArrayKlass::cast(k)); } else { @@ -178,7 +178,7 @@ KlassLUTEntry KlassLUTEntry::build_from_klass(const Klass* k) { k->name()->as_C_string(), not_encodable_reason); } } - return KlassLUTEntry(value); + return value; } #ifdef ASSERT diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index ebc51b9c5255e..222955562de11 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -129,6 +129,8 @@ class typeArrayOopDesc; // Implementation note: the value -1 (all bits 1) relies on the fact that a KlassKind of 7 (0b111) is invalid. We // don't use zero as "invalid entry" since zero would encode a valid Klass. +typedef uint32_t klute_raw_t; + class KlassLUTEntry { static constexpr int bits_total = 32; @@ -179,14 +181,14 @@ class KlassLUTEntry { }; union U { - uint32_t raw; + klute_raw_t raw; KE common; IKE ike; AKE ake; - U(uint32_t v) : raw(v) {} + U(klute_raw_t v) : raw(v) {} }; - U _v; + const U _v; // The limits to what we can numerically represent in an (InstanceKlass) Entry static constexpr size_t ik_wordsize_limit = nth_bit(bits_ik_wordsize); @@ -195,37 +197,30 @@ class KlassLUTEntry { static constexpr size_t ik_omb_offset_2_limit = nth_bit(bits_ik_omb_offset_2); static constexpr size_t ik_omb_count_2_limit = nth_bit(bits_ik_omb_count_2); - static uint32_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); - static uint32_t build_from_ak(const ArrayKlass* k); + static klute_raw_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); + static klute_raw_t build_from_ak(const ArrayKlass* k); public: // Invalid entries are entries that have not been set yet. // Note: cannot use "0" as invalid_entry, since 0 is valid (interface or abstract InstanceKlass, size = 0 and has no oop map) // We use kind=7=0b111 (invalid), and set the rest of the bits also to 1 - static constexpr uint32_t invalid_entry = 0xFFFFFFFF; + static constexpr klute_raw_t invalid_entry = 0xFFFFFFFF; - inline KlassLUTEntry() : _v(invalid_entry) {} - inline KlassLUTEntry(uint32_t v) : _v(v) {} + inline KlassLUTEntry(klute_raw_t v) : _v(v) {} inline KlassLUTEntry(const KlassLUTEntry& other) : _v(other._v) {} - inline KlassLUTEntry& operator=(uint32_t v) { _v = v; return *this; } - inline KlassLUTEntry& operator=(const KlassLUTEntry& other) { _v = other._v; return *this; } - - inline bool operator==(const KlassLUTEntry& other) const { return _v.raw == other._v.raw; } - inline bool operator!=(const KlassLUTEntry& other) const { return _v.raw != other._v.raw; } - // Note: all entries should be valid. An invalid entry indicates // an error somewhere. bool is_valid() const { return _v.raw != invalid_entry; } - static KlassLUTEntry build_from_klass(const Klass* k); + static klute_raw_t build_from_klass(const Klass* k); #ifdef ASSERT void verify_against_klass(const Klass* k) const; #endif - uint32_t value() const { return _v.raw; } + klute_raw_t value() const { return _v.raw; } inline unsigned kind() const { return _v.common.kind; } diff --git a/src/hotspot/share/oops/objArrayKlass.hpp b/src/hotspot/share/oops/objArrayKlass.hpp index dfaaabd21349d..d66d3e833fd93 100644 --- a/src/hotspot/share/oops/objArrayKlass.hpp +++ b/src/hotspot/share/oops/objArrayKlass.hpp @@ -121,15 +121,15 @@ class ObjArrayKlass : public ArrayKlass { // Iterate over oop elements and metadata. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute); // Iterate over oop elements and metadata. template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute); + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute); // Iterate over oop elements within mr, and metadata. template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute); + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute); // Iterate over oop elements within [start, end), and metadata. template diff --git a/src/hotspot/share/oops/objArrayKlass.inline.hpp b/src/hotspot/share/oops/objArrayKlass.inline.hpp index 1fdac791ec91e..5125a0b6348fe 100644 --- a/src/hotspot/share/oops/objArrayKlass.inline.hpp +++ b/src/hotspot/share/oops/objArrayKlass.inline.hpp @@ -69,7 +69,7 @@ void ObjArrayKlass::oop_oop_iterate_elements_bounded( } template -void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { assert(obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); @@ -81,7 +81,7 @@ void ObjArrayKlass::oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEn } template -void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) { +void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { assert(obj->is_array(), "obj must be array"); objArrayOop a = objArrayOop(obj); @@ -93,7 +93,7 @@ void ObjArrayKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, Me } template -void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) { +void ObjArrayKlass::oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) { // No reverse implementation ATM. oop_oop_iterate(obj, closure, klute); } diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 27bf15fc850a9..bb0f85f8045be 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -85,8 +85,8 @@ class oopDesc { inline void init_mark(); template - inline KlassLUTEntry get_klute() const; - inline KlassLUTEntry get_klute() const; + inline klute_raw_t get_klute() const; + inline klute_raw_t get_klute() const; template inline Klass* klass() const; diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 694329ce80cc8..654ea6b78a450 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -98,7 +98,7 @@ void oopDesc::init_mark() { } template -KlassLUTEntry oopDesc::get_klute() const { +klute_raw_t oopDesc::get_klute() const { switch (mode) { case HeaderMode::Compact: return KlassInfoLUT::lookup(mark().narrow_klass()); @@ -109,7 +109,7 @@ KlassLUTEntry oopDesc::get_klute() const { } } -KlassLUTEntry oopDesc::get_klute() const { +klute_raw_t oopDesc::get_klute() const { switch (ObjLayout::klass_mode()) { case HeaderMode::Compact: return get_klute(); case HeaderMode::Compressed: return get_klute(); @@ -264,13 +264,13 @@ bool oopDesc::is_stackChunk() const { return klass()->is_stack_chunk_instance_k bool oopDesc::is_array() const { return klass()->is_array_klass(); } bool oopDesc::is_objArray() const { - const bool rc = get_klute().is_obj_array(); + const bool rc = KlassLUTEntry(get_klute()).is_obj_array(); assert(rc == klass()->is_objArray_klass(), "Sanity"); return rc; } bool oopDesc::is_typeArray() const { - const bool rc = get_klute().is_type_array(); + const bool rc = KlassLUTEntry(get_klute()).is_type_array(); assert(rc == klass()->is_typeArray_klass(), "Sanity"); return rc; } diff --git a/src/hotspot/share/oops/typeArrayKlass.hpp b/src/hotspot/share/oops/typeArrayKlass.hpp index d312b9a46fd76..dc20a9f4a724b 100644 --- a/src/hotspot/share/oops/typeArrayKlass.hpp +++ b/src/hotspot/share/oops/typeArrayKlass.hpp @@ -85,11 +85,11 @@ class TypeArrayKlass : public ArrayKlass { // to be loaded by the boot class loader. template - static inline void oop_oop_iterate(oop obj, OopClosureType* closure, KlassLUTEntry klute) {} + static inline void oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) {} template - static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, KlassLUTEntry klute) {} + static inline void oop_oop_iterate_reverse(oop obj, OopClosureType* closure, klute_raw_t klute) {} template - static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, KlassLUTEntry klute) {} + static inline void oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) {} static TypeArrayKlass* cast(Klass* k) { return const_cast(cast(const_cast(k))); From c6e4a68d1dbb958d1e815729f2d74f56d4493f87 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Sat, 3 May 2025 07:11:59 +0200 Subject: [PATCH 087/101] fix non-cds build --- src/hotspot/share/oops/klassInfoLUT.inline.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 36ea060876bf2..9ad43834cf3d9 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -49,7 +49,7 @@ ALWAYSINLINE klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { return late_register_klass(nk); } #else - assert(e.is_valid(), "must never be invalid"); + assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); #endif #ifdef KLUT_ENABLE_EXPENSIVE_STATS @@ -57,7 +57,7 @@ ALWAYSINLINE klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { #endif #ifdef KLUT_ENABLE_EXPENSIVE_LOG - log_hit(e); + log_hit(klute); #endif return klute; From 311d364f69ff9edf5f573c6bdeda557089ed79b6 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 5 May 2025 06:47:27 +0200 Subject: [PATCH 088/101] scan CDS archive range instead of late register --- src/hotspot/share/cds/metaspaceShared.cpp | 9 ++++ src/hotspot/share/memory/metaspace.cpp | 4 ++ src/hotspot/share/memory/universe.cpp | 3 -- src/hotspot/share/oops/klass.cpp | 16 +++++- src/hotspot/share/oops/klass.hpp | 4 ++ src/hotspot/share/oops/klassInfoLUT.cpp | 52 +++++++++++++++++-- src/hotspot/share/oops/klassInfoLUT.hpp | 6 ++- .../share/oops/klassInfoLUT.inline.hpp | 24 ++++++--- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 2 + src/hotspot/share/oops/klassInfoLUTEntry.hpp | 2 + 10 files changed, 103 insertions(+), 19 deletions(-) diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp index ef2a6dcb8e63c..7ef19ce893176 100644 --- a/src/hotspot/share/cds/metaspaceShared.cpp +++ b/src/hotspot/share/cds/metaspaceShared.cpp @@ -75,6 +75,7 @@ #include "oops/compressedKlass.hpp" #include "oops/instanceMirrorKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayOop.hpp" #include "oops/oop.inline.hpp" #include "oops/oopHandle.hpp" @@ -1437,6 +1438,14 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File } CompressedKlassPointers::establish_protection_zone(encoding_base, prot_zone_size); + // Initialize KLUT + KlassInfoLUT::initialize(); + + // Scan the CDS klass range for already existing Klass structures + const address start = (address)align_up(archive_space_rs.base() + prot_zone_size, CompressedKlassPointers::klass_alignment_in_bytes()); + const address end = (address)archive_space_rs.base() + archive_space_rs.size(); + KlassInfoLUT::scan_klass_range_update_lut(start, end); + // map_or_load_heap_region() compares the current narrow oop and klass encodings // with the archived ones, so it must be done after all encodings are determined. static_mapinfo->map_or_load_heap_region(); diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp index ab7202d046ab2..8acfda642038b 100644 --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -53,6 +53,7 @@ #include "nmt/memTracker.hpp" #include "oops/compressedKlass.inline.hpp" #include "oops/compressedOops.hpp" +#include "oops/klassInfoLUT.hpp" #include "prims/jvmtiExport.hpp" #include "runtime/atomic.hpp" #include "runtime/globals_extension.hpp" @@ -810,6 +811,9 @@ void Metaspace::global_initialize() { // In CDS=off mode, we give the JVM some leeway to choose a favorable base/shift combination. CompressedKlassPointers::initialize((address)rs.base(), rs.size()); + // Initialize KLUT + KlassInfoLUT::initialize(); + // After narrowKlass encoding scheme is decided: if the encoding base points to class space start, // establish a protection zone. Accidentally decoding a zero nKlass ID and then using it will result // in an immediate segmentation fault instead of a delayed error much later. diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp index 00fbce16d419b..da531b9464071 100644 --- a/src/hotspot/share/memory/universe.cpp +++ b/src/hotspot/share/memory/universe.cpp @@ -887,9 +887,6 @@ jint universe_init() { Metaspace::global_initialize(); - // Initialize KLUT before starting to create any Klass - KlassInfoLUT::initialize(); - // Initialize performance counters for metaspaces MetaspaceCounters::initialize_performance_counters(); diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index a570cb6ebbd97..4c9c4a4eefaab 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -293,7 +293,17 @@ static markWord make_prototype(const Klass* kls) { return prototype; } -Klass::Klass() : _kind(UnknownKlassKind) { +static constexpr uint64_t stamp = 0x04f63b9b436c16d5ULL; + +static uint64_t calc_stamp(const void* p) { + return stamp; +} + +bool Klass::check_stamp() const { + return _stamp == stamp; +} + +Klass::Klass() : _kind(UnknownKlassKind), _stamp(calc_stamp(this)) { assert(CDSConfig::is_dumping_static_archive() || CDSConfig::is_using_archive(), "only for cds"); } @@ -303,7 +313,9 @@ Klass::Klass() : _kind(UnknownKlassKind) { // which doesn't zero out the memory before calling the constructor. Klass::Klass(KlassKind kind) : _kind(kind), _prototype_header(make_prototype(this)), - _shared_class_path_index(-1) { + _shared_class_path_index(-1), + _stamp(calc_stamp(this)) + { CDS_ONLY(_shared_class_flags = 0;) CDS_JAVA_HEAP_ONLY(_archived_mirror_index = -1;) _primary_supers[0] = this; diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index f4d21aefe6f1f..2a3ba99149b03 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -198,6 +198,8 @@ class Klass : public Metadata { // Keep it away from the beginning of a Klass to avoid cacheline // contention that may happen when a nearby object is modified. + const uint64_t _stamp; + CDS_JAVA_HEAP_ONLY(int _archived_mirror_index;) public: @@ -793,6 +795,8 @@ class Klass : public Metadata { // Returns true if this Klass needs to be addressable via narrow Klass ID. inline bool needs_narrow_id() const; + bool check_stamp() const; + }; #endif // SHARE_OOPS_KLASS_HPP diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index f34fdd0c4469c..d472b2ad10298 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -36,11 +36,12 @@ #include "utilities/ostream.hpp" ClassLoaderData* KlassInfoLUT::_common_loaders[4] = { nullptr }; +bool KlassInfoLUT::_initialized = false; klute_raw_t* KlassInfoLUT::_table = nullptr; unsigned KlassInfoLUT::_max_entries = -1; void KlassInfoLUT::initialize() { - + assert(!_initialized, "Only once"); if (UseCompactObjectHeaders) { // Init Lookup Table @@ -80,6 +81,7 @@ void KlassInfoLUT::initialize() { assert(_table[0] == KlassLUTEntry::invalid_entry, "Sanity"); // must be 0xffffffff } + _initialized = true; } static const char* common_loader_names[4] = { "other", "boot", "app", "platform" }; @@ -156,7 +158,7 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // Calculate klute from Klass properties and update the table value. const klute_raw_t klute = KlassLUTEntry::build_from_klass(k); if (add_to_table) { - _table[nk] = klute; + put(nk, klute); } log_klass_registration(k, nk, add_to_table, klute, "registered"); @@ -183,6 +185,45 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { } #if INCLUDE_CDS + +void KlassInfoLUT::scan_klass_range_update_lut(address from, address to) { + assert(_initialized, "not initialized"); + if (use_lookup_table()) { + log_info(klut)("Scanning CDS klass range: " RANGE2FMT, RANGE2FMTARGS(from, to)); + const size_t stepsize = CompressedKlassPointers::klass_alignment_in_bytes(); + assert(stepsize >= K, "only for COH and large alignments"); + assert(is_aligned(from, stepsize), "from address unaligned"); + assert(is_aligned(to, stepsize), "to address unaligned"); + assert(from < to, "invalid range"); + unsigned found = 0; + for (address here = from; here < to; here += stepsize) { + if (!os::is_readable_range(here, here + sizeof(Klass))) { + continue; + } + const Klass* const candidate = (Klass*)here; + if (!candidate->check_stamp()) { + continue; + } + const klute_raw_t klute = candidate->klute(); + if (klute == KlassLUTEntry::invalid_entry) { + continue; + } + // Above checks may of course, rarely, give false positives. That is absolutely fine: we + // then copy a "klute" from a "Klass" to the table that really isn't either klute nor Klass. + // Since that slot is not used anyway by a real Klass, nothing bad will happen. + // OTOH, *missing* to add a klute for a Klass that really exists would be really bad. Hence, + // err on the plus side. + const narrowKlass nk = CompressedKlassPointers::encode(const_cast(candidate)); + put(nk, klute); + log_info(klut)("Suspected Klass found at " PTR_FORMAT "; adding nk %u, klute: " INT32_FORMAT_X_0, + p2i(candidate), nk, klute); + found ++; + } + log_info(klut)("Found and registered %u possible Klass locations in CDS klass range " RANGE2FMT, + found, RANGE2FMTARGS(from, to)); + } +} + // We only tolerate this for CDS: // We currently have no simple way to iterate all Klass structures in a CDS/AOT archive // before the JVM starts calling methods on oops that refer to these classes. This is because @@ -194,6 +235,7 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // Unfortunately, this adds a branch into the very hot oop iteration path, albeit one that // would hopefully be mitigated by branch prediction since this should be exceedingly rare. klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { + assert(_initialized, "not initialized"); assert(nk != 0, "null narrow Klass - is this class encodable?"); const Klass* k = CompressedKlassPointers::decode(nk); assert(k->is_shared(), "Only for CDS classes"); @@ -204,7 +246,7 @@ klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { const klute_raw_t klute = k->klute(); const KlassLUTEntry klutehelper(klute); assert(klutehelper.is_valid(), "Must be a valid klute"); - _table[nk] = klutehelper.value(); + put(nk, klutehelper.value()); ClassLoaderData* const cld = k->class_loader_data(); if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS register_cld_if_needed(cld); @@ -336,11 +378,13 @@ void KlassInfoLUT::print_statistics(outputStream* st) { #ifdef KLUT_ENABLE_EXPENSIVE_STATS void KlassInfoLUT::update_hit_stats(klute_raw_t klute) { const KlassLUTEntry klutehelper(klute); + assert(klutehelper.is_valid(), "invalid klute"); switch (klutehelper.kind()) { #define XX(name, shortname) case name ## Kind: inc_hits_ ## shortname(); break; KLASSKIND_ALL_KINDS_DO(XX) #undef XX - default: ShouldNotReachHere(); + default: + fatal("invalid klute kind (%x)", klute); }; if (klutehelper.is_instance() && !klutehelper.ik_carries_infos()) { switch (klutehelper.kind()) { diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 9c8b154a5a564..2e57c8292d495 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -80,12 +80,12 @@ class ClassLoaderData; class KlassInfoLUT : public AllStatic { static ClassLoaderData* _common_loaders[4]; // See "loader" bits in Klute - - static unsigned _max_entries; static klute_raw_t* _table; + static bool _initialized; static inline unsigned max_entries() { return _max_entries; } static inline klute_raw_t at(unsigned index); + static inline void put(unsigned index, klute_raw_t klute); static void allocate_lookup_table(); // Klass registration statistics. These are not expensive and therefore @@ -143,6 +143,8 @@ class KlassInfoLUT : public AllStatic { static klute_raw_t register_klass(const Klass* k); static inline klute_raw_t lookup(narrowKlass k); + static void scan_klass_range_update_lut(address from, address to); + // ClassLoaderData handling static void register_cld_if_needed(ClassLoaderData* cld); static int index_for_cld(const ClassLoaderData* cld); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 9ad43834cf3d9..4267ea3b142c3 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -32,22 +32,30 @@ #include "utilities/globalDefinitions.hpp" #include "utilities/debug.hpp" -ALWAYSINLINE uint32_t KlassInfoLUT::at(unsigned index) { +inline uint32_t KlassInfoLUT::at(unsigned index) { + assert(_initialized, "not initialized"); assert(_table != nullptr, "LUT table does not exist"); assert(index < max_entries(), "oob (%x vs %x)", index, max_entries()); return _table[index]; } -ALWAYSINLINE klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { +inline void KlassInfoLUT::put(unsigned index, klute_raw_t klute) { + assert(_initialized, "not initialized"); + assert(_table != nullptr, "LUT table does not exist"); + assert(index < max_entries(), "oob (%x vs %x)", index, max_entries()); + _table[index] = klute; +} + +inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const klute_raw_t klute = at(nk); #if INCLUDE_CDS - if (klute == KlassLUTEntry::invalid_entry) { - // This branch, and the late_register_klass mechanic, only exists because it is - // so difficult to iterate CDS classes after loading CDS archives. See discussion - // surrounding 8353225. Hopefully we can remove this in the future. - return late_register_klass(nk); - } +// if (klute == KlassLUTEntry::invalid_entry) { +// // This branch, and the late_register_klass mechanic, only exists because it is +// // so difficult to iterate CDS classes after loading CDS archives. See discussion +// // surrounding 8353225. Hopefully we can remove this in the future. +// return late_register_klass(nk); +// } #else assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); #endif diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index bb2a6cbbc14f0..5cf77d09a7e53 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -185,6 +185,8 @@ klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { void KlassLUTEntry::verify_against_klass(const Klass* k) const { + assert(k->check_stamp(), "Stamp invalid"); + // General static asserts that need access to private members, but I don't want // to place them in a header STATIC_ASSERT(bits_common + bits_specific == bits_total); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 222955562de11..783c1c7dd1b6a 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -216,6 +216,8 @@ class KlassLUTEntry { static klute_raw_t build_from_klass(const Klass* k); + bool is_valid_for_klass(const Klass* k) const; + #ifdef ASSERT void verify_against_klass(const Klass* k) const; #endif From 17fb2c746e456c404f45f1734597c22df9a7428f Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 5 May 2025 17:17:48 +0200 Subject: [PATCH 089/101] IK metadata iteration: With AOT, CLD can be null :-( - dont call closure then --- src/hotspot/share/oops/instanceKlass.hpp | 3 +- .../share/oops/instanceKlass.inline.hpp | 28 ++++++++++--------- .../share/oops/klassInfoLUT.inline.hpp | 3 +- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index 230ba1bb69ef5..ab84a994bc8cd 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -1017,7 +1017,8 @@ class InstanceKlass: public Klass { template inline void oop_oop_iterate_oop_maps_bounded(oop obj, OopClosureType* closure, MemRegion mr); - static inline ClassLoaderData* cld_from_klut_or_klass(oop obj, klute_raw_t klute); + template + static inline void do_cld_from_klut_or_klass(oop obj, OopClosureType* closure, klute_raw_t klut); public: diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index eaa692ba1fab0..0ac6a06a650e2 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -27,6 +27,7 @@ #include "oops/instanceKlass.hpp" +#include "cds/aotLinkedClassBulkLoader.hpp" #include "memory/memRegion.hpp" #include "oops/fieldInfo.inline.hpp" #include "oops/klassInfoLUTEntry.inline.hpp" @@ -155,26 +156,28 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl } } -ALWAYSINLINE ClassLoaderData* InstanceKlass::cld_from_klut_or_klass(oop obj, klute_raw_t klute) { +template +inline void InstanceKlass::do_cld_from_klut_or_klass(oop obj, OopClosureType* closure, klute_raw_t klute) { + // Resolve the CLD from the klute. If that fails, resolve CLD from Klass. Call closure->do_cld. const unsigned perma_cld_index = KlassLUTEntry(klute).loader_index(); ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); - if (cld != nullptr) { -#ifdef ASSERT - const ClassLoaderData* const cld_real = obj->klass()->class_loader_data(); - assert(cld == cld_real, "CLD mismatch (" PTR_FORMAT " vs " PTR_FORMAT ")", p2i(cld_real), p2i(cld)); -#endif - return cld; + if (cld == nullptr) { + Klass* const k = obj->klass(); + cld = k->class_loader_data(); + if (cld == nullptr) { + // Unfortunately, this can now happen due to AOT, so we add another branch here + assert(AOTLinkedClassBulkLoader::is_pending_aot_linked_class(k), "sanity"); + return; + } } - // Rare path - return obj->klass()->class_loader_data(); + Devirtualizer::do_cld(closure, cld); } // Iterate over all oop fields and metadata. template ALWAYSINLINE void InstanceKlass::oop_oop_iterate(oop obj, OopClosureType* closure, klute_raw_t klute) { if (Devirtualizer::do_metadata(closure)) { - ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); - Devirtualizer::do_cld(closure, cld); + do_cld_from_klut_or_klass(obj, closure, klute); } const KlassLUTEntry klutehelper(klute); if (klutehelper.ik_carries_infos()) { @@ -220,8 +223,7 @@ template ALWAYSINLINE void InstanceKlass::oop_oop_iterate_bounded(oop obj, OopClosureType* closure, MemRegion mr, klute_raw_t klute) { if (Devirtualizer::do_metadata(closure)) { if (mr.contains(obj)) { - ClassLoaderData* cld = cld_from_klut_or_klass(obj, klute); - Devirtualizer::do_cld(closure, cld); + do_cld_from_klut_or_klass(obj, closure, klute); } } const KlassLUTEntry klutehelper(klute); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 4267ea3b142c3..4b440bcd10e23 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -74,7 +74,8 @@ inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(int index) { assert(index >= 0 && index <= 3, "Sanity"); ClassLoaderData* cld = _common_loaders[index]; - assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); + // Because of AOT, we can now encounter objects of Klasses that are not fully linked yet + // assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); return cld; } From a1efdbf79b818bcfa329d5488218c61407b4a7ff Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 6 May 2025 07:55:41 +0200 Subject: [PATCH 090/101] In Archivebuilder, move klute validity check up to before remove_unshareable_info --- src/hotspot/share/cds/archiveBuilder.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index b5a2c7d78cc99..fbaa4004c8124 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -772,17 +772,19 @@ void ArchiveBuilder::make_klasses_shareable() { int unreg_unlinked = 0; for (int i = 0; i < klasses()->length(); i++) { + Klass* k = get_buffered_addr(klasses()->at(i)); + + // Every archived Klass must carry a valid klute. That is because every archived Klass + // would have been created via the usual dynamic class loading or - generation, which should + // have registered the Klass with klut. + DEBUG_ONLY(KlassLUTEntry(k->klute()).verify_against_klass(k);) + // Some of the code in ConstantPool::remove_unshareable_info() requires the classes // to be in linked state, so it must be call here before the next loop, which returns // all classes to unlinked state. - Klass* k = get_buffered_addr(klasses()->at(i)); if (k->is_instance_klass()) { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } - // Every archived Klass must carry a valid klute. That is because every archived Klass - // would have been created via the usual dynamic class loading or - generation, which should - // have registered the Klass with klut. - DEBUG_ONLY(KlassLUTEntry(k->klute()).verify_against_klass(k);) } for (int i = 0; i < klasses()->length(); i++) { From 11ae2a3e8c46d9d6ea9d617cf0c21e9e979707df Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 6 May 2025 07:56:38 +0200 Subject: [PATCH 091/101] Static assert that oop closures used in oop iteration derive from OopIterateClosure --- src/hotspot/share/oops/instanceKlass.inline.hpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 0ac6a06a650e2..4bfe3976d106a 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -158,14 +158,21 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl template inline void InstanceKlass::do_cld_from_klut_or_klass(oop obj, OopClosureType* closure, klute_raw_t klute) { - // Resolve the CLD from the klute. If that fails, resolve CLD from Klass. Call closure->do_cld. + // Call closure->do_cld. The underlying assumption here is that if a closure subscribes via do_metadata(), + // it is really interested in the CLD instead of the Klass (does a do_cld(klass->class_loader_data()). + // That is true for all Closures that derive from OopIterateClosure. + static_assert(std::is_base_of::value, + "must inherit from OopIterateClosure"); + // ... and in that case we can fetch the CLD from the KLUT cld cache instead of letting the closure pull + // it from Klass. We don't even have to fetch and decode the narrowKlass. const unsigned perma_cld_index = KlassLUTEntry(klute).loader_index(); ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); if (cld == nullptr) { + // Rare path Klass* const k = obj->klass(); cld = k->class_loader_data(); if (cld == nullptr) { - // Unfortunately, this can now happen due to AOT, so we add another branch here + // See JDK-8342429. Unfortunately, this can now happen due to AOT. assert(AOTLinkedClassBulkLoader::is_pending_aot_linked_class(k), "sanity"); return; } From c291cb86835bae64f40438a0bc3498491c156e3c Mon Sep 17 00:00:00 2001 From: tstuefe Date: Tue, 6 May 2025 16:46:46 +0200 Subject: [PATCH 092/101] Rework KlassLUTEntry::verify_against_klass() to tolerate aot unlinked classes unconditionally --- src/hotspot/share/cds/archiveBuilder.cpp | 15 +-- src/hotspot/share/oops/klassInfoLUT.cpp | 10 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 97 ++++++++++++-------- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 6 +- 4 files changed, 78 insertions(+), 50 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index fbaa4004c8124..5e18b6716ed05 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -773,18 +773,21 @@ void ArchiveBuilder::make_klasses_shareable() { for (int i = 0; i < klasses()->length(); i++) { Klass* k = get_buffered_addr(klasses()->at(i)); - - // Every archived Klass must carry a valid klute. That is because every archived Klass - // would have been created via the usual dynamic class loading or - generation, which should - // have registered the Klass with klut. - DEBUG_ONLY(KlassLUTEntry(k->klute()).verify_against_klass(k);) - // Some of the code in ConstantPool::remove_unshareable_info() requires the classes // to be in linked state, so it must be call here before the next loop, which returns // all classes to unlinked state. if (k->is_instance_klass()) { InstanceKlass::cast(k)->constants()->remove_unshareable_info(); } + // Every archived Klass must carry a valid klute. That is because every archived Klass + // would have been created via the usual dynamic class loading or - generation, which should + // have registered the Klass with klut. +#ifdef ASSERT + // We can, unfortunately, encounter classes that are not linked yet at this point. + // Hopefully this will be resolved with JDK-8342429. + constexpr bool tolerate_aot_unlinked_classes = true; + KlassLUTEntry(k->klute()).verify_against_klass(k, tolerate_aot_unlinked_classes); +#endif } for (int i = 0; i < klasses()->length(); i++) { diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index d472b2ad10298..0064e374bb65b 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -131,7 +131,7 @@ static void log_klass_registration(const Klass* k, narrowKlass nk, bool added_to klute_raw_t klute, const char* message) { char tmp[1024]; const KlassLUTEntry klutehelper(klute); - log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u(%c), klute: " INT32_FORMAT_X_0 ": %s %s%s", + log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u(%c), klute: " KLUTE_FORMAT ": %s %s%s", p2i(k), common_loader_names[klutehelper.loader_index()], nk, (added_to_table ? '+' : '-'), klute, @@ -163,7 +163,7 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { log_klass_registration(k, nk, add_to_table, klute, "registered"); #ifdef ASSERT - KlassLUTEntry(klute).verify_against_klass(k); + KlassLUTEntry(klute).verify_against_klass(k, false /* tolerate_aot_unlinked_classes */); if (add_to_table) { KlassLUTEntry e2(at(nk)); assert(e2.value() == klute, "sanity"); @@ -215,7 +215,7 @@ void KlassInfoLUT::scan_klass_range_update_lut(address from, address to) { // err on the plus side. const narrowKlass nk = CompressedKlassPointers::encode(const_cast(candidate)); put(nk, klute); - log_info(klut)("Suspected Klass found at " PTR_FORMAT "; adding nk %u, klute: " INT32_FORMAT_X_0, + log_info(klut)("Suspected Klass found at " PTR_FORMAT "; adding nk %u, klute: " KLUTE_FORMAT, p2i(candidate), nk, klute); found ++; } @@ -250,11 +250,11 @@ klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { ClassLoaderData* const cld = k->class_loader_data(); if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS register_cld_if_needed(cld); - DEBUG_ONLY(klutehelper.verify_against_klass(k);) } else { // Note: cld may still be nullptr; in that case it will be initialized by CDS before the Klass // is used. At that point we may correct the klute entry to account for the new CDS. } + DEBUG_ONLY(klutehelper.verify_against_klass(k, (cld != nullptr) /* tolerate_aot_unlinked_classes */)); log_klass_registration(k, nk, true, klute, "late-registered"); return klute; } @@ -384,7 +384,7 @@ void KlassInfoLUT::update_hit_stats(klute_raw_t klute) { KLASSKIND_ALL_KINDS_DO(XX) #undef XX default: - fatal("invalid klute kind (%x)", klute); + fatal("invalid klute kind (" KLUTE_FORMAT ")", klute); }; if (klutehelper.is_instance() && !klutehelper.ik_carries_infos()) { switch (klutehelper.kind()) { diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 5cf77d09a7e53..f7fd5899f4f60 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -23,6 +23,7 @@ * */ +#include "cds/cdsConfig.hpp" #include "logging/log.hpp" #include "memory/resourceArea.hpp" #include "oops/instanceKlass.inline.hpp" @@ -182,8 +183,12 @@ klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { } #ifdef ASSERT +void KlassLUTEntry::verify_against_klass(const Klass* k, bool tolerate_aot_unlinked_classes) const { -void KlassLUTEntry::verify_against_klass(const Klass* k) const { +#define PREAMBLE_FORMAT "Klass: " PTR_FORMAT "(%s), klute " KLUTE_FORMAT ": " +#define PREAMBLE_ARGS p2i(k), k->external_name(), _v.raw +#define ASSERT_HERE(cond, msg) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS); +#define ASSERT_HERE2(cond, msg, ...) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS, __VA_ARGS__); assert(k->check_stamp(), "Stamp invalid"); @@ -200,40 +205,54 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { // Kind must fit into 3 bits STATIC_ASSERT(KlassKindCount < nth_bit(bits_kind)); - assert(is_valid(), "klute should be valid (%x)", _v.raw); + ASSERT_HERE(is_valid(), "klute invalid"); // kind const unsigned real_kind = (unsigned)k->kind(); const unsigned our_kind = kind(); const int real_lh = k->layout_helper(); - assert(our_kind == real_kind, "kind mismatch (%d vs %d) (%x)", real_kind, our_kind, _v.raw); + ASSERT_HERE2(our_kind == real_kind, "kind mismatch (%d vs %d)", real_kind, our_kind); const ClassLoaderData* const real_cld = k->class_loader_data(); const unsigned cld_index = loader_index(); - assert(cld_index < 4, "invalid loader index"); - if (cld_index > 0) { + ASSERT_HERE(cld_index < 4, "invalid loader index"); + // With AOT, we can encounter Klasses that have not set their CLD yet. Until that is solved (JDK-8342429), + // work around this problem. Dealing with unlinked classes is very frustrating. + const bool skip_cld_check = (real_cld == nullptr) && tolerate_aot_unlinked_classes; + if (!skip_cld_check) { + ASSERT_HERE(real_cld != nullptr, "Klass CLD is null?"); const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cld_index); - assert(cld_from_klute == real_cld, - "Different CLD (loader_index: %d, real CLD: " PTR_FORMAT ", from klute lookup table: " PTR_FORMAT ")?", - cld_index, p2i(real_cld), p2i(cld_from_klute)); - } else { - assert(!real_cld->is_permanent_class_loader_data(), "perma cld?"); + if (cld_index > 0) { + // We expect to get one of the three permanent class loaders, and for it to match the one in Klass + ASSERT_HERE2(cld_from_klute->is_permanent_class_loader_data(), "not perma cld (loader_index: %d, CLD: " PTR_FORMAT ")", + cld_index, p2i(cld_from_klute)); + ASSERT_HERE2(cld_from_klute == real_cld, + "Different CLD (loader_index: %d, real Klass CLD: " PTR_FORMAT ", from klute CLD lookup table: " PTR_FORMAT ")?", + cld_index, p2i(real_cld), p2i(cld_from_klute)); + } else { + // We expect to get a NULL from the CLD lookup table. + ASSERT_HERE2(cld_from_klute == nullptr, "CLD not null? (" PTR_FORMAT ")", p2i(cld_from_klute)); + // cld_index == 0 means "unknown CLD" and since we expect only to ever run with three permanent CLDs, + // that cld should not be permanent. + ASSERT_HERE2(!real_cld->is_permanent_class_loader_data(), + "Unregistered permanent CLD? (" PTR_FORMAT ")", p2i(cld_from_klute)); + } } if (k->is_array_klass()) { // compare our (truncated) lh with the real one const LayoutHelperHelper lhu = { (unsigned) real_lh }; - assert(lhu.bytes.lh_esz == ak_log2_elem_size() && - lhu.bytes.lh_hsz == ak_first_element_offset_in_bytes() && - ( (lhu.bytes.lh_tag == 0xC0 && real_kind == TypeArrayKlassKind) || - (lhu.bytes.lh_tag == 0x80 && real_kind == ObjArrayKlassKind) ), - "layouthelper mismatch (layouthelper: 0x%x, klute: 0x%x)", real_lh, _v.raw); + ASSERT_HERE2(lhu.bytes.lh_esz == ak_log2_elem_size() && + lhu.bytes.lh_hsz == ak_first_element_offset_in_bytes() && + ( (lhu.bytes.lh_tag == 0xC0 && real_kind == TypeArrayKlassKind) || + (lhu.bytes.lh_tag == 0x80 && real_kind == ObjArrayKlassKind) ), + "layouthelper mismatch (lh from Klass: 0x%x", real_lh); } else { - assert(k->is_instance_klass(), "unexpected"); + ASSERT_HERE(k->is_instance_klass(), "unexpected"); const InstanceKlass* const ik = InstanceKlass::cast(k); const int real_oop_map_count = ik->nonstatic_oop_map_count(); @@ -248,43 +267,47 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { if (ik_carries_infos()) { // check wordsize - assert(real_wordsize == ik_wordsize(), "wordsize mismatch? (%d vs %d) (%x)", real_wordsize, ik_wordsize(), _v.raw); + ASSERT_HERE2(real_wordsize == ik_wordsize(), "wordsize mismatch? (%d vs %d)", real_wordsize, ik_wordsize()); // check omb info switch (real_oop_map_count) { case 0: { - assert(ik_omb_offset_1() == 0 && ik_omb_count_1() == 0 && - ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, "omb should not be present (0x%x)", _v.raw); + ASSERT_HERE(ik_omb_offset_1() == 0 && ik_omb_count_1() == 0 && + ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, + "omb should not be present"); } break; case 1: { - assert(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch (0x%x)", _v.raw); - assert(ik_omb_count_1() == omb_count_1, "first omb count mismatch (0x%x)", _v.raw); - assert(ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, "second omb should not be present (0x%x)", _v.raw); + ASSERT_HERE(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch"); + ASSERT_HERE(ik_omb_count_1() == omb_count_1, "first omb count mismatch"); + ASSERT_HERE(ik_omb_offset_2() == 0 && ik_omb_count_2() == 0, "second omb should not be present"); } break; case 2: { - assert(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch (0x%x)", _v.raw); - assert(ik_omb_count_1() == omb_count_1, "first omb count mismatch (0x%x)", _v.raw); - assert(ik_omb_offset_2() * BytesPerHeapOop == omb_offset_2, "second omb offset mismatch (0x%x)", _v.raw); - assert(ik_omb_count_2() == omb_count_2, "second omb count mismatch (0x%x)", _v.raw); + ASSERT_HERE(ik_omb_offset_1() * BytesPerHeapOop == omb_offset_1, "first omb offset mismatch"); + ASSERT_HERE(ik_omb_count_1() == omb_count_1, "first omb count mismatch"); + ASSERT_HERE(ik_omb_offset_2() * BytesPerHeapOop == omb_offset_2, "second omb offset mismatch"); + ASSERT_HERE(ik_omb_count_2() == omb_count_2, "second omb count mismatch"); } break; - default: fatal("More than one oop maps, IKE should not be encodable"); + default: fatal("More than one oop maps, IKE should not have been fully encodable"); } } else { - // Check if this Klass should, in fact, have been encodable - assert( Klass::layout_helper_needs_slow_path(real_lh) || - (real_wordsize >= (int)ik_wordsize_limit) || - (real_oop_map_count > 2) || - ((size_t) omb_offset_1 >= ik_omb_offset_1_limit) || - ((size_t) omb_count_1 >= ik_omb_count_1_limit) || - ((size_t) omb_offset_2 >= ik_omb_offset_2_limit) || - ((size_t) omb_count_2 >= ik_omb_count_2_limit), - "Klass should have been encodable" ); + // Check if this Klass should, in fact, have been fully encodable + ASSERT_HERE(Klass::layout_helper_needs_slow_path(real_lh) || + (real_wordsize >= (int)ik_wordsize_limit) || + (real_oop_map_count > 2) || + ((size_t) omb_offset_1 >= ik_omb_offset_1_limit) || + ((size_t) omb_count_1 >= ik_omb_count_1_limit) || + ((size_t) omb_offset_2 >= ik_omb_offset_2_limit) || + ((size_t) omb_count_2 >= ik_omb_count_2_limit), + "Klass should have been encodable" ); } } - +#undef PREAMBLE_FORMAT +#undef PREAMBLE_ARGS +#undef ASSERT_HERE +#undef ASSERT2_HERE } // KlassLUTEntry::verify_against #endif // ASSERT diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 783c1c7dd1b6a..34fc338ebbe6e 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -219,8 +219,8 @@ class KlassLUTEntry { bool is_valid_for_klass(const Klass* k) const; #ifdef ASSERT - void verify_against_klass(const Klass* k) const; -#endif + void verify_against_klass(const Klass* k, bool tolerate_aot_unlinked_classes) const; +#endif // ASSERT klute_raw_t value() const { return _v.raw; } @@ -281,4 +281,6 @@ class KlassLUTEntry { }; // KlassInfoLUEntry +#define KLUTE_FORMAT INT32_FORMAT_X_0 + #endif // SHARE_OOPS_KLASSINFOLUTENTRY_HPP From a30333e22a861a03aaf0bda13f5d1e32304f21a9 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 7 May 2025 11:54:42 +0200 Subject: [PATCH 093/101] generally allow for Klass->CLD to be null --- src/hotspot/share/cds/archiveBuilder.cpp | 7 +- .../share/oops/instanceKlass.inline.hpp | 3 +- src/hotspot/share/oops/klass.cpp | 1 + src/hotspot/share/oops/klass.hpp | 2 +- src/hotspot/share/oops/klassInfoLUT.cpp | 78 ++++++++++------ src/hotspot/share/oops/klassInfoLUT.hpp | 14 +-- .../share/oops/klassInfoLUT.inline.hpp | 21 +++-- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 89 +++++++++++++------ src/hotspot/share/oops/klassInfoLUTEntry.hpp | 19 ++-- 9 files changed, 144 insertions(+), 90 deletions(-) diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp index 5e18b6716ed05..8311b3204e421 100644 --- a/src/hotspot/share/cds/archiveBuilder.cpp +++ b/src/hotspot/share/cds/archiveBuilder.cpp @@ -782,12 +782,7 @@ void ArchiveBuilder::make_klasses_shareable() { // Every archived Klass must carry a valid klute. That is because every archived Klass // would have been created via the usual dynamic class loading or - generation, which should // have registered the Klass with klut. -#ifdef ASSERT - // We can, unfortunately, encounter classes that are not linked yet at this point. - // Hopefully this will be resolved with JDK-8342429. - constexpr bool tolerate_aot_unlinked_classes = true; - KlassLUTEntry(k->klute()).verify_against_klass(k, tolerate_aot_unlinked_classes); -#endif + DEBUG_ONLY(KlassLUTEntry(k->klute()).verify_against_klass(k)); } for (int i = 0; i < klasses()->length(); i++) { diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 4bfe3976d106a..1a3124929d196 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -159,8 +159,7 @@ ALWAYSINLINE void InstanceKlass::oop_oop_iterate_oop_maps_bounded(oop obj, OopCl template inline void InstanceKlass::do_cld_from_klut_or_klass(oop obj, OopClosureType* closure, klute_raw_t klute) { // Call closure->do_cld. The underlying assumption here is that if a closure subscribes via do_metadata(), - // it is really interested in the CLD instead of the Klass (does a do_cld(klass->class_loader_data()). - // That is true for all Closures that derive from OopIterateClosure. + // it is interested in the CLD. That is true for all Closures that derive from OopIterateClosure. static_assert(std::is_base_of::value, "must inherit from OopIterateClosure"); // ... and in that case we can fetch the CLD from the KLUT cld cache instead of letting the closure pull diff --git a/src/hotspot/share/oops/klass.cpp b/src/hotspot/share/oops/klass.cpp index 4c9c4a4eefaab..484688db16dc2 100644 --- a/src/hotspot/share/oops/klass.cpp +++ b/src/hotspot/share/oops/klass.cpp @@ -823,6 +823,7 @@ void Klass::remove_unshareable_info() { // Null out class_loader_data because we don't share that yet. set_class_loader_data(nullptr); set_is_shared(); + KlassInfoLUT::shared_klass_cld_changed(this); // FIXME: validation in Klass::hash_secondary_supers() may fail for shared klasses. // Even though the bitmaps always match, the canonical order of elements in the table diff --git a/src/hotspot/share/oops/klass.hpp b/src/hotspot/share/oops/klass.hpp index 2a3ba99149b03..85f3e6e0e041e 100644 --- a/src/hotspot/share/oops/klass.hpp +++ b/src/hotspot/share/oops/klass.hpp @@ -220,7 +220,7 @@ class Klass : public Metadata { enum class PrivateLookupMode { find, skip }; // Klute handling - klute_raw_t klute() const { return _klute; } + klute_raw_t klute() const { return _klute; } void register_with_klut(); virtual bool is_klass() const { return true; } diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 0064e374bb65b..827122184dd29 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -88,8 +88,19 @@ static const char* common_loader_names[4] = { "other", "boot", "app", "platform" void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { +#ifdef INCLUDE_CDS + // Unfortunately, due to AOT delayed class linking (see JDK-8342429), we can + // encounter Klass that are unlinked and their CLD field is still nullptr. + // Until JDK-8342429 we must accept that. + if (cld == nullptr) { + return; + } +#else + assert(cld != nullptr, "CLD null"); +#endif + // We remember CLDs for the three permanent class loaders in a lookup array. - int index = -1; + unsigned index = 0; if (cld->is_permanent_class_loader_data()) { if (cld->is_the_null_class_loader_data()) { index = 1; @@ -100,7 +111,7 @@ void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { } } - if (index == -1) { + if (index == 0) { return; } @@ -117,14 +128,14 @@ void KlassInfoLUT::register_cld_if_needed(ClassLoaderData* cld) { assert(old_cld == cld || old_cld == nullptr, "Different CLD??"); } -int KlassInfoLUT::index_for_cld(const ClassLoaderData* cld) { - assert(cld != nullptr, "must not be null"); +unsigned KlassInfoLUT::index_for_cld(const ClassLoaderData* cld) { + assert(cld != nullptr, "CLD null?"); for (int i = 1; i <= 3; i++) { if (cld == _common_loaders[i]) { return i; } } - return 0; + return cld_index_unknown; } static void log_klass_registration(const Klass* k, narrowKlass nk, bool added_to_table, @@ -144,7 +155,6 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // First register the CLD in case we did not already do that ClassLoaderData* const cld = k->class_loader_data(); - assert(cld != nullptr, "Require CLD"); register_cld_if_needed(cld); // We calculate the klute that will be stored into the Klass. @@ -152,7 +162,7 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // We also add the klute to the lookup table iff we use a lookup table (we do if COH is enabled) // and if the Klass is in the narrowKlass encoding range. Interfaces and abstract classes are // not put there anymore since we don't need narrowKlass lookup for them. - const bool add_to_table = use_lookup_table() ? CompressedKlassPointers::is_encodable(k) : false; + const bool add_to_table = uses_lookup_table() ? CompressedKlassPointers::is_encodable(k) : false; const narrowKlass nk = add_to_table ? CompressedKlassPointers::encode(const_cast(k)) : 0; // Calculate klute from Klass properties and update the table value. @@ -163,7 +173,8 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { log_klass_registration(k, nk, add_to_table, klute, "registered"); #ifdef ASSERT - KlassLUTEntry(klute).verify_against_klass(k, false /* tolerate_aot_unlinked_classes */); + // Until See JDK-8342429 is solved + KlassLUTEntry(klute).verify_against_klass(k); if (add_to_table) { KlassLUTEntry e2(at(nk)); assert(e2.value() == klute, "sanity"); @@ -188,7 +199,7 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { void KlassInfoLUT::scan_klass_range_update_lut(address from, address to) { assert(_initialized, "not initialized"); - if (use_lookup_table()) { + if (uses_lookup_table()) { log_info(klut)("Scanning CDS klass range: " RANGE2FMT, RANGE2FMTARGS(from, to)); const size_t stepsize = CompressedKlassPointers::klass_alignment_in_bytes(); assert(stepsize >= K, "only for COH and large alignments"); @@ -208,11 +219,11 @@ void KlassInfoLUT::scan_klass_range_update_lut(address from, address to) { if (klute == KlassLUTEntry::invalid_entry) { continue; } - // Above checks may of course, rarely, give false positives. That is absolutely fine: we - // then copy a "klute" from a "Klass" to the table that really isn't either klute nor Klass. + // Above checks may of course, very rarely, result in false positives (locations wrongly + // identified as Klass locations). That is absolutely fine: we then copy a "klute" from + // that "Klass" to the table that really isn't either klute nor Klass. // Since that slot is not used anyway by a real Klass, nothing bad will happen. - // OTOH, *missing* to add a klute for a Klass that really exists would be really bad. Hence, - // err on the plus side. + // OTOH, *missing* to add a klute for a Klass that exists would be really bad. const narrowKlass nk = CompressedKlassPointers::encode(const_cast(candidate)); put(nk, klute); log_info(klut)("Suspected Klass found at " PTR_FORMAT "; adding nk %u, klute: " KLUTE_FORMAT, @@ -254,25 +265,36 @@ klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { // Note: cld may still be nullptr; in that case it will be initialized by CDS before the Klass // is used. At that point we may correct the klute entry to account for the new CDS. } - DEBUG_ONLY(klutehelper.verify_against_klass(k, (cld != nullptr) /* tolerate_aot_unlinked_classes */)); + DEBUG_ONLY(klutehelper.verify_against_klass(k)); log_klass_registration(k, nk, true, klute, "late-registered"); return klute; } void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { - // Called when a shared class gets its ClassLoaderData restored after being loaded. - // The function makes sure that the CLD bits in the Klass' klute match the new - // ClassLoaderData. - const klute_raw_t klute = k->klute(); - ClassLoaderData* cld = k->class_loader_data(); - assert(cld != nullptr, "must be"); - register_cld_if_needed(cld); - const int cld_index = index_for_cld(cld); - if (KlassLUTEntry(klute).loader_index() != cld_index) { - // for simplicity, just recalculate the klute and update the table. - log_debug(klut)("Re-registering Klass after CLD change"); - k->register_with_klut(); + // Called when the CLD field inside a Klass is changed by CDS. + // Recalculates the klute for this Klass (even though strictly speaking we + // only need to update the CLD index in the klute). + // + // This is necessary to prevent the klute and the Klass being out of sync. + // + // Two cases: + // - when the CLD is set to nullptr in the process of archive dumping (remove_unshareable_info), + // we set the klute.cld_index to 0 aka "unknown CLD". Any oop iteration over an object with a + // klute.cld_index of 0 will then retrieve the CLD from the Klass directly. + // - when the CLD is restored after the archive has been loaded, klute.cld_index is set to + // the value corresponding to that CLD. + assert(k->is_shared(), "Only for CDS classes"); + const klute_raw_t oldklute = k->klute(); + k->register_with_klut(); // re-register + const klute_raw_t newklute = k->klute(); + if (uses_lookup_table() && CompressedKlassPointers::is_encodable(k)) { + const narrowKlass nk = CompressedKlassPointers::encode(k); + put(nk, newklute); } + + log_debug(klut)("Updated klute for Klass " PTR_FORMAT " (%s) after CLD change:" + "old: " KLUTE_FORMAT ", new: " KLUTE_FORMAT, + p2i(k), k->external_name(), oldklute, newklute); } #endif // INCLUDE_CDS @@ -292,7 +314,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { st->print_cr("KLUT statistics:"); - if (use_lookup_table()) { + if (uses_lookup_table()) { st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(klute_raw_t)); } @@ -348,7 +370,7 @@ void KlassInfoLUT::print_statistics(outputStream* st) { ); #endif // KLUT_ENABLE_EXPENSIVE_STATS - if (use_lookup_table()) { + if (uses_lookup_table()) { // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) constexpr int chacheline_size = 64; constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 2e57c8292d495..433d0d1a0db52 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -83,10 +83,11 @@ class KlassInfoLUT : public AllStatic { static unsigned _max_entries; static klute_raw_t* _table; static bool _initialized; - static inline unsigned max_entries() { return _max_entries; } + static inline unsigned max_entries() { return _max_entries; } static inline klute_raw_t at(unsigned index); static inline void put(unsigned index, klute_raw_t klute); static void allocate_lookup_table(); + static bool uses_lookup_table() { return _table != nullptr; } // Klass registration statistics. These are not expensive and therefore // we carry them always. @@ -134,8 +135,6 @@ class KlassInfoLUT : public AllStatic { static klute_raw_t late_register_klass(narrowKlass nk); #endif - static bool use_lookup_table() { return _table != nullptr; } - public: static void initialize(); @@ -143,13 +142,14 @@ class KlassInfoLUT : public AllStatic { static klute_raw_t register_klass(const Klass* k); static inline klute_raw_t lookup(narrowKlass k); - static void scan_klass_range_update_lut(address from, address to); - // ClassLoaderData handling + static constexpr unsigned cld_index_unknown = 0; + static unsigned index_for_cld(const ClassLoaderData* cld); + static inline ClassLoaderData* lookup_cld(unsigned index); static void register_cld_if_needed(ClassLoaderData* cld); - static int index_for_cld(const ClassLoaderData* cld); - static inline ClassLoaderData* lookup_cld(int index); + #if INCLUDE_CDS + static void scan_klass_range_update_lut(address from, address to); static void shared_klass_cld_changed(Klass* k); #endif diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 4b440bcd10e23..343c22e81fcd3 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -49,13 +49,13 @@ inline void KlassInfoLUT::put(unsigned index, klute_raw_t klute) { inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const klute_raw_t klute = at(nk); -#if INCLUDE_CDS -// if (klute == KlassLUTEntry::invalid_entry) { -// // This branch, and the late_register_klass mechanic, only exists because it is -// // so difficult to iterate CDS classes after loading CDS archives. See discussion -// // surrounding 8353225. Hopefully we can remove this in the future. -// return late_register_klass(nk); -// } +#if 0 // INCLUDE_CDS + if (klute == KlassLUTEntry::invalid_entry) { + // This branch, and the late_register_klass mechanic, only exists because it is + // so difficult to iterate CDS classes after loading CDS archives. See discussion + // surrounding 8353225. Hopefully we can remove this in the future. + return late_register_klass(nk); + } #else assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); #endif @@ -71,11 +71,10 @@ inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { return klute; } -ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(int index) { - assert(index >= 0 && index <= 3, "Sanity"); +ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(unsigned index) { + assert(index <= 3, "Sanity"); ClassLoaderData* cld = _common_loaders[index]; - // Because of AOT, we can now encounter objects of Klasses that are not fully linked yet - // assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); + assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); return cld; } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index f7fd5899f4f60..8e4ffb8d99e85 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -32,6 +32,7 @@ #include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/klassKind.hpp" #include "utilities/debug.hpp" +#include "utilities/macros.hpp" // See klass.hpp union LayoutHelperHelper { @@ -58,26 +59,44 @@ static void read_and_check_omb_values(const OopMapBlock* omb, unsigned& omb_offs omb_count = count; } -klute_raw_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_encodable_reason) { +klute_raw_t KlassLUTEntry::build_from_common(const Klass* k) { + const int kind = k->kind(); + const ClassLoaderData* const cld = k->class_loader_data(); + unsigned cld_index = KlassInfoLUT::cld_index_unknown; + + if (cld == nullptr) { +#ifdef INCLUDE_CDS + // Unfortunately, due to AOT delayed class linking (see JDK-8342429), we can + // encounter Klass that are unlinked and their CLD field is still nullptr. + // Until JDK-8342429 we accept that and treat it the same as non-permanent CLDs: + // "unknown CLD, look CLD up in Klass directly". + // Later, when the CLD field in Klass is set in the course of class linking, we + // will recalculate the klute based on the new CLD. + cld_index = KlassInfoLUT::cld_index_unknown; +#else + fatal("CLD null for Klass " PTR_FORMAT, p2i(k)); +#endif + } else { + cld_index = KlassInfoLUT::index_for_cld(cld); + } + U value(0); + value.common.kind = kind; + value.common.cld_index = cld_index; - assert(ik->is_instance_klass(), "sanity"); + return value.raw; +} - const int kind = ik->kind(); - const int lh = ik->layout_helper(); - assert(ik->class_loader_data() != nullptr, "no associated class loader?"); - const int cld_index = KlassInfoLUT::index_for_cld(ik->class_loader_data()); +klute_raw_t KlassLUTEntry::build_from_ik(const InstanceKlass* ik, const char*& not_encodable_reason) { - U value(0); + assert(ik->is_instance_klass(), "sanity"); - // Set common bits, these are always present - assert(kind < 0b111, "sanity"); - value.common.kind = kind; - value.common.loader = cld_index; + U value(build_from_common(ik)); // We may not be able to encode the IK-specific info; if we can't, those bits are left zero // and we return an error string for logging #define NOPE(s) { not_encodable_reason = s; return value.raw; } + const int lh = ik->layout_helper(); if (Klass::layout_helper_needs_slow_path(lh)) { if (ik->is_abstract() || ik->is_interface()) { NOPE("klass is abstract or interface"); @@ -145,24 +164,21 @@ klute_raw_t KlassLUTEntry::build_from_ak(const ArrayKlass* ak) { assert(ak->is_array_klass(), "sanity"); - const int kind = ak->kind(); - const int lh = ak->layout_helper(); - const int cld_index = KlassInfoLUT::index_for_cld(ak->class_loader_data()); + U value(build_from_common(ak)); + + const int lh = ak->layout_helper(); assert(Klass::layout_helper_is_objArray(lh) || Klass::layout_helper_is_typeArray(lh), "unexpected"); LayoutHelperHelper lhu = { (unsigned) lh }; - U value(0); - value.common.kind = kind; - value.common.loader = cld_index; assert(lhu.bytes.lh_esz <= 3, "Sanity (%X)", lh); value.ake.l2esz = lhu.bytes.lh_esz; assert(lhu.bytes.lh_hsz >= 12 && lhu.bytes.lh_hsz <= 24, "Sanity"); value.ake.hsz = lhu.bytes.lh_hsz; - return value.raw; + return value.raw; } klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { @@ -183,7 +199,7 @@ klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { } #ifdef ASSERT -void KlassLUTEntry::verify_against_klass(const Klass* k, bool tolerate_aot_unlinked_classes) const { +void KlassLUTEntry::verify_against_klass(const Klass* k) const { #define PREAMBLE_FORMAT "Klass: " PTR_FORMAT "(%s), klute " KLUTE_FORMAT ": " #define PREAMBLE_ARGS p2i(k), k->external_name(), _v.raw @@ -217,13 +233,22 @@ void KlassLUTEntry::verify_against_klass(const Klass* k, bool tolerate_aot_unlin const ClassLoaderData* const real_cld = k->class_loader_data(); const unsigned cld_index = loader_index(); ASSERT_HERE(cld_index < 4, "invalid loader index"); - // With AOT, we can encounter Klasses that have not set their CLD yet. Until that is solved (JDK-8342429), - // work around this problem. Dealing with unlinked classes is very frustrating. - const bool skip_cld_check = (real_cld == nullptr) && tolerate_aot_unlinked_classes; - if (!skip_cld_check) { - ASSERT_HERE(real_cld != nullptr, "Klass CLD is null?"); + + if (real_cld == nullptr) { + // With AOT, we can encounter Klasses that have not set their CLD yet. Until that is solved (JDK-8342429), + // work around this problem. + constexpr bool tolerate_aot_unlinked_classes = CDS_ONLY(true) NOT_CDS(false); + if (tolerate_aot_unlinked_classes) { + // We expect the klute to show "unknown CLD" + ASSERT_HERE2(cld_index == KlassInfoLUT::cld_index_unknown, + "for CLD==nullptr cld_index is expected to be %u, was %u", + KlassInfoLUT::cld_index_unknown, cld_index); + } else { + ASSERT_HERE(real_cld != nullptr, "Klass CLD is null?"); + } + } else { const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cld_index); - if (cld_index > 0) { + if (cld_index != KlassInfoLUT::cld_index_unknown) { // We expect to get one of the three permanent class loaders, and for it to match the one in Klass ASSERT_HERE2(cld_from_klute->is_permanent_class_loader_data(), "not perma cld (loader_index: %d, CLD: " PTR_FORMAT ")", cld_index, p2i(cld_from_klute)); @@ -241,15 +266,13 @@ void KlassLUTEntry::verify_against_klass(const Klass* k, bool tolerate_aot_unlin } if (k->is_array_klass()) { - - // compare our (truncated) lh with the real one + // compare klute information with the information from the layouthelper const LayoutHelperHelper lhu = { (unsigned) real_lh }; ASSERT_HERE2(lhu.bytes.lh_esz == ak_log2_elem_size() && lhu.bytes.lh_hsz == ak_first_element_offset_in_bytes() && ( (lhu.bytes.lh_tag == 0xC0 && real_kind == TypeArrayKlassKind) || (lhu.bytes.lh_tag == 0x80 && real_kind == ObjArrayKlassKind) ), "layouthelper mismatch (lh from Klass: 0x%x", real_lh); - } else { ASSERT_HERE(k->is_instance_klass(), "unexpected"); @@ -312,6 +335,16 @@ void KlassLUTEntry::verify_against_klass(const Klass* k, bool tolerate_aot_unlin #endif // ASSERT +#if INCLUDE_CDS +klute_raw_t KlassLUTEntry::calculate_klute_with_new_cld_index(unsigned cld_index) const { + assert(cld_index < 3, "Sanity"); + U v(_v.raw); + v.common.cld_index = cld_index; + return v.raw; +} +#endif + + void KlassLUTEntry::print(outputStream* st) const { st->print("%X (Kind: %d Loader: %d)", value(), kind(), loader_index()); } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 34fc338ebbe6e..d67bba37946d8 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -137,15 +137,15 @@ class KlassLUTEntry { // All valid entries: KKKB ---- ---- ---- ---- ---- ---- ---- static constexpr int bits_kind = 3; - static constexpr int bits_loader = 2; - static constexpr int bits_common = bits_kind + bits_loader; + static constexpr int bits_cld_index = 2; + static constexpr int bits_common = bits_kind + bits_cld_index; static constexpr int bits_specific = bits_total - bits_common; // Bits valid for all entries, regardless of Klass kind struct KE { // lsb unsigned kind_specific_bits : bits_specific; - unsigned loader : bits_loader; + unsigned cld_index : bits_cld_index; unsigned kind : bits_kind; // msb }; @@ -197,6 +197,7 @@ class KlassLUTEntry { static constexpr size_t ik_omb_offset_2_limit = nth_bit(bits_ik_omb_offset_2); static constexpr size_t ik_omb_count_2_limit = nth_bit(bits_ik_omb_count_2); + static klute_raw_t build_from_common(const Klass* k); static klute_raw_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); static klute_raw_t build_from_ak(const ArrayKlass* k); @@ -214,20 +215,19 @@ class KlassLUTEntry { // an error somewhere. bool is_valid() const { return _v.raw != invalid_entry; } + // Given a Klass, construct a klute from it. static klute_raw_t build_from_klass(const Klass* k); bool is_valid_for_klass(const Klass* k) const; -#ifdef ASSERT - void verify_against_klass(const Klass* k, bool tolerate_aot_unlinked_classes) const; -#endif // ASSERT + DEBUG_ONLY(void verify_against_klass(const Klass* k) const;) klute_raw_t value() const { return _v.raw; } inline unsigned kind() const { return _v.common.kind; } // returns loader index (0 for unknown) - inline int loader_index() const { return _v.common.loader; } + inline int loader_index() const { return _v.common.cld_index; } bool is_array() const { return _v.common.kind >= TypeArrayKlassKind; } bool is_instance() const { return !is_array(); } @@ -279,6 +279,11 @@ class KlassLUTEntry { void print(outputStream* st) const; +#if INCLUDE_CDS + // Returns a newly calculated klute equal to this but with a different CLD index + klute_raw_t calculate_klute_with_new_cld_index(unsigned cld_index) const; +#endif + }; // KlassInfoLUEntry #define KLUTE_FORMAT INT32_FORMAT_X_0 From 36f1f6de9be27ae4f2f377dbb049f8c91dd03712 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 7 May 2025 15:46:57 +0200 Subject: [PATCH 094/101] remove late_register code since we now scan CDS range for Klasses eagerly --- src/hotspot/share/oops/klassInfoLUT.cpp | 35 ------------------- src/hotspot/share/oops/klassInfoLUT.hpp | 9 ++--- .../share/oops/klassInfoLUT.inline.hpp | 9 ----- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 10 ------ src/hotspot/share/oops/klassInfoLUTEntry.hpp | 20 ++--------- 5 files changed, 5 insertions(+), 78 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 827122184dd29..0c8133235a529 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -235,41 +235,6 @@ void KlassInfoLUT::scan_klass_range_update_lut(address from, address to) { } } -// We only tolerate this for CDS: -// We currently have no simple way to iterate all Klass structures in a CDS/AOT archive -// before the JVM starts calling methods on oops that refer to these classes. This is because -// these Klasses don't go through normal construction but are mapped into the address space -// when the CDS archive is mapped. -// So it can happen, early during CDS initialization, when CDS revives archived heap objects, -// that the entry in the KLUT table for this Klass is still uninitialized. If that happens, -// this function is called where we add the table entry on the fly. -// Unfortunately, this adds a branch into the very hot oop iteration path, albeit one that -// would hopefully be mitigated by branch prediction since this should be exceedingly rare. -klute_raw_t KlassInfoLUT::late_register_klass(narrowKlass nk) { - assert(_initialized, "not initialized"); - assert(nk != 0, "null narrow Klass - is this class encodable?"); - const Klass* k = CompressedKlassPointers::decode(nk); - assert(k->is_shared(), "Only for CDS classes"); - // Here we rely on the Klass itself carrying a valid klute already. No need to calculate it. - // That klute would have been pre-calculated during CDS dump time when the to-be-dumped Klass - // was dynamically constructed. - // We just copy that entry into the table slot. - const klute_raw_t klute = k->klute(); - const KlassLUTEntry klutehelper(klute); - assert(klutehelper.is_valid(), "Must be a valid klute"); - put(nk, klutehelper.value()); - ClassLoaderData* const cld = k->class_loader_data(); - if (cld != nullptr) { // May be too early; CLD may not yet been initialized by CDS - register_cld_if_needed(cld); - } else { - // Note: cld may still be nullptr; in that case it will be initialized by CDS before the Klass - // is used. At that point we may correct the klute entry to account for the new CDS. - } - DEBUG_ONLY(klutehelper.verify_against_klass(k)); - log_klass_registration(k, nk, true, klute, "late-registered"); - return klute; -} - void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { // Called when the CLD field inside a Klass is changed by CDS. // Recalculates the klute for this Klass (even though strictly speaking we diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 433d0d1a0db52..65189aa263bf4 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -73,9 +73,8 @@ class ClassLoaderData; // without going through the process of initialization. It also refers to that Klass via // a narrowKlass value that is the result of a precalculation during dump time (e.g. by // accessing obj->klass()->kind() on object that are re-animated from the CDS archive). -// Since there is no secure way to initialize the KLUT entry for these classes, we (for the -// moment) allow a form of "self-healing": if the KLUT entry for a class is requested but -// not yet added, we add it to the KLUT table on the fly. See KlassInfoLUT::late_register_klass(). +// These Klasses we register by scanning the CDS archive after it has been mapped into +// the Klass encoding range. class KlassInfoLUT : public AllStatic { @@ -131,10 +130,6 @@ class KlassInfoLUT : public AllStatic { static void log_hit(klute_raw_t klute); #endif -#if INCLUDE_CDS - static klute_raw_t late_register_klass(narrowKlass nk); -#endif - public: static void initialize(); diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index 343c22e81fcd3..b4dadec14efa3 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -49,16 +49,7 @@ inline void KlassInfoLUT::put(unsigned index, klute_raw_t klute) { inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { assert(nk != 0, "null narrow Klass - is this class encodable?"); const klute_raw_t klute = at(nk); -#if 0 // INCLUDE_CDS - if (klute == KlassLUTEntry::invalid_entry) { - // This branch, and the late_register_klass mechanic, only exists because it is - // so difficult to iterate CDS classes after loading CDS archives. See discussion - // surrounding 8353225. Hopefully we can remove this in the future. - return late_register_klass(nk); - } -#else assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); -#endif #ifdef KLUT_ENABLE_EXPENSIVE_STATS update_hit_stats(klute); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 8e4ffb8d99e85..11466e11a4675 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -335,16 +335,6 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { #endif // ASSERT -#if INCLUDE_CDS -klute_raw_t KlassLUTEntry::calculate_klute_with_new_cld_index(unsigned cld_index) const { - assert(cld_index < 3, "Sanity"); - U v(_v.raw); - v.common.cld_index = cld_index; - return v.raw; -} -#endif - - void KlassLUTEntry::print(outputStream* st) const { st->print("%X (Kind: %d Loader: %d)", value(), kind(), loader_index()); } diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index d67bba37946d8..6193c2174d184 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -113,18 +113,9 @@ class typeArrayOopDesc; // // It is the KLUT table entry initialization value (KLUT is zapped with '-1' on startup). // -// The "invalid" has two different uses: -// - if CDS is disabled at build time, it simply designates an invalid entry that should never be encountered -// at runtime. In other words, when doing a lookup with a narrowKlass value into the KLUT table, one should -// always find a valid klute, since a narrowKlass value can only result from a Klass that was loaded, and as -// part of Klass creation, the klute table entry is created. -// -// - if CDS is enabled at build time: unfortunately, CDS maps in archived Klass structures into memory and these -// Klass structures never go through a normal loading process; they just appear and then they are just there. -// These may be accessed via narrowKlass values that are the result of a precalculation during CDS archive dump -// time. -// in that case, an "invalid_entry" can mean a Klass that was loaded from CDS archive and for that no table -// entry in the KLUT exists yet. If we encounter such an entry, we generate it on the fly (see KlassInfoLUT::late_register_klass()). +// The "invalid" designates an invalid entry that should never be encountered at runtime. When doing a lookup +// with a narrowKlass value into the KLUT table, one should always find a valid klute, since a narrowKlass value +// can only result from a Klass that was loaded, and as part of Klass creation, the klute table entry is created. // // Implementation note: the value -1 (all bits 1) relies on the fact that a KlassKind of 7 (0b111) is invalid. We // don't use zero as "invalid entry" since zero would encode a valid Klass. @@ -279,11 +270,6 @@ class KlassLUTEntry { void print(outputStream* st) const; -#if INCLUDE_CDS - // Returns a newly calculated klute equal to this but with a different CLD index - klute_raw_t calculate_klute_with_new_cld_index(unsigned cld_index) const; -#endif - }; // KlassInfoLUEntry #define KLUTE_FORMAT INT32_FORMAT_X_0 From 70c71085ade1bbfc1fe2856c19626aa915cc0296 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Wed, 7 May 2025 16:12:05 +0200 Subject: [PATCH 095/101] renaming cdt_index --- .../share/oops/instanceKlass.inline.hpp | 4 +-- src/hotspot/share/oops/klassInfoLUT.cpp | 4 +-- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 26 +++++++++---------- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 14 +++++----- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/oops/instanceKlass.inline.hpp b/src/hotspot/share/oops/instanceKlass.inline.hpp index 1a3124929d196..80359229457b2 100644 --- a/src/hotspot/share/oops/instanceKlass.inline.hpp +++ b/src/hotspot/share/oops/instanceKlass.inline.hpp @@ -164,8 +164,8 @@ inline void InstanceKlass::do_cld_from_klut_or_klass(oop obj, OopClosureType* cl "must inherit from OopIterateClosure"); // ... and in that case we can fetch the CLD from the KLUT cld cache instead of letting the closure pull // it from Klass. We don't even have to fetch and decode the narrowKlass. - const unsigned perma_cld_index = KlassLUTEntry(klute).loader_index(); - ClassLoaderData* cld = KlassInfoLUT::lookup_cld(perma_cld_index); + const unsigned cldi = KlassLUTEntry(klute).cld_index(); + ClassLoaderData* cld = KlassInfoLUT::lookup_cld(cldi); if (cld == nullptr) { // Rare path Klass* const k = obj->klass(); diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 0c8133235a529..98f9c693be940 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -143,7 +143,7 @@ static void log_klass_registration(const Klass* k, narrowKlass nk, bool added_to char tmp[1024]; const KlassLUTEntry klutehelper(klute); log_debug(klut)("Klass " PTR_FORMAT ", cld: %s, nk %u(%c), klute: " KLUTE_FORMAT ": %s %s%s", - p2i(k), common_loader_names[klutehelper.loader_index()], nk, + p2i(k), common_loader_names[klutehelper.cld_index()], nk, (added_to_table ? '+' : '-'), klute, message, @@ -380,7 +380,7 @@ void KlassInfoLUT::update_hit_stats(klute_raw_t klute) { default: inc_noinfo_IK_other(); break; } } - switch (klutehelper.loader_index()) { + switch (klutehelper.cld_index()) { case 1: inc_hits_bootloader(); break; case 2: inc_hits_sysloader(); break; case 3: inc_hits_platformloader(); break; diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 11466e11a4675..e0906439ce90d 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -62,7 +62,7 @@ static void read_and_check_omb_values(const OopMapBlock* omb, unsigned& omb_offs klute_raw_t KlassLUTEntry::build_from_common(const Klass* k) { const int kind = k->kind(); const ClassLoaderData* const cld = k->class_loader_data(); - unsigned cld_index = KlassInfoLUT::cld_index_unknown; + unsigned cldi = KlassInfoLUT::cld_index_unknown; if (cld == nullptr) { #ifdef INCLUDE_CDS @@ -72,16 +72,16 @@ klute_raw_t KlassLUTEntry::build_from_common(const Klass* k) { // "unknown CLD, look CLD up in Klass directly". // Later, when the CLD field in Klass is set in the course of class linking, we // will recalculate the klute based on the new CLD. - cld_index = KlassInfoLUT::cld_index_unknown; + cldi = KlassInfoLUT::cld_index_unknown; #else fatal("CLD null for Klass " PTR_FORMAT, p2i(k)); #endif } else { - cld_index = KlassInfoLUT::index_for_cld(cld); + cldi = KlassInfoLUT::index_for_cld(cld); } U value(0); value.common.kind = kind; - value.common.cld_index = cld_index; + value.common.cld_index = cldi; return value.raw; } @@ -231,8 +231,8 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { ASSERT_HERE2(our_kind == real_kind, "kind mismatch (%d vs %d)", real_kind, our_kind); const ClassLoaderData* const real_cld = k->class_loader_data(); - const unsigned cld_index = loader_index(); - ASSERT_HERE(cld_index < 4, "invalid loader index"); + const unsigned cldi = cld_index(); + ASSERT_HERE(cldi < 4, "invalid loader index"); if (real_cld == nullptr) { // With AOT, we can encounter Klasses that have not set their CLD yet. Until that is solved (JDK-8342429), @@ -240,21 +240,21 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { constexpr bool tolerate_aot_unlinked_classes = CDS_ONLY(true) NOT_CDS(false); if (tolerate_aot_unlinked_classes) { // We expect the klute to show "unknown CLD" - ASSERT_HERE2(cld_index == KlassInfoLUT::cld_index_unknown, + ASSERT_HERE2(cldi == KlassInfoLUT::cld_index_unknown, "for CLD==nullptr cld_index is expected to be %u, was %u", - KlassInfoLUT::cld_index_unknown, cld_index); + KlassInfoLUT::cld_index_unknown, cldi); } else { ASSERT_HERE(real_cld != nullptr, "Klass CLD is null?"); } } else { - const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cld_index); - if (cld_index != KlassInfoLUT::cld_index_unknown) { + const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cldi); + if (cldi != KlassInfoLUT::cld_index_unknown) { // We expect to get one of the three permanent class loaders, and for it to match the one in Klass ASSERT_HERE2(cld_from_klute->is_permanent_class_loader_data(), "not perma cld (loader_index: %d, CLD: " PTR_FORMAT ")", - cld_index, p2i(cld_from_klute)); + cldi, p2i(cld_from_klute)); ASSERT_HERE2(cld_from_klute == real_cld, "Different CLD (loader_index: %d, real Klass CLD: " PTR_FORMAT ", from klute CLD lookup table: " PTR_FORMAT ")?", - cld_index, p2i(real_cld), p2i(cld_from_klute)); + cldi, p2i(real_cld), p2i(cld_from_klute)); } else { // We expect to get a NULL from the CLD lookup table. ASSERT_HERE2(cld_from_klute == nullptr, "CLD not null? (" PTR_FORMAT ")", p2i(cld_from_klute)); @@ -336,7 +336,7 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { #endif // ASSERT void KlassLUTEntry::print(outputStream* st) const { - st->print("%X (Kind: %d Loader: %d)", value(), kind(), loader_index()); + st->print("%X (Kind: %d Loader: %d)", value(), kind(), cld_index()); } // Helper function, prints current limits diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 6193c2174d184..6c7b691cf67ae 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -213,18 +213,18 @@ class KlassLUTEntry { DEBUG_ONLY(void verify_against_klass(const Klass* k) const;) - klute_raw_t value() const { return _v.raw; } + klute_raw_t value() const { return _v.raw; } - inline unsigned kind() const { return _v.common.kind; } + inline unsigned kind() const { return _v.common.kind; } // returns loader index (0 for unknown) - inline int loader_index() const { return _v.common.cld_index; } + inline int cld_index() const { return _v.common.cld_index; } - bool is_array() const { return _v.common.kind >= TypeArrayKlassKind; } - bool is_instance() const { return !is_array(); } + bool is_array() const { return _v.common.kind >= TypeArrayKlassKind; } + bool is_instance() const { return !is_array(); } - bool is_obj_array() const { return _v.common.kind == ObjArrayKlassKind; } - bool is_type_array() const { return _v.common.kind == TypeArrayKlassKind; } + bool is_obj_array() const { return _v.common.kind == ObjArrayKlassKind; } + bool is_type_array() const { return _v.common.kind == TypeArrayKlassKind; } // Following methods only if IK: From 58f44fef83965cc88c864a4d04eebf4704dc2692 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 8 May 2025 06:39:48 +0200 Subject: [PATCH 096/101] remove stray NULL -> nullptr --- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index e0906439ce90d..4d28760fb232f 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -256,7 +256,7 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { "Different CLD (loader_index: %d, real Klass CLD: " PTR_FORMAT ", from klute CLD lookup table: " PTR_FORMAT ")?", cldi, p2i(real_cld), p2i(cld_from_klute)); } else { - // We expect to get a NULL from the CLD lookup table. + // We expect to get a nullptr from the CLD lookup table. ASSERT_HERE2(cld_from_klute == nullptr, "CLD not null? (" PTR_FORMAT ")", p2i(cld_from_klute)); // cld_index == 0 means "unknown CLD" and since we expect only to ever run with three permanent CLDs, // that cld should not be permanent. From 17eb502666a845457766d932497268b1e275fb71 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 8 May 2025 06:57:25 +0200 Subject: [PATCH 097/101] log without need for resourcemark --- src/hotspot/share/oops/klassInfoLUT.cpp | 4 +++- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 98f9c693be940..705a02ff84992 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -32,6 +32,7 @@ #include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/klassKind.hpp" #include "runtime/atomic.hpp" + #include "utilities/debug.hpp" #include "utilities/ostream.hpp" @@ -257,9 +258,10 @@ void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { put(nk, newklute); } + char tmp[1024]; log_debug(klut)("Updated klute for Klass " PTR_FORMAT " (%s) after CLD change:" "old: " KLUTE_FORMAT ", new: " KLUTE_FORMAT, - p2i(k), k->external_name(), oldklute, newklute); + p2i(k), k->name()->as_C_string(tmp, sizeof(tmp)), oldklute, newklute); } #endif // INCLUDE_CDS diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 4d28760fb232f..455a280c9e578 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -200,9 +200,9 @@ klute_raw_t KlassLUTEntry::build_from_klass(const Klass* k) { #ifdef ASSERT void KlassLUTEntry::verify_against_klass(const Klass* k) const { - + char tmp[1024]; #define PREAMBLE_FORMAT "Klass: " PTR_FORMAT "(%s), klute " KLUTE_FORMAT ": " -#define PREAMBLE_ARGS p2i(k), k->external_name(), _v.raw +#define PREAMBLE_ARGS p2i(k), k->name()->as_C_string(tmp, sizeof(tmp)), _v.raw #define ASSERT_HERE(cond, msg) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS); #define ASSERT_HERE2(cond, msg, ...) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS, __VA_ARGS__); From b807fbaede38ad1f2b817e611016a4575a7f6707 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 8 May 2025 14:33:49 +0200 Subject: [PATCH 098/101] registration stats --- .../share/classfile/classLoaderData.cpp | 2 + src/hotspot/share/oops/klassInfoLUT.cpp | 298 +++++++++++------- src/hotspot/share/oops/klassInfoLUT.hpp | 118 ++++--- .../share/oops/klassInfoLUT.inline.hpp | 18 +- src/hotspot/share/oops/klassInfoLUTEntry.cpp | 19 +- src/hotspot/share/oops/klassInfoLUTEntry.hpp | 15 +- 6 files changed, 286 insertions(+), 184 deletions(-) diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index 825072cb13bef..94cf6a787ca7d 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -763,6 +763,8 @@ ClassLoaderData::~ClassLoaderData() { if (_name_and_id != nullptr) { _name_and_id->decrement_refcount(); } + +memset((char*)this, 0x17, sizeof(ClassLoaderData)); } // Returns true if this class loader data is for the app class loader diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 705a02ff84992..14782101afc59 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -41,6 +41,14 @@ bool KlassInfoLUT::_initialized = false; klute_raw_t* KlassInfoLUT::_table = nullptr; unsigned KlassInfoLUT::_max_entries = -1; +#ifdef KLUT_ENABLE_REGISTRATION_STATS +KlassInfoLUT::Counters KlassInfoLUT::_registration_counters; +#endif + +#ifdef KLUT_ENABLE_HIT_STATS +KlassInfoLUT::Counters KlassInfoLUT::_hit_counters; +#endif + void KlassInfoLUT::initialize() { assert(!_initialized, "Only once"); if (UseCompactObjectHeaders) { @@ -168,11 +176,18 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { // Calculate klute from Klass properties and update the table value. const klute_raw_t klute = KlassLUTEntry::build_from_klass(k); + const klute_raw_t oldklute = k->klute(); if (add_to_table) { put(nk, klute); } log_klass_registration(k, nk, add_to_table, klute, "registered"); +#ifdef KLUT_ENABLE_REGISTRATION_STATS + if (oldklute != klute) { + update_registration_counters(k, klute); + } +#endif // KLUT_ENABLE_REGISTRATION_STATS + #ifdef ASSERT // Until See JDK-8342429 is solved KlassLUTEntry(klute).verify_against_klass(k); @@ -182,17 +197,6 @@ klute_raw_t KlassInfoLUT::register_klass(const Klass* k) { } #endif // ASSERT - // update register stats - switch (k->kind()) { -#define WHAT(name, shorthand) case name ## Kind : inc_registered_ ## shorthand(); break; - KLASSKIND_ALL_KINDS_DO(WHAT) -#undef WHAT - default: ShouldNotReachHere(); - }; - if (k->is_abstract() || k->is_interface()) { - inc_registered_IK_for_abstract_or_interface(); - } - return klute; } @@ -265,134 +269,204 @@ void KlassInfoLUT::shared_klass_cld_changed(Klass* k) { } #endif // INCLUDE_CDS -// Counters and incrementors -#define XX(xx) \ -volatile uint64_t counter_##xx = 0; \ -void KlassInfoLUT::inc_##xx() { \ - Atomic::inc(&counter_##xx); \ + +// Statistics + +#if defined(KLUT_ENABLE_REGISTRATION_STATS) || defined(KLUT_ENABLE_HIT_STATS) + + void KlassInfoLUT::Counter::inc() { + Atomic::inc(&_v); } -REGISTER_STATS_DO(XX) -#ifdef KLUT_ENABLE_EXPENSIVE_STATS -HIT_STATS_DO(XX) -#endif // KLUT_ENABLE_EXPENSIVE_STATS -#undef XX -void KlassInfoLUT::print_statistics(outputStream* st) { +void KlassInfoLUT::update_counters(Counters& counters, const Klass* k, klute_raw_t klute) { + const KlassLUTEntry kle(klute); - st->print_cr("KLUT statistics:"); + switch (k->kind()) { +#define WHAT(name, shorthand) \ + case name ## Kind : counters.counter_ ## shorthand.inc(); break; + KLASSKIND_ALL_KINDS_DO(WHAT) +#undef WHAT + default: ShouldNotReachHere(); + }; - if (uses_lookup_table()) { - st->print_cr("Lookup Table Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(klute_raw_t)); + if (kle.is_instance()) { + const InstanceKlass* const ik = InstanceKlass::cast(k); + + if (!kle.ik_carries_infos()) { + counters.counter_IK_no_info.inc(); + const InstanceKlass* const ik = InstanceKlass::cast(k); + } + + if (ik->is_abstract() || ik->is_interface()) { + counters.counter_IK_no_info_abstract_or_interface.inc(); + } + + const int lh = ik->layout_helper(); + if (!Klass::layout_helper_needs_slow_path(lh)) { + const size_t wordsize = Klass::layout_helper_to_size_helper(ik->layout_helper()); + if (wordsize >= KlassLUTEntry::ik_wordsize_limit) { + counters.counter_IK_no_info_too_large.inc(); + } + } + + switch (ik->nonstatic_oop_map_count()) { + case 0: counters.counter_IK_zero_oopmapentries.inc(); break; + case 1: counters.counter_IK_one_oopmapentries.inc(); break; + case 2: counters.counter_IK_two_oopmapentries.inc(); break; + default: counters.counter_IK_no_info_too_many_oopmapentries.inc(); break; + } + } + + switch (kle.cld_index()) { + case cld_index_unknown: { + if (k->class_loader_data() != nullptr) { + counters.counter_from_unknown_cld.inc(); + } else { + counters.counter_from_null_cld.inc(); + } + break; + } + case 1: counters.counter_from_boot_cld.inc(); break; + case 2: counters.counter_from_system_cld.inc(); break; + case 3: counters.counter_from_platform_cld.inc(); break; + default: ShouldNotReachHere(); } +} - const uint64_t registered_all = counter_registered_IK + counter_registered_IRK + counter_registered_IMK + - counter_registered_ICLK + counter_registered_ISCK + counter_registered_TAK + counter_registered_OAK; +static void print_part_counter(outputStream* st, const char* prefix1, const char* prefix2, uint64_t v, uint64_t total) { + st->print("%s %s: ", prefix1, prefix2); + st->fill_to(32); + st->print_cr(UINT64_FORMAT " (%.2f%%)", v, ((double)v * 100.0f) / total); +} -#define PERCENTAGE_OF(x, x100) ( ((double)x * 100.0f) / x100 ) -#define PRINT_WITH_PERCENTAGE(title, x, x100) \ - st->print(" " title ": "); \ - st->fill_to(24); \ - st->print_cr(" " UINT64_FORMAT " (%.2f%%)", x, PERCENTAGE_OF(x, x100)); +void KlassInfoLUT::print_counters(outputStream* st, const Counters& counters, const char* prefix) { - st->print_cr(" Registered classes, total: " UINT64_FORMAT, registered_all); -#define XX(name, shortname) PRINT_WITH_PERCENTAGE("Registered, " #shortname, counter_registered_##shortname, registered_all); + // All Klasses + const uint64_t all = +#define XX(name, shortname) counters.counter_ ## shortname.get() + KLASSKIND_ALL_KINDS_DO(XX) #undef XX + 0; - const uint64_t registered_AK = counter_registered_OAK - counter_registered_TAK; - const uint64_t registered_IK = registered_all - registered_AK; - PRINT_WITH_PERCENTAGE("Registered classes, IK (all)", registered_IK, registered_all); - PRINT_WITH_PERCENTAGE("Registered classes, AK (all)", registered_AK, registered_all); + print_part_counter(st, prefix, "(all)", all, all); - PRINT_WITH_PERCENTAGE("Registered classes, IK, for abstract/interface", counter_registered_IK_for_abstract_or_interface, registered_all); + const uint64_t registered_AK = counters.counter_TAK.get() + counters.counter_OAK.get(); + const uint64_t registered_IK = all - registered_AK; + print_part_counter(st, prefix, "IK (all)", registered_IK, all); + print_part_counter(st, prefix, "AK (all)", registered_AK, all); -#ifdef KLUT_ENABLE_EXPENSIVE_STATS - const uint64_t hits = counter_hits_IK + counter_hits_IRK + counter_hits_IMK + - counter_hits_ICLK + counter_hits_ISCK + counter_hits_TAK + counter_hits_OAK; - - st->print_cr(" Hits, total: " UINT64_FORMAT, hits); -#define XX(name, shortname) PRINT_WITH_PERCENTAGE("Hits, " #shortname, counter_hits_##shortname, hits); +#define XX(name, shortname) \ + print_part_counter(st, prefix, ""# shortname, counters.counter_ ## shortname.get(), all); KLASSKIND_ALL_KINDS_DO(XX) #undef XX - const uint64_t hits_ak = counter_hits_OAK + counter_hits_TAK; - const uint64_t hits_ik = hits - hits_ak; - const uint64_t no_info_hits = counter_noinfo_ICLK + counter_noinfo_IMK + counter_noinfo_IK_other; + print_part_counter(st, prefix, "IK (no info)", counters.counter_IK_no_info.get(), all); + print_part_counter(st, prefix, "IK (no info, abstract or interface)", counters.counter_IK_no_info_abstract_or_interface.get(), all); + print_part_counter(st, prefix, "IK (no info, too many oopmap entries)", counters.counter_IK_no_info_too_many_oopmapentries.get(), all); + print_part_counter(st, prefix, "IK (no info, obj size too large)", counters.counter_IK_no_info_too_large.get(), all); - PRINT_WITH_PERCENTAGE("Hits, IK (all)", hits_ik, hits); - PRINT_WITH_PERCENTAGE("Hits, AK (all)", hits_ak, hits); + print_part_counter(st, prefix, "IK (0 oopmap entries)", counters.counter_IK_zero_oopmapentries.get(), all); + print_part_counter(st, prefix, "IK (1 oopmap entry)", counters.counter_IK_one_oopmapentries.get(), all); + print_part_counter(st, prefix, "IK (2 oopmap entries)", counters.counter_IK_two_oopmapentries.get(), all); - PRINT_WITH_PERCENTAGE("Hits, all for bootloader", counter_hits_bootloader, hits); - PRINT_WITH_PERCENTAGE("Hits, all for systemloader", counter_hits_sysloader, hits); - PRINT_WITH_PERCENTAGE("Hits, all for platformloader", counter_hits_platformloader, hits); + print_part_counter(st, prefix, "boot cld", counters.counter_from_boot_cld.get(), all); + print_part_counter(st, prefix, "system cld", counters.counter_from_system_cld.get(), all); + print_part_counter(st, prefix, "platform cld", counters.counter_from_platform_cld.get(), all); + print_part_counter(st, prefix, "unknown cld", counters.counter_from_unknown_cld.get(), all); + print_part_counter(st, prefix, "null cld", counters.counter_from_null_cld.get(), all); - st->print_cr(" IK details missing for " UINT64_FORMAT " hits (%.2f%%) due to: " - "IMK " UINT64_FORMAT " (%.2f%%) " - "ICLK " UINT64_FORMAT " (%.2f%%) " - "other " UINT64_FORMAT " (%.2f%%)", - no_info_hits, PERCENTAGE_OF(no_info_hits, hits), - counter_noinfo_IMK, PERCENTAGE_OF(counter_noinfo_IMK, hits), - counter_noinfo_ICLK, PERCENTAGE_OF(counter_noinfo_ICLK, hits), - counter_noinfo_IK_other, PERCENTAGE_OF(counter_noinfo_IK_other, hits) - ); -#endif // KLUT_ENABLE_EXPENSIVE_STATS +} - if (uses_lookup_table()) { - // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) - constexpr int chacheline_size = 64; - constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); - const int num_cachelines = max_entries() / slots_per_cacheline; - int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; - for (int i = 0; i < num_cachelines; i++) { - int n = 0; - for (int j = 0; j < slots_per_cacheline; j++) { - KlassLUTEntry e(at((i * slots_per_cacheline) + j)); - const bool fully_valid = e.is_valid() && (e.is_array() || e.ik_carries_infos()); - if (fully_valid) { - n++; +void KlassInfoLUT::print_statistics(outputStream* st) { + StreamAutoIndentor indent(st); + + st->print_cr("KLUT"); + + st->print_cr("Klass registrations:"); + { + streamIndentor si(st, 4); +#ifdef KLUT_ENABLE_REGISTRATION_STATS + print_counters(st, _registration_counters, "registrations"); +#else + st->print_cr("Not available"); +#endif + } + + st->print_cr("Hits:"); + { + streamIndentor si(st, 4); +#ifdef KLUT_ENABLE_HIT_STATS + if (uses_lookup_table()) { + print_counters(st, _hit_counters, "hits"); + } else { + st->print_cr("Not available (COH disabled)"); + } +#else + st->print_cr("Not available"); +#endif + } + + st->print_cr("Lookup Table:"); + { + streamIndentor si(st, 4); + if (uses_lookup_table()) { + + st->print_cr("Size: %u slots (%zu bytes)", _max_entries, _max_entries * sizeof(klute_raw_t)); + + // Hit density per cacheline distribution (How well are narrow Klass IDs clustered to give us good local density) + constexpr int chacheline_size = 64; + constexpr int slots_per_cacheline = chacheline_size / sizeof(KlassLUTEntry); + const int num_cachelines = max_entries() / slots_per_cacheline; + int valid_hits_per_cacheline_distribution[slots_per_cacheline + 1] = { 0 }; + for (int i = 0; i < num_cachelines; i++) { + int n = 0; + for (int j = 0; j < slots_per_cacheline; j++) { + KlassLUTEntry e(at((i * slots_per_cacheline) + j)); + const bool fully_valid = e.is_valid() && (e.is_array() || e.ik_carries_infos()); + if (fully_valid) { + n++; + } } + assert(n <= slots_per_cacheline, "Sanity"); + valid_hits_per_cacheline_distribution[n]++; } - assert(n <= slots_per_cacheline, "Sanity"); - valid_hits_per_cacheline_distribution[n]++; - } - st->print_cr("LUT valid hit density over cacheline size:"); - for (int i = 0; i <= slots_per_cacheline; i++) { - st->print_cr("%d valid entries per cacheline: %d", i, valid_hits_per_cacheline_distribution[i]); + st->print_cr("LUT valid hit density over cacheline size:"); + { + streamIndentor si(st, 4); + for (int i = 0; i <= slots_per_cacheline; i++) { + st->print_cr("%d valid entries per cacheline: %d", i, valid_hits_per_cacheline_distribution[i]); + } + } + } else { + st->print_cr("Not available (COH disabled)"); } } - // Just for info, print limits - KlassLUTEntry::print_limits(st); -} -#ifdef KLUT_ENABLE_EXPENSIVE_STATS -void KlassInfoLUT::update_hit_stats(klute_raw_t klute) { - const KlassLUTEntry klutehelper(klute); - assert(klutehelper.is_valid(), "invalid klute"); - switch (klutehelper.kind()) { -#define XX(name, shortname) case name ## Kind: inc_hits_ ## shortname(); break; - KLASSKIND_ALL_KINDS_DO(XX) -#undef XX - default: - fatal("invalid klute kind (" KLUTE_FORMAT ")", klute); - }; - if (klutehelper.is_instance() && !klutehelper.ik_carries_infos()) { - switch (klutehelper.kind()) { - case InstanceClassLoaderKlassKind: inc_noinfo_ICLK(); break; - case InstanceMirrorKlassKind: inc_noinfo_IMK(); break; - default: inc_noinfo_IK_other(); break; - } + st->print_cr("Limits:"); + { + streamIndentor si(st, 4); + st->print_cr("max instance size: %zu words", KlassLUTEntry::ik_wordsize_limit); + st->print_cr("max oopmap block 1 count: %zu", KlassLUTEntry::ik_omb_count_1_limit); + st->print_cr("max oopmap block 1 offset: %zu oops", KlassLUTEntry::ik_omb_offset_1_limit); + st->print_cr("max oopmap block 2 count: %zu", KlassLUTEntry::ik_omb_count_2_limit); + st->print_cr("max oopmap block 2 offset: %zu oops", KlassLUTEntry::ik_omb_offset_2_limit); } - switch (klutehelper.cld_index()) { - case 1: inc_hits_bootloader(); break; - case 2: inc_hits_sysloader(); break; - case 3: inc_hits_platformloader(); break; - }; + st->cr(); } -#endif // KLUT_ENABLE_EXPENSIVE_STATS +#endif // (KLUT_ENABLE_REGISTRATION_STATS) || defined(KLUT_ENABLE_HIT_STATS) -#ifdef KLUT_ENABLE_EXPENSIVE_LOG -void KlassInfoLUT::log_hit(klute_raw_t klute) { - //log_debug(klut)("retrieval: klute: name: %s kind: %d", k->name()->as_C_string(), k->kind()); + +#ifdef KLUT_ENABLE_REGISTRATION_STATS +void KlassInfoLUT::update_registration_counters(const Klass* k, klute_raw_t klute) { + update_counters(_registration_counters, k, klute); } -#endif +#endif // KLUT_ENABLE_REGISTRATION_STATS + +#ifdef KLUT_ENABLE_HIT_STATS +void KlassInfoLUT::update_hit_counters(const Klass* k, klute_raw_t klute) { + update_counters(_hit_counters, k, klute); +} + + +#endif // KLUT_ENABLE_HIT_STATS diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 65189aa263bf4..54c650732607d 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -34,9 +34,14 @@ class Klass; class ClassLoaderData; +// Unexpensive stats +#define KLUT_ENABLE_REGISTRATION_STATS + #ifdef ASSERT -#define KLUT_ENABLE_EXPENSIVE_STATS -//#define KLUT_ENABLE_EXPENSIVE_LOG +// expensive stats +#define KLUT_ENABLE_HIT_STATS +// very expensive tests +#define KLUT_SUPER_PARANOID #endif // The Klass Info Lookup Table (KLUT) is a table of 32-bit values. Each value represents @@ -88,47 +93,76 @@ class KlassInfoLUT : public AllStatic { static void allocate_lookup_table(); static bool uses_lookup_table() { return _table != nullptr; } - // Klass registration statistics. These are not expensive and therefore - // we carry them always. -#define REGISTER_STATS_DO(f) \ - f(registered_IK) \ - f(registered_IRK) \ - f(registered_IMK) \ - f(registered_ICLK) \ - f(registered_ISCK) \ - f(registered_TAK) \ - f(registered_OAK) \ - f(registered_IK_for_abstract_or_interface) -#define XX(xx) static void inc_##xx(); - REGISTER_STATS_DO(XX) -#undef XX - - // Statistics about the hit rate of the lookup table. These are - // expensive and only enabled when needed. -#ifdef KLUT_ENABLE_EXPENSIVE_STATS -#define HIT_STATS_DO(f) \ - f(hits_IK) \ - f(hits_IRK) \ - f(hits_IMK) \ - f(hits_ICLK) \ - f(hits_ISCK) \ - f(hits_TAK) \ - f(hits_OAK) \ - f(hits_bootloader) \ - f(hits_sysloader) \ - f(hits_platformloader) \ - f(noinfo_IMK) \ - f(noinfo_ICLK) \ - f(noinfo_IK_other) -#define XX(xx) static void inc_##xx(); - HIT_STATS_DO(XX) -#undef XX - static void update_hit_stats(klute_raw_t klute); -#endif // KLUT_ENABLE_EXPENSIVE_STATS - -#ifdef KLUT_ENABLE_EXPENSIVE_LOG - static void log_hit(klute_raw_t klute); +#if defined(KLUT_ENABLE_REGISTRATION_STATS) || defined(KLUT_ENABLE_HIT_STATS) + class Counter { + volatile uint64_t _v; + public: + Counter() : _v(0) {} + void inc(); + uint64_t get() const { return _v; } + }; + struct Counters { + // How many InstanceKlass (excluding sub types) + Counter counter_IK; + // How many InstanceRefKlass + Counter counter_IRK; + // How many InstanceMirrorKlas + Counter counter_IMK; + // How many InstanceClassLoaderKlass + Counter counter_ICLK; + // How many InstanceStackChunkKlass + Counter counter_ISCK; + // How many TypeArrayKlass + Counter counter_TAK; + // How many ObjectArrayKlass + Counter counter_OAK; + + // Of InstanceKlass registrations: + // How many were not fully representable + Counter counter_IK_no_info; + // Of InstanceKlass registrations: + // How many were from abstract/interface klasses (hence not fully representable) + Counter counter_IK_no_info_abstract_or_interface; + // Of InstanceKlass registrations: + // How many had more than two oopmap entries (hence not fully representable) + Counter counter_IK_no_info_too_many_oopmapentries; + // Of InstanceKlass registrations: + // How many were larger than 64 heap words (hence not fully representable) + Counter counter_IK_no_info_too_large; + + // Of InstanceKlass: How many had zero oopmap entries + Counter counter_IK_zero_oopmapentries; + // Of InstanceKlass: How many had one oopmap entries + Counter counter_IK_one_oopmapentries; + // Of InstanceKlass: How many had two oopmap entries + Counter counter_IK_two_oopmapentries; + + // Of Klass: How many were tied to the permanent boot class loader CLD + Counter counter_from_boot_cld; + // Of Klass: How many were tied to the permanent sys class loader CLD + Counter counter_from_system_cld; + // Of Klass: How many were tied to the permanent platform class loader CLD + Counter counter_from_platform_cld; + // Of all Klass registrations: + // How many were tied to an unknown CLD + Counter counter_from_unknown_cld; + // Of all Klass registrations: + // How many were tied to a CLD that was nullptr at + // registration time (AOT unlinked class) + Counter counter_from_null_cld; + }; + static void update_counters(Counters& counters, const Klass* k, klute_raw_t klute); + static void print_counters(outputStream* st, const Counters& counters, const char* prefix); +#ifdef KLUT_ENABLE_REGISTRATION_STATS + static Counters _registration_counters; + static void update_registration_counters(const Klass* k, klute_raw_t klute); +#endif + +#ifdef KLUT_ENABLE_HIT_STATS + static Counters _hit_counters; + static void update_hit_counters(const Klass* k, klute_raw_t klute); #endif +#endif // (KLUT_ENABLE_REGISTRATION_STATS) || defined(KLUT_ENABLE_HIT_STATS) public: diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index b4dadec14efa3..b2e866e22d815 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -31,6 +31,7 @@ #include "oops/klassInfoLUTEntry.inline.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/debug.hpp" +#include "runtime/atomic.hpp" inline uint32_t KlassInfoLUT::at(unsigned index) { assert(_initialized, "not initialized"); @@ -47,22 +48,29 @@ inline void KlassInfoLUT::put(unsigned index, klute_raw_t klute) { } inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { + // This function is extremely hot. Be careful when adding/modifying. assert(nk != 0, "null narrow Klass - is this class encodable?"); const klute_raw_t klute = at(nk); assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); -#ifdef KLUT_ENABLE_EXPENSIVE_STATS - update_hit_stats(klute); +#ifdef KLUT_ENABLE_HIT_STATS + { + const Klass* const k = CompressedKlassPointers::decode(nk); + update_hit_counters(k, klute); + } #endif -#ifdef KLUT_ENABLE_EXPENSIVE_LOG - log_hit(klute); +#ifdef KLUT_SUPER_PARANOID + { + const Klass* const k = CompressedKlassPointers::decode(nk); + KlassLUTEntry(klute).verify_against_klass(k); + } #endif return klute; } -ALWAYSINLINE ClassLoaderData* KlassInfoLUT::lookup_cld(unsigned index) { +inline ClassLoaderData* KlassInfoLUT::lookup_cld(unsigned index) { assert(index <= 3, "Sanity"); ClassLoaderData* cld = _common_loaders[index]; assert(index == 0 || cld != nullptr, "CLD for index %d not yet registered?", index); diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.cpp b/src/hotspot/share/oops/klassInfoLUTEntry.cpp index 455a280c9e578..5659b48d2b492 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.cpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.cpp @@ -205,7 +205,6 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { #define PREAMBLE_ARGS p2i(k), k->name()->as_C_string(tmp, sizeof(tmp)), _v.raw #define ASSERT_HERE(cond, msg) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS); #define ASSERT_HERE2(cond, msg, ...) assert( (cond), PREAMBLE_FORMAT msg, PREAMBLE_ARGS, __VA_ARGS__); - assert(k->check_stamp(), "Stamp invalid"); // General static asserts that need access to private members, but I don't want @@ -243,25 +242,21 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { ASSERT_HERE2(cldi == KlassInfoLUT::cld_index_unknown, "for CLD==nullptr cld_index is expected to be %u, was %u", KlassInfoLUT::cld_index_unknown, cldi); + // Note: don't test anything that relies on OopStorage here, since this function is + // called during GCs. E.g. don't call CLD::is_permanent_class_loader_data() since that + // needs to resolve the class loader oop handle inside CLD. } else { ASSERT_HERE(real_cld != nullptr, "Klass CLD is null?"); } } else { const ClassLoaderData* const cld_from_klute = KlassInfoLUT::lookup_cld(cldi); if (cldi != KlassInfoLUT::cld_index_unknown) { - // We expect to get one of the three permanent class loaders, and for it to match the one in Klass - ASSERT_HERE2(cld_from_klute->is_permanent_class_loader_data(), "not perma cld (loader_index: %d, CLD: " PTR_FORMAT ")", - cldi, p2i(cld_from_klute)); ASSERT_HERE2(cld_from_klute == real_cld, "Different CLD (loader_index: %d, real Klass CLD: " PTR_FORMAT ", from klute CLD lookup table: " PTR_FORMAT ")?", cldi, p2i(real_cld), p2i(cld_from_klute)); } else { // We expect to get a nullptr from the CLD lookup table. ASSERT_HERE2(cld_from_klute == nullptr, "CLD not null? (" PTR_FORMAT ")", p2i(cld_from_klute)); - // cld_index == 0 means "unknown CLD" and since we expect only to ever run with three permanent CLDs, - // that cld should not be permanent. - ASSERT_HERE2(!real_cld->is_permanent_class_loader_data(), - "Unregistered permanent CLD? (" PTR_FORMAT ")", p2i(cld_from_klute)); } } @@ -338,11 +333,3 @@ void KlassLUTEntry::verify_against_klass(const Klass* k) const { void KlassLUTEntry::print(outputStream* st) const { st->print("%X (Kind: %d Loader: %d)", value(), kind(), cld_index()); } - -// Helper function, prints current limits -void KlassLUTEntry::print_limits(outputStream* st) { - st->print_cr("IKE Limits: instance byte size %zu, omb1 count: %zu, omb1 byte offset: %zu, omb2 oop count: %zu, omb2 byte offset: %zu", - ik_wordsize_limit * BytesPerWord, - ik_omb_count_1_limit, ik_omb_offset_1_limit * BytesPerHeapOop, - ik_omb_count_2_limit, ik_omb_offset_2_limit * BytesPerHeapOop); -} diff --git a/src/hotspot/share/oops/klassInfoLUTEntry.hpp b/src/hotspot/share/oops/klassInfoLUTEntry.hpp index 6c7b691cf67ae..f0dcc7862edce 100644 --- a/src/hotspot/share/oops/klassInfoLUTEntry.hpp +++ b/src/hotspot/share/oops/klassInfoLUTEntry.hpp @@ -181,6 +181,12 @@ class KlassLUTEntry { const U _v; + static klute_raw_t build_from_common(const Klass* k); + static klute_raw_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); + static klute_raw_t build_from_ak(const ArrayKlass* k); + +public: + // The limits to what we can numerically represent in an (InstanceKlass) Entry static constexpr size_t ik_wordsize_limit = nth_bit(bits_ik_wordsize); static constexpr size_t ik_omb_offset_1_limit = nth_bit(bits_ik_omb_offset_1); @@ -188,12 +194,6 @@ class KlassLUTEntry { static constexpr size_t ik_omb_offset_2_limit = nth_bit(bits_ik_omb_offset_2); static constexpr size_t ik_omb_count_2_limit = nth_bit(bits_ik_omb_count_2); - static klute_raw_t build_from_common(const Klass* k); - static klute_raw_t build_from_ik(const InstanceKlass* k, const char*& not_encodable_reason); - static klute_raw_t build_from_ak(const ArrayKlass* k); - -public: - // Invalid entries are entries that have not been set yet. // Note: cannot use "0" as invalid_entry, since 0 is valid (interface or abstract InstanceKlass, size = 0 and has no oop map) // We use kind=7=0b111 (invalid), and set the rest of the bits also to 1 @@ -265,9 +265,6 @@ class KlassLUTEntry { template inline size_t tak_calculate_wordsize_given_oop_fast(typeArrayOopDesc* obj) const; - // Helper function, prints current limits - static void print_limits(outputStream* st); - void print(outputStream* st) const; }; // KlassInfoLUEntry From e533f4784111d3e10a96752760b6dc7455d8115a Mon Sep 17 00:00:00 2001 From: tstuefe Date: Thu, 8 May 2025 18:24:01 +0200 Subject: [PATCH 099/101] oopDesc::klute() should assert for invalid klute also if not using COH --- src/hotspot/share/classfile/classLoaderData.cpp | 2 -- src/hotspot/share/oops/klassInfoLUT.inline.hpp | 1 - src/hotspot/share/oops/oop.inline.hpp | 3 +++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/classfile/classLoaderData.cpp b/src/hotspot/share/classfile/classLoaderData.cpp index 94cf6a787ca7d..825072cb13bef 100644 --- a/src/hotspot/share/classfile/classLoaderData.cpp +++ b/src/hotspot/share/classfile/classLoaderData.cpp @@ -763,8 +763,6 @@ ClassLoaderData::~ClassLoaderData() { if (_name_and_id != nullptr) { _name_and_id->decrement_refcount(); } - -memset((char*)this, 0x17, sizeof(ClassLoaderData)); } // Returns true if this class loader data is for the app class loader diff --git a/src/hotspot/share/oops/klassInfoLUT.inline.hpp b/src/hotspot/share/oops/klassInfoLUT.inline.hpp index b2e866e22d815..9db6d838cc77d 100644 --- a/src/hotspot/share/oops/klassInfoLUT.inline.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.inline.hpp @@ -51,7 +51,6 @@ inline klute_raw_t KlassInfoLUT::lookup(narrowKlass nk) { // This function is extremely hot. Be careful when adding/modifying. assert(nk != 0, "null narrow Klass - is this class encodable?"); const klute_raw_t klute = at(nk); - assert(klute != KlassLUTEntry::invalid_entry, "must never be invalid"); #ifdef KLUT_ENABLE_HIT_STATS { diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 654ea6b78a450..c5ea807031a18 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -99,6 +99,7 @@ void oopDesc::init_mark() { template klute_raw_t oopDesc::get_klute() const { + klute_raw_t klute; switch (mode) { case HeaderMode::Compact: return KlassInfoLUT::lookup(mark().narrow_klass()); @@ -107,6 +108,8 @@ klute_raw_t oopDesc::get_klute() const { default: return _metadata._klass->klute(); } + assert(klute != KlassLUTEntry::invalid_entry, "invalid klute"); + return klute; } klute_raw_t oopDesc::get_klute() const { From d6ed1c33d80464b13d0b6d73511d30b490effd23 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 12 May 2025 06:45:11 +0200 Subject: [PATCH 100/101] GC.class_statistics should print more Klass information --- src/hotspot/share/memory/heapInspection.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/memory/heapInspection.cpp b/src/hotspot/share/memory/heapInspection.cpp index 867ccc6106d8e..e6ce4581b0904 100644 --- a/src/hotspot/share/memory/heapInspection.cpp +++ b/src/hotspot/share/memory/heapInspection.cpp @@ -97,21 +97,24 @@ const char* KlassInfoEntry::name() const { void KlassInfoEntry::print_on(outputStream* st) const { ResourceMark rm; + const narrowKlass nk = CompressedKlassPointers::is_encodable(_klass) ? CompressedKlassPointers::encode(_klass) : 0; // simplify the formatting (ILP32 vs LP64) - always cast the numbers to 64-bit ModuleEntry* module = _klass->module(); if (module->is_named()) { - st->print_cr(INT64_FORMAT_W(13) " " UINT64_FORMAT_W(13) " %s (%s%s%s)", + st->print_cr(INT64_FORMAT_W(13) " " UINT64_FORMAT_W(13) " " PTR_FORMAT " %u %s (%s%s%s)", (int64_t)_instance_count, (uint64_t)_instance_words * HeapWordSize, + p2i(_klass), nk, name(), module->name()->as_C_string(), module->version() != nullptr ? "@" : "", module->version() != nullptr ? module->version()->as_C_string() : ""); } else { - st->print_cr(INT64_FORMAT_W(13) " " UINT64_FORMAT_W(13) " %s", + st->print_cr(INT64_FORMAT_W(13) " " UINT64_FORMAT_W(13) " " PTR_FORMAT " %u %s", (int64_t)_instance_count, (uint64_t)_instance_words * HeapWordSize, + p2i(_klass), nk, name()); } } @@ -492,8 +495,9 @@ void KlassHierarchy::print_class(outputStream* st, KlassInfoEntry* cie, bool pri } void KlassInfoHisto::print_histo_on(outputStream* st) { - st->print_cr(" num #instances #bytes class name (module)"); - st->print_cr("-------------------------------------------------------"); + int len_divider = 0; + st->print_cr(" num #instances #bytes Klass nKlass class name (module)"); + st->print_cr(" ---------------------------------------------------------------------------------------"); print_elements(st); } From 2c7e952d796b0a99211f2cffced93dcc6a2b8585 Mon Sep 17 00:00:00 2001 From: tstuefe Date: Mon, 12 May 2025 14:08:55 +0200 Subject: [PATCH 101/101] preferred classes (CDS off only for now) --- src/hotspot/share/memory/allocation.cpp | 4 +- src/hotspot/share/memory/metaspace.cpp | 54 ++++++++++++++++++++-- src/hotspot/share/memory/metaspace.hpp | 4 +- src/hotspot/share/oops/array.inline.hpp | 2 +- src/hotspot/share/oops/arrayKlass.cpp | 4 +- src/hotspot/share/oops/arrayKlass.hpp | 2 +- src/hotspot/share/oops/compressedKlass.hpp | 4 ++ src/hotspot/share/oops/instanceKlass.cpp | 18 +++++--- src/hotspot/share/oops/instanceKlass.hpp | 2 +- src/hotspot/share/oops/klassInfoLUT.cpp | 48 ++++++++++++++++++- src/hotspot/share/oops/klassInfoLUT.hpp | 5 ++ src/hotspot/share/oops/objArrayKlass.cpp | 7 ++- src/hotspot/share/oops/typeArrayKlass.cpp | 6 ++- 13 files changed, 137 insertions(+), 23 deletions(-) diff --git a/src/hotspot/share/memory/allocation.cpp b/src/hotspot/share/memory/allocation.cpp index 13280006fe692..d5241988c8384 100644 --- a/src/hotspot/share/memory/allocation.cpp +++ b/src/hotspot/share/memory/allocation.cpp @@ -75,7 +75,7 @@ void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data, MetaspaceObj::Type type, TRAPS) throw() { // Klass has its own operator new assert(type != ClassType, "class has its own operator new"); - return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false, THREAD); + return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false, /* preferred */ false, THREAD); } void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data, @@ -83,7 +83,7 @@ void* MetaspaceObj::operator new(size_t size, ClassLoaderData* loader_data, MetaspaceObj::Type type) throw() { assert(!Thread::current()->is_Java_thread(), "only allowed by non-Java thread"); assert(type != ClassType, "class has its own operator new"); - return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false); + return Metaspace::allocate(loader_data, word_size, type, /*use_class_space*/ false, /* preferred */ false); } bool MetaspaceObj::is_valid(const MetaspaceObj* p) { diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp index 8acfda642038b..6bac025999c3c 100644 --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -66,6 +66,7 @@ #include "utilities/globalDefinitions.hpp" #include "utilities/ostream.hpp" +using metaspace::Metachunk; using metaspace::ChunkManager; using metaspace::CommitLimiter; using metaspace::MetaspaceContext; @@ -73,6 +74,43 @@ using metaspace::MetaspaceReporter; using metaspace::RunningCounters; using metaspace::VirtualSpaceList; +class PreferredKlassSlots : public CHeapObj { + static constexpr unsigned slots = 16; + static constexpr size_t slot_size = (size_t)CompressedKlassPointers::klass_alignment_in_words_coh; + static constexpr size_t backing_storage_size = slot_size * slots; + + Metachunk* const _chunk; // An immortal chunk + + static Metachunk* allocate_chunk() { + const metaspace::chunklevel_t lvl = metaspace::chunklevel::level_fitting_word_size(backing_storage_size); + metaspace::Metachunk* const chunk = MetaspaceContext::context_class()->cm()->get_chunk(lvl); + assert(chunk->word_size() == backing_storage_size, "must be"); + assert(is_aligned(chunk->base(), slot_size), "must be"); + return chunk; // Note: not yet committed + } + +public: + + PreferredKlassSlots() : _chunk(allocate_chunk()) { + log_info(metaspace)("Reserved chunk for preferred classes " METACHUNK_FORMAT, + METACHUNK_FORMAT_ARGS(_chunk)); + } + + MetaWord* allocate(size_t word_size) { + assert(UseCompactObjectHeaders, "Sanity"); + assert(word_size <= slot_size, "Klass too large? Does not fit into one slot (%zu)", word_size); + _chunk->ensure_committed(backing_storage_size); + // Should not fire unless we added too many preferred classes + assert(_chunk->free_below_committed_words() >= slot_size, "Too many preferred classes?"); + MetaWord* const result = _chunk->allocate(slot_size); + assert(is_aligned(result, slot_size), "must be"); + log_debug(metaspace)("Reserved preferred Klass slot at " PTR_FORMAT, p2i(result)); + return result; + } +}; + +static PreferredKlassSlots* preferred_klass_slots = nullptr; + size_t MetaspaceUtils::used_words() { return RunningCounters::used_words(); } @@ -835,6 +873,10 @@ void Metaspace::global_initialize() { assert(CompressedKlassPointers::base() == nullptr, "Zero-based encoding expected"); } + // Initialize storage for preferred Klasses + if (UseCompactObjectHeaders) { + preferred_klass_slots = new PreferredKlassSlots(); + } } #endif // _LP64 @@ -871,7 +913,7 @@ size_t Metaspace::max_allocation_word_size() { // is suitable for calling from non-Java threads. // Callers are responsible for checking null. MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, - MetaspaceObj::Type type, bool use_class_space) { + MetaspaceObj::Type type, bool use_class_space, bool preferred) { assert(word_size <= Metaspace::max_allocation_word_size(), "allocation size too large (%zu)", word_size); @@ -905,7 +947,7 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, } MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, - MetaspaceObj::Type type, bool use_class_space, TRAPS) { + MetaspaceObj::Type type, bool use_class_space, bool preferred, TRAPS) { if (HAS_PENDING_EXCEPTION) { assert(false, "Should not allocate with exception pending"); @@ -913,7 +955,13 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, } assert(!THREAD->owns_locks(), "allocating metaspace while holding mutex"); - MetaWord* result = allocate(loader_data, word_size, type, use_class_space); + MetaWord* result = nullptr; + if (preferred && preferred_klass_slots != nullptr) { + result = preferred_klass_slots->allocate(word_size); + } + if (result == nullptr) { + result = allocate(loader_data, word_size, type, use_class_space, preferred); + } if (result == nullptr) { MetadataType mdtype = use_class_space ? ClassType : NonClassType; diff --git a/src/hotspot/share/memory/metaspace.hpp b/src/hotspot/share/memory/metaspace.hpp index 293782c0d758a..c5e288e163483 100644 --- a/src/hotspot/share/memory/metaspace.hpp +++ b/src/hotspot/share/memory/metaspace.hpp @@ -120,12 +120,12 @@ class Metaspace : public AllStatic { static constexpr size_t min_allocation_word_size = min_allocation_alignment_words; static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size, - MetaspaceObj::Type type, bool use_class_space, TRAPS); + MetaspaceObj::Type type, bool use_class_space, bool preferred, TRAPS); // Non-TRAPS version of allocate which can be called by a non-Java thread, that returns // null on failure. static MetaWord* allocate(ClassLoaderData* loader_data, size_t word_size, - MetaspaceObj::Type type, bool use_class_space); + MetaspaceObj::Type type, bool use_class_space, bool preferred); // Returns true if the pointer points into class space, non-class metaspace, or the // metadata portion of the CDS archive. diff --git a/src/hotspot/share/oops/array.inline.hpp b/src/hotspot/share/oops/array.inline.hpp index 28f8a35dc0d4a..e69f71bc666d0 100644 --- a/src/hotspot/share/oops/array.inline.hpp +++ b/src/hotspot/share/oops/array.inline.hpp @@ -34,7 +34,7 @@ template inline void* Array::operator new(size_t size, ClassLoaderData* loader_data, int length, TRAPS) throw() { size_t word_size = Array::size(length); return (void*) Metaspace::allocate(loader_data, word_size, - MetaspaceObj::array_type(sizeof(T)), false, THREAD); + MetaspaceObj::array_type(sizeof(T)), false, false, THREAD); } #endif // SHARE_OOPS_ARRAY_INLINE_HPP diff --git a/src/hotspot/share/oops/arrayKlass.cpp b/src/hotspot/share/oops/arrayKlass.cpp index 8b6a97fd62d68..1d38a02e3ee54 100644 --- a/src/hotspot/share/oops/arrayKlass.cpp +++ b/src/hotspot/share/oops/arrayKlass.cpp @@ -41,8 +41,8 @@ #include "oops/oop.inline.hpp" #include "runtime/handles.inline.hpp" -void* ArrayKlass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw() { - return Metaspace::allocate(loader_data, word_size, MetaspaceObj::ClassType, true, THREAD); +void* ArrayKlass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, bool preferred, TRAPS) throw() { + return Metaspace::allocate(loader_data, word_size, MetaspaceObj::ClassType, true, preferred, THREAD); } ArrayKlass::ArrayKlass() { diff --git a/src/hotspot/share/oops/arrayKlass.hpp b/src/hotspot/share/oops/arrayKlass.hpp index da06dbe464cd6..1fded3f7a9ecb 100644 --- a/src/hotspot/share/oops/arrayKlass.hpp +++ b/src/hotspot/share/oops/arrayKlass.hpp @@ -50,7 +50,7 @@ class ArrayKlass: public Klass { ArrayKlass(Symbol* name, KlassKind kind); ArrayKlass(); - void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, TRAPS) throw(); + void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, bool preferred, TRAPS) throw(); public: // Testing operation diff --git a/src/hotspot/share/oops/compressedKlass.hpp b/src/hotspot/share/oops/compressedKlass.hpp index 5cd2a3431b72b..2c5f92c2e0373 100644 --- a/src/hotspot/share/oops/compressedKlass.hpp +++ b/src/hotspot/share/oops/compressedKlass.hpp @@ -226,6 +226,10 @@ class CompressedKlassPointers : public AllStatic { static int klass_alignment_in_bytes() { return nth_bit(MAX2(3, _shift)); } static int klass_alignment_in_words() { return klass_alignment_in_bytes() / BytesPerWord; } + // Returns the alignment a Klass* is guaranteed to have, for COH + static constexpr int klass_alignment_in_bytes_coh = nth_bit(max_shift_coh); + static constexpr int klass_alignment_in_words_coh = nth_bit(max_shift_coh) / BytesPerWord; + // Returns the highest possible narrowKlass value given the current Klass range static narrowKlass highest_valid_narrow_klass_id() { return _highest_valid_narrow_klass_id; } diff --git a/src/hotspot/share/oops/instanceKlass.cpp b/src/hotspot/share/oops/instanceKlass.cpp index 8c0b10986c389..2f1be34ff9467 100644 --- a/src/hotspot/share/oops/instanceKlass.cpp +++ b/src/hotspot/share/oops/instanceKlass.cpp @@ -456,8 +456,8 @@ const char* InstanceKlass::nest_host_error() { } void* InstanceKlass::operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, - bool use_class_space, TRAPS) throw() { - return Metaspace::allocate(loader_data, word_size, ClassType, use_class_space, THREAD); + bool use_class_space, bool preferred, TRAPS) throw() { + return Metaspace::allocate(loader_data, word_size, ClassType, use_class_space, preferred, THREAD); } InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& parser, TRAPS) { @@ -473,23 +473,27 @@ InstanceKlass* InstanceKlass::allocate_instance_klass(const ClassFileParser& par InstanceKlass* ik; const bool use_class_space = parser.klass_needs_narrow_id(); + const bool preferred = + UseCompactObjectHeaders && + loader_data->is_the_null_class_loader_data() && + KlassInfoLUT::is_preferred_instanceklass(parser.class_name()); // Allocation if (parser.is_instance_ref_klass()) { // java.lang.ref.Reference - ik = new (loader_data, size, use_class_space, THREAD) InstanceRefKlass(parser); + ik = new (loader_data, size, use_class_space, preferred, THREAD) InstanceRefKlass(parser); } else if (class_name == vmSymbols::java_lang_Class()) { // mirror - java.lang.Class - ik = new (loader_data, size, use_class_space, THREAD) InstanceMirrorKlass(parser); + ik = new (loader_data, size, use_class_space, preferred, THREAD) InstanceMirrorKlass(parser); } else if (is_stack_chunk_class(class_name, loader_data)) { // stack chunk - ik = new (loader_data, size, use_class_space, THREAD) InstanceStackChunkKlass(parser); + ik = new (loader_data, size, use_class_space, preferred, THREAD) InstanceStackChunkKlass(parser); } else if (is_class_loader(class_name, parser)) { // class loader - java.lang.ClassLoader - ik = new (loader_data, size, use_class_space, THREAD) InstanceClassLoaderKlass(parser); + ik = new (loader_data, size, use_class_space, preferred, THREAD) InstanceClassLoaderKlass(parser); } else { // normal - ik = new (loader_data, size, use_class_space, THREAD) InstanceKlass(parser); + ik = new (loader_data, size, use_class_space, preferred, THREAD) InstanceKlass(parser); } if (ik != nullptr && UseCompressedClassPointers && use_class_space) { diff --git a/src/hotspot/share/oops/instanceKlass.hpp b/src/hotspot/share/oops/instanceKlass.hpp index ab84a994bc8cd..8972c921e2fe1 100644 --- a/src/hotspot/share/oops/instanceKlass.hpp +++ b/src/hotspot/share/oops/instanceKlass.hpp @@ -145,7 +145,7 @@ class InstanceKlass: public Klass { protected: InstanceKlass(const ClassFileParser& parser, KlassKind kind = Kind, ReferenceType reference_type = REF_NONE); - void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, bool use_class_space, TRAPS) throw(); + void* operator new(size_t size, ClassLoaderData* loader_data, size_t word_size, bool use_class_space, bool preferred, TRAPS) throw(); public: InstanceKlass(); diff --git a/src/hotspot/share/oops/klassInfoLUT.cpp b/src/hotspot/share/oops/klassInfoLUT.cpp index 14782101afc59..0482cdb23f00a 100644 --- a/src/hotspot/share/oops/klassInfoLUT.cpp +++ b/src/hotspot/share/oops/klassInfoLUT.cpp @@ -31,6 +31,7 @@ #include "oops/klassInfoLUT.inline.hpp" #include "oops/klassInfoLUTEntry.inline.hpp" #include "oops/klassKind.hpp" +#include "oops/symbol.hpp" #include "runtime/atomic.hpp" #include "utilities/debug.hpp" @@ -466,7 +467,52 @@ void KlassInfoLUT::update_registration_counters(const Klass* k, klute_raw_t klut void KlassInfoLUT::update_hit_counters(const Klass* k, klute_raw_t klute) { update_counters(_hit_counters, k, klute); } +#endif // KLUT_ENABLE_HIT_STATS + +static bool matches_one_of(const Symbol* name, const char* names[]) { + char tmp[256]; + name->as_C_string(tmp, sizeof(tmp)); + for (int i = 0; names[i] != nullptr; i++) { + if (strcmp(names[i], tmp) == 0) { + return true; + } + } + return false; +} +bool KlassInfoLUT::is_preferred_instanceklass(const Symbol* name) { + static const char* names[] = { + "java/lang/String", // 1 + "java/lang/Object", // 2 + "java/util/HashMap$Node", // 3 + "java/util/concurrent/ConcurrentHashMap$Node", // 4 + "java/util/LinkedHashMap$Entry", // 5 + "java/lang/Long", // 6 + "java/lang/Integer", // 7 + "java/math/BigInteger", // 8 + "java/util/concurrent/locks/ReentrantLock", // 9 + "java/util/concurrent/locks/ReentrantLock$NonfairSync", // 10 + nullptr + }; + return matches_one_of(name, names); +} +bool KlassInfoLUT::is_preferred_objectarrayklass(const Symbol* element_class_name) { + static const char* names[] = { + "java/lang/Object", // 11 + "java/lang/String", // 12 + "java/util/HashMap$Node", // 13 + "java/util/concurrent/ConcurrentHashMap$Node", // 14 + nullptr + }; + return matches_one_of(element_class_name, names); +} -#endif // KLUT_ENABLE_HIT_STATS +bool KlassInfoLUT::is_preferred_typearrayklass(const Symbol* name) { + static const char* names[] = { + "[B", // 15 Note: [B but not filler element + "[I", // 16 + nullptr + }; + return matches_one_of(name, names); +} diff --git a/src/hotspot/share/oops/klassInfoLUT.hpp b/src/hotspot/share/oops/klassInfoLUT.hpp index 54c650732607d..be37d46c1b474 100644 --- a/src/hotspot/share/oops/klassInfoLUT.hpp +++ b/src/hotspot/share/oops/klassInfoLUT.hpp @@ -33,6 +33,7 @@ class Klass; class ClassLoaderData; +class Symbol; // Unexpensive stats #define KLUT_ENABLE_REGISTRATION_STATS @@ -183,6 +184,10 @@ class KlassInfoLUT : public AllStatic { #endif static void print_statistics(outputStream* out); + + static bool is_preferred_instanceklass(const Symbol* name); + static bool is_preferred_objectarrayklass(const Symbol* element_class_name); + static bool is_preferred_typearrayklass(const Symbol* name); }; #endif // SHARE_OOPS_KLASSINFOLUT_HPP diff --git a/src/hotspot/share/oops/objArrayKlass.cpp b/src/hotspot/share/oops/objArrayKlass.cpp index bd8dfafe33b43..a06c7df826eb1 100644 --- a/src/hotspot/share/oops/objArrayKlass.cpp +++ b/src/hotspot/share/oops/objArrayKlass.cpp @@ -36,6 +36,7 @@ #include "oops/arrayKlass.hpp" #include "oops/instanceKlass.hpp" #include "oops/klass.inline.hpp" +#include "oops/klassInfoLUT.hpp" #include "oops/objArrayKlass.inline.hpp" #include "oops/objArrayOop.inline.hpp" #include "oops/oop.inline.hpp" @@ -49,8 +50,12 @@ ObjArrayKlass* ObjArrayKlass::allocate(ClassLoaderData* loader_data, int n, Klas "array klasses must be same size as InstanceKlass"); int size = ArrayKlass::static_size(ObjArrayKlass::header_size()); + const bool preferred = + UseCompactObjectHeaders && + loader_data->is_the_null_class_loader_data() && + KlassInfoLUT::is_preferred_objectarrayklass(k->name()); - ObjArrayKlass* oak = new (loader_data, size, THREAD) ObjArrayKlass(n, k, name); + ObjArrayKlass* oak = new (loader_data, size, preferred, THREAD) ObjArrayKlass(n, k, name); { char tmp[1024]; diff --git a/src/hotspot/share/oops/typeArrayKlass.cpp b/src/hotspot/share/oops/typeArrayKlass.cpp index aed7e84422b20..f294adcd4c736 100644 --- a/src/hotspot/share/oops/typeArrayKlass.cpp +++ b/src/hotspot/share/oops/typeArrayKlass.cpp @@ -55,9 +55,10 @@ TypeArrayKlass* TypeArrayKlass::create_klass(BasicType type, { char tmp[1024]; const int size = ArrayKlass::static_size(TypeArrayKlass::header_size()); - log_debug(metaspace)("Returning new TAK @" PTR_FORMAT " for %s, nKlass=%u, word size=%d", + log_debug(metaspace)("Returning new TAK @" PTR_FORMAT " for %s, BasicType %d, nKlass=%u, word size=%d", p2i(ak), ak->name()->as_C_string(tmp, sizeof(tmp)), + (int)type, CompressedKlassPointers::encode(ak), size); } @@ -83,8 +84,9 @@ TypeArrayKlass* TypeArrayKlass::allocate(ClassLoaderData* loader_data, BasicType "array klasses must be same size as InstanceKlass"); int size = ArrayKlass::static_size(TypeArrayKlass::header_size()); + const bool preferred = UseCompactObjectHeaders && KlassInfoLUT::is_preferred_typearrayklass(name); - return new (loader_data, size, THREAD) TypeArrayKlass(type, name); + return new (loader_data, size, preferred, THREAD) TypeArrayKlass(type, name); } u2 TypeArrayKlass::compute_modifier_flags() const {