diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
index fd73b725a1289..f9da082795083 100644
--- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
+++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp
@@ -85,7 +85,6 @@
#include "gc/shared/isGCActiveMark.hpp"
#include "gc/shared/locationPrinter.inline.hpp"
#include "gc/shared/oopStorageParState.hpp"
-#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referenceProcessor.inline.hpp"
#include "gc/shared/suspendibleThreadSet.hpp"
#include "gc/shared/taskqueue.inline.hpp"
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
index 32a56d7120569..4819cbfed7074 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.cpp
@@ -105,7 +105,6 @@ G1GCPhaseTimes::G1GCPhaseTimes(STWGCTimer* gc_timer, uint max_gc_threads) :
_gc_par_phases[UpdateDerivedPointers] = new WorkerDataArray<double>("UpdateDerivedPointers", "Update Derived Pointers (ms):", max_gc_threads);
#endif
_gc_par_phases[EagerlyReclaimHumongousObjects] = new WorkerDataArray<double>("EagerlyReclaimHumongousObjects", "Eagerly Reclaim Humongous Objects (ms):", max_gc_threads);
- _gc_par_phases[RestorePreservedMarks] = new WorkerDataArray<double>("RestorePreservedMarks", "Restore Preserved Marks (ms):", max_gc_threads);
_gc_par_phases[ProcessEvacuationFailedRegions] = new WorkerDataArray<double>("ProcessEvacuationFailedRegions", "Process Evacuation Failed Regions (ms):", max_gc_threads);
_gc_par_phases[ScanHR]->create_thread_work_items("Scanned Cards:", ScanHRScannedCards);
@@ -512,7 +511,6 @@ double G1GCPhaseTimes::print_post_evacuate_collection_set(bool evacuation_failed
debug_time("Post Evacuate Cleanup 2", _cur_post_evacuate_cleanup_2_time_ms);
if (evacuation_failed) {
debug_phase(_gc_par_phases[RecalculateUsed], 1);
- debug_phase(_gc_par_phases[RestorePreservedMarks], 1);
debug_phase(_gc_par_phases[ProcessEvacuationFailedRegions], 1);
}
#if COMPILER2_OR_JVMCI
diff --git a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
index 40abfd605339f..171efa7be778d 100644
--- a/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
+++ b/src/hotspot/share/gc/g1/g1GCPhaseTimes.hpp
@@ -87,7 +87,6 @@ class G1GCPhaseTimes : public CHeapObj<mtGC> {
UpdateDerivedPointers,
#endif
EagerlyReclaimHumongousObjects,
- RestorePreservedMarks,
ProcessEvacuationFailedRegions,
ResetMarkingState,
NoteStartOfMark,
diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
index 26cc88de0beb5..5b3bbedfeb289 100644
--- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
+++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
@@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
oop forwardee;
markWord m = obj->mark();
if (m.is_forwarded()) {
- forwardee = m.forwardee();
+ forwardee = obj->forwardee(m);
} else {
forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
}
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
index f81c3241a1a90..7bd05bc66ebd1 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
@@ -37,7 +37,6 @@
#include "gc/shared/continuationGCSupport.inline.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStepper.inline.hpp"
-#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskqueue.inline.hpp"
#include "memory/allocation.inline.hpp"
@@ -59,7 +58,6 @@
G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
G1RedirtyCardsQueueSet* rdcqs,
- PreservedMarks* preserved_marks,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
@@ -90,7 +88,6 @@ G1ParScanThreadState::G1ParScanThreadState(G1CollectedHeap* g1h,
_numa(g1h->numa()),
_obj_alloc_stat(nullptr),
ALLOCATION_FAILURE_INJECTOR_ONLY(_allocation_failure_inject_counter(0) COMMA)
- _preserved_marks(preserved_marks),
_evacuation_failed_info(),
_evac_failure_regions(evac_failure_regions),
_evac_failure_enqueued_cards(0)
@@ -216,7 +213,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
markWord m = obj->mark();
if (m.is_forwarded()) {
- obj = m.forwardee();
+ obj = obj->forwardee(m);
} else {
obj = do_copy_to_survivor_space(region_attr, obj, m);
}
@@ -595,7 +592,6 @@ G1ParScanThreadState* G1ParScanThreadStateSet::state_for_worker(uint worker_id)
if (_states[worker_id] == nullptr) {
_states[worker_id] =
new G1ParScanThreadState(_g1h, rdcqs(),
- _preserved_marks_set.get(worker_id),
worker_id,
_num_workers,
_collection_set,
@@ -655,7 +651,7 @@ NOINLINE
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz, bool cause_pinned) {
assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
- oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
+ oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
if (forward_ptr == nullptr) {
// Forward-to-self succeeded. We are the "owner" of the object.
G1HeapRegion* r = _g1h->heap_region_containing(old);
@@ -668,8 +664,6 @@ oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, siz
// evacuation failure recovery.
_g1h->mark_evac_failure_object(_worker_id, old, word_sz);
- _preserved_marks->push_if_necessary(old, m);
-
ContinuationGCSupport::transform_stack_chunk(old);
_evacuation_failed_info.register_copy_failure(word_sz);
@@ -727,7 +721,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_g1h(g1h),
_collection_set(collection_set),
_rdcqs(G1BarrierSet::dirty_card_queue_set().allocator()),
- _preserved_marks_set(true /* in_c_heap */),
_states(NEW_C_HEAP_ARRAY(G1ParScanThreadState*, num_workers, mtGC)),
_rdc_buffers(NEW_C_HEAP_ARRAY(BufferNodeList, num_workers, mtGC)),
_surviving_young_words_total(NEW_C_HEAP_ARRAY(size_t, collection_set->young_region_length() + 1, mtGC)),
@@ -736,7 +729,6 @@ G1ParScanThreadStateSet::G1ParScanThreadStateSet(G1CollectedHeap* g1h,
_evac_failure_regions(evac_failure_regions),
_partial_array_state_allocator(num_workers)
{
- _preserved_marks_set.init(num_workers);
for (uint i = 0; i < num_workers; ++i) {
_states[i] = nullptr;
_rdc_buffers[i] = BufferNodeList();
@@ -749,5 +741,4 @@ G1ParScanThreadStateSet::~G1ParScanThreadStateSet() {
FREE_C_HEAP_ARRAY(G1ParScanThreadState*, _states);
FREE_C_HEAP_ARRAY(size_t, _surviving_young_words_total);
FREE_C_HEAP_ARRAY(BufferNodeList, _rdc_buffers);
- _preserved_marks_set.reclaim();
}
diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
index 1cfd6fca08a6f..561d7e7dc8581 100644
--- a/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
+++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.hpp
@@ -34,7 +34,6 @@
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/partialArrayState.hpp"
#include "gc/shared/partialArrayTaskStepper.hpp"
-#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/taskqueue.hpp"
#include "memory/allocation.hpp"
@@ -48,8 +47,6 @@ class G1EvacuationRootClosures;
class G1OopStarChunkedList;
class G1PLABAllocator;
class G1HeapRegion;
-class PreservedMarks;
-class PreservedMarksSet;
class outputStream;
class G1ParScanThreadState : public CHeapObj<mtGC> {
@@ -108,7 +105,6 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
// Per-thread evacuation failure data structures.
ALLOCATION_FAILURE_INJECTOR_ONLY(size_t _allocation_failure_inject_counter;)
- PreservedMarks* _preserved_marks;
EvacuationFailedInfo _evacuation_failed_info;
G1EvacFailureRegions* _evac_failure_regions;
// Number of additional cards into evacuation failed regions enqueued into
@@ -127,7 +123,6 @@ class G1ParScanThreadState : public CHeapObj<mtGC> {
public:
G1ParScanThreadState(G1CollectedHeap* g1h,
G1RedirtyCardsQueueSet* rdcqs,
- PreservedMarks* preserved_marks,
uint worker_id,
uint num_workers,
G1CollectionSet* collection_set,
@@ -248,7 +243,6 @@ class G1ParScanThreadStateSet : public StackObj {
G1CollectedHeap* _g1h;
G1CollectionSet* _collection_set;
G1RedirtyCardsQueueSet _rdcqs;
- PreservedMarksSet _preserved_marks_set;
G1ParScanThreadState** _states;
BufferNodeList* _rdc_buffers;
size_t* _surviving_young_words_total;
@@ -266,7 +260,6 @@ class G1ParScanThreadStateSet : public StackObj {
G1RedirtyCardsQueueSet* rdcqs() { return &_rdcqs; }
BufferNodeList* rdc_buffers() { return _rdc_buffers; }
- PreservedMarksSet* preserved_marks_set() { return &_preserved_marks_set; }
void flush_stats();
void record_unused_optional_region(G1HeapRegion* hr);
diff --git a/src/hotspot/share/gc/g1/g1YoungCollector.cpp b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
index f2fe93015c532..f3590aa2ff696 100644
--- a/src/hotspot/share/gc/g1/g1YoungCollector.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungCollector.cpp
@@ -53,7 +53,6 @@
#include "gc/shared/gcTraceTime.inline.hpp"
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gc_globals.hpp"
-#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/referenceProcessor.hpp"
#include "gc/shared/weakProcessor.inline.hpp"
#include "gc/shared/workerPolicy.hpp"
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
index a0e9a9b1569ab..6b218d3971e8b 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.cpp
@@ -42,7 +42,6 @@
#include "gc/g1/g1RemSet.hpp"
#include "gc/g1/g1YoungGCPostEvacuateTasks.hpp"
#include "gc/shared/bufferNode.hpp"
-#include "gc/shared/preservedMarks.inline.hpp"
#include "jfr/jfrEvents.hpp"
#include "oops/access.inline.hpp"
#include "oops/compressedOops.inline.hpp"
@@ -252,7 +251,7 @@ class G1PostEvacuateCollectionSetCleanupTask1::RestoreEvacFailureRegionsTask : p
{
// Process marked object.
assert(obj->is_forwarded() && obj->forwardee() == obj, "must be self-forwarded");
- obj->init_mark();
+ obj->unset_self_forwarded();
hr->update_bot_for_block(obj_addr, obj_end_addr);
// Statistics
@@ -477,27 +476,6 @@ class G1PostEvacuateCollectionSetCleanupTask2::EagerlyReclaimHumongousObjectsTas
}
};
-class G1PostEvacuateCollectionSetCleanupTask2::RestorePreservedMarksTask : public G1AbstractSubTask {
- PreservedMarksSet* _preserved_marks;
- WorkerTask* _task;
-
-public:
- RestorePreservedMarksTask(PreservedMarksSet* preserved_marks) :
- G1AbstractSubTask(G1GCPhaseTimes::RestorePreservedMarks),
- _preserved_marks(preserved_marks),
- _task(preserved_marks->create_task()) { }
-
- virtual ~RestorePreservedMarksTask() {
- delete _task;
- }
-
- double worker_cost() const override {
- return _preserved_marks->num();
- }
-
- void do_work(uint worker_id) override { _task->work(worker_id); }
-};
-
class RedirtyLoggedCardTableEntryClosure : public G1CardTableEntryClosure {
size_t _num_dirtied;
G1CollectedHeap* _g1h;
@@ -979,7 +957,6 @@ G1PostEvacuateCollectionSetCleanupTask2::G1PostEvacuateCollectionSetCleanupTask2
}
if (evac_failure_regions->has_regions_evac_failed()) {
- add_parallel_task(new RestorePreservedMarksTask(per_thread_states->preserved_marks_set()));
add_parallel_task(new ProcessEvacuationFailedRegionsTask(evac_failure_regions));
}
add_parallel_task(new RedirtyLoggedCardsTask(evac_failure_regions,
diff --git a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp
index 868ab788b534a..c79a8dd8eb8a1 100644
--- a/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp
+++ b/src/hotspot/share/gc/g1/g1YoungGCPostEvacuateTasks.hpp
@@ -56,7 +56,6 @@ class G1PostEvacuateCollectionSetCleanupTask1 : public G1BatchedTask {
// - Update Derived Pointers (s)
// - Clear Retained Region Data (on evacuation failure)
// - Redirty Logged Cards
-// - Restore Preserved Marks (on evacuation failure)
// - Free Collection Set
// - Resize TLABs
class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
@@ -67,7 +66,6 @@ class G1PostEvacuateCollectionSetCleanupTask2 : public G1BatchedTask {
class ProcessEvacuationFailedRegionsTask;
class RedirtyLoggedCardsTask;
- class RestorePreservedMarksTask;
class FreeCollectionSetTask;
class ResizeTLABsTask;
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
index 19f688852385b..b055e0a4d1fac 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp
@@ -319,7 +319,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
// this started. If it is the same (i.e., no forwarding
// pointer has been installed), then this thread owns
// it.
- if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
+ if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
// We won any races, we "own" this object.
assert(obj == obj->forwardee(), "Sanity");
diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
index 390dea4976d19..f152b225bb5a3 100644
--- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
+++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
@@ -149,7 +149,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
return copy_unmarked_to_survivor_space<promote_immediately>(o, m);
} else {
// Return the already installed forwardee.
- return m.forwardee();
+ return o->forwardee(m);
}
}
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.cpp b/src/hotspot/share/gc/serial/defNewGeneration.cpp
index 047171a5eb364..0a10998d6f8b2 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.cpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp
@@ -39,7 +39,6 @@
#include "gc/shared/gcTimer.hpp"
#include "gc/shared/gcTrace.hpp"
#include "gc/shared/gcTraceTime.inline.hpp"
-#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/referenceProcessorPhaseTimes.hpp"
#include "gc/shared/space.hpp"
@@ -227,7 +226,6 @@ DefNewGeneration::DefNewGeneration(ReservedSpace rs,
const char* policy)
: Generation(rs, initial_size),
_promotion_failed(false),
- _preserved_marks_set(false /* in_c_heap */),
_promo_failure_drain_in_progress(false),
_string_dedup_requests()
{
@@ -609,8 +607,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) {
age_table()->clear();
to()->clear(SpaceDecorator::Mangle);
- // The preserved marks should be empty at the start of the GC.
- _preserved_marks_set.init(1);
YoungGenScanClosure young_gen_cl(this);
OldGenScanClosure old_gen_cl(this);
@@ -681,8 +677,6 @@ bool DefNewGeneration::collect(bool clear_all_soft_refs) {
// Reset the PromotionFailureALot counters.
NOT_PRODUCT(heap->reset_promotion_should_fail();)
}
- // We should have processed and cleared all the preserved marks.
- _preserved_marks_set.reclaim();
heap->trace_heap_after_gc(_gc_tracer);
@@ -706,19 +700,13 @@ void DefNewGeneration::remove_forwarding_pointers() {
// starts. (The mark word is overloaded: `is_marked()` == `is_forwarded()`.)
struct ResetForwardedMarkWord : ObjectClosure {
void do_object(oop obj) override {
- if (obj->is_forwarded()) {
- obj->init_mark();
+ if (obj->is_self_forwarded()) {
+ obj->unset_self_forwarded();
}
}
} cl;
eden()->object_iterate(&cl);
from()->object_iterate(&cl);
-
- restore_preserved_marks();
-}
-
-void DefNewGeneration::restore_preserved_marks() {
- _preserved_marks_set.restore(nullptr);
}
void DefNewGeneration::handle_promotion_failure(oop old) {
@@ -726,12 +714,11 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
_promotion_failed = true;
_promotion_failed_info.register_copy_failure(old->size());
- _preserved_marks_set.get()->push_if_necessary(old, old->mark());
ContinuationGCSupport::transform_stack_chunk(old);
// forward to self
- old->forward_to(old);
+ old->forward_to_self();
_promo_failure_scan_stack.push(old);
diff --git a/src/hotspot/share/gc/serial/defNewGeneration.hpp b/src/hotspot/share/gc/serial/defNewGeneration.hpp
index c5b7c095ac4e6..ee85ec5cd89f8 100644
--- a/src/hotspot/share/gc/serial/defNewGeneration.hpp
+++ b/src/hotspot/share/gc/serial/defNewGeneration.hpp
@@ -32,7 +32,6 @@
#include "gc/shared/copyFailedInfo.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/generationCounters.hpp"
-#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/stringdedup/stringDedup.hpp"
#include "gc/shared/tlab_globals.hpp"
#include "utilities/align.hpp"
@@ -99,11 +98,6 @@ class DefNewGeneration: public Generation {
// therefore we must remove their forwarding pointers.
void remove_forwarding_pointers();
- virtual void restore_preserved_marks();
-
- // Preserved marks
- PreservedMarksSet _preserved_marks_set;
-
Stack<oop, mtGC> _promo_failure_scan_stack;
void drain_promo_failure_scan_stack(void);
bool _promo_failure_drain_in_progress;
diff --git a/src/hotspot/share/oops/markWord.hpp b/src/hotspot/share/oops/markWord.hpp
index 92577a8b40b01..1de05c184389c 100644
--- a/src/hotspot/share/oops/markWord.hpp
+++ b/src/hotspot/share/oops/markWord.hpp
@@ -37,11 +37,11 @@
//
// 32 bits:
// --------
-// hash:25 ------------>| age:4 unused_gap:1 lock:2 (normal object)
+// hash:25 ------------>| age:4 self-fwd:1 lock:2 (normal object)
//
// 64 bits:
// --------
-// unused:25 hash:31 -->| unused_gap:1 age:4 unused_gap:1 lock:2 (normal object)
+// unused:25 hash:31 -->| unused_gap:1 age:4 self-fwd:1 lock:2 (normal object)
//
// - hash contains the identity hash value: largest value is
// 31 bits, see os::random(). Also, 64-bit vm's require
@@ -103,17 +103,20 @@ class markWord {
// Constants
static const int age_bits = 4;
static const int lock_bits = 2;
- static const int first_unused_gap_bits = 1;
- static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits;
+ static const int self_fwd_bits = 1;
+ static const int max_hash_bits = BitsPerWord - age_bits - lock_bits - self_fwd_bits;
static const int hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits;
- static const int second_unused_gap_bits = LP64_ONLY(1) NOT_LP64(0);
+ static const int unused_gap_bits = LP64_ONLY(1) NOT_LP64(0);
static const int lock_shift = 0;
- static const int age_shift = lock_bits + first_unused_gap_bits;
- static const int hash_shift = age_shift + age_bits + second_unused_gap_bits;
+ static const int self_fwd_shift = lock_shift + lock_bits;
+ static const int age_shift = self_fwd_shift + self_fwd_bits;
+ static const int hash_shift = age_shift + age_bits + unused_gap_bits;
static const uintptr_t lock_mask = right_n_bits(lock_bits);
static const uintptr_t lock_mask_in_place = lock_mask << lock_shift;
+ static const uintptr_t self_fwd_mask = right_n_bits(self_fwd_bits);
+ static const uintptr_t self_fwd_mask_in_place = self_fwd_mask << self_fwd_shift;
static const uintptr_t age_mask = right_n_bits(age_bits);
static const uintptr_t age_mask_in_place = age_mask << age_shift;
static const uintptr_t hash_mask = right_n_bits(hash_bits);
@@ -143,9 +146,11 @@ class markWord {
bool is_marked() const {
return (mask_bits(value(), lock_mask_in_place) == marked_value);
}
- bool is_forwarded() const {
- return (mask_bits(value(), lock_mask_in_place) == marked_value);
+ bool is_forwarded() const {
+ // Returns true for normal forwarded (0b011) and self-forwarded (0b1xx).
+ return mask_bits(value(), lock_mask_in_place | self_fwd_mask_in_place) >= static_cast<intptr_t>(marked_value);
}
+
bool is_neutral() const { // Not locked, or marked - a "clean" neutral state
return (mask_bits(value(), lock_mask_in_place) == unlocked_value);
}
@@ -273,6 +278,21 @@ class markWord {
// Recover address of oop from encoded form used in mark
inline void* decode_pointer() const { return (void*)clear_lock_bits().value(); }
+ inline bool is_self_forwarded() const {
+ NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
+ return mask_bits(value(), self_fwd_mask_in_place) != 0;
+ }
+
+ inline markWord set_self_forwarded() const {
+ NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
+ return markWord(value() | self_fwd_mask_in_place);
+ }
+
+ inline markWord unset_self_forwarded() const {
+ NOT_LP64(assert(LockingMode != LM_LEGACY, "incorrect with LM_LEGACY on 32 bit");)
+ return markWord(value() & ~self_fwd_mask_in_place);
+ }
+
inline oop forwardee() const {
return cast_to_oop(decode_pointer());
}
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index 2d827699b41fa..b84530700f68e 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -62,6 +62,8 @@ class oopDesc {
// make use of the C++ copy/assign incorrect.
NONCOPYABLE(oopDesc);
+ inline oop cas_set_forwardee(markWord new_mark, markWord old_mark, atomic_memory_order order);
+
public:
// Must be trivial; see verifying static assert after the class.
oopDesc() = default;
@@ -258,16 +260,22 @@ class oopDesc {
// Forward pointer operations for scavenge
inline bool is_forwarded() const;
+ inline bool is_self_forwarded() const;
inline void forward_to(oop p);
+ inline void forward_to_self();
// Like "forward_to", but inserts the forwarding pointer atomically.
// Exactly one thread succeeds in inserting the forwarding pointer, and
// this call returns null for that thread; any other thread has the
// value of the forwarding pointer returned and does not modify "this".
inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
+ inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
inline oop forwardee() const;
+ inline oop forwardee(markWord header) const;
+
+ inline void unset_self_forwarded();
// Age of object during scavenge
inline uint age() const;
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index ffcc9af24c328..eea6ee604019a 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -267,6 +267,10 @@ bool oopDesc::is_forwarded() const {
return mark().is_forwarded();
}
+bool oopDesc::is_self_forwarded() const {
+ return mark().is_self_forwarded();
+}
+
// Used by scavengers
void oopDesc::forward_to(oop p) {
markWord m = markWord::encode_pointer_as_mark(p);
@@ -274,14 +278,38 @@ void oopDesc::forward_to(oop p) {
set_mark(m);
}
-oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
- markWord m = markWord::encode_pointer_as_mark(p);
- assert(m.decode_pointer() == p, "encoding must be reversible");
- markWord old_mark = cas_set_mark(m, compare, order);
+void oopDesc::forward_to_self() {
+ set_mark(mark().set_self_forwarded());
+}
+
+oop oopDesc::cas_set_forwardee(markWord new_mark, markWord compare, atomic_memory_order order) {
+ markWord old_mark = cas_set_mark(new_mark, compare, order);
if (old_mark == compare) {
return nullptr;
} else {
- return cast_to_oop(old_mark.decode_pointer());
+ assert(old_mark.is_forwarded(), "must be forwarded here");
+ return forwardee(old_mark);
+ }
+}
+
+oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
+ markWord m = markWord::encode_pointer_as_mark(p);
+ assert(forwardee(m) == p, "encoding must be reversible");
+ return cas_set_forwardee(m, compare, order);
+}
+
+oop oopDesc::forward_to_self_atomic(markWord old_mark, atomic_memory_order order) {
+ markWord new_mark = old_mark.set_self_forwarded();
+ assert(forwardee(new_mark) == cast_to_oop(this), "encoding must be reversible");
+ return cas_set_forwardee(new_mark, old_mark, order);
+}
+
+oop oopDesc::forwardee(markWord mark) const {
+ assert(mark.is_forwarded(), "only decode when actually forwarded");
+ if (mark.is_self_forwarded()) {
+ return cast_to_oop(this);
+ } else {
+ return mark.forwardee();
}
}
@@ -289,7 +317,11 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
// The forwardee is used when copying during scavenge and mark-sweep.
// It does need to clear the low two locking- and GC-related bits.
oop oopDesc::forwardee() const {
- return mark().forwardee();
+ return forwardee(mark());
+}
+
+void oopDesc::unset_self_forwarded() {
+ set_mark(mark().unset_self_forwarded());
}
// The following method needs to be MT safe.
diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp
index 5f9a361105ec7..7700a44e2cd92 100644
--- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp
+++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp
@@ -29,7 +29,11 @@
// Class to create a "fake" oop with a mark that will
// return true for calls to must_be_preserved().
class FakeOop {
- oopDesc _oop;
+ // Align at least to 8 bytes, otherwise the lowest address bit
+ // could be interpreted as 'self-forwarded' when encoded as
+ // forwardee in the mark-word. This is required on 32 bit builds,
+ // where the oop could be 4-byte-aligned.
+ alignas(BytesPerLong) oopDesc _oop;
public:
FakeOop() : _oop() { _oop.set_mark(originalMark()); }
diff --git a/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java b/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java
index d37bf56738143..e407895107186 100644
--- a/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java
+++ b/test/hotspot/jtreg/gc/g1/TestGCLogMessages.java
@@ -259,7 +259,6 @@ private void testConcurrentRefinementLogs() throws Exception {
LogMessageWithLevel exhFailureMessages[] = new LogMessageWithLevel[] {
new LogMessageWithLevel("Recalculate Used Memory \\(ms\\):", Level.DEBUG),
- new LogMessageWithLevel("Restore Preserved Marks \\(ms\\):", Level.DEBUG),
new LogMessageWithLevel("Restore Evacuation Failed Regions \\(ms\\):", Level.DEBUG),
new LogMessageWithLevel("Process Evacuation Failed Regions \\(ms\\):", Level.DEBUG),
new LogMessageWithLevel("Evacuation Failed Regions:", Level.DEBUG),