8305898: Alternative self-forwarding mechanism

src/hotspot/share/gc/g1/g1OopClosures.inline.hpp

@@ -228,7 +228,7 @@ void G1ParCopyClosure<barrier, should_mark>::do_oop_work(T* p) {
     oop forwardee;
     markWord m = obj->mark();
     if (m.is_marked()) {
-      forwardee = cast_to_oop(m.decode_pointer());
+      forwardee = obj->forwardee();
     } else {
       forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
     }

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -212,7 +212,7 @@ void G1ParScanThreadState::do_oop_evac(T* p) {
 
   markWord m = obj->mark();
   if (m.is_marked()) {
-    obj = cast_to_oop(m.decode_pointer());
+    obj = obj->forwardee();
   } else {
     obj = do_copy_to_survivor_space(region_attr, obj, m);
   }
@@ -627,7 +627,7 @@ NOINLINE
 oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markWord m, size_t word_sz) {
   assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));
 
-  oop forward_ptr = old->forward_to_atomic(old, m, memory_order_relaxed);
+  oop forward_ptr = old->forward_to_self_atomic(m, memory_order_relaxed);
   if (forward_ptr == nullptr) {
     // Forward-to-self succeeded. We are the "owner" of the object.
     HeapRegion* r = _g1h->heap_region_containing(old);

src/hotspot/share/gc/parallel/psPromotionManager.cpp

@@ -325,7 +325,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markWord obj_mark) {
   // this started.  If it is the same (i.e., no forwarding
   // pointer has been installed), then this thread owns
   // it.
-  if (obj->forward_to_atomic(obj, obj_mark) == nullptr) {
+  if (obj->forward_to_self_atomic(obj_mark) == nullptr) {
     // We won any races, we "own" this object.
     assert(obj == obj->forwardee(), "Sanity");
 

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -152,7 +152,7 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
     // other thread.
     OrderAccess::acquire();
     // Return the already installed forwardee.
-    return cast_to_oop(m.decode_pointer());
+    return o->forwardee(m);
   }
 }
 

src/hotspot/share/gc/serial/defNewGeneration.cpp

@@ -900,7 +900,7 @@ void DefNewGeneration::handle_promotion_failure(oop old) {
   ContinuationGCSupport::transform_stack_chunk(old);
 
   // forward to self
-  old->forward_to(old);
+  old->forward_to_self();
 
   _promo_failure_scan_stack.push(old);
 

src/hotspot/share/oops/markWord.hpp

@@ -37,11 +37,11 @@
 //
 //  32 bits:
 //  --------
-//             hash:25 ------------>| age:4  unused_gap:1  lock:2 (normal object)
+//             hash:25 ------------>| age:4  self-fwd:1  lock:2 (normal object)
 //
 //  64 bits:
 //  --------
-//  unused:25 hash:31 -->| unused_gap:1  age:4  unused_gap:1  lock:2 (normal object)
+//  unused:25 hash:31 -->| unused_gap:1  age:4  self-fwd:1  lock:2 (normal object)
 //
 //  - hash contains the identity hash value: largest value is
 //    31 bits, see os::random().  Also, 64-bit vm's require
@@ -103,17 +103,20 @@ class markWord {
   // Constants
   static const int age_bits                       = 4;
   static const int lock_bits                      = 2;
-  static const int first_unused_gap_bits          = 1;
-  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - first_unused_gap_bits;
+  static const int self_fwd_bits                  = 1;
+  static const int max_hash_bits                  = BitsPerWord - age_bits - lock_bits - self_fwd_bits;
   static const int hash_bits                      = max_hash_bits > 31 ? 31 : max_hash_bits;
-  static const int second_unused_gap_bits         = LP64_ONLY(1) NOT_LP64(0);
+  static const int unused_gap_bits                = LP64_ONLY(1) NOT_LP64(0);
 
   static const int lock_shift                     = 0;
-  static const int age_shift                      = lock_bits + first_unused_gap_bits;
-  static const int hash_shift                     = age_shift + age_bits + second_unused_gap_bits;
+  static const int self_fwd_shift                 = lock_shift + lock_bits;
+  static const int age_shift                      = self_fwd_shift + self_fwd_bits;
+  static const int hash_shift                     = age_shift + age_bits + unused_gap_bits;
 
   static const uintptr_t lock_mask                = right_n_bits(lock_bits);
   static const uintptr_t lock_mask_in_place       = lock_mask << lock_shift;
+  static const uintptr_t self_fwd_mask            = right_n_bits(self_fwd_bits);
+  static const uintptr_t self_fwd_mask_in_place   = self_fwd_mask << self_fwd_shift;
   static const uintptr_t age_mask                 = right_n_bits(age_bits);
   static const uintptr_t age_mask_in_place        = age_mask << age_shift;
   static const uintptr_t hash_mask                = right_n_bits(hash_bits);
@@ -143,6 +146,9 @@ class markWord {
   bool is_marked()   const {
     return (mask_bits(value(), lock_mask_in_place) == marked_value);
   }
+  bool is_forwarded() const {
+    return is_marked();
+  }
   bool is_neutral()  const {
     return (mask_bits(value(), lock_mask_in_place) == unlocked_value);
   }
@@ -260,6 +266,14 @@ class markWord {
 
   // Recover address of oop from encoded form used in mark
   inline void* decode_pointer() { return (void*)clear_lock_bits().value(); }
+
+  inline bool self_forwarded() const {
+    return mask_bits(value(), self_fwd_mask_in_place) != 0;
+  }
+
+  inline markWord set_self_forwarded() const {
+    return markWord(value() | self_fwd_mask_in_place | marked_value);
+  }
 };
 
 // Support atomic operations.

src/hotspot/share/oops/oop.hpp

@@ -64,7 +64,9 @@ class oopDesc {
   // make use of the C++ copy/assign incorrect.
   NONCOPYABLE(oopDesc);
 
- public:
+  inline oop cas_set_forwardee(markWord new_mark, markWord old_mark, atomic_memory_order order);
+
+public:
   // Must be trivial; see verifying static assert after the class.
   oopDesc() = default;
 
@@ -260,14 +262,17 @@ class oopDesc {
   inline bool is_forwarded() const;
 
   inline void forward_to(oop p);
+  inline void forward_to_self();
 
   // Like "forward_to", but inserts the forwarding pointer atomically.
   // Exactly one thread succeeds in inserting the forwarding pointer, and
   // this call returns null for that thread; any other thread has the
   // value of the forwarding pointer returned and does not modify "this".
   inline oop forward_to_atomic(oop p, markWord compare, atomic_memory_order order = memory_order_conservative);
+  inline oop forward_to_self_atomic(markWord compare, atomic_memory_order order = memory_order_conservative);
 
   inline oop forwardee() const;
+  inline oop forwardee(markWord header) const;
 
   // Age of object during scavenge
   inline uint age() const;

src/hotspot/share/oops/oop.inline.hpp

@@ -263,9 +263,7 @@ bool oopDesc::is_gc_marked() const {
 
 // Used by scavengers
 bool oopDesc::is_forwarded() const {
-  // The extra heap check is needed since the obj might be locked, in which case the
-  // mark would point to a stack location and have the sentinel bit cleared
-  return mark().is_marked();
+  return mark().is_forwarded();
 }
 
 // Used by scavengers
@@ -275,14 +273,38 @@ void oopDesc::forward_to(oop p) {
   set_mark(m);
 }
 
-oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
-  markWord m = markWord::encode_pointer_as_mark(p);
-  assert(m.decode_pointer() == p, "encoding must be reversible");
-  markWord old_mark = cas_set_mark(m, compare, order);
+void oopDesc::forward_to_self() {
+  set_mark(markWord::prototype().set_self_forwarded());
+}
+
+oop oopDesc::cas_set_forwardee(markWord new_mark, markWord compare, atomic_memory_order order) {
+  markWord old_mark = cas_set_mark(new_mark, compare, order);
   if (old_mark == compare) {
     return nullptr;
   } else {
-    return cast_to_oop(old_mark.decode_pointer());
+    assert(old_mark.is_forwarded(), "must be forwarded here");
+    return forwardee(old_mark);
+  }
+}
+
+oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order order) {
+  markWord m = markWord::encode_pointer_as_mark(p);
+  assert(forwardee(m) == p, "encoding must be reversible");
+  return cas_set_forwardee(m, compare, order);
+}
+
+oop oopDesc::forward_to_self_atomic(markWord compare, atomic_memory_order order) {
+  markWord new_mark = markWord::prototype().set_self_forwarded();
+  assert(forwardee(new_mark) == cast_to_oop(this), "encoding must be reversible");
+  return cas_set_forwardee(new_mark, compare, order);
+}
+
+oop oopDesc::forwardee(markWord mark) const {
+  assert(mark.is_forwarded(), "only decode when actually forwarded");
+  if (mark.self_forwarded()) {
+    return cast_to_oop(this);
+  } else {
+    return cast_to_oop(mark.decode_pointer());
   }
 }
 
@@ -291,7 +313,7 @@ oop oopDesc::forward_to_atomic(oop p, markWord compare, atomic_memory_order orde
 // It does need to clear the low two locking- and GC-related bits.
 oop oopDesc::forwardee() const {
   assert(is_forwarded(), "only decode when actually forwarded");
-  return cast_to_oop(mark().decode_pointer());
+  return forwardee(mark());
 }
 
 // The following method needs to be MT safe.

test/hotspot/gtest/gc/shared/test_preservedMarks.cpp

@@ -29,7 +29,10 @@
 // Class to create a "fake" oop with a mark that will
 // return true for calls to must_be_preserved().
 class FakeOop {
-  oopDesc _oop;
+  // Align at least to 8 bytes, otherwise the lowest address bit
+  // could be interpreted as 'self-forwarded' when encoded as
+  // forwardee in the mark-word.
+  alignas(BytesPerLong) oopDesc _oop;
 
 public:
   FakeOop() : _oop() { _oop.set_mark(originalMark()); }