8346971: [ubsan] psCardTable.cpp:131:24: runtime error: large index is out of bounds

Reviewed-by: ayang, tschatzl

Author: Kim Barrett

src/hotspot/share/gc/parallel/psCardTable.cpp

@@ -122,13 +122,38 @@ class PSStripeShadowCardTable {
   const uint _card_shift;
   const uint _card_size;
   CardValue _table[PSCardTable::num_cards_in_stripe];
-  const CardValue* _table_base;
+  uintptr_t _table_base;
+
+  // Avoid UB pointer operations by using integers internally.
+
+  static_assert(sizeof(uintptr_t) == sizeof(CardValue*), "simplifying assumption");
+  static_assert(sizeof(CardValue) == 1, "simplifying assumption");
+
+  static uintptr_t iaddr(const void* p) {
+    return reinterpret_cast<uintptr_t>(p);
+  }
+
+  uintptr_t compute_table_base(HeapWord* start) const {
+    uintptr_t offset = iaddr(start) >> _card_shift;
+    return iaddr(_table) - offset;
+  }
+
+  void verify_card_inclusive(const CardValue* card) const {
+    assert(iaddr(card) >= iaddr(_table), "out of bounds");
+    assert(iaddr(card) <= (iaddr(_table) + sizeof(_table)), "out of bounds");
+  }
+
+  void verify_card_exclusive(const CardValue* card) const {
+    assert(iaddr(card) >= iaddr(_table), "out of bounds");
+    assert(iaddr(card) < (iaddr(_table) + sizeof(_table)), "out of bounds");
+  }
 
 public:
   PSStripeShadowCardTable(PSCardTable* pst, HeapWord* const start, HeapWord* const end) :
     _card_shift(CardTable::card_shift()),
     _card_size(CardTable::card_size()),
-    _table_base(_table - (uintptr_t(start) >> _card_shift)) {
+    _table_base(compute_table_base(start))
+  {
     size_t stripe_byte_size = pointer_delta(end, start) * HeapWordSize;
     size_t copy_length = align_up(stripe_byte_size, _card_size) >> _card_shift;
     // The end of the last stripe may not be card aligned as it is equal to old
@@ -143,20 +168,24 @@ class PSStripeShadowCardTable {
   }
 
   HeapWord* addr_for(const CardValue* const card) {
-    assert(card >= _table && card <=  &_table[PSCardTable::num_cards_in_stripe], "out of bounds");
-    return (HeapWord*) ((card - _table_base) << _card_shift);
+    verify_card_inclusive(card);
+    uintptr_t addr = (iaddr(card) - _table_base) << _card_shift;
+    return reinterpret_cast<HeapWord*>(addr);
   }
 
   const CardValue* card_for(HeapWord* addr) {
-    return &_table_base[uintptr_t(addr) >> _card_shift];
+    uintptr_t icard = _table_base + (iaddr(addr) >> _card_shift);
+    const CardValue* card = reinterpret_cast<const CardValue*>(icard);
+    verify_card_inclusive(card);
+    return card;
   }
 
   bool is_dirty(const CardValue* const card) {
     return !is_clean(card);
   }
 
   bool is_clean(const CardValue* const card) {
-    assert(card >= _table && card <  &_table[PSCardTable::num_cards_in_stripe], "out of bounds");
+    verify_card_exclusive(card);
     return *card == PSCardTable::clean_card_val();
   }