8340490: Shenandoah: Optimize ShenandoahPacer

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -693,7 +693,7 @@ void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) {
   if (ShenandoahPacing) {
     control_thread()->pacing_notify_alloc(words);
     if (waste) {
-      pacer()->claim_for_alloc(words, true);
+      pacer()->claim_for_alloc<true>(words);
     }
   }
 }

src/hotspot/share/gc/shenandoah/shenandoahPacer.cpp

@@ -188,8 +188,8 @@ void ShenandoahPacer::restart_with(size_t non_taxable_bytes, double tax_rate) {
   // Shake up stalled waiters after budget update.
   _need_notify_waiters.try_set();
 }
-
-bool ShenandoahPacer::claim_for_alloc(size_t words, bool force) {
+template<bool FORCE>
+bool ShenandoahPacer::claim_for_alloc(size_t words) {
   assert(ShenandoahPacing, "Only be here when pacing is enabled");
 
   intptr_t tax = MAX2<intptr_t>(1, words * Atomic::load(&_tax_rate));
@@ -198,15 +198,19 @@ bool ShenandoahPacer::claim_for_alloc(size_t words, bool force) {
   intptr_t new_val = 0;
   do {
     cur = Atomic::load(&_budget);
-    if (cur < tax && !force) {
+    if (cur < tax && !FORCE) {
       // Progress depleted, alas.
       return false;
     }
     new_val = cur - tax;
-  } while (Atomic::cmpxchg(&_budget, cur, new_val, memory_order_relaxed) != cur);
+  } while (Atomic::load(&_budget) == cur &&
+           Atomic::cmpxchg(&_budget, cur, new_val, memory_order_relaxed) != cur);
   return true;
 }
 
+template bool ShenandoahPacer::claim_for_alloc<true>(size_t words);
+template bool ShenandoahPacer::claim_for_alloc<false>(size_t words);
+
 void ShenandoahPacer::unpace_for_alloc(intptr_t epoch, size_t words) {
   assert(ShenandoahPacing, "Only be here when pacing is enabled");
 
@@ -227,18 +231,11 @@ void ShenandoahPacer::pace_for_alloc(size_t words) {
   assert(ShenandoahPacing, "Only be here when pacing is enabled");
 
   // Fast path: try to allocate right away
-  bool claimed = claim_for_alloc(words, false);
+  bool claimed = claim_for_alloc<false>(words);
   if (claimed) {
     return;
   }
 
-  // Forcefully claim the budget: it may go negative at this point, and
-  // GC should replenish for this and subsequent allocations. After this claim,
-  // we would wait a bit until our claim is matched by additional progress,
-  // or the time budget depletes.
-  claimed = claim_for_alloc(words, true);
-  assert(claimed, "Should always succeed");
-
   // Threads that are attaching should not block at all: they are not
   // fully initialized yet. Blocking them would be awkward.
   // This is probably the path that allocates the thread oop itself.
@@ -249,32 +246,30 @@ void ShenandoahPacer::pace_for_alloc(size_t words) {
   JavaThread* current = JavaThread::current();
   if (current->is_attaching_via_jni() ||
       !current->is_active_Java_thread()) {
+    claim_for_alloc<true>(words);
     return;
   }
 
-  double start = os::elapsedTime();
-
-  size_t max_ms = ShenandoahPacingMaxDelay;
-  size_t total_ms = 0;
+  double const max_delay = static_cast<double>(ShenandoahPacingMaxDelay) / 1000;
+  double total_delay = 0;
 
-  while (true) {
+  double start = os::elapsedTime();
+  while (!claimed) {
     // We could instead assist GC, but this would suffice for now.
-    size_t cur_ms = (max_ms > total_ms) ? (max_ms - total_ms) : 1;
-    wait(cur_ms);
-
-    double end = os::elapsedTime();
-    total_ms = (size_t)((end - start) * 1000);
-
-    if (total_ms > max_ms || Atomic::load(&_budget) >= 0) {
-      // Exiting if either:
-      //  a) Spent local time budget to wait for enough GC progress.
-      //     Breaking out and allocating anyway, which may mean we outpace GC,
-      //     and start Degenerated GC cycle.
-      //  b) The budget had been replenished, which means our claim is satisfied.
-      ShenandoahThreadLocalData::add_paced_time(JavaThread::current(), end - start);
-      break;
+    wait(1);
+    total_delay = os::elapsedTime() - start;
+    if (total_delay > max_delay) {
+      // Exiting if spent local time budget to wait for enough GC progress.
+      // Breaking out and allocating anyway, which may mean we outpace GC,
+      // and start Degenerated GC cycle.
+      claim_for_alloc<true>(words);
+      ShenandoahThreadLocalData::add_paced_time(JavaThread::current(), total_delay);
+      return;
     }
+    claimed = claim_for_alloc<false>(words);
   }
+  assert(claimed, "Should always succeed");
+  ShenandoahThreadLocalData::add_paced_time(JavaThread::current(), total_delay);
 }
 
 void ShenandoahPacer::wait(size_t time_ms) {

src/hotspot/share/gc/shenandoah/shenandoahPacer.hpp

@@ -106,8 +106,9 @@ class ShenandoahPacer : public CHeapObj<mtGC> {
   inline void report_updaterefs(size_t words);
 
   inline void report_alloc(size_t words);
+  template<bool FORCE>
+  bool claim_for_alloc(size_t words);
 
-  bool claim_for_alloc(size_t words, bool force);
   void pace_for_alloc(size_t words);
   void unpace_for_alloc(intptr_t epoch, size_t words);