Add extra option for forcing earlier eviction

and add a simple unit test

Author: igchor

Diff for: MultiTierDataMovement.md

@@ -103,6 +103,8 @@ and `minAcAllocationWatermark`: then extra checks (described below) are performe
 
 By default, allocation will always be performed from the upper tier.
 
+- `acTopTierEvictionWatermark`: If there is less that this percent of free memory in topmost tier, cachelib will attempt to evict from top tier. This option takes precedence before allocationWatermarks.
+
 ### Extra policies (used only when  percentage of free AllocationClasses is between `maxAcAllocationWatermark`
 and `minAcAllocationWatermark`)
 - `sizeThresholdPolicy`: If item is smaller than this value, always allocate it in upper tier.

Diff for: cachelib/allocator/CacheAllocator-inl.h

@@ -419,7 +419,19 @@ CacheAllocator<CacheTrait>::allocateInternalTier(TierId tid,
   // TODO: per-tier
   (*stats_.allocAttempts)[pid][cid].inc();
 
-  void* memory = allocator_[tid]->allocate(pid, requiredSize);
+  void *memory = nullptr;
+
+  if (tid == 0 && config_.acTopTierEvictionWatermark > 0.0
+    && getAllocationClassStats(tid, pid, cid)
+      .approxFreePercent < config_.acTopTierEvictionWatermark) {
+    memory = findEviction(tid, pid, cid);
+  } 
+  
+  if (memory == nullptr) {
+    // TODO: should we try allocate item even if this will result in violating
+    // acTopTierEvictionWatermark?
+    memory = allocator_[tid]->allocate(pid, requiredSize);
+  }
 
   if (backgroundEvictor_.size() && !fromEvictorThread && (memory == nullptr || shouldWakeupBgEvictor(tid, pid, cid))) {
     backgroundEvictor_[backgroundWorkerId(tid, pid, cid, backgroundEvictor_.size())]->wakeUp();
@@ -1580,7 +1592,7 @@ CacheAllocator<CacheTrait>::tryEvictToNextMemoryTier(
 
     if (newItemHdl) {
       XDCHECK_EQ(newItemHdl->getSize(), item.getSize());
-      if (tryMovingForSlabRelease(item, newItemHdl, itemMovingPredicate)) {
+      if (tryMovingItem(item, newItemHdl, itemMovingPredicate)) {
         return acquire(&item);
       }
     }
@@ -1612,7 +1624,7 @@ CacheAllocator<CacheTrait>::tryPromoteToNextMemoryTier(
         // if inclusive cache is allowed, replace even if there are active users.
         return config_.numDuplicateElements > 0 || item.getRefCount() == 0;
       };
-      if (tryMovingForSlabRelease(item, newItemHdl, std::move(predicate))) {
+      if (tryMovingItem(item, newItemHdl, std::move(predicate))) {
         return true;
       }
     }
@@ -2728,7 +2740,7 @@ bool CacheAllocator<CacheTrait>::moveForSlabRelease(
 
     // if we have a valid handle, try to move, if not, we retry.
     if (newItemHdl) {
-      isMoved = tryMovingForSlabRelease(oldItem, newItemHdl, itemMovingPredicate);
+      isMoved = tryMovingItem(oldItem, newItemHdl, itemMovingPredicate);
       if (isMoved) {
         break;
       }
@@ -2852,7 +2864,7 @@ CacheAllocator<CacheTrait>::allocateNewItemForOldItem(const Item& oldItem) {
 
 template <typename CacheTrait>
 template <typename Predicate>
-bool CacheAllocator<CacheTrait>::tryMovingForSlabRelease(
+bool CacheAllocator<CacheTrait>::tryMovingItem(
     Item& oldItem, ItemHandle& newItemHdl, Predicate&& predicate) {
   // By holding onto a user-level synchronization object, we ensure moving
   // a regular item or chained item is synchronized with any potential

Diff for: cachelib/allocator/CacheAllocator.h

@@ -1769,7 +1769,7 @@ class CacheAllocator : public CacheBase {
   // @return    true  if the item has been moved
   //            false if we have exhausted moving attempts
   template <typename Predicate>
-  bool tryMovingForSlabRelease(Item& item, ItemHandle& newItemHdl, Predicate &&predicate);
+  bool tryMovingItem(Item& item, ItemHandle& newItemHdl, Predicate &&predicate);
 
   // Evict an item from access and mm containers and
   // ensure it is safe for freeing.

Diff for: cachelib/allocator/CacheAllocatorConfig.h

@@ -625,6 +625,7 @@ class CacheAllocatorConfig {
   double highEvictionAcWatermark{5.0};
   double minAcAllocationWatermark{0.0};
   double maxAcAllocationWatermark{0.0};
+  double acTopTierEvictionWatermark{0.0}; // TODO: make it per TIER?
   uint64_t sizeThresholdPolicy{0};   
   double defaultTierChancePercentage{50.0};
   // TODO: default could be based on ratio

Diff for: cachelib/allocator/tests/AllocatorMemoryTiersTest.cpp

@@ -28,6 +28,7 @@ TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValid) { this->testMultiTiersValid
 TEST_F(LruAllocatorMemoryTiersTest, MultiTiersValidMixed) { this->testMultiTiersValidMixed(); }
 TEST_F(LruAllocatorMemoryTiersTest, MultiTiersForceTierAllocation) { this->testMultiTiersForceTierAllocation(); }
 TEST_F(LruAllocatorMemoryTiersTest, MultiTiersWatermarkTierAllocation) { this->testMultiTiersWatermarkAllocation(); }
+TEST_F(LruAllocatorMemoryTiersTest, MultiTiersSyncEviction) { this->testSyncEviction(); }
 
 } // end of namespace tests
 } // end of namespace cachelib

Diff for: cachelib/allocator/tests/AllocatorMemoryTiersTest.h

@@ -123,6 +123,52 @@ class AllocatorMemoryTiersTest : public AllocatorTest<AllocatorT> {
     }
   }
 
+  void testSyncEviction() {
+    auto config = makeDefaultConfig();
+    config.setCacheSize(10 * Slab::kSize);
+    config.acTopTierEvictionWatermark = 99.0;
+
+    {
+      AllocatorT alloc(AllocatorT::SharedMemNew, config);
+      auto pool = alloc.addPool("default", alloc.getCacheMemoryStats().cacheSize);
+
+      {
+        // should be allocated in upper tier.
+        auto handle = alloc.allocate(pool, "key", Slab::kSize / 2);
+        ASSERT_NE(handle, nullptr);
+        std::string data = "some data";
+        std::memcpy(handle->getMemory(), data.data(), data.size());
+
+        alloc.insertOrReplace(handle);
+
+        auto found = alloc.find("key");
+        ASSERT_NE(found, nullptr);
+        ASSERT_EQ(found->getSize(), Slab::kSize / 2);
+        ASSERT_EQ(std::string(reinterpret_cast<const char*>(found->getMemory()), data.size()), data);
+      }
+
+      auto toptier_free = alloc.getCacheMemoryStats().slabsApproxFreePercentages[0];
+      ASSERT_NE(toptier_free, 100.0);
+      ASSERT_EQ(alloc.getCacheMemoryStats().slabsApproxFreePercentages[1], 100.0);
+
+      auto handle2 = alloc.allocate(pool, "key2", Slab::kSize / 2);
+      ASSERT_NE(handle2, nullptr);
+      std::string data2 = "other data";
+      std::memcpy(reinterpret_cast<char*>(handle2->getMemory()), data2.data(), data2.size());
+
+      alloc.insertOrReplace(handle2);
+
+      auto found2 = alloc.find("key2");
+      ASSERT_NE(found2, nullptr);
+      ASSERT_EQ(found2->getSize(), Slab::kSize / 2);
+      ASSERT_EQ(std::string(reinterpret_cast<const char*>(found2->getMemory()), data2.size()), data2);
+
+      // previous data should be evicted, and replaced by new one
+      ASSERT_EQ(toptier_free, alloc.getCacheMemoryStats().slabsApproxFreePercentages[0]);
+      ASSERT_NE(alloc.getCacheMemoryStats().slabsApproxFreePercentages[1], 100.0);
+    }
+  }
+
   void testMultiTiersWatermarkAllocation() {
     auto config = makeDefaultConfig();
 

Diff for: cachelib/cachebench/cache/Cache-inl.h

@@ -292,6 +292,7 @@ Cache<Allocator>::Cache(const CacheConfig& config,
   allocatorConfig_.numDuplicateElements = config_.numDuplicateElements;
   allocatorConfig_.syncPromotion = config_.syncPromotion;
   allocatorConfig_.promotionAcWatermark = config_.promotionAcWatermark;
+  allocatorConfig_.acTopTierEvictionWatermark = config_.acTopTierEvictionWatermark;
 
   if (!allocatorConfig_.cacheDir.empty()) {
     cache_ =

Diff for: cachelib/cachebench/util/CacheConfig.cpp

@@ -105,6 +105,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, highEvictionAcWatermark);
   JSONSetVal(configJson, minAcAllocationWatermark);
   JSONSetVal(configJson, maxAcAllocationWatermark);
+  JSONSetVal(configJson, acTopTierEvictionWatermark);
   JSONSetVal(configJson, sizeThresholdPolicy);
   JSONSetVal(configJson, defaultTierChancePercentage);
   JSONSetVal(configJson, numDuplicateElements);
@@ -131,7 +132,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   // if you added new fields to the configuration, update the JSONSetVal
   // to make them available for the json configs and increment the size
   // below
-  checkCorrectSize<CacheConfig, 936>();
+  checkCorrectSize<CacheConfig, 944>();
 
   if (numPools != poolSizes.size()) {
     throw std::invalid_argument(folly::sformat(

Diff for: cachelib/cachebench/util/CacheConfig.h

@@ -298,6 +298,7 @@ struct CacheConfig : public JSONConfig {
   double highEvictionAcWatermark{5.0};
   double minAcAllocationWatermark{0.0};
   double maxAcAllocationWatermark{0.0};
+  double acTopTierEvictionWatermark{0.0};
   uint64_t sizeThresholdPolicy{0};   
   double defaultTierChancePercentage{50.0};
   // TODO: default could be based on ratio