Skip to content

Fix unaccessed RI key data lost after second checkpoint purge #1756

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
tiagonapoli wants to merge 3 commits into
base: dev
Choose a base branch
from
+169 −0
Draft

Fix unaccessed RI key data lost after second checkpoint purge #1756

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d5e7d4
Add failing test: unaccessed RI key lost after second checkpoint
Apr 29, 2026
57e0607
Add file-level assertions to unaccessed key checkpoint test
Apr 30, 2026
21ae868
Add file assertions and second-recovery test for unaccessed RI key purge
Apr 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 169 additions & 0 deletions test/Garnet.test/RespRangeIndexTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@
// Licensed under the MIT license.

using System;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using Allure.NUnit;
using Garnet.server;
using NUnit.Framework;
using NUnit.Framework.Legacy;
using StackExchange.Redis;
Expand Down Expand Up @@ -1954,6 +1956,173 @@ public void RIConcurrentOpsWithCheckpointTest()
}
}

/// <summary>
/// After checkpoint recovery, if an RI key is never accessed before a second
/// checkpoint, <c>PurgeOldCheckpointSnapshots</c> deletes the only snapshot file.
/// The tree was never restored (not in <c>liveIndexes</c>), so no new snapshot
/// was created. Accessing the key after the purge should still return data.
/// </summary>
[Test]
public void RIUnaccessedKeyAfterRecoveryAndSecondCheckpointTest()
{
server.Dispose();
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true);
server.Start();

var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager;
var rangeIndexDir = Path.Combine(TestUtils.MethodTestDir, "checkpoints", "rangeindex");
var idx1Dir = Path.Combine(rangeIndexDir, RangeIndexManager.HashKeyToDirectoryName("idx1"u8));
var idx2Dir = Path.Combine(rangeIndexDir, RangeIndexManager.HashKeyToDirectoryName("idx2"u8));

using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)))
{
var db = redis.GetDatabase(0);

// Create two RI keys with data
db.Execute("RI.CREATE", "idx1", "DISK", "CACHESIZE", "65536", "MINRECORD", "8");
db.Execute("RI.SET", "idx1", "key-a", "val-a");

db.Execute("RI.CREATE", "idx2", "DISK", "CACHESIZE", "65536", "MINRECORD", "8");
db.Execute("RI.SET", "idx2", "key-b", "val-b");

// First checkpoint — both trees get snapshot files
db.Execute("SAVE");
}

// Both key directories should have data.bftree + snapshot file
ClassicAssert.IsTrue(Directory.Exists(idx1Dir), "idx1 directory should exist after first checkpoint");
ClassicAssert.IsTrue(Directory.Exists(idx2Dir), "idx2 directory should exist after first checkpoint");
ClassicAssert.IsTrue(File.Exists(Path.Combine(idx1Dir, "data.bftree")), "idx1 data.bftree should exist");
ClassicAssert.IsTrue(File.Exists(Path.Combine(idx2Dir, "data.bftree")), "idx2 data.bftree should exist");
var idx1Snapshots = Directory.GetFiles(idx1Dir, "snapshot.*.bftree");
var idx2Snapshots = Directory.GetFiles(idx2Dir, "snapshot.*.bftree");
ClassicAssert.AreEqual(1, idx1Snapshots.Length, "idx1 should have 1 snapshot after first checkpoint");
ClassicAssert.AreEqual(1, idx2Snapshots.Length, "idx2 should have 1 snapshot after first checkpoint");

// Recover — both stubs get FlagRecovered=true, TreeHandle=0
server.Dispose();
server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true);
server.Start();

rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager;
ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "No trees should be live right after recovery");

using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)))
{
var db = redis.GetDatabase(0);

// Access idx1 only — this restores it and registers in liveIndexes
var val = db.Execute("RI.GET", "idx1", "key-a");
ClassicAssert.AreEqual("val-a", (string)val);
ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Only idx1 should be live after restore");

// Do NOT access idx2 — it stays unrestored (TreeHandle=0, FlagRecovered=true)

// Second checkpoint: idx1 gets a new snapshot, idx2 does not.
// PurgeOldCheckpointSnapshots deletes the old checkpoint snapshot files.
db.Execute("SAVE");

// idx1: old snapshot purged, new snapshot created
idx1Snapshots = Directory.GetFiles(idx1Dir, "snapshot.*.bftree");
ClassicAssert.AreEqual(1, idx1Snapshots.Length, "idx1 should have 1 snapshot after second checkpoint (old purged, new created)");

// idx2: old snapshot purged, NO new snapshot created (tree was never restored)
idx2Snapshots = Directory.GetFiles(idx2Dir, "snapshot.*.bftree");
ClassicAssert.AreEqual(0, idx2Snapshots.Length,
"idx2 old snapshot should be purged and no new snapshot created (tree was never restored)");

// idx2 data.bftree should still exist (working file from initial creation)
ClassicAssert.IsTrue(File.Exists(Path.Combine(idx2Dir, "data.bftree")),
"idx2 data.bftree should still exist (not deleted by purge)");

// Now access idx2 — its old checkpoint snapshot was purged,
// and no flush.bftree was ever written.
val = db.Execute("RI.GET", "idx2", "key-b");
ClassicAssert.AreEqual("val-b", (string)val,
"Unaccessed RI key should still be readable after second checkpoint purges old snapshots");
}
}

/// <summary>
/// After checkpoint + recovery, accessing only one RI key and taking a second
/// checkpoint causes <c>PurgeOldCheckpointSnapshots</c> to delete the unaccessed
/// key's only snapshot (no new one was created because it was never restored).
/// A second recovery then loses the unaccessed key entirely.
/// </summary>
[Test]
public void RIUnaccessedKeyLostAfterSecondRecoveryTest()
{
server.Dispose();
TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true);
server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true);
server.Start();

var rangeIndexDir = Path.Combine(TestUtils.MethodTestDir, "checkpoints", "rangeindex");
var idx1Dir = Path.Combine(rangeIndexDir, RangeIndexManager.HashKeyToDirectoryName("idx1"u8));
var idx2Dir = Path.Combine(rangeIndexDir, RangeIndexManager.HashKeyToDirectoryName("idx2"u8));

// Step 1-3: Create two RI keys with data, then checkpoint
using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)))
{
var db = redis.GetDatabase(0);

db.Execute("RI.CREATE", "idx1", "DISK", "CACHESIZE", "65536", "MINRECORD", "8");
db.Execute("RI.SET", "idx1", "key-a", "val-a");

db.Execute("RI.CREATE", "idx2", "DISK", "CACHESIZE", "65536", "MINRECORD", "8");
db.Execute("RI.SET", "idx2", "key-b", "val-b");

db.Execute("SAVE");
}

// Both keys have snapshots
ClassicAssert.AreEqual(1, Directory.GetFiles(idx1Dir, "snapshot.*.bftree").Length, "idx1 should have 1 snapshot");
ClassicAssert.AreEqual(1, Directory.GetFiles(idx2Dir, "snapshot.*.bftree").Length, "idx2 should have 1 snapshot");

// Step 4: Restart and recover from checkpoint
server.Dispose();
server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true);
server.Start();

// Step 5: Access only idx1, then take a second checkpoint
using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)))
{
var db = redis.GetDatabase(0);

var val = db.Execute("RI.GET", "idx1", "key-a");
ClassicAssert.AreEqual("val-a", (string)val);

// Do NOT access idx2

db.Execute("SAVE");
}

// idx1 has a new snapshot; idx2's old snapshot was purged with no replacement
ClassicAssert.AreEqual(1, Directory.GetFiles(idx1Dir, "snapshot.*.bftree").Length,
"idx1 should have 1 snapshot after second checkpoint");
ClassicAssert.AreEqual(0, Directory.GetFiles(idx2Dir, "snapshot.*.bftree").Length,
"idx2 snapshot should have been purged with no replacement");

// Step 6: Second restart and recover
server.Dispose();
server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true);
server.Start();

// Step 7: idx1 should exist, idx2 should be lost
using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()))
{
var db = redis.GetDatabase(0);

var val = db.Execute("RI.GET", "idx1", "key-a");
ClassicAssert.AreEqual("val-a", (string)val, "idx1 should survive second recovery");

val = db.Execute("RI.GET", "idx2", "key-b");
ClassicAssert.AreEqual("val-b", (string)val,
"idx2 should survive second recovery (unaccessed key must not be lost by snapshot purge)");
}
}

/// <summary>
/// Verifies pure AOF-only recovery (no checkpoint). RI.CREATE is replayed to
/// recreate the BfTree, then RI.SET/RI.DEL operations rebuild the data.
Expand Down