Skip to content

Commit 9ae71e5

Browse files
BorysTheDevBorysTheDev
committed
feat: add TTL
1 parent a713ff9 commit 9ae71e5

File tree

2 files changed

+117
-50
lines changed

2 files changed

+117
-50
lines changed

‎src/core/intrusive_string_set.h‎

Lines changed: 80 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ namespace dfly {
2121
class ISLEntry {
2222
friend class IntrusiveStringList;
2323

24+
// we can assume that high 12 bits of user address space
25+
// can be used for tagging. At most 52 bits of address are reserved for
26+
// some configurations, and usually it's 48 bits.
27+
// https://docs.kernel.org/arch/arm64/memory.html
28+
static constexpr size_t kTtlBit = 1ULL << 55;
29+
static constexpr size_t kTagMask = 4095ULL << 52; // we reserve 12 high bits.
30+
2431
public:
2532
ISLEntry() = default;
2633

@@ -36,50 +43,95 @@ class ISLEntry {
3643
return {GetKeyData(), GetKeySize()};
3744
}
3845

46+
bool HasExpiry() const {
47+
return HasTtl();
48+
}
49+
50+
// returns the expiry time of the current entry or UINT32_MAX if no ttl is set.
51+
uint32_t ExpiryTime() const {
52+
std::uint32_t res = UINT32_MAX;
53+
if (HasTtl()) {
54+
std::memcpy(&res, Raw() + sizeof(ISLEntry*), sizeof(res));
55+
}
56+
return res;
57+
}
58+
3959
private:
40-
static ISLEntry Create(std::string_view key) {
60+
static ISLEntry Create(std::string_view key, uint32_t ttl_sec = UINT32_MAX) {
4161
char* next = nullptr;
4262
uint32_t key_size = key.size();
4363

44-
auto size = sizeof(next) + sizeof(key_size) + key_size;
64+
bool has_ttl = ttl_sec != UINT32_MAX;
65+
66+
auto size = sizeof(next) + sizeof(key_size) + key_size + has_ttl * sizeof(ttl_sec);
4567

4668
char* data = (char*)malloc(size);
69+
ISLEntry res(data);
4770

4871
std::memcpy(data, &next, sizeof(next));
4972

50-
auto* key_size_pos = data + sizeof(next);
73+
auto* ttl_pos = data + sizeof(next);
74+
if (has_ttl) {
75+
res.SetTtlBit(true);
76+
std::memcpy(ttl_pos, &ttl_sec, sizeof(ttl_sec));
77+
}
78+
79+
auto* key_size_pos = ttl_pos + res.GetTtlSize();
5180
std::memcpy(key_size_pos, &key_size, sizeof(key_size));
5281

5382
auto* key_pos = key_size_pos + sizeof(key_size);
5483
std::memcpy(key_pos, key.data(), key_size);
5584

56-
return ISLEntry(data);
85+
return res;
5786
}
5887

5988
static void Destroy(ISLEntry entry) {
60-
free(entry.data_);
89+
free(entry.Raw());
6190
}
6291

6392
ISLEntry Next() const {
6493
ISLEntry next;
65-
std::memcpy(&next.data_, data_, sizeof(next));
94+
std::memcpy(&next.data_, Raw(), sizeof(next));
6695
return next;
6796
}
6897

6998
void SetNext(ISLEntry next) {
70-
std::memcpy(data_, &next, sizeof(next));
99+
std::memcpy(Raw(), &next, sizeof(next));
71100
}
72101

73102
const char* GetKeyData() const {
74-
return data_ + sizeof(ISLEntry*) + sizeof(uint32_t);
103+
return Raw() + sizeof(ISLEntry*) + sizeof(uint32_t) + GetTtlSize();
75104
}
76105

77106
uint32_t GetKeySize() const {
78107
uint32_t size = 0;
79-
std::memcpy(&size, data_ + sizeof(ISLEntry*), sizeof(size));
108+
std::memcpy(&size, Raw() + sizeof(ISLEntry*) + GetTtlSize(), sizeof(size));
80109
return size;
81110
}
82111

112+
uint64_t uptr() const {
113+
return uint64_t(data_);
114+
}
115+
116+
char* Raw() const {
117+
return (char*)(uptr() & ~kTagMask);
118+
}
119+
120+
void SetTtlBit(bool b) {
121+
if (b)
122+
data_ = (char*)(uptr() | kTtlBit);
123+
else
124+
data_ = (char*)(uptr() & (~kTtlBit));
125+
}
126+
127+
bool HasTtl() const {
128+
return (uptr() & kTtlBit) != 0;
129+
}
130+
131+
std::uint32_t GetTtlSize() const {
132+
return HasTtl() ? sizeof(std::uint32_t) : 0;
133+
}
134+
83135
// TODO consider use SDS strings or other approach
84136
// TODO add optimization for big keys
85137
// memory daya layout [ISLEntry*, key_size, key]
@@ -117,8 +169,8 @@ class IntrusiveStringList {
117169
return res;
118170
}
119171

120-
ISLEntry Emplace(std::string_view key) {
121-
return Insert(ISLEntry::Create(key));
172+
ISLEntry Emplace(std::string_view key, uint32_t ttl_sec = UINT32_MAX) {
173+
return Insert(ISLEntry::Create(key, ttl_sec));
122174
}
123175

124176
ISLEntry Find(std::string_view str) const {
@@ -189,7 +241,7 @@ class IntrusiveStringSet {
189241
ISLEntry AddUnique(std::string_view str, IntrusiveStringList& bucket,
190242
uint32_t ttl_sec = UINT32_MAX) {
191243
++size_;
192-
return bucket.Emplace(str);
244+
return bucket.Emplace(str, ttl_sec);
193245
}
194246

195247
unsigned AddMany(absl::Span<std::string_view> span, uint32_t ttl_sec, bool keepttl) {
@@ -219,7 +271,12 @@ class IntrusiveStringSet {
219271
if (entries_.empty())
220272
return {};
221273
auto bucket_id = BucketId(Hash(member));
222-
return entries_[bucket_id].Find(member);
274+
auto res = entries_[bucket_id].Find(member);
275+
if (!res) {
276+
bucket_id = BucketId(Hash(member));
277+
res = entries_[bucket_id].Find(member);
278+
}
279+
return res;
223280
}
224281

225282
bool Contains(std::string_view member) const {
@@ -240,6 +297,15 @@ class IntrusiveStringSet {
240297
return 1 << capacity_log_;
241298
}
242299

300+
// set an abstract time that allows expiry.
301+
void set_time(uint32_t val) {
302+
time_now_ = val;
303+
}
304+
305+
uint32_t time_now() const {
306+
return time_now_;
307+
}
308+
243309
private:
244310
// was Grow in StringSet
245311
void Rehash(size_t prev_size) {
@@ -265,6 +331,7 @@ class IntrusiveStringSet {
265331
private:
266332
std::uint32_t capacity_log_ = 1;
267333
std::uint32_t size_ = 0; // number of elements in the set.
334+
std::uint32_t time_now_ = 0;
268335

269336
static_assert(sizeof(IntrusiveStringList) == sizeof(void*),
270337
"IntrusiveStringList should be just a pointer");

‎src/core/intrusive_string_set_test.cc‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -207,46 +207,46 @@ TEST_F(IntrusiveStringSetTest, DisplacedBug) {
207207
ss_->Add("HPq");
208208
}
209209

210-
// static string random_string(mt19937& rand, unsigned len) {
211-
// const string_view alpanum = "1234567890abcdefghijklmnopqrstuvwxyz";
212-
// string ret;
213-
// ret.reserve(len);
210+
static string random_string(mt19937& rand, unsigned len) {
211+
const string_view alpanum = "1234567890abcdefghijklmnopqrstuvwxyz";
212+
string ret;
213+
ret.reserve(len);
214214

215-
// for (size_t i = 0; i < len; ++i) {
216-
// ret += alpanum[rand() % alpanum.size()];
217-
// }
215+
for (size_t i = 0; i < len; ++i) {
216+
ret += alpanum[rand() % alpanum.size()];
217+
}
218218

219-
// return ret;
220-
// }
219+
return ret;
220+
}
221221

222-
// TEST_F(IntrusiveStringSetTest, Resizing) {
223-
// constexpr size_t num_strs = 4096;
224-
// unordered_set<string> strs;
225-
// while (strs.size() != num_strs) {
226-
// auto str = random_string(generator_, 10);
227-
// strs.insert(str);
228-
// }
229-
230-
// unsigned size = 0;
231-
// for (auto it = strs.begin(); it != strs.end(); ++it) {
232-
// const auto& str = *it;
233-
// EXPECT_TRUE(ss_->Add(str, 1));
234-
// EXPECT_EQ(ss_->UpperBoundSize(), size + 1);
235-
236-
// // make sure we haven't lost any items after a grow
237-
// // which happens every power of 2
238-
// if ((size & (size - 1)) == 0) {
239-
// for (auto j = strs.begin(); j != it; ++j) {
240-
// const auto& str = *j;
241-
// auto it = ss_->Find(str);
242-
// ASSERT_TRUE(it != ss_->end());
243-
// EXPECT_TRUE(it.HasExpiry());
244-
// EXPECT_EQ(it.ExpiryTime(), ss_->time_now() + 1);
245-
// }
246-
// }
247-
// ++size;
248-
// }
249-
// }
222+
TEST_F(IntrusiveStringSetTest, Resizing) {
223+
constexpr size_t num_strs = 4096;
224+
unordered_set<string> strs;
225+
while (strs.size() != num_strs) {
226+
auto str = random_string(generator_, 10);
227+
strs.insert(str);
228+
}
229+
230+
unsigned size = 0;
231+
for (auto it = strs.begin(); it != strs.end(); ++it) {
232+
const auto& str = *it;
233+
EXPECT_TRUE(ss_->Add(str, 1));
234+
EXPECT_EQ(ss_->UpperBoundSize(), size + 1);
235+
236+
// make sure we haven't lost any items after a grow
237+
// which happens every power of 2
238+
if ((size & (size - 1)) == 0) {
239+
for (auto j = strs.begin(); j != it; ++j) {
240+
const auto& str = *j;
241+
auto it = ss_->Find(str);
242+
ASSERT_TRUE(it);
243+
EXPECT_TRUE(it.HasExpiry());
244+
EXPECT_EQ(it.ExpiryTime(), ss_->time_now() + 1);
245+
}
246+
}
247+
++size;
248+
}
249+
}
250250

251251
// TEST_F(IntrusiveStringSetTest, SimpleScan) {
252252
// unordered_set<string_view> info = {"foo", "bar"};

0 commit comments

Comments
 (0)