Skip to content

Commit 9db1a44

Browse files
alesforzcason
alesforz
and
cason
authoredAug 9, 2024··
Reinstate BoltDB and ClevelDB as backend DBs (#177)
* Revert "remove deprecated boltdb and cleveldb (#155)" This reverts commit badc0b8. We decided to reinstate boltDB and clevelDB and mark them as deprecated until a future version of CometBFT in which we'll drop cometbft-db and support only 1 backend DB. * updated cleveldb Iterator API docs to conform to the changes made in #168 * updated go.mod * updated boltDB Iterator APIs to conform to the changes made in #168 * added changelog entry * Formatting Co-authored-by: Daniel <[email protected]> * formatting to please the linter * added additional context in the docs of backend types constants --------- Co-authored-by: Daniel <[email protected]>
·
v1.0.4v0.14.0
1 parent a79d349 commit 9db1a44

15 files changed

+1054
-3
lines changed
 

‎.changelog/unreleased/dependencies/177-reinstate-boltdb-cleveldb.md

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
- reinstate BoltDB and ClevelDB as backend DBs
2+
([\#177](https://github.com/cometbft/cometbft-db/pull/177))

‎Makefile

+12-2
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,21 @@ test:
1919
@go test $(PACKAGES) -v
2020
.PHONY: test
2121

22+
test-cleveldb:
23+
@echo "--> Running go test"
24+
@go test $(PACKAGES) -tags cleveldb -v
25+
.PHONY: test-cleveldb
26+
2227
test-rocksdb:
2328
@echo "--> Running go test"
2429
@go test $(PACKAGES) -tags rocksdb -v
2530
.PHONY: test-rocksdb
2631

32+
test-boltdb:
33+
@echo "--> Running go test"
34+
@go test $(PACKAGES) -tags boltdb -v
35+
.PHONY: test-boltdb
36+
2737
test-badgerdb:
2838
@echo "--> Running go test"
2939
@go test $(PACKAGES) -tags badgerdb -v
@@ -35,7 +45,7 @@ test-pebble:
3545

3646
test-all:
3747
@echo "--> Running go test"
38-
@go test $(PACKAGES) -tags rocksdb,grocksdb_clean_link,badgerdb,pebbledb -v
48+
@go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb -v
3949
.PHONY: test-all
4050

4151
test-all-with-coverage:
@@ -46,7 +56,7 @@ test-all-with-coverage:
4656
-race \
4757
-coverprofile=coverage.txt \
4858
-covermode=atomic \
49-
-tags=memdb,goleveldb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb \
59+
-tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb \
5060
-v
5161
.PHONY: test-all-with-coverage
5262

‎README.md

+13
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,19 @@ Go 1.22+
3333
sets, and tests. Used for [IAVL](https://github.com/tendermint/iavl) working
3434
sets when the pruning strategy allows it.
3535

36+
- **[LevelDB](https://github.com/google/leveldb) [DEPRECATED]:** A [Go
37+
wrapper](https://github.com/jmhodges/levigo) around
38+
[LevelDB](https://github.com/google/leveldb). Uses LSM-trees for on-disk
39+
storage, which have good performance for write-heavy workloads, particularly
40+
on spinning disks, but requires periodic compaction to maintain decent read
41+
performance and reclaim disk space. Does not support transactions.
42+
43+
- **[BoltDB](https://github.com/etcd-io/bbolt) [DEPRECATED]:** A
44+
[fork](https://github.com/etcd-io/bbolt) of
45+
[BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage,
46+
which have good performance for read-heavy workloads and range scans. Supports
47+
serializable ACID transactions.
48+
3649
- **[RocksDB](https://github.com/linxGnu/grocksdb) [experimental]:** A [Go
3750
wrapper](https://github.com/linxGnu/grocksdb) around
3851
[RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees

‎boltdb.go

+213
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
//go:build boltdb
2+
// +build boltdb
3+
4+
package db
5+
6+
import (
7+
"errors"
8+
"fmt"
9+
"os"
10+
"path/filepath"
11+
12+
"go.etcd.io/bbolt"
13+
)
14+
15+
var bucket = []byte("tm")
16+
17+
func init() {
18+
registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) {
19+
return NewBoltDB(name, dir)
20+
})
21+
}
22+
23+
// BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt).
24+
//
25+
// NOTE: All operations (including Set, Delete) are synchronous by default. One
26+
// can globally turn it off by using NoSync config option (not recommended).
27+
//
28+
// A single bucket ([]byte("tm")) is used per a database instance. This could
29+
// lead to performance issues when/if there will be lots of keys.
30+
type BoltDB struct {
31+
db *bbolt.DB
32+
}
33+
34+
var _ DB = (*BoltDB)(nil)
35+
36+
// NewBoltDB returns a BoltDB with default options.
37+
//
38+
// Deprecated: boltdb is deprecated and will be removed in the future.
39+
func NewBoltDB(name, dir string) (DB, error) {
40+
return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions)
41+
}
42+
43+
// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not
44+
// supported because NewBoltDBWithOpts creates a global bucket.
45+
func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) {
46+
if opts.ReadOnly {
47+
return nil, errors.New("ReadOnly: true is not supported")
48+
}
49+
50+
dbPath := filepath.Join(dir, name+".db")
51+
db, err := bbolt.Open(dbPath, os.ModePerm, opts)
52+
if err != nil {
53+
return nil, err
54+
}
55+
56+
// create a global bucket
57+
err = db.Update(func(tx *bbolt.Tx) error {
58+
_, err := tx.CreateBucketIfNotExists(bucket)
59+
return err
60+
})
61+
if err != nil {
62+
return nil, err
63+
}
64+
65+
return &BoltDB{db: db}, nil
66+
}
67+
68+
// Get implements DB.
69+
func (bdb *BoltDB) Get(key []byte) (value []byte, err error) {
70+
if len(key) == 0 {
71+
return nil, errKeyEmpty
72+
}
73+
err = bdb.db.View(func(tx *bbolt.Tx) error {
74+
b := tx.Bucket(bucket)
75+
if v := b.Get(key); v != nil {
76+
value = append([]byte{}, v...)
77+
}
78+
return nil
79+
})
80+
if err != nil {
81+
return nil, err
82+
}
83+
return
84+
}
85+
86+
// Has implements DB.
87+
func (bdb *BoltDB) Has(key []byte) (bool, error) {
88+
bytes, err := bdb.Get(key)
89+
if err != nil {
90+
return false, err
91+
}
92+
return bytes != nil, nil
93+
}
94+
95+
// Set implements DB.
96+
func (bdb *BoltDB) Set(key, value []byte) error {
97+
if len(key) == 0 {
98+
return errKeyEmpty
99+
}
100+
if value == nil {
101+
return errValueNil
102+
}
103+
err := bdb.db.Update(func(tx *bbolt.Tx) error {
104+
b := tx.Bucket(bucket)
105+
return b.Put(key, value)
106+
})
107+
if err != nil {
108+
return err
109+
}
110+
return nil
111+
}
112+
113+
// SetSync implements DB.
114+
func (bdb *BoltDB) SetSync(key, value []byte) error {
115+
return bdb.Set(key, value)
116+
}
117+
118+
// Delete implements DB.
119+
func (bdb *BoltDB) Delete(key []byte) error {
120+
if len(key) == 0 {
121+
return errKeyEmpty
122+
}
123+
err := bdb.db.Update(func(tx *bbolt.Tx) error {
124+
return tx.Bucket(bucket).Delete(key)
125+
})
126+
if err != nil {
127+
return err
128+
}
129+
return nil
130+
}
131+
132+
// DeleteSync implements DB.
133+
func (bdb *BoltDB) DeleteSync(key []byte) error {
134+
return bdb.Delete(key)
135+
}
136+
137+
// Close implements DB.
138+
func (bdb *BoltDB) Close() error {
139+
return bdb.db.Close()
140+
}
141+
142+
// Print implements DB.
143+
func (bdb *BoltDB) Print() error {
144+
stats := bdb.db.Stats()
145+
fmt.Printf("%v\n", stats)
146+
147+
err := bdb.db.View(func(tx *bbolt.Tx) error {
148+
tx.Bucket(bucket).ForEach(func(k, v []byte) error {
149+
fmt.Printf("[%X]:\t[%X]\n", k, v)
150+
return nil
151+
})
152+
return nil
153+
})
154+
if err != nil {
155+
return err
156+
}
157+
return nil
158+
}
159+
160+
// Stats implements DB.
161+
func (bdb *BoltDB) Stats() map[string]string {
162+
stats := bdb.db.Stats()
163+
m := make(map[string]string)
164+
165+
// Freelist stats
166+
m["FreePageN"] = fmt.Sprintf("%v", stats.FreePageN)
167+
m["PendingPageN"] = fmt.Sprintf("%v", stats.PendingPageN)
168+
m["FreeAlloc"] = fmt.Sprintf("%v", stats.FreeAlloc)
169+
m["FreelistInuse"] = fmt.Sprintf("%v", stats.FreelistInuse)
170+
171+
// Transaction stats
172+
m["TxN"] = fmt.Sprintf("%v", stats.TxN)
173+
m["OpenTxN"] = fmt.Sprintf("%v", stats.OpenTxN)
174+
175+
return m
176+
}
177+
178+
// NewBatch implements DB.
179+
func (bdb *BoltDB) NewBatch() Batch {
180+
return newBoltDBBatch(bdb)
181+
}
182+
183+
// WARNING: Any concurrent writes or reads will block until the iterator is
184+
// closed.
185+
func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) {
186+
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
187+
return nil, errKeyEmpty
188+
}
189+
tx, err := bdb.db.Begin(false)
190+
if err != nil {
191+
return nil, err
192+
}
193+
return newBoltDBIterator(tx, start, end, false), nil
194+
}
195+
196+
// WARNING: Any concurrent writes or reads will block until the iterator is
197+
// closed.
198+
func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) {
199+
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
200+
return nil, errKeyEmpty
201+
}
202+
tx, err := bdb.db.Begin(false)
203+
if err != nil {
204+
return nil, err
205+
}
206+
return newBoltDBIterator(tx, start, end, true), nil
207+
}
208+
209+
func (bdb *BoltDB) Compact(start, end []byte) error {
210+
// There is no explicit CompactRange support in BoltDB, only a function that copies the
211+
// entire DB from one place to another while doing deletions. Hence we do not support it.
212+
return nil
213+
}

‎boltdb_batch.go

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
//go:build boltdb
2+
// +build boltdb
3+
4+
package db
5+
6+
import "go.etcd.io/bbolt"
7+
8+
// boltDBBatch stores operations internally and dumps them to BoltDB on Write().
9+
type boltDBBatch struct {
10+
db *BoltDB
11+
ops []operation
12+
}
13+
14+
var _ Batch = (*boltDBBatch)(nil)
15+
16+
func newBoltDBBatch(db *BoltDB) *boltDBBatch {
17+
return &boltDBBatch{
18+
db: db,
19+
ops: []operation{},
20+
}
21+
}
22+
23+
// Set implements Batch.
24+
func (b *boltDBBatch) Set(key, value []byte) error {
25+
if len(key) == 0 {
26+
return errKeyEmpty
27+
}
28+
if value == nil {
29+
return errValueNil
30+
}
31+
if b.ops == nil {
32+
return errBatchClosed
33+
}
34+
b.ops = append(b.ops, operation{opTypeSet, key, value})
35+
return nil
36+
}
37+
38+
// Delete implements Batch.
39+
func (b *boltDBBatch) Delete(key []byte) error {
40+
if len(key) == 0 {
41+
return errKeyEmpty
42+
}
43+
if b.ops == nil {
44+
return errBatchClosed
45+
}
46+
b.ops = append(b.ops, operation{opTypeDelete, key, nil})
47+
return nil
48+
}
49+
50+
// Write implements Batch.
51+
func (b *boltDBBatch) Write() error {
52+
if b.ops == nil {
53+
return errBatchClosed
54+
}
55+
err := b.db.db.Batch(func(tx *bbolt.Tx) error {
56+
bkt := tx.Bucket(bucket)
57+
for _, op := range b.ops {
58+
switch op.opType {
59+
case opTypeSet:
60+
if err := bkt.Put(op.key, op.value); err != nil {
61+
return err
62+
}
63+
case opTypeDelete:
64+
if err := bkt.Delete(op.key); err != nil {
65+
return err
66+
}
67+
}
68+
}
69+
return nil
70+
})
71+
if err != nil {
72+
return err
73+
}
74+
// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
75+
return b.Close()
76+
}
77+
78+
// WriteSync implements Batch.
79+
func (b *boltDBBatch) WriteSync() error {
80+
return b.Write()
81+
}
82+
83+
// Close implements Batch.
84+
func (b *boltDBBatch) Close() error {
85+
b.ops = nil
86+
return nil
87+
}

‎boltdb_iterator.go

+142
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
//go:build boltdb
2+
// +build boltdb
3+
4+
package db
5+
6+
import (
7+
"bytes"
8+
9+
"go.etcd.io/bbolt"
10+
)
11+
12+
// boltDBIterator allows you to iterate on range of keys/values given some
13+
// start / end keys (nil & nil will result in doing full scan).
14+
type boltDBIterator struct {
15+
tx *bbolt.Tx
16+
17+
itr *bbolt.Cursor
18+
start []byte
19+
end []byte
20+
21+
currentKey []byte
22+
currentValue []byte
23+
24+
isInvalid bool
25+
isReverse bool
26+
}
27+
28+
var _ Iterator = (*boltDBIterator)(nil)
29+
30+
// newBoltDBIterator creates a new boltDBIterator.
31+
func newBoltDBIterator(tx *bbolt.Tx, start, end []byte, isReverse bool) *boltDBIterator {
32+
itr := tx.Bucket(bucket).Cursor()
33+
34+
var ck, cv []byte
35+
if isReverse {
36+
switch {
37+
case end == nil:
38+
ck, cv = itr.Last()
39+
default:
40+
_, _ = itr.Seek(end) // after key
41+
ck, cv = itr.Prev() // return to end key
42+
}
43+
} else {
44+
switch {
45+
case start == nil:
46+
ck, cv = itr.First()
47+
default:
48+
ck, cv = itr.Seek(start)
49+
}
50+
}
51+
52+
return &boltDBIterator{
53+
tx: tx,
54+
itr: itr,
55+
start: start,
56+
end: end,
57+
currentKey: ck,
58+
currentValue: cv,
59+
isReverse: isReverse,
60+
isInvalid: false,
61+
}
62+
}
63+
64+
// Domain implements Iterator.
65+
func (itr *boltDBIterator) Domain() ([]byte, []byte) {
66+
return itr.start, itr.end
67+
}
68+
69+
// Valid implements Iterator.
70+
func (itr *boltDBIterator) Valid() bool {
71+
if itr.isInvalid {
72+
return false
73+
}
74+
75+
if itr.Error() != nil {
76+
itr.isInvalid = true
77+
return false
78+
}
79+
80+
// iterated to the end of the cursor
81+
if itr.currentKey == nil {
82+
itr.isInvalid = true
83+
return false
84+
}
85+
86+
if itr.isReverse {
87+
if itr.start != nil && bytes.Compare(itr.currentKey, itr.start) < 0 {
88+
itr.isInvalid = true
89+
return false
90+
}
91+
} else {
92+
if itr.end != nil && bytes.Compare(itr.end, itr.currentKey) <= 0 {
93+
itr.isInvalid = true
94+
return false
95+
}
96+
}
97+
98+
// Valid
99+
return true
100+
}
101+
102+
// Next implements Iterator.
103+
func (itr *boltDBIterator) Next() {
104+
itr.assertIsValid()
105+
if itr.isReverse {
106+
itr.currentKey, itr.currentValue = itr.itr.Prev()
107+
} else {
108+
itr.currentKey, itr.currentValue = itr.itr.Next()
109+
}
110+
}
111+
112+
// Key implements Iterator.
113+
// The caller should not modify the contents of the returned slice.
114+
// Instead, the caller should make a copy and work on the copy.
115+
func (itr *boltDBIterator) Key() []byte {
116+
itr.assertIsValid()
117+
return itr.currentKey
118+
}
119+
120+
// Value implements Iterator.
121+
// The caller should not modify the contents of the returned slice.
122+
// Instead, the caller should make a copy and work on the copy.
123+
func (itr *boltDBIterator) Value() []byte {
124+
itr.assertIsValid()
125+
return itr.currentValue
126+
}
127+
128+
// Error implements Iterator.
129+
func (itr *boltDBIterator) Error() error {
130+
return nil
131+
}
132+
133+
// Close implements Iterator.
134+
func (itr *boltDBIterator) Close() error {
135+
return itr.tx.Rollback()
136+
}
137+
138+
func (itr *boltDBIterator) assertIsValid() {
139+
if !itr.Valid() {
140+
panic("iterator is invalid")
141+
}
142+
}

‎boltdb_test.go

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
//go:build boltdb
2+
// +build boltdb
3+
4+
package db
5+
6+
import (
7+
"fmt"
8+
"os"
9+
"testing"
10+
11+
"github.com/stretchr/testify/require"
12+
)
13+
14+
func TestBoltDBNewBoltDB(t *testing.T) {
15+
name := fmt.Sprintf("test_%x", randStr(12))
16+
dir := os.TempDir()
17+
defer cleanupDBDir(dir, name)
18+
19+
db, err := NewBoltDB(name, dir)
20+
require.NoError(t, err)
21+
db.Close()
22+
}
23+
24+
func BenchmarkBoltDBRandomReadsWrites(b *testing.B) {
25+
name := fmt.Sprintf("test_%x", randStr(12))
26+
db, err := NewBoltDB(name, "")
27+
if err != nil {
28+
b.Fatal(err)
29+
}
30+
defer func() {
31+
db.Close()
32+
cleanupDBDir("", name)
33+
}()
34+
35+
benchmarkRandomReadsWrites(b, db)
36+
}

‎cleveldb.go

+205
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
//go:build cleveldb
2+
// +build cleveldb
3+
4+
package db
5+
6+
import (
7+
"fmt"
8+
"path/filepath"
9+
10+
"github.com/jmhodges/levigo"
11+
)
12+
13+
func init() {
14+
dbCreator := func(name string, dir string) (DB, error) {
15+
return NewCLevelDB(name, dir)
16+
}
17+
registerDBCreator(CLevelDBBackend, dbCreator)
18+
}
19+
20+
// CLevelDB uses the C LevelDB database via a Go wrapper.
21+
type CLevelDB struct {
22+
db *levigo.DB
23+
ro *levigo.ReadOptions
24+
wo *levigo.WriteOptions
25+
woSync *levigo.WriteOptions
26+
}
27+
28+
var _ DB = (*CLevelDB)(nil)
29+
30+
// NewCLevelDB creates a new CLevelDB.
31+
//
32+
// Deprecated: cleveldb is deprecated and will be removed in the future.
33+
func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
34+
dbPath := filepath.Join(dir, name+".db")
35+
36+
opts := levigo.NewOptions()
37+
opts.SetCache(levigo.NewLRUCache(1 << 30))
38+
opts.SetCreateIfMissing(true)
39+
db, err := levigo.Open(dbPath, opts)
40+
if err != nil {
41+
return nil, err
42+
}
43+
ro := levigo.NewReadOptions()
44+
wo := levigo.NewWriteOptions()
45+
woSync := levigo.NewWriteOptions()
46+
woSync.SetSync(true)
47+
database := &CLevelDB{
48+
db: db,
49+
ro: ro,
50+
wo: wo,
51+
woSync: woSync,
52+
}
53+
return database, nil
54+
}
55+
56+
// Get implements DB.
57+
func (db *CLevelDB) Get(key []byte) ([]byte, error) {
58+
if len(key) == 0 {
59+
return nil, errKeyEmpty
60+
}
61+
res, err := db.db.Get(db.ro, key)
62+
if err != nil {
63+
return nil, err
64+
}
65+
return res, nil
66+
}
67+
68+
// Has implements DB.
69+
func (db *CLevelDB) Has(key []byte) (bool, error) {
70+
bytes, err := db.Get(key)
71+
if err != nil {
72+
return false, err
73+
}
74+
return bytes != nil, nil
75+
}
76+
77+
// Set implements DB.
78+
func (db *CLevelDB) Set(key []byte, value []byte) error {
79+
if len(key) == 0 {
80+
return errKeyEmpty
81+
}
82+
if value == nil {
83+
return errValueNil
84+
}
85+
if err := db.db.Put(db.wo, key, value); err != nil {
86+
return err
87+
}
88+
return nil
89+
}
90+
91+
// SetSync implements DB.
92+
func (db *CLevelDB) SetSync(key []byte, value []byte) error {
93+
if len(key) == 0 {
94+
return errKeyEmpty
95+
}
96+
if value == nil {
97+
return errValueNil
98+
}
99+
if err := db.db.Put(db.woSync, key, value); err != nil {
100+
return err
101+
}
102+
return nil
103+
}
104+
105+
// Delete implements DB.
106+
func (db *CLevelDB) Delete(key []byte) error {
107+
if len(key) == 0 {
108+
return errKeyEmpty
109+
}
110+
if err := db.db.Delete(db.wo, key); err != nil {
111+
return err
112+
}
113+
return nil
114+
}
115+
116+
// DeleteSync implements DB.
117+
func (db *CLevelDB) DeleteSync(key []byte) error {
118+
if len(key) == 0 {
119+
return errKeyEmpty
120+
}
121+
if err := db.db.Delete(db.woSync, key); err != nil {
122+
return err
123+
}
124+
return nil
125+
}
126+
127+
// Compact implements DB and compacts the given range of the DB
128+
func (db *CLevelDB) Compact(start, end []byte) error {
129+
// CompactRange of clevelDB does not return anything
130+
db.db.CompactRange(levigo.Range{Start: start, Limit: end})
131+
return nil
132+
}
133+
134+
// FIXME This should not be exposed
135+
func (db *CLevelDB) DB() *levigo.DB {
136+
return db.db
137+
}
138+
139+
// Close implements DB.
140+
func (db *CLevelDB) Close() error {
141+
db.db.Close()
142+
db.ro.Close()
143+
db.wo.Close()
144+
db.woSync.Close()
145+
return nil
146+
}
147+
148+
// Print implements DB.
149+
func (db *CLevelDB) Print() error {
150+
itr, err := db.Iterator(nil, nil)
151+
if err != nil {
152+
return err
153+
}
154+
defer itr.Close()
155+
for ; itr.Valid(); itr.Next() {
156+
key := itr.Key()
157+
value := itr.Value()
158+
fmt.Printf("[%X]:\t[%X]\n", key, value)
159+
}
160+
return nil
161+
}
162+
163+
// Stats implements DB.
164+
func (db *CLevelDB) Stats() map[string]string {
165+
keys := []string{
166+
"leveldb.aliveiters",
167+
"leveldb.alivesnaps",
168+
"leveldb.blockpool",
169+
"leveldb.cachedblock",
170+
"leveldb.num-files-at-level{n}",
171+
"leveldb.openedtables",
172+
"leveldb.sstables",
173+
"leveldb.stats",
174+
}
175+
176+
stats := make(map[string]string, len(keys))
177+
for _, key := range keys {
178+
str := db.db.PropertyValue(key)
179+
stats[key] = str
180+
}
181+
return stats
182+
}
183+
184+
// NewBatch implements DB.
185+
func (db *CLevelDB) NewBatch() Batch {
186+
return newCLevelDBBatch(db)
187+
}
188+
189+
// Iterator implements DB.
190+
func (db *CLevelDB) Iterator(start, end []byte) (Iterator, error) {
191+
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
192+
return nil, errKeyEmpty
193+
}
194+
itr := db.db.NewIterator(db.ro)
195+
return newCLevelDBIterator(itr, start, end, false), nil
196+
}
197+
198+
// ReverseIterator implements DB.
199+
func (db *CLevelDB) ReverseIterator(start, end []byte) (Iterator, error) {
200+
if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) {
201+
return nil, errKeyEmpty
202+
}
203+
itr := db.db.NewIterator(db.ro)
204+
return newCLevelDBIterator(itr, start, end, true), nil
205+
}

‎cleveldb_batch.go

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
//go:build cleveldb
2+
// +build cleveldb
3+
4+
package db
5+
6+
import "github.com/jmhodges/levigo"
7+
8+
// cLevelDBBatch is a LevelDB batch.
9+
type cLevelDBBatch struct {
10+
db *CLevelDB
11+
batch *levigo.WriteBatch
12+
}
13+
14+
func newCLevelDBBatch(db *CLevelDB) *cLevelDBBatch {
15+
return &cLevelDBBatch{
16+
db: db,
17+
batch: levigo.NewWriteBatch(),
18+
}
19+
}
20+
21+
// Set implements Batch.
22+
func (b *cLevelDBBatch) Set(key, value []byte) error {
23+
if len(key) == 0 {
24+
return errKeyEmpty
25+
}
26+
if value == nil {
27+
return errValueNil
28+
}
29+
if b.batch == nil {
30+
return errBatchClosed
31+
}
32+
b.batch.Put(key, value)
33+
return nil
34+
}
35+
36+
// Delete implements Batch.
37+
func (b *cLevelDBBatch) Delete(key []byte) error {
38+
if len(key) == 0 {
39+
return errKeyEmpty
40+
}
41+
if b.batch == nil {
42+
return errBatchClosed
43+
}
44+
b.batch.Delete(key)
45+
return nil
46+
}
47+
48+
// Write implements Batch.
49+
func (b *cLevelDBBatch) Write() error {
50+
if b.batch == nil {
51+
return errBatchClosed
52+
}
53+
err := b.db.db.Write(b.db.wo, b.batch)
54+
if err != nil {
55+
return err
56+
}
57+
// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
58+
return b.Close()
59+
}
60+
61+
// WriteSync implements Batch.
62+
func (b *cLevelDBBatch) WriteSync() error {
63+
if b.batch == nil {
64+
return errBatchClosed
65+
}
66+
err := b.db.db.Write(b.db.woSync, b.batch)
67+
if err != nil {
68+
return err
69+
}
70+
// Make sure batch cannot be used afterwards. Callers should still call Close(), for errors.
71+
b.Close()
72+
return nil
73+
}
74+
75+
// Close implements Batch.
76+
func (b *cLevelDBBatch) Close() error {
77+
if b.batch != nil {
78+
b.batch.Close()
79+
b.batch = nil
80+
}
81+
return nil
82+
}

‎cleveldb_iterator.go

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
//go:build cleveldb
2+
// +build cleveldb
3+
4+
package db
5+
6+
import (
7+
"bytes"
8+
9+
"github.com/jmhodges/levigo"
10+
)
11+
12+
// cLevelDBIterator is a cLevelDB iterator.
13+
type cLevelDBIterator struct {
14+
source *levigo.Iterator
15+
start, end []byte
16+
isReverse bool
17+
isInvalid bool
18+
}
19+
20+
var _ Iterator = (*cLevelDBIterator)(nil)
21+
22+
func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator {
23+
if isReverse {
24+
if end == nil || len(end) == 0 {
25+
source.SeekToLast()
26+
} else {
27+
source.Seek(end)
28+
if source.Valid() {
29+
eoakey := source.Key() // end or after key
30+
if bytes.Compare(end, eoakey) <= 0 {
31+
source.Prev()
32+
}
33+
} else {
34+
source.SeekToLast()
35+
}
36+
}
37+
} else {
38+
if start == nil || len(start) == 0 {
39+
source.SeekToFirst()
40+
} else {
41+
source.Seek(start)
42+
}
43+
}
44+
return &cLevelDBIterator{
45+
source: source,
46+
start: start,
47+
end: end,
48+
isReverse: isReverse,
49+
isInvalid: false,
50+
}
51+
}
52+
53+
// Domain implements Iterator.
54+
func (itr cLevelDBIterator) Domain() ([]byte, []byte) {
55+
return itr.start, itr.end
56+
}
57+
58+
// Valid implements Iterator.
59+
func (itr cLevelDBIterator) Valid() bool {
60+
// Once invalid, forever invalid.
61+
if itr.isInvalid {
62+
return false
63+
}
64+
65+
// If source errors, invalid.
66+
if itr.source.GetError() != nil {
67+
itr.isInvalid = true
68+
return false
69+
}
70+
71+
// If source is invalid, invalid.
72+
if !itr.source.Valid() {
73+
itr.isInvalid = true
74+
return false
75+
}
76+
77+
// If key is end or past it, invalid.
78+
start := itr.start
79+
end := itr.end
80+
key := itr.source.Key()
81+
if itr.isReverse {
82+
if start != nil && bytes.Compare(key, start) < 0 {
83+
itr.isInvalid = true
84+
return false
85+
}
86+
} else {
87+
if end != nil && bytes.Compare(end, key) <= 0 {
88+
itr.isInvalid = true
89+
return false
90+
}
91+
}
92+
93+
// It's valid.
94+
return true
95+
}
96+
97+
// Key implements Iterator.
98+
// The caller should not modify the contents of the returned slice.
99+
// Instead, the caller should make a copy and work on the copy.
100+
func (itr cLevelDBIterator) Key() []byte {
101+
itr.assertIsValid()
102+
return itr.source.Key()
103+
}
104+
105+
// Value implements Iterator.
106+
// The caller should not modify the contents of the returned slice.
107+
// Instead, the caller should make a copy and work on the copy.
108+
func (itr cLevelDBIterator) Value() []byte {
109+
itr.assertIsValid()
110+
return itr.source.Value()
111+
}
112+
113+
// Next implements Iterator.
114+
func (itr cLevelDBIterator) Next() {
115+
itr.assertIsValid()
116+
if itr.isReverse {
117+
itr.source.Prev()
118+
} else {
119+
itr.source.Next()
120+
}
121+
}
122+
123+
// Error implements Iterator.
124+
func (itr cLevelDBIterator) Error() error {
125+
return itr.source.GetError()
126+
}
127+
128+
// Close implements Iterator.
129+
func (itr cLevelDBIterator) Close() error {
130+
itr.source.Close()
131+
return nil
132+
}
133+
134+
func (itr cLevelDBIterator) assertIsValid() {
135+
if !itr.Valid() {
136+
panic("iterator is invalid")
137+
}
138+
}

‎cleveldb_test.go

+101
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
//go:build cleveldb
2+
// +build cleveldb
3+
4+
package db
5+
6+
import (
7+
"bytes"
8+
"fmt"
9+
"math/rand"
10+
"os"
11+
"testing"
12+
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
func BenchmarkRandomReadsWrites2(b *testing.B) {
18+
b.StopTimer()
19+
20+
numItems := int64(1000000)
21+
internal := map[int64]int64{}
22+
for i := 0; i < int(numItems); i++ {
23+
internal[int64(i)] = int64(0)
24+
}
25+
db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "")
26+
if err != nil {
27+
b.Fatal(err.Error())
28+
return
29+
}
30+
31+
b.StartTimer()
32+
33+
for i := 0; i < b.N; i++ {
34+
// Write something
35+
{
36+
idx := (int64(rand.Int()) % numItems)
37+
internal[idx]++
38+
val := internal[idx]
39+
idxBytes := int642Bytes(int64(idx))
40+
valBytes := int642Bytes(int64(val))
41+
db.Set(
42+
idxBytes,
43+
valBytes,
44+
)
45+
}
46+
// Read something
47+
{
48+
idx := (int64(rand.Int()) % numItems)
49+
val := internal[idx]
50+
idxBytes := int642Bytes(int64(idx))
51+
valBytes, err := db.Get(idxBytes)
52+
if err != nil {
53+
b.Error(err)
54+
}
55+
if val == 0 {
56+
if !bytes.Equal(valBytes, nil) {
57+
b.Errorf("Expected %v for %v, got %X",
58+
nil, idx, valBytes)
59+
break
60+
}
61+
} else {
62+
if len(valBytes) != 8 {
63+
b.Errorf("Expected length 8 for %v, got %X",
64+
idx, valBytes)
65+
break
66+
}
67+
valGot := bytes2Int64(valBytes)
68+
if val != valGot {
69+
b.Errorf("Expected %v for %v, got %v",
70+
val, idx, valGot)
71+
break
72+
}
73+
}
74+
}
75+
}
76+
77+
db.Close()
78+
}
79+
80+
func TestCLevelDBBackend(t *testing.T) {
81+
name := fmt.Sprintf("test_%x", randStr(12))
82+
// Can't use "" (current directory) or "./" here because levigo.Open returns:
83+
// "Error initializing DB: IO error: test_XXX.db: Invalid argument"
84+
dir := os.TempDir()
85+
db, err := NewDB(name, CLevelDBBackend, dir)
86+
require.NoError(t, err)
87+
defer cleanupDBDir(dir, name)
88+
89+
_, ok := db.(*CLevelDB)
90+
assert.True(t, ok)
91+
}
92+
93+
func TestCLevelDBStats(t *testing.T) {
94+
name := fmt.Sprintf("test_%x", randStr(12))
95+
dir := os.TempDir()
96+
db, err := NewDB(name, CLevelDBBackend, dir)
97+
require.NoError(t, err)
98+
defer cleanupDBDir(dir, name)
99+
100+
assert.NotEmpty(t, db.Stats())
101+
}

‎db.go

+12-1
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,22 @@ const (
1313
// popular implementation)
1414
// - pure go
1515
// - stable
16-
// - unmaintained
16+
// - unmaintaned
1717
GoLevelDBBackend BackendType = "goleveldb"
18+
// CLevelDBBackend represents cleveldb (uses levigo wrapper)
19+
// - fast
20+
// - requires gcc
21+
// - use cleveldb build tag (go build -tags cleveldb)
22+
CLevelDBBackend BackendType = "cleveldb"
1823
// MemDBBackend represents in-memory key value store, which is mostly used
1924
// for testing.
2025
MemDBBackend BackendType = "memdb"
26+
// BoltDBBackend represents bolt (uses etcd's fork of bolt -
27+
// github.com/etcd-io/bbolt)
28+
// - EXPERIMENTAL
29+
// - may be faster is some use-cases (random reads - indexer)
30+
// - use boltdb build tag (go build -tags boltdb)
31+
BoltDBBackend BackendType = "boltdb"
2132
// RocksDBBackend represents rocksdb (uses github.com/tecbot/gorocksdb)
2233
// - EXPERIMENTAL
2334
// - requires gcc

‎go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ require (
66
github.com/cockroachdb/pebble v1.1.1
77
github.com/dgraph-io/badger/v4 v4.2.0
88
github.com/google/btree v1.1.2
9+
github.com/jmhodges/levigo v1.0.0
910
github.com/linxGnu/grocksdb v1.8.14
1011
github.com/stretchr/testify v1.9.0
1112
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
13+
go.etcd.io/bbolt v1.3.10
1214
)
1315

1416
require (

‎go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
173173
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
174174
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
175175
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
176+
github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
177+
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
176178
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
177179
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
178180
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -268,6 +270,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
268270
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
269271
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
270272
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
273+
go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0=
274+
go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ=
271275
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
272276
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
273277
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

‎util_test.go

+5
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ func TestPrefixIteratorNoMatchNil(t *testing.T) {
2525
// Empty iterator for db populated after iterator created.
2626
func TestPrefixIteratorNoMatch1(t *testing.T) {
2727
for backend := range backends {
28+
if backend == BoltDBBackend {
29+
t.Log("bolt does not support concurrent writes while iterating")
30+
continue
31+
}
32+
2833
t.Run(fmt.Sprintf("Prefix w/ backend %s", backend), func(t *testing.T) {
2934
db, dir := newTempDB(t, backend)
3035
defer os.RemoveAll(dir)

0 commit comments

Comments
 (0)
Please sign in to comment.