zolfa
/
tendermint
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 221 Wiki Activity
Browse Source

Fix GoLevelDB Iterator which needs to copy a temp []byte

pull/1842/head
Jae Kwon 7 years ago
parent
93c05aa8c0
commit
b31397aff5
6 changed files with 129 additions and 52 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +14
    -1
      db/c_level_db.go
  2. +1
    -1
      db/db.go
  3. +37
    -12
      db/go_level_db.go
  4. +48
    -23
      db/mem_db.go
  5. +14
    -6
      db/types.go
  6. +15
    -9
      glide.lock

+ 14
- 1
db/c_level_db.go View File

@ -50,6 +50,7 @@ func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
return database, nil
}
// Implements DB.
func (db *CLevelDB) Get(key []byte) []byte {
key = nonNilBytes(key)
res, err := db.db.Get(db.ro, key)
@ -59,10 +60,12 @@ func (db *CLevelDB) Get(key []byte) []byte {
return res
}
// Implements DB.
func (db *CLevelDB) Has(key []byte) bool {
return db.Get(key) != nil
}
// Implements DB.
func (db *CLevelDB) Set(key []byte, value []byte) {
key = nonNilBytes(key)
value = nonNilBytes(value)
@ -72,6 +75,7 @@ func (db *CLevelDB) Set(key []byte, value []byte) {
}
}
// Implements DB.
func (db *CLevelDB) SetSync(key []byte, value []byte) {
key = nonNilBytes(key)
value = nonNilBytes(value)
@ -81,6 +85,7 @@ func (db *CLevelDB) SetSync(key []byte, value []byte) {
}
}
// Implements DB.
func (db *CLevelDB) Delete(key []byte) {
key = nonNilBytes(key)
err := db.db.Delete(db.wo, key)
@ -89,6 +94,7 @@ func (db *CLevelDB) Delete(key []byte) {
}
}
// Implements DB.
func (db *CLevelDB) DeleteSync(key []byte) {
key = nonNilBytes(key)
err := db.db.Delete(db.woSync, key)
@ -101,6 +107,7 @@ func (db *CLevelDB) DB() *levigo.DB {
return db.db
}
// Implements DB.
func (db *CLevelDB) Close() {
db.db.Close()
db.ro.Close()
@ -108,6 +115,7 @@ func (db *CLevelDB) Close() {
db.woSync.Close()
}
// Implements DB.
func (db *CLevelDB) Print() {
itr := db.Iterator(nil, nil)
defer itr.Close()
@ -118,6 +126,7 @@ func (db *CLevelDB) Print() {
}
}
// Implements DB.
func (db *CLevelDB) Stats() map[string]string {
// TODO: Find the available properties for the C LevelDB implementation
keys := []string{}
@ -133,6 +142,7 @@ func (db *CLevelDB) Stats() map[string]string {
//----------------------------------------
// Batch
// Implements DB.
func (db *CLevelDB) NewBatch() Batch {
batch := levigo.NewWriteBatch()
return &cLevelDBBatch{db, batch}
@ -143,14 +153,17 @@ type cLevelDBBatch struct {
batch *levigo.WriteBatch
}
// Implements Batch.
func (mBatch *cLevelDBBatch) Set(key, value []byte) {
mBatch.batch.Put(key, value)
}
// Implements Batch.
func (mBatch *cLevelDBBatch) Delete(key []byte) {
mBatch.batch.Delete(key)
}
// Implements Batch.
func (mBatch *cLevelDBBatch) Write() {
err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch)
if err != nil {
@ -204,7 +217,7 @@ func (itr cLevelDBIterator) Domain() ([]byte, []byte) {
}
func (itr cLevelDBIterator) Valid() bool {
// Once invalid, forever invalid.
if itr.isInvalid {
return false


+ 1
- 1
db/db.go View File

@ -2,7 +2,7 @@ package db
import "fmt"
//-----------------------------------------------------------------------------
//----------------------------------------
// Main entry
const (


+ 37
- 12
db/go_level_db.go View File

@ -39,6 +39,7 @@ func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
return database, nil
}
// Implements DB.
func (db *GoLevelDB) Get(key []byte) []byte {
key = nonNilBytes(key)
res, err := db.db.Get(key, nil)
@ -52,10 +53,12 @@ func (db *GoLevelDB) Get(key []byte) []byte {
return res
}
// Implements DB.
func (db *GoLevelDB) Has(key []byte) bool {
return db.Get(key) != nil
}
// Implements DB.
func (db *GoLevelDB) Set(key []byte, value []byte) {
key = nonNilBytes(key)
value = nonNilBytes(value)
@ -65,6 +68,7 @@ func (db *GoLevelDB) Set(key []byte, value []byte) {
}
}
// Implements DB.
func (db *GoLevelDB) SetSync(key []byte, value []byte) {
key = nonNilBytes(key)
value = nonNilBytes(value)
@ -74,6 +78,7 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) {
}
}
// Implements DB.
func (db *GoLevelDB) Delete(key []byte) {
key = nonNilBytes(key)
err := db.db.Delete(key, nil)
@ -82,6 +87,7 @@ func (db *GoLevelDB) Delete(key []byte) {
}
}
// Implements DB.
func (db *GoLevelDB) DeleteSync(key []byte) {
key = nonNilBytes(key)
err := db.db.Delete(key, &opt.WriteOptions{Sync: true})
@ -94,10 +100,12 @@ func (db *GoLevelDB) DB() *leveldb.DB {
return db.db
}
// Implements DB.
func (db *GoLevelDB) Close() {
db.db.Close()
}
// Implements DB.
func (db *GoLevelDB) Print() {
str, _ := db.db.GetProperty("leveldb.stats")
fmt.Printf("%v\n", str)
@ -110,6 +118,7 @@ func (db *GoLevelDB) Print() {
}
}
// Implements DB.
func (db *GoLevelDB) Stats() map[string]string {
keys := []string{
"leveldb.num-files-at-level{n}",
@ -135,6 +144,7 @@ func (db *GoLevelDB) Stats() map[string]string {
//----------------------------------------
// Batch
// Implements DB.
func (db *GoLevelDB) NewBatch() Batch {
batch := new(leveldb.Batch)
return &goLevelDBBatch{db, batch}
@ -145,18 +155,21 @@ type goLevelDBBatch struct {
batch *leveldb.Batch
}
// Implements Batch.
func (mBatch *goLevelDBBatch) Set(key, value []byte) {
mBatch.batch.Put(key, value)
}
// Implements Batch.
func (mBatch *goLevelDBBatch) Delete(key []byte) {
mBatch.batch.Delete(key)
}
// Implements Batch.
func (mBatch *goLevelDBBatch) Write() {
err := mBatch.db.db.Write(mBatch.batch, nil)
if err != nil {
PanicCrisis(err)
panic(err)
}
}
@ -165,6 +178,17 @@ func (mBatch *goLevelDBBatch) Write() {
// NOTE This is almost identical to db/c_level_db.Iterator
// Before creating a third version, refactor.
// Implements DB.
func (db *GoLevelDB) Iterator(start, end []byte) Iterator {
itr := db.db.NewIterator(nil, nil)
return newGoLevelDBIterator(itr, start, end, false)
}
// Implements DB.
func (db *GoLevelDB) ReverseIterator(start, end []byte) Iterator {
panic("not implemented yet") // XXX
}
type goLevelDBIterator struct {
source iterator.Iterator
start []byte
@ -189,19 +213,12 @@ func newGoLevelDBIterator(source iterator.Iterator, start, end []byte, isReverse
}
}
func (db *GoLevelDB) Iterator(start, end []byte) Iterator {
itr := db.db.NewIterator(nil, nil)
return newGoLevelDBIterator(itr, start, end, false)
}
func (db *GoLevelDB) ReverseIterator(start, end []byte) Iterator {
panic("not implemented yet") // XXX
}
// Implements Iterator.
func (itr *goLevelDBIterator) Domain() ([]byte, []byte) {
return itr.start, itr.end
}
// Implements Iterator.
func (itr *goLevelDBIterator) Valid() bool {
// Once invalid, forever invalid.
@ -230,24 +247,32 @@ func (itr *goLevelDBIterator) Valid() bool {
return true
}
// Implements Iterator.
func (itr *goLevelDBIterator) Key() []byte {
// Key returns a copy of the current key.
// See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88
itr.assertNoError()
itr.assertIsValid()
return itr.source.Key()
return cp(itr.source.Key())
}
// Implements Iterator.
func (itr *goLevelDBIterator) Value() []byte {
// Value returns a copy of the current value.
// See https://github.com/syndtr/goleveldb/blob/52c212e6c196a1404ea59592d3f1c227c9f034b2/leveldb/iterator/iter.go#L88
itr.assertNoError()
itr.assertIsValid()
return itr.source.Value()
return cp(itr.source.Value())
}
// Implements Iterator.
func (itr *goLevelDBIterator) Next() {
itr.assertNoError()
itr.assertIsValid()
itr.source.Next()
}
// Implements Iterator.
func (itr *goLevelDBIterator) Close() {
itr.source.Release()
}


+ 48
- 23
db/mem_db.go View File

@ -26,14 +26,16 @@ func NewMemDB() *MemDB {
return database
}
// Implements DB.
func (db *MemDB) Get(key []byte) []byte {
db.mtx.Lock()
defer db.mtx.Unlock()
key = nonNilBytes(key)
return db.db[string(key)]
}
// Implements DB.
func (db *MemDB) Has(key []byte) bool {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -43,6 +45,7 @@ func (db *MemDB) Has(key []byte) bool {
return ok
}
// Implements DB.
func (db *MemDB) Set(key []byte, value []byte) {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -50,6 +53,7 @@ func (db *MemDB) Set(key []byte, value []byte) {
db.SetNoLock(key, value)
}
// Implements DB.
func (db *MemDB) SetSync(key []byte, value []byte) {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -57,7 +61,7 @@ func (db *MemDB) SetSync(key []byte, value []byte) {
db.SetNoLock(key, value)
}
// NOTE: Implements atomicSetDeleter
// Implements atomicSetDeleter.
func (db *MemDB) SetNoLock(key []byte, value []byte) {
key = nonNilBytes(key)
value = nonNilBytes(value)
@ -65,6 +69,7 @@ func (db *MemDB) SetNoLock(key []byte, value []byte) {
db.db[string(key)] = value
}
// Implements DB.
func (db *MemDB) Delete(key []byte) {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -72,6 +77,7 @@ func (db *MemDB) Delete(key []byte) {
db.DeleteNoLock(key)
}
// Implements DB.
func (db *MemDB) DeleteSync(key []byte) {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -79,13 +85,14 @@ func (db *MemDB) DeleteSync(key []byte) {
db.DeleteNoLock(key)
}
// NOTE: Implements atomicSetDeleter
// Implements atomicSetDeleter.
func (db *MemDB) DeleteNoLock(key []byte) {
key = nonNilBytes(key)
delete(db.db, string(key))
}
// Implements DB.
func (db *MemDB) Close() {
// Close is a noop since for an in-memory
// database, we don't have a destination
@ -94,6 +101,7 @@ func (db *MemDB) Close() {
// See the discussion in https://github.com/tendermint/tmlibs/pull/56
}
// Implements DB.
func (db *MemDB) Print() {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -103,6 +111,7 @@ func (db *MemDB) Print() {
}
}
// Implements DB.
func (db *MemDB) Stats() map[string]string {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -113,6 +122,10 @@ func (db *MemDB) Stats() map[string]string {
return stats
}
//----------------------------------------
// Batch
// Implements DB.
func (db *MemDB) NewBatch() Batch {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -125,7 +138,9 @@ func (db *MemDB) Mutex() *sync.Mutex {
}
//----------------------------------------
// Iterator
// Implements DB.
func (db *MemDB) Iterator(start, end []byte) Iterator {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -134,6 +149,7 @@ func (db *MemDB) Iterator(start, end []byte) Iterator {
return newMemDBIterator(db, keys, start, end)
}
// Implements DB.
func (db *MemDB) ReverseIterator(start, end []byte) Iterator {
db.mtx.Lock()
defer db.mtx.Unlock()
@ -142,25 +158,6 @@ func (db *MemDB) ReverseIterator(start, end []byte) Iterator {
return newMemDBIterator(db, keys, start, end)
}
func (db *MemDB) getSortedKeys(start, end []byte, reverse bool) []string {
keys := []string{}
for key, _ := range db.db {
if IsKeyInDomain([]byte(key), start, end, false) {
keys = append(keys, key)
}
}
sort.Strings(keys)
if reverse {
nkeys := len(keys)
for i := 0; i < nkeys/2; i++ {
keys[i] = keys[nkeys-i-1]
}
}
return keys
}
var _ Iterator = (*memDBIterator)(nil)
// We need a copy of all of the keys.
// Not the best, but probably not a bottleneck depending.
type memDBIterator struct {
@ -171,6 +168,8 @@ type memDBIterator struct {
end []byte
}
var _ Iterator = (*memDBIterator)(nil)
// Keys is expected to be in reverse order for reverse iterators.
func newMemDBIterator(db DB, keys []string, start, end []byte) *memDBIterator {
return &memDBIterator{
@ -182,30 +181,36 @@ func newMemDBIterator(db DB, keys []string, start, end []byte) *memDBIterator {
}
}
// Implements Iterator.
func (itr *memDBIterator) Domain() ([]byte, []byte) {
return itr.start, itr.end
}
// Implements Iterator.
func (itr *memDBIterator) Valid() bool {
return 0 <= itr.cur && itr.cur < len(itr.keys)
}
// Implements Iterator.
func (itr *memDBIterator) Next() {
itr.assertIsValid()
itr.cur++
}
// Implements Iterator.
func (itr *memDBIterator) Key() []byte {
itr.assertIsValid()
return []byte(itr.keys[itr.cur])
}
// Implements Iterator.
func (itr *memDBIterator) Value() []byte {
itr.assertIsValid()
key := []byte(itr.keys[itr.cur])
return itr.db.Get(key)
}
// Implements Iterator.
func (itr *memDBIterator) Close() {
itr.keys = nil
itr.db = nil
@ -215,4 +220,24 @@ func (itr *memDBIterator) assertIsValid() {
if !itr.Valid() {
panic("memDBIterator is invalid")
}
}
}
//----------------------------------------
// Misc.
func (db *MemDB) getSortedKeys(start, end []byte, reverse bool) []string {
keys := []string{}
for key, _ := range db.db {
if IsKeyInDomain([]byte(key), start, end, false) {
keys = append(keys, key)
}
}
sort.Strings(keys)
if reverse {
nkeys := len(keys)
for i := 0; i < nkeys/2; i++ {
keys[i] = keys[nkeys-i-1]
}
}
return keys
}

+ 14
- 6
db/types.go View File

@ -4,19 +4,23 @@ type DB interface {
// Get returns nil iff key doesn't exist.
// A nil key is interpreted as an empty byteslice.
// CONTRACT: key, value readonly []byte
Get([]byte) []byte
// Has checks if a key exists.
// A nil key is interpreted as an empty byteslice.
// CONTRACT: key, value readonly []byte
Has(key []byte) bool
// Set sets the key.
// A nil key is interpreted as an empty byteslice.
// CONTRACT: key, value readonly []byte
Set([]byte, []byte)
SetSync([]byte, []byte)
// Delete deletes the key.
// A nil key is interpreted as an empty byteslice.
// CONTRACT: key readonly []byte
Delete([]byte)
DeleteSync([]byte)
@ -25,6 +29,7 @@ type DB interface {
// A nil start is interpreted as an empty byteslice.
// If end is nil, iterates up to the last item (inclusive).
// CONTRACT: No writes may happen within a domain while an iterator exists over it.
// CONTRACT: start, end readonly []byte
Iterator(start, end []byte) Iterator
// Iterate over a domain of keys in descending order. End is exclusive.
@ -32,6 +37,7 @@ type DB interface {
// If start is nil, iterates from the last/greatest item (inclusive).
// If end is nil, iterates up to the first/least item (iclusive).
// CONTRACT: No writes may happen within a domain while an iterator exists over it.
// CONTRACT: start, end readonly []byte
ReverseIterator(start, end []byte) Iterator
// Closes the connection.
@ -56,11 +62,12 @@ type Batch interface {
}
type SetDeleter interface {
Set(key, value []byte)
Delete(key []byte)
Set(key, value []byte) // CONTRACT: key, value readonly []byte
Delete(key []byte) // CONTRACT: key readonly []byte
}
//----------------------------------------
// Iterator
/*
Usage:
@ -83,6 +90,7 @@ type Iterator interface {
//
// The smallest key is the empty byte array []byte{} - see BeginningKey().
// The largest key is the nil byte array []byte(nil) - see EndingKey().
// CONTRACT: start, end readonly []byte
Domain() (start []byte, end []byte)
// Valid returns whether the current position is valid.
@ -96,14 +104,14 @@ type Iterator interface {
Next()
// Key returns the key of the cursor.
//
// If Valid returns false, this method will panic.
Key() []byte
// CONTRACT: key readonly []byte
Key() (key []byte)
// Value returns the value of the cursor.
//
// If Valid returns false, this method will panic.
Value() []byte
// CONTRACT: value readonly []byte
Value() (value []byte)
// Close releases the Iterator.
Close()


+ 15
- 9
glide.lock View File

@ -1,10 +1,10 @@
hash: 325b2f9c7e84696f88fa88126a22eb1e1e91c2be5f60402d17bfaad6713b33c2
updated: 2017-12-25T17:45:52.357002873-08:00
updated: 2017-12-28T18:27:21.247160207-08:00
imports:
- name: github.com/fsnotify/fsnotify
version: 4da3e2cfbabc9f751898f250b49f2439785783a1
- name: github.com/go-kit/kit
version: e3b2152e0063c5f05efea89ecbe297852af2a92d
version: e2b298466b32c7cd5579a9b9b07e968fc9d9452c
subpackages:
- log
- log/level
@ -12,7 +12,13 @@ imports:
- name: github.com/go-logfmt/logfmt
version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
- name: github.com/go-stack/stack
version: 259ab82a6cad3992b4e21ff5cac294ccb06474bc
version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf
- name: github.com/gogo/protobuf
version: 342cbe0a04158f6dcb03ca0079991a51a4248c02
subpackages:
- gogoproto
- proto
- protoc-gen-gogo/descriptor
- name: github.com/golang/snappy
version: 553a641470496b2327abcac10b36396bd98e45c9
- name: github.com/hashicorp/hcl
@ -45,7 +51,7 @@ imports:
- name: github.com/pelletier/go-toml
version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a
- name: github.com/pkg/errors
version: f15c970de5b76fac0b59abb32d62c17cc7bed265
version: 645ef00459ed84a119197bfb8d8205042c6df63d
- name: github.com/spf13/afero
version: 5660eeed305fe5f69c8fc6cf899132a459a97064
subpackages:
@ -57,11 +63,11 @@ imports:
- name: github.com/spf13/jwalterweatherman
version: 12bd96e66386c1960ab0f74ced1362f66f552f7b
- name: github.com/spf13/pflag
version: 4c012f6dcd9546820e378d0bdda4d8fc772cdfea
version: 97afa5e7ca8a08a383cb259e06636b5e2cc7897f
- name: github.com/spf13/viper
version: 8ef37cbca71638bf32f3d5e194117d4cb46da163
- name: github.com/syndtr/goleveldb
version: adf24ef3f94bd13ec4163060b21a5678f22b429b
version: b89cc31ef7977104127d34c1bd31ebd1a9db2199
subpackages:
- leveldb
- leveldb/cache
@ -76,7 +82,7 @@ imports:
- leveldb/table
- leveldb/util
- name: github.com/tendermint/go-wire
version: 5ab49b4c6ad674da6b81442911cf713ef0afb544
version: 27be46e25124ddf775e23317a83647ce62a93f6b
subpackages:
- data
- data/base58
@ -85,7 +91,7 @@ imports:
subpackages:
- term
- name: golang.org/x/crypto
version: 94eea52f7b742c7cbe0b03b22f0c4c8631ece122
version: edd5e9b0879d13ee6970a50153d85b8fec9f7686
subpackages:
- ripemd160
- name: golang.org/x/sys
@ -93,7 +99,7 @@ imports:
subpackages:
- unix
- name: golang.org/x/text
version: 75cc3cad82b5f47d3fb229ddda8c5167da14f294
version: c01e4764d870b77f8abe5096ee19ad20d80e8075
subpackages:
- transform
- unicode/norm


Write Preview
Loading…
Cancel
Save