diff --git a/c_level_db.go b/c_level_db.go new file mode 100644 index 000000000..6c87c2949 --- /dev/null +++ b/c_level_db.go @@ -0,0 +1,134 @@ +// +build gcc + +package db + +import ( + "fmt" + "path" + + "github.com/jmhodges/levigo" + + . "github.com/tendermint/go-common" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewCLevelDB(name, dir) + } + registerDBCreator(LevelDBBackendStr, dbCreator, true) + registerDBCreator(CLevelDBBackendStr, dbCreator, false) +} + +type CLevelDB struct { + db *levigo.DB + ro *levigo.ReadOptions + wo *levigo.WriteOptions + woSync *levigo.WriteOptions +} + +func NewCLevelDB(name string, dir string) (*CLevelDB, error) { + dbPath := path.Join(dir, name+".db") + + opts := levigo.NewOptions() + opts.SetCache(levigo.NewLRUCache(1 << 30)) + opts.SetCreateIfMissing(true) + db, err := levigo.Open(dbPath, opts) + if err != nil { + return nil, err + } + ro := levigo.NewReadOptions() + wo := levigo.NewWriteOptions() + woSync := levigo.NewWriteOptions() + woSync.SetSync(true) + database := &CLevelDB{ + db: db, + ro: ro, + wo: wo, + woSync: woSync, + } + return database, nil +} + +func (db *CLevelDB) Get(key []byte) []byte { + res, err := db.db.Get(db.ro, key) + if err != nil { + PanicCrisis(err) + } + return res +} + +func (db *CLevelDB) Set(key []byte, value []byte) { + err := db.db.Put(db.wo, key, value) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) SetSync(key []byte, value []byte) { + err := db.db.Put(db.woSync, key, value) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) Delete(key []byte) { + err := db.db.Delete(db.wo, key) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) DeleteSync(key []byte) { + err := db.db.Delete(db.woSync, key) + if err != nil { + PanicCrisis(err) + } +} + +func (db *CLevelDB) DB() *levigo.DB { + return db.db +} + +func (db *CLevelDB) Close() { + db.db.Close() + db.ro.Close() + db.wo.Close() + db.woSync.Close() +} + +func (db *CLevelDB) Print() { + iter := db.db.NewIterator(db.ro) + defer iter.Close() + for iter.Seek(nil); iter.Valid(); iter.Next() { + key := iter.Key() + value := iter.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +func (db *CLevelDB) NewBatch() Batch { + batch := levigo.NewWriteBatch() + return &cLevelDBBatch{db, batch} +} + +//-------------------------------------------------------------------------------- + +type cLevelDBBatch struct { + db *CLevelDB + batch *levigo.WriteBatch +} + +func (mBatch *cLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +func (mBatch *cLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +func (mBatch *cLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.db.wo, mBatch.batch) + if err != nil { + PanicCrisis(err) + } +} diff --git a/c_level_db_test.go b/c_level_db_test.go new file mode 100644 index 000000000..3baa8ba4c --- /dev/null +++ b/c_level_db_test.go @@ -0,0 +1,86 @@ +// +build gcc + +package db + +import ( + "bytes" + "fmt" + "testing" + + . "github.com/tendermint/go-common" +) + +func BenchmarkRandomReadsWrites2(b *testing.B) { + b.StopTimer() + + numItems := int64(1000000) + internal := map[int64]int64{} + for i := 0; i < int(numItems); i++ { + internal[int64(i)] = int64(0) + } + db, err := NewGoLevelDB(Fmt("test_%x", RandStr(12)), "") + if err != nil { + b.Fatal(err.Error()) + return + } + + fmt.Println("ok, starting") + b.StartTimer() + + for i := 0; i < b.N; i++ { + // Write something + { + idx := (int64(RandInt()) % numItems) + internal[idx] += 1 + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := int642Bytes(int64(val)) + //fmt.Printf("Set %X -> %X\n", idxBytes, valBytes) + db.Set( + idxBytes, + valBytes, + ) + } + // Read something + { + idx := (int64(RandInt()) % numItems) + val := internal[idx] + idxBytes := int642Bytes(int64(idx)) + valBytes := db.Get(idxBytes) + //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) + if val == 0 { + if !bytes.Equal(valBytes, nil) { + b.Errorf("Expected %X for %v, got %X", + nil, idx, valBytes) + break + } + } else { + if len(valBytes) != 8 { + b.Errorf("Expected length 8 for %v, got %X", + idx, valBytes) + break + } + valGot := bytes2Int64(valBytes) + if val != valGot { + b.Errorf("Expected %v for %v, got %v", + val, idx, valGot) + break + } + } + } + } + + db.Close() +} + +/* +func int642Bytes(i int64) []byte { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, uint64(i)) + return buf +} + +func bytes2Int64(buf []byte) int64 { + return int64(binary.BigEndian.Uint64(buf)) +} +*/ diff --git a/db.go b/db.go index 6bb1efae7..8ab1c43be 100644 --- a/db.go +++ b/db.go @@ -1,10 +1,6 @@ package db -import ( - "path" - - . "github.com/tendermint/go-common" -) +import . "github.com/tendermint/go-common" type DB interface { Get([]byte) []byte @@ -13,30 +9,43 @@ type DB interface { Delete([]byte) DeleteSync([]byte) Close() + NewBatch() Batch // For debugging Print() } +type Batch interface { + Set(key, value []byte) + Delete(key []byte) + Write() +} + //----------------------------------------------------------------------------- -// Database types -const DBBackendMemDB = "memdb" -const DBBackendLevelDB = "leveldb" +const ( + LevelDBBackendStr = "leveldb" // legacy, defaults to goleveldb. + CLevelDBBackendStr = "cleveldb" + GoLevelDBBackendStr = "goleveldb" + MemDBBackendStr = "memdb" +) + +type dbCreator func(name string, dir string) (DB, error) + +var backends = map[string]dbCreator{} + +func registerDBCreator(backend string, creator dbCreator, force bool) { + _, ok := backends[backend] + if !force && ok { + return + } + backends[backend] = creator +} func NewDB(name string, backend string, dir string) DB { - switch backend { - case DBBackendMemDB: - db := NewMemDB() - return db - case DBBackendLevelDB: - db, err := NewLevelDB(path.Join(dir, name+".db")) - if err != nil { - PanicCrisis(err) - } - return db - default: - PanicSanity(Fmt("Unknown DB backend: %v", backend)) + db, err := backends[backend](name, dir) + if err != nil { + PanicSanity(Fmt("Error initializing DB: %v", err)) } - return nil + return db } diff --git a/go_level_db.go b/go_level_db.go new file mode 100644 index 000000000..a16c5d9e0 --- /dev/null +++ b/go_level_db.go @@ -0,0 +1,118 @@ +package db + +import ( + "fmt" + "path" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/opt" + + . "github.com/tendermint/go-common" +) + +func init() { + dbCreator := func(name string, dir string) (DB, error) { + return NewGoLevelDB(name, dir) + } + registerDBCreator(LevelDBBackendStr, dbCreator, false) + registerDBCreator(GoLevelDBBackendStr, dbCreator, false) +} + +type GoLevelDB struct { + db *leveldb.DB +} + +func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) { + dbPath := path.Join(dir, name+".db") + db, err := leveldb.OpenFile(dbPath, nil) + if err != nil { + return nil, err + } + database := &GoLevelDB{db: db} + return database, nil +} + +func (db *GoLevelDB) Get(key []byte) []byte { + res, err := db.db.Get(key, nil) + if err != nil { + if err == errors.ErrNotFound { + return nil + } else { + PanicCrisis(err) + } + } + return res +} + +func (db *GoLevelDB) Set(key []byte, value []byte) { + err := db.db.Put(key, value, nil) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) SetSync(key []byte, value []byte) { + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) Delete(key []byte) { + err := db.db.Delete(key, nil) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) DeleteSync(key []byte) { + err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) + if err != nil { + PanicCrisis(err) + } +} + +func (db *GoLevelDB) DB() *leveldb.DB { + return db.db +} + +func (db *GoLevelDB) Close() { + db.db.Close() +} + +func (db *GoLevelDB) Print() { + iter := db.db.NewIterator(nil, nil) + for iter.Next() { + key := iter.Key() + value := iter.Value() + fmt.Printf("[%X]:\t[%X]\n", key, value) + } +} + +func (db *GoLevelDB) NewBatch() Batch { + batch := new(leveldb.Batch) + return &goLevelDBBatch{db, batch} +} + +//-------------------------------------------------------------------------------- + +type goLevelDBBatch struct { + db *GoLevelDB + batch *leveldb.Batch +} + +func (mBatch *goLevelDBBatch) Set(key, value []byte) { + mBatch.batch.Put(key, value) +} + +func (mBatch *goLevelDBBatch) Delete(key []byte) { + mBatch.batch.Delete(key) +} + +func (mBatch *goLevelDBBatch) Write() { + err := mBatch.db.db.Write(mBatch.batch, nil) + if err != nil { + PanicCrisis(err) + } +} diff --git a/level_db_test.go b/go_level_db_test.go similarity index 96% rename from level_db_test.go rename to go_level_db_test.go index 2a8e9ac1e..b5dad1163 100644 --- a/level_db_test.go +++ b/go_level_db_test.go @@ -17,7 +17,7 @@ func BenchmarkRandomReadsWrites(b *testing.B) { for i := 0; i < int(numItems); i++ { internal[int64(i)] = int64(0) } - db, err := NewLevelDB(Fmt("test_%x", RandStr(12))) + db, err := NewCLevelDB(Fmt("test_%x", RandStr(12)), "") if err != nil { b.Fatal(err.Error()) return diff --git a/level_db.go b/level_db.go deleted file mode 100644 index dee57a321..000000000 --- a/level_db.go +++ /dev/null @@ -1,83 +0,0 @@ -package db - -import ( - "fmt" - "path" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - - . "github.com/tendermint/go-common" -) - -type LevelDB struct { - db *leveldb.DB -} - -func NewLevelDB(name string) (*LevelDB, error) { - dbPath := path.Join(name) - db, err := leveldb.OpenFile(dbPath, nil) - if err != nil { - return nil, err - } - database := &LevelDB{db: db} - return database, nil -} - -func (db *LevelDB) Get(key []byte) []byte { - res, err := db.db.Get(key, nil) - if err != nil { - if err == errors.ErrNotFound { - return nil - } else { - PanicCrisis(err) - } - } - return res -} - -func (db *LevelDB) Set(key []byte, value []byte) { - err := db.db.Put(key, value, nil) - if err != nil { - PanicCrisis(err) - } -} - -func (db *LevelDB) SetSync(key []byte, value []byte) { - err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) - if err != nil { - PanicCrisis(err) - } -} - -func (db *LevelDB) Delete(key []byte) { - err := db.db.Delete(key, nil) - if err != nil { - PanicCrisis(err) - } -} - -func (db *LevelDB) DeleteSync(key []byte) { - err := db.db.Delete(key, &opt.WriteOptions{Sync: true}) - if err != nil { - PanicCrisis(err) - } -} - -func (db *LevelDB) DB() *leveldb.DB { - return db.db -} - -func (db *LevelDB) Close() { - db.db.Close() -} - -func (db *LevelDB) Print() { - iter := db.db.NewIterator(nil, nil) - for iter.Next() { - key := iter.Key() - value := iter.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } -} diff --git a/mem_db.go b/mem_db.go index b7d8918d4..eb1e54b56 100644 --- a/mem_db.go +++ b/mem_db.go @@ -2,10 +2,18 @@ package db import ( "fmt" + "sync" ) +func init() { + registerDBCreator(MemDBBackendStr, func(name string, dir string) (DB, error) { + return NewMemDB(), nil + }, false) +} + type MemDB struct { - db map[string][]byte + mtx sync.Mutex + db map[string][]byte } func NewMemDB() *MemDB { @@ -14,31 +22,91 @@ func NewMemDB() *MemDB { } func (db *MemDB) Get(key []byte) []byte { + db.mtx.Lock() + defer db.mtx.Unlock() return db.db[string(key)] } func (db *MemDB) Set(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() db.db[string(key)] = value } func (db *MemDB) SetSync(key []byte, value []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() db.db[string(key)] = value } func (db *MemDB) Delete(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() delete(db.db, string(key)) } func (db *MemDB) DeleteSync(key []byte) { + db.mtx.Lock() + defer db.mtx.Unlock() delete(db.db, string(key)) } func (db *MemDB) Close() { + db.mtx.Lock() + defer db.mtx.Unlock() db = nil } func (db *MemDB) Print() { + db.mtx.Lock() + defer db.mtx.Unlock() for key, value := range db.db { fmt.Printf("[%X]:\t[%X]\n", []byte(key), value) } } + +func (db *MemDB) NewBatch() Batch { + return &memDBBatch{db, nil} +} + +//-------------------------------------------------------------------------------- + +type memDBBatch struct { + db *MemDB + ops []operation +} + +type opType int + +const ( + opTypeSet = 1 + opTypeDelete = 2 +) + +type operation struct { + opType + key []byte + value []byte +} + +func (mBatch *memDBBatch) Set(key, value []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeSet, key, value}) +} + +func (mBatch *memDBBatch) Delete(key []byte) { + mBatch.ops = append(mBatch.ops, operation{opTypeDelete, key, nil}) +} + +func (mBatch *memDBBatch) Write() { + mBatch.db.mtx.Lock() + defer mBatch.db.mtx.Unlock() + + for _, op := range mBatch.ops { + if op.opType == opTypeSet { + mBatch.db.db[string(op.key)] = op.value + } else if op.opType == opTypeDelete { + delete(mBatch.db.db, string(op.key)) + } + } + +}