Browse Source

persistence

pull/9/head
Jae Kwon 10 years ago
parent
commit
cb56808a9c
10 changed files with 361 additions and 137 deletions
  1. +0
    -2
      README.md
  2. +13
    -13
      db/level_db.go
  3. +32
    -0
      db/mem_db.go
  4. +52
    -0
      merkle/binary.go
  5. +137
    -16
      merkle/iavl.go
  6. +31
    -16
      merkle/iavl_test.go
  7. +59
    -69
      merkle/int.go
  8. +11
    -15
      merkle/string.go
  9. +10
    -6
      merkle/types.go
  10. +16
    -0
      merkle/util.go

+ 0
- 2
README.md View File

@ -1,3 +1 @@
TenderMint - proof of concept
* merklelized AVL tree based on [timtadh's code](https://github.com/timtadh/data-structures)

merkle/db.go → db/level_db.go View File


+ 32
- 0
db/mem_db.go View File

@ -0,0 +1,32 @@
package db
import (
"fmt"
)
type MemDB struct {
db map[string][]byte
}
func NewMemDB() (*MemDB) {
database := &MemDB{db:make(map[string][]byte)}
return database
}
func (db *MemDB) Put(key []byte, value []byte) {
db.db[string(key)] = value
}
func (db *MemDB) Get(key []byte) ([]byte) {
return db.db[string(key)]
}
func (db *MemDB) Delete(key []byte) {
delete(db.db, string(key))
}
func (db *MemDB) Print() {
for key, value := range db.db {
fmt.Printf("[%x]:\t[%x]", []byte(key), value)
}
}

+ 52
- 0
merkle/binary.go View File

@ -0,0 +1,52 @@
package merkle
const (
TYPE_BYTE = byte(0x00)
TYPE_INT8 = byte(0x02)
TYPE_UINT8 = byte(0x03)
TYPE_INT16 = byte(0x04)
TYPE_UINT16 = byte(0x05)
TYPE_INT32 = byte(0x06)
TYPE_UINT32 = byte(0x07)
TYPE_INT64 = byte(0x08)
TYPE_UINT64 = byte(0x09)
TYPE_STRING = byte(0x10)
TYPE_BYTESLICE = byte(0x11)
)
func GetBinaryType(o Binary) byte {
switch o.(type) {
case Byte: return TYPE_BYTE
case Int8: return TYPE_INT8
case UInt8: return TYPE_UINT8
case Int16: return TYPE_INT16
case UInt16: return TYPE_UINT16
case Int32: return TYPE_INT32
case UInt32: return TYPE_UINT32
case Int64: return TYPE_INT64
case UInt64: return TYPE_UINT64
case Int: panic("Int not supported")
case UInt: panic("UInt not supported")
case String: return TYPE_STRING
case ByteSlice: return TYPE_BYTESLICE
default: panic("Unsupported type")
}
}
func LoadBinary(buf []byte, start int) (Binary, int) {
typeByte := buf[start]
switch typeByte {
case TYPE_BYTE: return LoadByte(buf[start+1:]), start+2
case TYPE_INT8: return LoadInt8(buf[start+1:]), start+2
case TYPE_UINT8: return LoadUInt8(buf[start+1:]), start+2
case TYPE_INT16: return LoadInt16(buf[start+1:]), start+3
case TYPE_UINT16: return LoadUInt16(buf[start+1:]), start+3
case TYPE_INT32: return LoadInt32(buf[start+1:]), start+5
case TYPE_UINT32: return LoadUInt32(buf[start+1:]), start+5
case TYPE_INT64: return LoadInt64(buf[start+1:]), start+9
case TYPE_UINT64: return LoadUInt64(buf[start+1:]), start+9
case TYPE_STRING: return LoadString(buf, start+1)
case TYPE_BYTESLICE:return LoadByteSlice(buf, start+1)
default: panic("Unsupported type")
}
}

+ 137
- 16
merkle/iavl.go View File

@ -9,11 +9,21 @@ const HASH_BYTE_SIZE int = 4+32
// Immutable AVL Tree (wraps the Node root)
type IAVLTree struct {
root *IAVLNode
db Db
root *IAVLNode
}
func NewIAVLTree() *IAVLTree {
return &IAVLTree{}
func NewIAVLTree(db Db) *IAVLTree {
return &IAVLTree{db:db, root:nil}
}
func NewIAVLTreeFromHash(db Db, hash ByteSlice) *IAVLTree {
root := &IAVLNode{
hash: hash,
flags: IAVLNODE_FLAG_PERSISTED | IAVLNODE_FLAG_PLACEHOLDER,
}
root.fill(db)
return &IAVLTree{db:db, root:root}
}
func (self *IAVLTree) Root() Node {
@ -29,23 +39,30 @@ func (self *IAVLTree) Height() uint8 {
}
func (self *IAVLTree) Has(key Key) bool {
return self.root.Has(nil, key)
return self.root.Has(self.db, key)
}
func (self *IAVLTree) Put(key Key, value Value) {
self.root, _ = self.root.Put(nil, key, value)
self.root, _ = self.root.Put(self.db, key, value)
}
func (self *IAVLTree) Hash() (ByteSlice, uint64) {
return self.root.Hash()
}
func (self *IAVLTree) Save() {
if self.root.hash == nil {
self.root.Hash()
}
self.root.Save(self.db)
}
func (self *IAVLTree) Get(key Key) (value Value) {
return self.root.Get(nil, key)
return self.root.Get(self.db, key)
}
func (self *IAVLTree) Remove(key Key) (value Value, err error) {
new_root, value, err := self.root.Remove(nil, key)
new_root, value, err := self.root.Remove(self.db, key)
if err != nil {
return nil, err
}
@ -60,7 +77,7 @@ type IAVLNode struct {
value Value
size uint64
height uint8
hash []byte
hash ByteSlice
left *IAVLNode
right *IAVLNode
@ -68,6 +85,15 @@ type IAVLNode struct {
flags byte
}
const (
IAVLNODE_FLAG_PERSISTED = byte(0x01)
IAVLNODE_FLAG_PLACEHOLDER = byte(0x02)
IAVLNODE_DESC_HAS_VALUE = byte(0x01)
IAVLNODE_DESC_HAS_LEFT = byte(0x02)
IAVLNODE_DESC_HAS_RIGHT = byte(0x04)
)
func (self *IAVLNode) Copy() *IAVLNode {
if self == nil {
return nil
@ -84,6 +110,14 @@ func (self *IAVLNode) Copy() *IAVLNode {
}
}
func (self *IAVLNode) Equals(other Binary) bool {
if o, ok := other.(*IAVLNode); ok {
return self.hash.Equals(o.hash)
} else {
return false
}
}
func (self *IAVLNode) Key() Key {
return self.key
}
@ -150,6 +184,31 @@ func (self *IAVLNode) Hash() (ByteSlice, uint64) {
return self.hash, hashCount+1
}
func (self *IAVLNode) Save(db Db) {
if self == nil {
return
} else if self.hash == nil {
panic("savee.hash can't be nil")
}
if self.flags & IAVLNODE_FLAG_PERSISTED > 0 ||
self.flags & IAVLNODE_FLAG_PLACEHOLDER > 0 {
return
}
// save self
buf := make([]byte, self.ByteSize(), self.ByteSize())
self.SaveTo(buf)
db.Put([]byte(self.hash), buf)
// save left
self.left.Save(db)
// save right
self.right.Save(db)
self.flags |= IAVLNODE_FLAG_PERSISTED
}
// TODO: don't clear the hash if the value hasn't changed.
func (self *IAVLNode) Put(db Db, key Key, value Value) (_ *IAVLNode, updated bool) {
if self == nil {
@ -235,12 +294,17 @@ func (self *IAVLNode) ByteSize() int {
// 1 byte node neight
// 8 bytes node size
size := 10
// key
size += 1 // type info
size += self.key.ByteSize()
// value
if self.value != nil {
size += 1 // type info
size += self.value.ByteSize()
} else {
size += 1
}
// children
if self.left != nil {
size += HASH_BYTE_SIZE
}
@ -261,9 +325,9 @@ func (self *IAVLNode) saveToCountHashes(buf []byte) (int, uint64) {
// node descriptor
nodeDesc := byte(0)
if self.value != nil { nodeDesc |= 0x01 }
if self.left != nil { nodeDesc |= 0x02 }
if self.right != nil { nodeDesc |= 0x04 }
if self.value != nil { nodeDesc |= IAVLNODE_DESC_HAS_VALUE }
if self.left != nil { nodeDesc |= IAVLNODE_DESC_HAS_LEFT }
if self.right != nil { nodeDesc |= IAVLNODE_DESC_HAS_RIGHT }
cur += UInt8(nodeDesc).SaveTo(buf[cur:])
// node height & size
@ -271,13 +335,15 @@ func (self *IAVLNode) saveToCountHashes(buf []byte) (int, uint64) {
cur += UInt64(self.size).SaveTo(buf[cur:])
// node key
buf[cur] = GetBinaryType(self.key)
cur += 1
cur += self.key.SaveTo(buf[cur:])
// node value
if self.value != nil {
buf[cur] = GetBinaryType(self.value)
cur += 1
cur += self.value.SaveTo(buf[cur:])
} else {
cur += UInt8(0).SaveTo(buf[cur:])
}
// left child
@ -297,13 +363,68 @@ func (self *IAVLNode) saveToCountHashes(buf []byte) (int, uint64) {
return cur, hashCount
}
// Given a placeholder node which has only the hash set,
// load the rest of the data from db.
// Not threadsafe.
func (self *IAVLNode) fill(db Db) {
if self == nil {
panic("placeholder can't be nil")
} else if self.hash == nil {
panic("placeholder.hash can't be nil")
}
buf := db.Get(self.hash)
cur := 0
// node header
nodeDesc := byte(LoadUInt8(buf))
self.height = uint8(LoadUInt8(buf[1:]))
self.size = uint64(LoadUInt64(buf[2:]))
// key
key, cur := LoadBinary(buf, 10)
self.key = key.(Key)
// value
if nodeDesc & IAVLNODE_DESC_HAS_VALUE > 0 {
self.value, cur = LoadBinary(buf, cur)
}
// children
if nodeDesc & IAVLNODE_DESC_HAS_LEFT > 0 {
var leftHash ByteSlice
leftHash, cur = LoadByteSlice(buf, cur)
self.left = &IAVLNode{
hash: leftHash,
flags: IAVLNODE_FLAG_PERSISTED | IAVLNODE_FLAG_PLACEHOLDER,
}
}
if nodeDesc & IAVLNODE_DESC_HAS_RIGHT > 0 {
var rightHash ByteSlice
rightHash, cur = LoadByteSlice(buf, cur)
self.right = &IAVLNode{
hash: rightHash,
flags: IAVLNODE_FLAG_PERSISTED | IAVLNODE_FLAG_PLACEHOLDER,
}
}
if cur != len(buf) {
panic("buf not all consumed")
}
self.flags &= ^IAVLNODE_FLAG_PLACEHOLDER
}
func (self *IAVLNode) leftFilled(db Db) *IAVLNode {
// XXX
if self.left == nil {
return nil
}
if self.left.flags & IAVLNODE_FLAG_PLACEHOLDER > 0 {
self.left.fill(db)
}
return self.left
}
func (self *IAVLNode) rightFilled(db Db) *IAVLNode {
// XXX
if self.right == nil {
return nil
}
if self.right.flags & IAVLNODE_FLAG_PLACEHOLDER > 0 {
self.right.fill(db)
}
return self.right
}
@ -315,7 +436,7 @@ func (self *IAVLNode) popNode(db Db, node *IAVLNode) (newSelf, new_node *IAVLNod
} else if node == nil {
panic("node can't be nil")
} else if node.left != nil && node.right != nil {
panic("node must not have both left and right")
panic("node hnot have both left and right")
}
if self == node {


+ 31
- 16
merkle/iavl_test.go View File

@ -8,6 +8,7 @@ import (
"bytes"
"math/rand"
"encoding/binary"
"github.com/tendermint/tendermint/db"
)
@ -26,20 +27,6 @@ func init() {
}
}
func randstr(length int) String {
if urandom, err := os.Open("/dev/urandom"); err != nil {
panic(err)
} else {
slice := make([]byte, length)
if _, err := urandom.Read(slice); err != nil {
panic(err)
}
urandom.Close()
return String(slice)
}
panic("unreachable")
}
func TestImmutableAvlPutHasGetRemove(t *testing.T) {
type record struct {
@ -121,7 +108,7 @@ func BenchmarkImmutableAvlTree(b *testing.B) {
return &record{ randstr(32), randstr(32) }
}
t := NewIAVLTree()
t := NewIAVLTree(nil)
for i:=0; i<1000000; i++ {
r := randomRecord()
t.Put(r.key, r.value)
@ -159,7 +146,7 @@ func TestTraversals(t *testing.T) {
j += 1
}
}
test(NewIAVLTree())
test(NewIAVLTree(nil))
}
// from http://stackoverflow.com/questions/3955680/how-to-check-if-my-avl-tree-implementation-is-correct
@ -279,5 +266,33 @@ func TestGriffin(t *testing.T) {
if P(n6) != "((1 2 (- 3 4)) 5 ((6 7 -) 8 (9 10 (- 11 12))))" { t.Fatalf("Got %v", P(n6)) }
expectRemove(n6, 1, "(((2 3 4) 5 (6 7 -)) 8 (9 10 (- 11 12)))", 4)
}
func TestPersistence(t *testing.T) {
db := db.NewMemDB()
// Create some random key value pairs
records := make(map[String]String)
for i:=0; i<10000; i++ {
records[String(randstr(20))] = String(randstr(20))
}
// Construct some tree and save it
t1 := NewIAVLTree(db)
for key, value := range records {
t1.Put(key, value)
}
t1.Save()
hash, _ := t1.Hash()
// Load a tree
t2 := NewIAVLTreeFromHash(db, hash)
for key, value := range records {
t2value := t2.Get(key)
if !t2value.Equals(value) {
t.Fatalf("Invalid value. Expected %v, got %v", value, t2value)
}
}
}

+ 59
- 69
merkle/int.go View File

@ -4,6 +4,7 @@ import (
"encoding/binary"
)
type Byte byte
type Int8 int8
type UInt8 uint8
type Int16 int16
@ -16,21 +17,46 @@ type Int int
type UInt uint
// Int8
// Byte
func (self Int8) Equals(other Key) bool {
if o, ok := other.(Int8); ok {
return self == o
func (self Byte) Equals(other Binary) bool {
return self == other
}
func (self Byte) Less(other Key) bool {
if o, ok := other.(Byte); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
func (self Byte) ByteSize() int {
return 1
}
func (self Byte) SaveTo(b []byte) int {
if cap(b) < 1 { panic("buf too small") }
b[0] = byte(self)
return 1
}
func LoadByte(bytes []byte) Byte {
return Byte(bytes[0])
}
// Int8
func (self Int8) Equals(other Binary) bool {
return self == other
}
func (self Int8) Less(other Key) bool {
if o, ok := other.(Int8); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -51,19 +77,15 @@ func LoadInt8(bytes []byte) Int8 {
// UInt8
func (self UInt8) Equals(other Key) bool {
if o, ok := other.(UInt8); ok {
return self == o
} else {
return false
}
func (self UInt8) Equals(other Binary) bool {
return self == other
}
func (self UInt8) Less(other Key) bool {
if o, ok := other.(UInt8); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -84,19 +106,15 @@ func LoadUInt8(bytes []byte) UInt8 {
// Int16
func (self Int16) Equals(other Key) bool {
if o, ok := other.(Int16); ok {
return self == o
} else {
return false
}
func (self Int16) Equals(other Binary) bool {
return self == other
}
func (self Int16) Less(other Key) bool {
if o, ok := other.(Int16); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -117,19 +135,15 @@ func LoadInt16(bytes []byte) Int16 {
// UInt16
func (self UInt16) Equals(other Key) bool {
if o, ok := other.(UInt16); ok {
return self == o
} else {
return false
}
func (self UInt16) Equals(other Binary) bool {
return self == other
}
func (self UInt16) Less(other Key) bool {
if o, ok := other.(UInt16); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -150,19 +164,15 @@ func LoadUInt16(bytes []byte) UInt16 {
// Int32
func (self Int32) Equals(other Key) bool {
if o, ok := other.(Int32); ok {
return self == o
} else {
return false
}
func (self Int32) Equals(other Binary) bool {
return self == other
}
func (self Int32) Less(other Key) bool {
if o, ok := other.(Int32); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -183,19 +193,15 @@ func LoadInt32(bytes []byte) Int32 {
// UInt32
func (self UInt32) Equals(other Key) bool {
if o, ok := other.(UInt32); ok {
return self == o
} else {
return false
}
func (self UInt32) Equals(other Binary) bool {
return self == other
}
func (self UInt32) Less(other Key) bool {
if o, ok := other.(UInt32); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -216,19 +222,15 @@ func LoadUInt32(bytes []byte) UInt32 {
// Int64
func (self Int64) Equals(other Key) bool {
if o, ok := other.(Int64); ok {
return self == o
} else {
return false
}
func (self Int64) Equals(other Binary) bool {
return self == other
}
func (self Int64) Less(other Key) bool {
if o, ok := other.(Int64); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -249,19 +251,15 @@ func LoadInt64(bytes []byte) Int64 {
// UInt64
func (self UInt64) Equals(other Key) bool {
if o, ok := other.(UInt64); ok {
return self == o
} else {
return false
}
func (self UInt64) Equals(other Binary) bool {
return self == other
}
func (self UInt64) Less(other Key) bool {
if o, ok := other.(UInt64); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -282,19 +280,15 @@ func LoadUInt64(bytes []byte) UInt64 {
// Int
func (self Int) Equals(other Key) bool {
if o, ok := other.(Int); ok {
return self == o
} else {
return false
}
func (self Int) Equals(other Binary) bool {
return self == other
}
func (self Int) Less(other Key) bool {
if o, ok := other.(Int); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -314,19 +308,15 @@ func LoadInt(bytes []byte) Int {
// UInt
func (self UInt) Equals(other Key) bool {
if o, ok := other.(UInt); ok {
return self == o
} else {
return false
}
func (self UInt) Equals(other Binary) bool {
return self == other
}
func (self UInt) Less(other Key) bool {
if o, ok := other.(UInt); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}


+ 11
- 15
merkle/string.go View File

@ -7,19 +7,15 @@ type ByteSlice []byte
// String
func (self String) Equals(other Key) bool {
if o, ok := other.(String); ok {
return self == o
} else {
return false
}
func (self String) Equals(other Binary) bool {
return self == other
}
func (self String) Less(other Key) bool {
if o, ok := other.(String); ok {
return self < o
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -34,15 +30,15 @@ func (self String) SaveTo(buf []byte) int {
return len(self)+4
}
func LoadString(bytes []byte) String {
length := LoadUInt32(bytes)
return String(bytes[4:4+length])
func LoadString(bytes []byte, start int) (String, int) {
length := int(LoadUInt32(bytes[start:]))
return String(bytes[start+4:start+4+length]), start+4+length
}
// ByteSlice
func (self ByteSlice) Equals(other Key) bool {
func (self ByteSlice) Equals(other Binary) bool {
if o, ok := other.(ByteSlice); ok {
return bytes.Equal(self, o)
} else {
@ -54,7 +50,7 @@ func (self ByteSlice) Less(other Key) bool {
if o, ok := other.(ByteSlice); ok {
return bytes.Compare(self, o) < 0 // -1 if a < b
} else {
return false
panic("Cannot compare unequal types")
}
}
@ -69,7 +65,7 @@ func (self ByteSlice) SaveTo(buf []byte) int {
return len(self)+4
}
func LoadByteSlice(bytes []byte) ByteSlice {
length := LoadUInt32(bytes)
return ByteSlice(bytes[4:4+length])
func LoadByteSlice(bytes []byte, start int) (ByteSlice, int) {
length := int(LoadUInt32(bytes[start:]))
return ByteSlice(bytes[start+4:start+4+length]), start+4+length
}

+ 10
- 6
merkle/types.go View File

@ -7,6 +7,7 @@ import (
type Binary interface {
ByteSize() int
SaveTo([]byte) int
Equals(Binary) bool
}
type Value interface {
@ -16,10 +17,14 @@ type Value interface {
type Key interface {
Binary
Equals(b Key) bool
Less(b Key) bool
}
type Db interface {
Get([]byte) []byte
Put([]byte, []byte)
}
type Tree interface {
Root() Node
@ -27,17 +32,14 @@ type Tree interface {
Height() uint8
Has(key Key) bool
Get(key Key) Value
Hash() (ByteSlice, uint64)
Save()
Put(Key, Value)
Remove(Key) (Value, error)
}
type Db interface {
Get([]byte) []byte
Put([]byte, []byte)
}
type Node interface {
Binary
@ -50,7 +52,9 @@ type Node interface {
Height() uint8
Has(Db, Key) bool
Get(Db, Key) Value
Hash() (ByteSlice, uint64)
Save(Db)
Put(Db, Key, Value) (*IAVLNode, bool)
Remove(Db, Key) (*IAVLNode, Value, error)


+ 16
- 0
merkle/util.go View File

@ -1,6 +1,7 @@
package merkle
import (
"os"
"fmt"
)
@ -52,3 +53,18 @@ func printIAVLNode(node *IAVLNode, indent int) {
printIAVLNode(node.rightFilled(nil), indent+1)
}
}
func randstr(length int) String {
if urandom, err := os.Open("/dev/urandom"); err != nil {
panic(err)
} else {
slice := make([]byte, length)
if _, err := urandom.Read(slice); err != nil {
panic(err)
}
urandom.Close()
return String(slice)
}
panic("unreachable")
}

Loading…
Cancel
Save