From f66c552ef25eb141737259534ca17cd20f86dba8 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Wed, 21 May 2014 21:48:41 -0700 Subject: [PATCH] lazifying --- merkle/db.go | 54 +++++++++++++++ merkle/iavl.go | 160 ++++++++++++++++++++++---------------------- merkle/iavl_test.go | 18 +++-- merkle/types.go | 51 +++++++------- 4 files changed, 173 insertions(+), 110 deletions(-) create mode 100644 merkle/db.go diff --git a/merkle/db.go b/merkle/db.go new file mode 100644 index 000000000..89f3c05cb --- /dev/null +++ b/merkle/db.go @@ -0,0 +1,54 @@ +package merkle + +import ( + "fmt" + "github.com/syndtr/goleveldb/leveldb" + "path" +) + +type LDBDatabase struct { + db *leveldb.DB +} + +func NewLDBDatabase(name string) (*LDBDatabase, error) { + dbPath := path.Join(name) + db, err := leveldb.OpenFile(dbPath, nil) + if err != nil { + return nil, err + } + database := &LDBDatabase{db: db} + return database, nil +} + +func (db *LDBDatabase) Put(key []byte, value []byte) { + err := db.db.Put(key, value, nil) + if err != nil { + fmt.Println("Error put", err) + } +} + +func (db *LDBDatabase) Get(key []byte) ([]byte, error) { + return db.db.Get(key, nil) +} + +func (db *LDBDatabase) Delete(key []byte) error { + return db.db.Delete(key, nil) +} + +func (db *LDBDatabase) Db() *leveldb.DB { + return db.db +} + +func (db *LDBDatabase) Close() { + db.db.Close() +} + +func (db *LDBDatabase) Print() { + iter := db.db.NewIterator(nil, nil) + for iter.Next() { + key := iter.Key() + value := iter.Value() + fmt.Printf("%x(%d): %v ", key, len(key), value) + } +} + diff --git a/merkle/iavl.go b/merkle/iavl.go index ad66b3369..000326b8d 100644 --- a/merkle/iavl.go +++ b/merkle/iavl.go @@ -1,9 +1,9 @@ package merkle import ( - //"fmt" + "bytes" "math" - //"hash" + "io" "crypto/sha256" ) @@ -18,13 +18,17 @@ func NewIAVLTree() *IAVLTree { } func (self *IAVLTree) Root() Node { - return self.root.Copy(true) + return self.root } -func (self *IAVLTree) Size() int { +func (self *IAVLTree) Size() uint64 { return self.root.Size() } +func (self *IAVLTree) Height() uint8 { + return self.root.Height() +} + func (self *IAVLTree) Has(key Key) bool { return self.root.Has(key) } @@ -34,7 +38,7 @@ func (self *IAVLTree) Put(key Key, value Value) (err error) { return nil } -func (self *IAVLTree) Hash() ([]byte, int) { +func (self *IAVLTree) Hash() ([]byte, uint64) { return self.root.Hash() } @@ -56,27 +60,29 @@ func (self *IAVLTree) Remove(key Key) (value Value, err error) { type IAVLNode struct { key Key value Value - height int + size uint64 + height uint8 hash []byte left *IAVLNode right *IAVLNode + + // volatile + flags byte } -func (self *IAVLNode) Copy(copyHash bool) *IAVLNode { +func (self *IAVLNode) Copy() *IAVLNode { if self == nil { return nil } - var hash []byte - if copyHash { - hash = self.hash - } return &IAVLNode{ key: self.key, value: self.value, + size: self.size, height: self.height, - hash: hash, left: self.left, right: self.right, + hash: nil, + flags: byte(0), } } @@ -98,11 +104,11 @@ func (self *IAVLNode) Right() Node { return self.right } -func (self *IAVLNode) Size() int { +func (self *IAVLNode) Size() uint64 { if self == nil { return 0 } - return 1 + self.left.Size() + self.right.Size() + return self.size } func (self *IAVLNode) Has(key Key) (has bool) { @@ -131,54 +137,74 @@ func (self *IAVLNode) Get(key Key) (value Value, err error) { } } -func (self *IAVLNode) Hash() ([]byte, int) { +func (self *IAVLNode) Bytes() []byte { + b := new(bytes.Buffer) + self.WriteTo(b) + return b.Bytes() +} + +func (self *IAVLNode) Hash() ([]byte, uint64) { if self == nil { return nil, 0 } if self.hash != nil { return self.hash, 0 } + hasher := sha256.New() - hashCount := 1 + _, hashCount, err := self.WriteTo(hasher) + if err != nil { panic(err) } + self.hash = hasher.Sum(nil) + + return self.hash, hashCount +} + +func (self *IAVLNode) WriteTo(writer io.Writer) (written int64, hashCount uint64, err error) { + + write := func(bytes []byte) { + if err == nil { + var n int + n, err = writer.Write(bytes) + written += int64(n) + } + } // node descriptor nodeDesc := byte(0) if self.value != nil { nodeDesc |= 0x01 } if self.left != nil { nodeDesc |= 0x02 } if self.right != nil { nodeDesc |= 0x04 } - hasher.Write([]byte{nodeDesc}) + write([]byte{nodeDesc}) // node key keyBytes := self.key.Bytes() if len(keyBytes) > 255 { panic("key is too long") } - hasher.Write([]byte{byte(len(keyBytes))}) - hasher.Write(keyBytes) + write([]byte{byte(len(keyBytes))}) + write(keyBytes) // node value if self.value != nil { valueBytes := self.value.Bytes() if len(valueBytes) > math.MaxUint32 { panic("value is too long") } - hasher.Write([]byte{byte(len(valueBytes))}) - hasher.Write(valueBytes) + write([]byte{byte(len(valueBytes))}) + write(valueBytes) } // left child if self.left != nil { leftHash, leftCount := self.left.Hash() hashCount += leftCount - hasher.Write(leftHash) + write(leftHash) } // right child if self.right != nil { rightHash, rightCount := self.right.Hash() hashCount += rightCount - hasher.Write(rightHash) + write(rightHash) } - self.hash = hasher.Sum(nil) - - return self.hash, hashCount + return written, hashCount+1, err } // Returns a new tree (unless node is the root) & a copy of the popped node. @@ -202,84 +228,64 @@ func (self *IAVLNode) pop_node(node *IAVLNode) (new_self, new_node *IAVLNode) { } else { n = nil } - node = node.Copy(false) + node = node.Copy() node.left = nil node.right = nil + node.calc_height_and_size() return n, node } else { - self = self.Copy(false) + self = self.Copy() if node.key.Less(self.key) { self.left, node = self.left.pop_node(node) } else { self.right, node = self.right.pop_node(node) } - self.calc_height() + self.calc_height_and_size() return self, node } } -// Pushes the node to the tree, returns a new tree -func (self *IAVLNode) push_node(node *IAVLNode) *IAVLNode { - if node == nil { - panic("node can't be nil") - } else if node.left != nil || node.right != nil { - panic("node must now be a leaf") - } - - self = self.Copy(false) - - if self == nil { - node.height = 1 - return node - } else if node.key.Less(self.key) { - self.left = self.left.push_node(node) - } else { - self.right = self.right.push_node(node) - } - self.calc_height() - return self -} - func (self *IAVLNode) rotate_right() *IAVLNode { - self = self.Copy(false) - sl := self.left.Copy(false) + self = self.Copy() + sl := self.left.Copy() slr := sl.right sl.right = self self.left = slr - self.calc_height() - sl.calc_height() + self.calc_height_and_size() + sl.calc_height_and_size() return sl } func (self *IAVLNode) rotate_left() *IAVLNode { - self = self.Copy(false) - sr := self.right.Copy(false) + self = self.Copy() + sr := self.right.Copy() srl := sr.left sr.left = self self.right = srl - self.calc_height() - sr.calc_height() + self.calc_height_and_size() + sr.calc_height_and_size() return sr } -func (self *IAVLNode) calc_height() { - self.height = max(self.left.Height(), self.right.Height()) + 1 +func (self *IAVLNode) calc_height_and_size() { + self.height = maxUint8(self.left.Height(), self.right.Height()) + 1 + self.size = self.left.Size() + self.right.Size() + 1 } func (self *IAVLNode) calc_balance() int { if self == nil { return 0 } - return self.left.Height() - self.right.Height() + return int(self.left.Height()) - int(self.right.Height()) } func (self *IAVLNode) balance() (new_self *IAVLNode) { @@ -290,8 +296,9 @@ func (self *IAVLNode) balance() (new_self *IAVLNode) { return self.rotate_right() } else { // Left Right Case - self = self.Copy(false) + self = self.Copy() self.left = self.left.rotate_left() + //self.calc_height_and_size() return self.rotate_right() } } @@ -301,8 +308,9 @@ func (self *IAVLNode) balance() (new_self *IAVLNode) { return self.rotate_left() } else { // Right Left Case - self = self.Copy(false) + self = self.Copy() self.right = self.right.rotate_right() + //self.calc_height_and_size() return self.rotate_left() } } @@ -313,10 +321,10 @@ func (self *IAVLNode) balance() (new_self *IAVLNode) { // TODO: don't clear the hash if the value hasn't changed. func (self *IAVLNode) Put(key Key, value Value) (_ *IAVLNode, updated bool) { if self == nil { - return &IAVLNode{key: key, value: value, height: 1, hash: nil}, false + return &IAVLNode{key: key, value: value, height: 1, size: 1, hash: nil}, false } - self = self.Copy(false) + self = self.Copy() if self.key.Equals(key) { self.value = value @@ -331,7 +339,7 @@ func (self *IAVLNode) Put(key Key, value Value) (_ *IAVLNode, updated bool) { if updated { return self, updated } else { - self.calc_height() + self.calc_height_and_size() return self.balance(), updated } } @@ -350,6 +358,7 @@ func (self *IAVLNode) Remove(key Key) (new_self *IAVLNode, value Value, err erro } new_self.left = self.left new_self.right = self.right + new_self.calc_height_and_size() return new_self, self.value, nil } else if self.left == nil { return self.right, self.value, nil @@ -371,7 +380,7 @@ func (self *IAVLNode) Remove(key Key) (new_self *IAVLNode, value Value, err erro } else if err != nil { // some other error return self, value, err } - self = self.Copy(false) + self = self.Copy() self.left = new_left } else { if self.right == nil { @@ -384,14 +393,14 @@ func (self *IAVLNode) Remove(key Key) (new_self *IAVLNode, value Value, err erro } else if err != nil { // some other error return self, value, err } - self = self.Copy(false) + self = self.Copy() self.right = new_right } - self.calc_height() + self.calc_height_and_size() return self.balance(), value, err } -func (self *IAVLNode) Height() int { +func (self *IAVLNode) Height() uint8 { if self == nil { return 0 } @@ -418,14 +427,7 @@ func (self *IAVLNode) rmd() (*IAVLNode) { return self._md(func(node *IAVLNode)*IAVLNode { return node.right }) } -func abs(i int) int { - if i < 0 { - return -i - } - return i -} - -func max(a, b int) int { +func maxUint8(a, b uint8) uint8 { if a > b { return a } diff --git a/merkle/iavl_test.go b/merkle/iavl_test.go index 26d3b8e9c..d57d5c139 100644 --- a/merkle/iavl_test.go +++ b/merkle/iavl_test.go @@ -68,7 +68,7 @@ func TestImmutableAvlPutHasGetRemove(t *testing.T) { if !updated { t.Error("should have been updated") } - if tree.Size() != (i+1) { + if tree.Size() != uint64(i+1) { t.Error("size was wrong", tree.Size(), i+1) } } @@ -106,7 +106,7 @@ func TestImmutableAvlPutHasGetRemove(t *testing.T) { t.Error("wrong value") } } - if tree.Size() != (len(records) - (i+1)) { + if tree.Size() != uint64(len(records) - (i+1)) { t.Error("size was wrong", tree.Size(), (len(records) - (i+1))) } } @@ -173,8 +173,12 @@ func TestGriffin(t *testing.T) { // Convenience for a new node N := func(l *IAVLNode, i int, r *IAVLNode) *IAVLNode { - n := &IAVLNode{Int32(i), nil, -1, nil, l, r} - n.calc_height() + n := &IAVLNode{ + key: Int32(i), + left: l, + right: r, + } + n.calc_height_and_size() n.Hash() return n } @@ -193,7 +197,7 @@ func TestGriffin(t *testing.T) { } } - expectHash := func(n2 *IAVLNode, hashCount int) { + expectHash := func(n2 *IAVLNode, hashCount uint64) { // ensure number of new hash calculations is as expected. hash, count := n2.Hash() if count != hashCount { @@ -213,7 +217,7 @@ func TestGriffin(t *testing.T) { } } - expectPut := func(n *IAVLNode, i int, repr string, hashCount int) { + expectPut := func(n *IAVLNode, i int, repr string, hashCount uint64) { n2, updated := n.Put(Int32(i), nil) // ensure node was added & structure is as expected. if updated == true || P(n2) != repr { @@ -224,7 +228,7 @@ func TestGriffin(t *testing.T) { expectHash(n2, hashCount) } - expectRemove := func(n *IAVLNode, i int, repr string, hashCount int) { + expectRemove := func(n *IAVLNode, i int, repr string, hashCount uint64) { n2, value, err := n.Remove(Int32(i)) // ensure node was added & structure is as expected. if value != nil || err != nil || P(n2) != repr { diff --git a/merkle/types.go b/merkle/types.go index aa1567fa1..4f99383ac 100644 --- a/merkle/types.go +++ b/merkle/types.go @@ -5,43 +5,46 @@ import ( ) type Value interface { - Bytes() []byte + Bytes() []byte } type Key interface { - Equals(b Key) bool - Less(b Key) bool - Bytes() []byte + Equals(b Key) bool + Less(b Key) bool + Bytes() []byte } type Tree interface { - Root() Node + Root() Node - Size() int - Has(key Key) bool - Get(key Key) (value Value, err error) - Hash() ([]byte, int) + Size() uint64 + Height() uint8 + Has(key Key) bool + Get(key Key) (Value, error) + Hash() ([]byte, uint64) - Put(key Key, value Value) (err error) - Remove(key Key) (value Value, err error) + Put(Key, Value) (err error) + Remove(Key) (Value, error) } type Node interface { - Key() Key - Value() Value - Left() Node - Right() Node - - Size() int - Has(key Key) bool - Get(key Key) (value Value, err error) - Hash() ([]byte, int) - - Put(key Key, value Value) (_ *IAVLNode, updated bool) - Remove(key Key) (_ *IAVLNode, value Value, err error) + Key() Key + Value() Value + Left() Node + Right() Node + + Size() uint64 + Height() uint8 + Has(Key) bool + Get(Key) (Value, error) + Hash() ([]byte, uint64) + Bytes() []byte + + Put(Key, Value) (*IAVLNode, bool) + Remove(Key) (*IAVLNode, Value, error) } -type NodeIterator func() (node Node) +type NodeIterator func() Node func NotFound(key Key) error { return fmt.Errorf("Key was not found.")