From 36d7180ca216f0d7ff62851fa441de2b0371d699 Mon Sep 17 00:00:00 2001 From: Yumin Xia Date: Sun, 10 Mar 2019 00:46:32 -0800 Subject: [PATCH 1/2] libs/db: close batch (#3397) ClevelDB requires closing when WriteBatch is no longer needed, https://godoc.org/github.com/jmhodges/levigo#WriteBatch.Close Fixes the memory leak in https://github.com/cosmos/cosmos-sdk/issues/3842 --- libs/db/c_level_db.go | 5 +++++ libs/db/debug_db.go | 5 +++++ libs/db/go_level_db.go | 4 ++++ libs/db/mem_batch.go | 4 ++++ libs/db/prefix_db.go | 4 ++++ libs/db/remotedb/grpcdb/server.go | 1 + libs/db/remotedb/remotedb.go | 4 ++++ libs/db/types.go | 2 ++ lite/dbprovider.go | 1 + state/txindex/kv/kv.go | 2 ++ 10 files changed, 32 insertions(+) diff --git a/libs/db/c_level_db.go b/libs/db/c_level_db.go index 7f74b2a71..decb1af51 100644 --- a/libs/db/c_level_db.go +++ b/libs/db/c_level_db.go @@ -179,6 +179,11 @@ func (mBatch *cLevelDBBatch) WriteSync() { } } +// Implements Batch. +func (mBatch *cLevelDBBatch) Close() { + mBatch.batch.Close() +} + //---------------------------------------- // Iterator // NOTE This is almost identical to db/go_level_db.Iterator diff --git a/libs/db/debug_db.go b/libs/db/debug_db.go index bb361a266..658cd0555 100644 --- a/libs/db/debug_db.go +++ b/libs/db/debug_db.go @@ -250,3 +250,8 @@ func (dbch debugBatch) WriteSync() { fmt.Printf("%v.batch.WriteSync()\n", dbch.label) dbch.bch.WriteSync() } + +// Implements Batch. +func (dbch debugBatch) Close() { + dbch.bch.Close() +} diff --git a/libs/db/go_level_db.go b/libs/db/go_level_db.go index fd487a4dd..9a4358f60 100644 --- a/libs/db/go_level_db.go +++ b/libs/db/go_level_db.go @@ -184,6 +184,10 @@ func (mBatch *goLevelDBBatch) WriteSync() { } } +// Implements Batch. +// Close is no-op for goLevelDBBatch. +func (mBatch *goLevelDBBatch) Close() {} + //---------------------------------------- // Iterator // NOTE This is almost identical to db/c_level_db.Iterator diff --git a/libs/db/mem_batch.go b/libs/db/mem_batch.go index 5c5d0c13a..ebba43f54 100644 --- a/libs/db/mem_batch.go +++ b/libs/db/mem_batch.go @@ -46,6 +46,10 @@ func (mBatch *memBatch) WriteSync() { mBatch.write(true) } +func (mBatch *memBatch) Close() { + mBatch.ops = nil +} + func (mBatch *memBatch) write(doSync bool) { if mtx := mBatch.db.Mutex(); mtx != nil { mtx.Lock() diff --git a/libs/db/prefix_db.go b/libs/db/prefix_db.go index 40d72560c..0dd06ef9d 100644 --- a/libs/db/prefix_db.go +++ b/libs/db/prefix_db.go @@ -248,6 +248,10 @@ func (pb prefixBatch) WriteSync() { pb.source.WriteSync() } +func (pb prefixBatch) Close() { + pb.source.Close() +} + //---------------------------------------- // prefixIterator diff --git a/libs/db/remotedb/grpcdb/server.go b/libs/db/remotedb/grpcdb/server.go index 3a9955ddf..bfe65e610 100644 --- a/libs/db/remotedb/grpcdb/server.go +++ b/libs/db/remotedb/grpcdb/server.go @@ -180,6 +180,7 @@ func (s *server) BatchWriteSync(c context.Context, b *protodb.Batch) (*protodb.N func (s *server) batchWrite(c context.Context, b *protodb.Batch, sync bool) (*protodb.Nothing, error) { bat := s.db.NewBatch() + defer bat.Close() for _, op := range b.Ops { switch op.Type { case protodb.Operation_SET: diff --git a/libs/db/remotedb/remotedb.go b/libs/db/remotedb/remotedb.go index 2b60d8159..c70d54b9e 100644 --- a/libs/db/remotedb/remotedb.go +++ b/libs/db/remotedb/remotedb.go @@ -260,3 +260,7 @@ func (bat *batch) WriteSync() { panic(fmt.Sprintf("RemoteDB.BatchWriteSync: %v", err)) } } + +func (bat *batch) Close() { + bat.ops = nil +} diff --git a/libs/db/types.go b/libs/db/types.go index 9b9c6d0b9..30f8afd18 100644 --- a/libs/db/types.go +++ b/libs/db/types.go @@ -57,10 +57,12 @@ type DB interface { //---------------------------------------- // Batch +// Batch Close must be called when the program no longer needs the object. type Batch interface { SetDeleter Write() WriteSync() + Close() } type SetDeleter interface { diff --git a/lite/dbprovider.go b/lite/dbprovider.go index ef1b2a598..9a3636d57 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -54,6 +54,7 @@ func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { dbp.logger.Info("DBProvider.SaveFullCommit()...", "fc", fc) batch := dbp.db.NewBatch() + defer batch.Close() // Save the fc.validators. // We might be overwriting what we already have, but diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 93249b7f9..84208b8c1 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -78,6 +78,7 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { // AddBatch indexes a batch of transactions using the given list of tags. func (txi *TxIndex) AddBatch(b *txindex.Batch) error { storeBatch := txi.store.NewBatch() + defer storeBatch.Close() for _, result := range b.Ops { hash := result.Tx.Hash() @@ -109,6 +110,7 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // Index indexes a single transaction using the given list of tags. func (txi *TxIndex) Index(result *types.TxResult) error { b := txi.store.NewBatch() + defer b.Close() hash := result.Tx.Hash() From f996b10f479d7c9a6d81cca5a02c47b29a52b3f3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sun, 10 Mar 2019 13:06:34 +0400 Subject: [PATCH 2/2] update changelog and bump version to 0.30.2 --- CHANGELOG.md | 19 +++++++++++++++++++ version/version.go | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42e8761aa..44ecdf388 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## v0.30.2 + +*March 10th, 2019* + +This release fixes a CLevelDB memory leak. It was happening because we were not +closing the WriteBatch object after use. See [levigo's +godoc](https://godoc.org/github.com/jmhodges/levigo#WriteBatch.Close) for the +Close method. Special thanks goes to @Stumble who both reported an issue in +[cosmos-sdk](https://github.com/cosmos/cosmos-sdk/issues/3842) and provided a +fix here. + +### BREAKING CHANGES: + +* Go API +- [libs/db] [\#3842](https://github.com/cosmos/cosmos-sdk/issues/3842) Add Close() method to Batch interface (@Stumble) + +### BUG FIXES: +- [libs/db] [\#3842](https://github.com/cosmos/cosmos-sdk/issues/3842) Fix CLevelDB memory leak (@Stumble) + ## v0.30.1 *February 20th, 2019* diff --git a/version/version.go b/version/version.go index 1f30978c1..1b0a36ae1 100644 --- a/version/version.go +++ b/version/version.go @@ -20,7 +20,7 @@ const ( // Must be a string because scripts like dist.sh read this file. // XXX: Don't change the name of this variable or you will break // automation :) - TMCoreSemVer = "0.30.1" + TMCoreSemVer = "0.30.2" // ABCISemVer is the semantic version of the ABCI library ABCISemVer = "0.15.0"