diff --git a/CHANGELOG.md b/CHANGELOG.md index e36a02d9a..c380fdcd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.4.0 (October 26, 2017) + +BREAKING: + - [common] GoPath is now a function + - [db] `DB` and `Iterator` interfaces have new methods to better support iteration + +FEATURES: + - [autofile] `Read([]byte)` and `Write([]byte)` methods on `Group` to support binary WAL + - [common] `Kill()` sends SIGTERM to the current process + +IMPROVEMENTS: + - comments and linting + +BUG FIXES: + - [events] fix allocation error prefixing cache with 1000 empty events + ## 0.3.2 (October 2, 2017) BUG FIXES: diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..d2dddf85a --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,3 @@ +* @melekes @ebuchman +*.md @zramsay +*.rst @zramsay diff --git a/Makefile b/Makefile index 8e43dd11a..25773ed36 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,15 @@ .PHONY: all test get_vendor_deps ensure_tools GOTOOLS = \ - github.com/Masterminds/glide + github.com/Masterminds/glide \ + github.com/alecthomas/gometalinter + REPO:=github.com/tendermint/tmlibs all: test +NOVENDOR = go list github.com/tendermint/tmlibs/... | grep -v /vendor/ + test: go test `glide novendor` @@ -16,3 +20,37 @@ get_vendor_deps: ensure_tools ensure_tools: go get $(GOTOOLS) + +metalinter: ensure_tools + @gometalinter --install + gometalinter --vendor --deadline=600s --enable-all --disable=lll ./... + +metalinter_test: ensure_tools + @gometalinter --install + gometalinter --vendor --deadline=600s --disable-all \ + --enable=deadcode \ + --enable=gas \ + --enable=goconst \ + --enable=gosimple \ + --enable=ineffassign \ + --enable=interfacer \ + --enable=megacheck \ + --enable=misspell \ + --enable=staticcheck \ + --enable=safesql \ + --enable=structcheck \ + --enable=unconvert \ + --enable=unused \ + --enable=varcheck \ + --enable=vetshadow \ + --enable=vet \ + ./... + + #--enable=aligncheck \ + #--enable=dupl \ + #--enable=errcheck \ + #--enable=gocyclo \ + #--enable=goimports \ + #--enable=golint \ <== comments on anything exported + #--enable=gotype \ + #--enable=unparam \ diff --git a/autofile/autofile_test.go b/autofile/autofile_test.go index 8f8017e1b..8f453dd07 100644 --- a/autofile/autofile_test.go +++ b/autofile/autofile_test.go @@ -1,20 +1,20 @@ package autofile import ( - . "github.com/tendermint/tmlibs/common" "os" "sync/atomic" "syscall" "testing" "time" + + cmn "github.com/tendermint/tmlibs/common" ) func TestSIGHUP(t *testing.T) { // First, create an AutoFile writing to a tempfile dir - file, name := Tempfile("sighup_test") - err := file.Close() - if err != nil { + file, name := cmn.Tempfile("sighup_test") + if err := file.Close(); err != nil { t.Fatalf("Error creating tempfile: %v", err) } // Here is the actual AutoFile @@ -57,17 +57,15 @@ func TestSIGHUP(t *testing.T) { if err != nil { t.Fatalf("Error writing to autofile: %v", err) } - err = af.Close() - if err != nil { + if err := af.Close(); err != nil { t.Fatalf("Error closing autofile") } // Both files should exist - if body := MustReadFile(name + "_old"); string(body) != "Line 1\nLine 2\n" { + if body := cmn.MustReadFile(name + "_old"); string(body) != "Line 1\nLine 2\n" { t.Errorf("Unexpected body %s", body) } - if body := MustReadFile(name); string(body) != "Line 3\nLine 4\n" { + if body := cmn.MustReadFile(name); string(body) != "Line 3\nLine 4\n" { t.Errorf("Unexpected body %s", body) } - } diff --git a/autofile/group.go b/autofile/group.go index ce3e30009..bbf77d27e 100644 --- a/autofile/group.go +++ b/autofile/group.go @@ -18,6 +18,13 @@ import ( . "github.com/tendermint/tmlibs/common" ) +const ( + groupCheckDuration = 5000 * time.Millisecond + defaultHeadSizeLimit = 10 * 1024 * 1024 // 10MB + defaultTotalSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB + maxFilesToRemove = 4 // needs to be greater than 1 +) + /* You can open a Group to keep restrictions on an AutoFile, like the maximum size of each chunk, and/or the total amount of bytes @@ -25,33 +32,27 @@ stored in the group. The first file to be written in the Group.Dir is the head file. - Dir/ - - + Dir/ + - Once the Head file reaches the size limit, it will be rotated. - Dir/ - - .000 // First rolled file - - // New head path, starts empty. - // The implicit index is 001. + Dir/ + - .000 // First rolled file + - // New head path, starts empty. + // The implicit index is 001. As more files are written, the index numbers grow... - Dir/ - - .000 // First rolled file - - .001 // Second rolled file - - ... - - // New head path + Dir/ + - .000 // First rolled file + - .001 // Second rolled file + - ... + - // New head path The Group can also be used to binary-search for some line, assuming that marker lines are written occasionally. */ - -const groupCheckDuration = 5000 * time.Millisecond -const defaultHeadSizeLimit = 10 * 1024 * 1024 // 10MB -const defaultTotalSizeLimit = 1 * 1024 * 1024 * 1024 // 1GB -const maxFilesToRemove = 4 // needs to be greater than 1 - type Group struct { BaseService @@ -107,40 +108,63 @@ func (g *Group) OnStart() error { func (g *Group) OnStop() { g.BaseService.OnStop() g.ticker.Stop() - return } +// SetHeadSizeLimit allows you to overwrite default head size limit - 10MB. func (g *Group) SetHeadSizeLimit(limit int64) { g.mtx.Lock() g.headSizeLimit = limit g.mtx.Unlock() } +// HeadSizeLimit returns the current head size limit. func (g *Group) HeadSizeLimit() int64 { g.mtx.Lock() defer g.mtx.Unlock() return g.headSizeLimit } +// SetTotalSizeLimit allows you to overwrite default total size limit of the +// group - 1GB. func (g *Group) SetTotalSizeLimit(limit int64) { g.mtx.Lock() g.totalSizeLimit = limit g.mtx.Unlock() } +// TotalSizeLimit returns total size limit of the group. func (g *Group) TotalSizeLimit() int64 { g.mtx.Lock() defer g.mtx.Unlock() return g.totalSizeLimit } +// MaxIndex returns index of the last file in the group. func (g *Group) MaxIndex() int { g.mtx.Lock() defer g.mtx.Unlock() return g.maxIndex } -// Auto appends "\n" +// MinIndex returns index of the first file in the group. +func (g *Group) MinIndex() int { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.minIndex +} + +// Write writes the contents of p into the current head of the group. It +// returns the number of bytes written. If nn < len(p), it also returns an +// error explaining why the write is short. +// NOTE: Writes are buffered so they don't write synchronously +// TODO: Make it halt if space is unavailable +func (g *Group) Write(p []byte) (nn int, err error) { + g.mtx.Lock() + defer g.mtx.Unlock() + return g.headBuf.Write(p) +} + +// WriteLine writes line into the current head of the group. It also appends "\n". // NOTE: Writes are buffered so they don't write synchronously // TODO: Make it halt if space is unavailable func (g *Group) WriteLine(line string) error { @@ -150,6 +174,8 @@ func (g *Group) WriteLine(line string) error { return err } +// Flush writes any buffered data to the underlying file and commits the +// current content of the file to stable storage. func (g *Group) Flush() error { g.mtx.Lock() defer g.mtx.Unlock() @@ -224,6 +250,8 @@ func (g *Group) checkTotalSizeLimit() { } } +// RotateFile causes group to close the current head and assign it some index. +// Note it does not create a new head. func (g *Group) RotateFile() { g.mtx.Lock() defer g.mtx.Unlock() @@ -242,8 +270,8 @@ func (g *Group) RotateFile() { g.maxIndex += 1 } -// NOTE: if error, returns no GroupReader. -// CONTRACT: Caller must close the returned GroupReader +// NewReader returns a new group reader. +// CONTRACT: Caller must close the returned GroupReader. func (g *Group) NewReader(index int) (*GroupReader, error) { r := newGroupReader(g) err := r.SetIndex(index) @@ -424,14 +452,15 @@ GROUP_LOOP: return } +// GroupInfo holds information about the group. type GroupInfo struct { - MinIndex int - MaxIndex int - TotalSize int64 - HeadSize int64 + MinIndex int // index of the first file in the group, including head + MaxIndex int // index of the last file in the group, including head + TotalSize int64 // total size of the group + HeadSize int64 // size of the head } -// Returns info after scanning all files in g.Head's dir +// Returns info after scanning all files in g.Head's dir. func (g *Group) ReadGroupInfo() GroupInfo { g.mtx.Lock() defer g.mtx.Unlock() @@ -506,6 +535,7 @@ func filePathForIndex(headPath string, index int, maxIndex int) string { //-------------------------------------------------------------------------------- +// GroupReader provides an interface for reading from a Group. type GroupReader struct { *Group mtx sync.Mutex @@ -525,6 +555,7 @@ func newGroupReader(g *Group) *GroupReader { } } +// Close closes the GroupReader by closing the cursor file. func (gr *GroupReader) Close() error { gr.mtx.Lock() defer gr.mtx.Unlock() @@ -541,7 +572,48 @@ func (gr *GroupReader) Close() error { } } -// Reads a line (without delimiter) +// Read implements io.Reader, reading bytes from the current Reader +// incrementing index until enough bytes are read. +func (gr *GroupReader) Read(p []byte) (n int, err error) { + lenP := len(p) + if lenP == 0 { + return 0, errors.New("given empty slice") + } + + gr.mtx.Lock() + defer gr.mtx.Unlock() + + // Open file if not open yet + if gr.curReader == nil { + if err = gr.openFile(gr.curIndex); err != nil { + return 0, err + } + } + + // Iterate over files until enough bytes are read + var nn int + for { + nn, err = gr.curReader.Read(p[n:]) + n += nn + if err == io.EOF { + // Open the next file + if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { + return n, err1 + } + if n >= lenP { + return n, nil + } else { + continue + } + } else if err != nil { + return n, err + } else if nn == 0 { // empty file + return n, err + } + } +} + +// ReadLine reads a line (without delimiter). // just return io.EOF if no new lines found. func (gr *GroupReader) ReadLine() (string, error) { gr.mtx.Lock() @@ -568,9 +640,8 @@ func (gr *GroupReader) ReadLine() (string, error) { bytesRead, err := gr.curReader.ReadBytes('\n') if err == io.EOF { // Open the next file - err := gr.openFile(gr.curIndex + 1) - if err != nil { - return "", err + if err1 := gr.openFile(gr.curIndex + 1); err1 != nil { + return "", err1 } if len(bytesRead) > 0 && bytesRead[len(bytesRead)-1] == byte('\n') { return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil @@ -615,6 +686,9 @@ func (gr *GroupReader) openFile(index int) error { return nil } +// PushLine makes the given line the current one, so the next time somebody +// calls ReadLine, this line will be returned. +// panics if called twice without calling ReadLine. func (gr *GroupReader) PushLine(line string) { gr.mtx.Lock() defer gr.mtx.Unlock() @@ -626,13 +700,15 @@ func (gr *GroupReader) PushLine(line string) { } } -// Cursor's file index. +// CurIndex returns cursor's file index. func (gr *GroupReader) CurIndex() int { gr.mtx.Lock() defer gr.mtx.Unlock() return gr.curIndex } +// SetIndex sets the cursor's file index to index by opening a file at this +// position. func (gr *GroupReader) SetIndex(index int) error { gr.mtx.Lock() defer gr.mtx.Unlock() diff --git a/autofile/group_test.go b/autofile/group_test.go index 92e259701..398ea3ae9 100644 --- a/autofile/group_test.go +++ b/autofile/group_test.go @@ -1,6 +1,7 @@ package autofile import ( + "bytes" "errors" "io" "io/ioutil" @@ -77,8 +78,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) // Write 1000 more bytes. - err := g.WriteLine(RandStr(999)) - if err != nil { + if err := g.WriteLine(RandStr(999)); err != nil { t.Fatal("Error appending to head", err) } g.Flush() @@ -88,8 +88,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0) // Write 1000 more bytes. - err = g.WriteLine(RandStr(999)) - if err != nil { + if err := g.WriteLine(RandStr(999)); err != nil { t.Fatal("Error appending to head", err) } g.Flush() @@ -100,8 +99,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { // Write 1000 bytes 999 times. for i := 0; i < 999; i++ { - err := g.WriteLine(RandStr(999)) - if err != nil { + if err := g.WriteLine(RandStr(999)); err != nil { t.Fatal("Error appending to head", err) } } @@ -113,7 +111,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2000000, 0) // Write 1000 more bytes. - _, err = g.Head.Write([]byte(RandStr(999) + "\n")) + _, err := g.Head.Write([]byte(RandStr(999) + "\n")) if err != nil { t.Fatal("Error appending to head", err) } @@ -403,3 +401,93 @@ func TestFindLast4(t *testing.T) { // Cleanup destroyTestGroup(t, g) } + +func TestWrite(t *testing.T) { + g := createTestGroup(t, 0) + + written := []byte("Medusa") + g.Write(written) + g.Flush() + + read := make([]byte, len(written)) + gr, err := g.NewReader(0) + if err != nil { + t.Fatalf("Failed to create reader: %v", err) + } + _, err = gr.Read(read) + if err != nil { + t.Fatalf("Failed to read data: %v", err) + } + + if !bytes.Equal(written, read) { + t.Errorf("%s, %s should be equal", string(written), string(read)) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestGroupReaderRead(t *testing.T) { + g := createTestGroup(t, 0) + + professor := []byte("Professor Monster") + g.Write(professor) + g.Flush() + g.RotateFile() + frankenstein := []byte("Frankenstein's Monster") + g.Write(frankenstein) + g.Flush() + + totalWrittenLength := len(professor) + len(frankenstein) + read := make([]byte, totalWrittenLength) + gr, err := g.NewReader(0) + if err != nil { + t.Fatalf("Failed to create reader: %v", err) + } + n, err := gr.Read(read) + if err != nil { + t.Fatalf("Failed to read data: %v", err) + } + if n != totalWrittenLength { + t.Errorf("Failed to read enough bytes: wanted %d, but read %d", totalWrittenLength, n) + } + + professorPlusFrankenstein := professor + professorPlusFrankenstein = append(professorPlusFrankenstein, frankenstein...) + if !bytes.Equal(read, professorPlusFrankenstein) { + t.Errorf("%s, %s should be equal", string(professorPlusFrankenstein), string(read)) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestMinIndex(t *testing.T) { + g := createTestGroup(t, 0) + + if g.MinIndex() != 0 { + t.Error("MinIndex should be zero at the beginning") + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestMaxIndex(t *testing.T) { + g := createTestGroup(t, 0) + + if g.MaxIndex() != 0 { + t.Error("MaxIndex should be zero at the beginning") + } + + g.WriteLine("Line 1") + g.Flush() + g.RotateFile() + + if g.MaxIndex() != 1 { + t.Error("MaxIndex should point to the last file") + } + + // Cleanup + destroyTestGroup(t, g) +} diff --git a/autofile/sighup_watcher.go b/autofile/sighup_watcher.go index facc238d5..56fbd4d86 100644 --- a/autofile/sighup_watcher.go +++ b/autofile/sighup_watcher.go @@ -22,7 +22,7 @@ func initSighupWatcher() { signal.Notify(c, syscall.SIGHUP) go func() { - for _ = range c { + for range c { sighupWatchers.closeAll() atomic.AddInt32(&sighupCounter, 1) } diff --git a/circle.yml b/circle.yml index 23ac4bd9f..3dba976be 100644 --- a/circle.yml +++ b/circle.yml @@ -1,11 +1,9 @@ machine: environment: - GOPATH: /home/ubuntu/.go_workspace + GOPATH: "${HOME}/.go_workspace" PROJECT_PARENT_PATH: "$GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME" PROJECT_PATH: $GOPATH/src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME - GO15VENDOREXPERIMENT: 1 hosts: - circlehost: 127.0.0.1 localhost: 127.0.0.1 dependencies: @@ -17,5 +15,7 @@ dependencies: test: override: - - "go version" - - "cd $PROJECT_PATH && make get_vendor_deps && make test" + - cd $PROJECT_PATH && make get_vendor_deps && make metalinter_test && bash ./test.sh + post: + - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt + - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}" diff --git a/cli/flags/log_level_test.go b/cli/flags/log_level_test.go index 458a9e24d..faf9b19db 100644 --- a/cli/flags/log_level_test.go +++ b/cli/flags/log_level_test.go @@ -49,8 +49,6 @@ func TestParseLogLevel(t *testing.T) { t.Fatal(err) } - logger = logger - buf.Reset() logger.With("module", "wire").Debug("Kingpin") diff --git a/clist/clist_test.go b/clist/clist_test.go index ab5cf4b26..2063cf465 100644 --- a/clist/clist_test.go +++ b/clist/clist_test.go @@ -55,6 +55,7 @@ func TestSmall(t *testing.T) { This test is quite hacky because it relies on SetFinalizer which isn't guaranteed to run at all. */ +// nolint: megacheck func _TestGCFifo(t *testing.T) { const numElements = 1000000 @@ -102,6 +103,7 @@ func _TestGCFifo(t *testing.T) { This test is quite hacky because it relies on SetFinalizer which isn't guaranteed to run at all. */ +// nolint: megacheck func _TestGCRandom(t *testing.T) { const numElements = 1000000 @@ -132,7 +134,7 @@ func _TestGCRandom(t *testing.T) { for _, i := range rand.Perm(numElements) { el := els[i] l.Remove(el) - el = el.Next() + _ = el.Next() } runtime.GC() @@ -153,7 +155,7 @@ func TestScanRightDeleteRandom(t *testing.T) { l := New() stop := make(chan struct{}) - els := make([]*CElement, numElements, numElements) + els := make([]*CElement, numElements) for i := 0; i < numElements; i++ { el := l.PushBack(i) els[i] = el diff --git a/common/cmap.go b/common/cmap.go index 5de6fa2fa..e2a140dd0 100644 --- a/common/cmap.go +++ b/common/cmap.go @@ -10,7 +10,7 @@ type CMap struct { func NewCMap() *CMap { return &CMap{ - m: make(map[string]interface{}, 0), + m: make(map[string]interface{}), } } @@ -48,7 +48,7 @@ func (cm *CMap) Size() int { func (cm *CMap) Clear() { cm.l.Lock() defer cm.l.Unlock() - cm.m = make(map[string]interface{}, 0) + cm.m = make(map[string]interface{}) } func (cm *CMap) Values() []interface{} { diff --git a/common/errors.go b/common/errors.go index 3a1b09542..039342a67 100644 --- a/common/errors.go +++ b/common/errors.go @@ -21,7 +21,7 @@ func (se StackError) Error() string { // panic wrappers // A panic resulting from a sanity check means there is a programmer error -// and some gaurantee is not satisfied. +// and some guarantee is not satisfied. func PanicSanity(v interface{}) { panic(Fmt("Panicked on a Sanity Check: %v", v)) } diff --git a/common/http_test.go b/common/http_test.go index 73761fb1e..4272f6062 100644 --- a/common/http_test.go +++ b/common/http_test.go @@ -95,7 +95,7 @@ func TestWriteCode(t *testing.T) { common.WriteCode(w, &marshalFailer{}, code) wantCode := http.StatusBadRequest assert.Equal(t, w.Code, wantCode, "#%d", i) - assert.True(t, strings.Contains(string(w.Body.Bytes()), errFooFailed.Error()), + assert.True(t, strings.Contains(w.Body.String(), errFooFailed.Error()), "#%d: expected %q in the error message", i, errFooFailed) } } diff --git a/common/os.go b/common/os.go index 71ee88422..81f703c7d 100644 --- a/common/os.go +++ b/common/os.go @@ -6,19 +6,38 @@ import ( "io" "io/ioutil" "os" + "os/exec" "os/signal" "path/filepath" "strings" + "syscall" ) -var ( - GoPath = os.Getenv("GOPATH") -) +var gopath string + +// GoPath returns GOPATH env variable value. If it is not set, this function +// will try to call `go env GOPATH` subcommand. +func GoPath() string { + if gopath != "" { + return gopath + } + + path := os.Getenv("GOPATH") + if len(path) == 0 { + goCmd := exec.Command("go", "env", "GOPATH") + out, err := goCmd.Output() + if err != nil { + panic(fmt.Sprintf("failed to determine gopath: %v", err)) + } + path = string(out) + } + gopath = path + return path +} func TrapSignal(cb func()) { c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - signal.Notify(c, os.Kill) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) go func() { for sig := range c { fmt.Printf("captured %v, exiting...\n", sig) @@ -31,6 +50,12 @@ func TrapSignal(cb func()) { select {} } +// Kill the running process by sending itself SIGTERM +func Kill() error { + pid := os.Getpid() + return syscall.Kill(pid, syscall.SIGTERM) +} + func Exit(s string) { fmt.Printf(s + "\n") os.Exit(1) @@ -84,12 +109,7 @@ func MustReadFile(filePath string) []byte { } func WriteFile(filePath string, contents []byte, mode os.FileMode) error { - err := ioutil.WriteFile(filePath, contents, mode) - if err != nil { - return err - } - // fmt.Printf("File written to %v.\n", filePath) - return nil + return ioutil.WriteFile(filePath, contents, mode) } func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { diff --git a/common/os_test.go b/common/os_test.go index 05359e36e..126723aa6 100644 --- a/common/os_test.go +++ b/common/os_test.go @@ -27,3 +27,43 @@ func TestWriteFileAtomic(t *testing.T) { t.Fatal(err) } } + +func TestGoPath(t *testing.T) { + // restore original gopath upon exit + path := os.Getenv("GOPATH") + defer func() { + _ = os.Setenv("GOPATH", path) + }() + + err := os.Setenv("GOPATH", "~/testgopath") + if err != nil { + t.Fatal(err) + } + path = GoPath() + if path != "~/testgopath" { + t.Fatalf("should get GOPATH env var value, got %v", path) + } + os.Unsetenv("GOPATH") + + path = GoPath() + if path != "~/testgopath" { + t.Fatalf("subsequent calls should return the same value, got %v", path) + } +} + +func TestGoPathWithoutEnvVar(t *testing.T) { + // restore original gopath upon exit + path := os.Getenv("GOPATH") + defer func() { + _ = os.Setenv("GOPATH", path) + }() + + os.Unsetenv("GOPATH") + // reset cache + gopath = "" + + path = GoPath() + if path == "" || path == "~/testgopath" { + t.Fatalf("should get nonempty result of calling go env GOPATH, got %v", path) + } +} diff --git a/common/service.go b/common/service.go index 71fc03cb9..8d4de30a8 100644 --- a/common/service.go +++ b/common/service.go @@ -140,18 +140,16 @@ func (bs *BaseService) OnStop() {} // Implements Service func (bs *BaseService) Reset() (bool, error) { - if atomic.CompareAndSwapUint32(&bs.stopped, 1, 0) { - // whether or not we've started, we can reset - atomic.CompareAndSwapUint32(&bs.started, 1, 0) - - bs.Quit = make(chan struct{}) - return true, bs.impl.OnReset() - } else { + if !atomic.CompareAndSwapUint32(&bs.stopped, 1, 0) { bs.Logger.Debug(Fmt("Can't reset %v. Not stopped", bs.name), "impl", bs.impl) return false, nil } - // never happens - return false, nil + + // whether or not we've started, we can reset + atomic.CompareAndSwapUint32(&bs.started, 1, 0) + + bs.Quit = make(chan struct{}) + return true, bs.impl.OnReset() } // Implements Service diff --git a/common/string.go b/common/string.go index 2818f5ed5..1ab91f15a 100644 --- a/common/string.go +++ b/common/string.go @@ -31,10 +31,7 @@ func LeftPadString(s string, totalLength int) string { func IsHex(s string) bool { if len(s) > 2 && s[:2] == "0x" { _, err := hex.DecodeString(s[2:]) - if err != nil { - return false - } - return true + return err == nil } return false } diff --git a/db/c_level_db_test.go b/db/c_level_db_test.go index 0ee6d6414..e7336cc5f 100644 --- a/db/c_level_db_test.go +++ b/db/c_level_db_test.go @@ -50,7 +50,7 @@ func BenchmarkRandomReadsWrites2(b *testing.B) { //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) if val == 0 { if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %X for %v, got %X", + b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) break } diff --git a/db/db.go b/db/db.go index aa8ff48a8..8156c1e92 100644 --- a/db/db.go +++ b/db/db.go @@ -10,10 +10,11 @@ type DB interface { DeleteSync([]byte) Close() NewBatch() Batch + Iterator() Iterator + IteratorPrefix([]byte) Iterator // For debugging Print() - Iterator() Iterator Stats() map[string]string } @@ -28,6 +29,9 @@ type Iterator interface { Key() []byte Value() []byte + + Release() + Error() error } //----------------------------------------------------------------------------- diff --git a/db/go_level_db.go b/db/go_level_db.go index 54ae1149f..4abd76112 100644 --- a/db/go_level_db.go +++ b/db/go_level_db.go @@ -6,7 +6,9 @@ import ( "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" + "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" + "github.com/syndtr/goleveldb/leveldb/util" . "github.com/tendermint/tmlibs/common" ) @@ -115,8 +117,46 @@ func (db *GoLevelDB) Stats() map[string]string { return stats } +type goLevelDBIterator struct { + source iterator.Iterator +} + +// Key returns a copy of the current key. +func (it *goLevelDBIterator) Key() []byte { + key := it.source.Key() + k := make([]byte, len(key)) + copy(k, key) + + return k +} + +// Value returns a copy of the current value. +func (it *goLevelDBIterator) Value() []byte { + val := it.source.Value() + v := make([]byte, len(val)) + copy(v, val) + + return v +} + +func (it *goLevelDBIterator) Error() error { + return it.source.Error() +} + +func (it *goLevelDBIterator) Next() bool { + return it.source.Next() +} + +func (it *goLevelDBIterator) Release() { + it.source.Release() +} + func (db *GoLevelDB) Iterator() Iterator { - return db.db.NewIterator(nil, nil) + return &goLevelDBIterator{db.db.NewIterator(nil, nil)} +} + +func (db *GoLevelDB) IteratorPrefix(prefix []byte) Iterator { + return &goLevelDBIterator{db.db.NewIterator(util.BytesPrefix(prefix), nil)} } func (db *GoLevelDB) NewBatch() Batch { diff --git a/db/go_level_db_test.go b/db/go_level_db_test.go index 0603b2d4f..2cd3192c3 100644 --- a/db/go_level_db_test.go +++ b/db/go_level_db_test.go @@ -49,7 +49,7 @@ func BenchmarkRandomReadsWrites(b *testing.B) { //fmt.Printf("Get %X -> %X\n", idxBytes, valBytes) if val == 0 { if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %X for %v, got %X", + b.Errorf("Expected %v for %v, got %X", nil, idx, valBytes) break } diff --git a/db/mem_db.go b/db/mem_db.go index 58e74895b..077427509 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "strings" "sync" ) @@ -99,7 +100,20 @@ func (it *memDBIterator) Value() []byte { return it.db.Get(it.Key()) } +func (it *memDBIterator) Release() { + it.db = nil + it.keys = nil +} + +func (it *memDBIterator) Error() error { + return nil +} + func (db *MemDB) Iterator() Iterator { + return db.IteratorPrefix([]byte{}) +} + +func (db *MemDB) IteratorPrefix(prefix []byte) Iterator { it := newMemDBIterator() it.db = db it.last = -1 @@ -109,7 +123,9 @@ func (db *MemDB) Iterator() Iterator { // unfortunately we need a copy of all of the keys for key, _ := range db.db { - it.keys = append(it.keys, key) + if strings.HasPrefix(key, string(prefix)) { + it.keys = append(it.keys, key) + } } return it } diff --git a/events/event_cache.go b/events/event_cache.go index 905f1096a..f508e873d 100644 --- a/events/event_cache.go +++ b/events/event_cache.go @@ -1,9 +1,5 @@ package events -const ( - eventsBufferSize = 1000 -) - // An EventCache buffers events for a Fireable // All events are cached. Filtering happens on Flush type EventCache struct { @@ -14,8 +10,7 @@ type EventCache struct { // Create a new EventCache with an EventSwitch as backend func NewEventCache(evsw Fireable) *EventCache { return &EventCache{ - evsw: evsw, - events: make([]eventInfo, eventsBufferSize), + evsw: evsw, } } @@ -27,7 +22,7 @@ type eventInfo struct { // Cache an event to be fired upon finality. func (evc *EventCache) FireEvent(event string, data EventData) { - // append to list + // append to list (go will grow our backing array exponentially) evc.events = append(evc.events, eventInfo{event, data}) } @@ -37,5 +32,6 @@ func (evc *EventCache) Flush() { for _, ei := range evc.events { evc.evsw.FireEvent(ei.event, ei.data) } - evc.events = make([]eventInfo, eventsBufferSize) + // Clear the buffer, since we only add to it with append it's safe to just set it to nil and maybe safe an allocation + evc.events = nil } diff --git a/events/event_cache_test.go b/events/event_cache_test.go new file mode 100644 index 000000000..ab321da3a --- /dev/null +++ b/events/event_cache_test.go @@ -0,0 +1,35 @@ +package events + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEventCache_Flush(t *testing.T) { + evsw := NewEventSwitch() + evsw.Start() + evsw.AddListenerForEvent("nothingness", "", func(data EventData) { + // Check we are not initialising an empty buffer full of zeroed eventInfos in the EventCache + require.FailNow(t, "We should never receive a message on this switch since none are fired") + }) + evc := NewEventCache(evsw) + evc.Flush() + // Check after reset + evc.Flush() + fail := true + pass := false + evsw.AddListenerForEvent("somethingness", "something", func(data EventData) { + if fail { + require.FailNow(t, "Shouldn't see a message until flushed") + } + pass = true + }) + evc.FireEvent("something", struct{ int }{1}) + evc.FireEvent("something", struct{ int }{2}) + evc.FireEvent("something", struct{ int }{3}) + fail = false + evc.Flush() + assert.True(t, pass) +} diff --git a/events/events_test.go b/events/events_test.go index c1b48b16f..dee50e5bd 100644 --- a/events/events_test.go +++ b/events/events_test.go @@ -14,7 +14,7 @@ import ( func TestAddListenerForEventFireOnce(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } messages := make(chan EventData) @@ -34,7 +34,7 @@ func TestAddListenerForEventFireOnce(t *testing.T) { func TestAddListenerForEventFireMany(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } doneSum := make(chan uint64) @@ -63,7 +63,7 @@ func TestAddListenerForEventFireMany(t *testing.T) { func TestAddListenerForDifferentEvents(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } doneSum := make(chan uint64) @@ -108,7 +108,7 @@ func TestAddListenerForDifferentEvents(t *testing.T) { func TestAddDifferentListenerForDifferentEvents(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } doneSum1 := make(chan uint64) @@ -168,7 +168,7 @@ func TestAddDifferentListenerForDifferentEvents(t *testing.T) { func TestAddAndRemoveListener(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } doneSum1 := make(chan uint64) @@ -213,7 +213,7 @@ func TestAddAndRemoveListener(t *testing.T) { func TestRemoveListener(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } count := 10 @@ -266,7 +266,7 @@ func TestRemoveListener(t *testing.T) { func TestRemoveListenersAsync(t *testing.T) { evsw := NewEventSwitch() started, err := evsw.Start() - if started == false || err != nil { + if !started || err != nil { t.Errorf("Failed to start EventSwitch, error: %v", err) } doneSum1 := make(chan uint64) @@ -377,5 +377,4 @@ func fireEvents(evsw EventSwitch, event string, doneChan chan uint64, } doneChan <- sentSum close(doneChan) - return } diff --git a/flowrate/io_test.go b/flowrate/io_test.go index 6d4934a8a..db40337c9 100644 --- a/flowrate/io_test.go +++ b/flowrate/io_test.go @@ -171,10 +171,7 @@ func statusesAreEqual(s1 *Status, s2 *Status) bool { } func durationsAreEqual(d1 time.Duration, d2 time.Duration, maxDeviation time.Duration) bool { - if d2-d1 <= maxDeviation { - return true - } - return false + return d2-d1 <= maxDeviation } func ratesAreEqual(r1 int64, r2 int64, maxDeviation int64) bool { diff --git a/log/filter_test.go b/log/filter_test.go index fafafacb0..8d8b3b27c 100644 --- a/log/filter_test.go +++ b/log/filter_test.go @@ -73,8 +73,7 @@ func TestVariousLevels(t *testing.T) { func TestLevelContext(t *testing.T) { var buf bytes.Buffer - var logger log.Logger - logger = log.NewTMJSONLogger(&buf) + logger := log.NewTMJSONLogger(&buf) logger = log.NewFilter(logger, log.AllowError()) logger = logger.With("context", "value") @@ -93,8 +92,7 @@ func TestLevelContext(t *testing.T) { func TestVariousAllowWith(t *testing.T) { var buf bytes.Buffer - var logger log.Logger - logger = log.NewTMJSONLogger(&buf) + logger := log.NewTMJSONLogger(&buf) logger1 := log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("context", "value")) logger1.With("context", "value").Info("foo", "bar", "baz") diff --git a/log/tmfmt_logger.go b/log/tmfmt_logger.go index 14028d756..2b464a6b0 100644 --- a/log/tmfmt_logger.go +++ b/log/tmfmt_logger.go @@ -49,9 +49,10 @@ func (l tmfmtLogger) Log(keyvals ...interface{}) error { enc.Reset() defer tmfmtEncoderPool.Put(enc) + const unknown = "unknown" lvl := "none" - msg := "unknown" - module := "unknown" + msg := unknown + module := unknown // indexes of keys to skip while encoding later excludeIndexes := make([]int, 0) @@ -90,7 +91,7 @@ func (l tmfmtLogger) Log(keyvals ...interface{}) error { // Stopping ... - message enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s ", lvl[0]-32, time.Now().UTC().Format("01-02|15:04:05.000"), msg)) - if module != "unknown" { + if module != unknown { enc.buf.WriteString("module=" + module + " ") } diff --git a/log/tracing_logger_test.go b/log/tracing_logger_test.go index 584b34bef..6b0838ca8 100644 --- a/log/tracing_logger_test.go +++ b/log/tracing_logger_test.go @@ -14,8 +14,7 @@ import ( func TestTracingLogger(t *testing.T) { var buf bytes.Buffer - var logger log.Logger - logger = log.NewTMJSONLogger(&buf) + logger := log.NewTMJSONLogger(&buf) logger1 := log.NewTracingLogger(logger) err1 := errors.New("Courage is grace under pressure.") diff --git a/merkle/simple_tree.go b/merkle/simple_tree.go index b5520f723..8106246d6 100644 --- a/merkle/simple_tree.go +++ b/merkle/simple_tree.go @@ -31,8 +31,8 @@ import ( "golang.org/x/crypto/ripemd160" - . "github.com/tendermint/tmlibs/common" "github.com/tendermint/go-wire" + . "github.com/tendermint/tmlibs/common" ) func SimpleHashFromTwoHashes(left []byte, right []byte) []byte { diff --git a/process/util.go b/process/util.go index b3e0aef11..24cf35280 100644 --- a/process/util.go +++ b/process/util.go @@ -15,8 +15,8 @@ func Run(dir string, command string, args []string) (string, bool, error) { <-proc.WaitCh if proc.ExitState.Success() { - return string(outFile.Bytes()), true, nil + return outFile.String(), true, nil } else { - return string(outFile.Bytes()), false, nil + return outFile.String(), false, nil } } diff --git a/pubsub/query/query.peg.go b/pubsub/query/query.peg.go index 37ce75cd9..c86e4a47f 100644 --- a/pubsub/query/query.peg.go +++ b/pubsub/query/query.peg.go @@ -1,3 +1,4 @@ +// nolint package query import ( diff --git a/test.sh b/test.sh new file mode 100755 index 000000000..012162b07 --- /dev/null +++ b/test.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e +echo "" > coverage.txt + +for d in $(go list ./... | grep -v vendor); do + go test -race -coverprofile=profile.out -covermode=atomic "$d" + if [ -f profile.out ]; then + cat profile.out >> coverage.txt + rm profile.out + fi +done diff --git a/version/version.go b/version/version.go index 77580b5ad..c1635d202 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "0.3.2" +const Version = "0.4.0"