zolfa
/
tendermint
1
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 221 Wiki Activity
Browse Source

libs: remove useless code in group (#3504) 

* lib: remove useless code in group
* update change log
* Update CHANGELOG_PENDING.md

Co-Authored-By: guagualvcha <baifudong@lancai.cn>
pull/3438/head
zjubfd 6 years ago
committed by Alexander Simmerl
parent
9199f3f613
commit
2233dd45bd
3 changed files with 1 additions and 450 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      CHANGELOG_PENDING.md
  2. +0
    -251
      libs/autofile/group.go
  3. +0
    -199
      libs/autofile/group_test.go

+ 1
- 0
CHANGELOG_PENDING.md View File

@ -9,6 +9,7 @@
* Apps
* Go API
- [libs/autofile] \#3504 Remove unused code in autofile package. Deleted functions: `Group.Search`, `Group.FindLast`, `GroupReader.ReadLine`, `GroupReader.PushLine`, `MakeSimpleSearchFunc` (@guagualvcha)
* Blockchain Protocol


+ 0
- 251
libs/autofile/group.go View File

@ -331,172 +331,6 @@ func (g *Group) NewReader(index int) (*GroupReader, error) {
return r, nil
}
// Returns -1 if line comes after, 0 if found, 1 if line comes before.
type SearchFunc func(line string) (int, error)
// Searches for the right file in Group, then returns a GroupReader to start
// streaming lines.
// Returns true if an exact match was found, otherwise returns the next greater
// line that starts with prefix.
// CONTRACT: Caller must close the returned GroupReader
func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error) {
g.mtx.Lock()
minIndex, maxIndex := g.minIndex, g.maxIndex
g.mtx.Unlock()
// Now minIndex/maxIndex may change meanwhile,
// but it shouldn't be a big deal
// (maybe we'll want to limit scanUntil though)
for {
curIndex := (minIndex + maxIndex + 1) / 2
// Base case, when there's only 1 choice left.
if minIndex == maxIndex {
r, err := g.NewReader(maxIndex)
if err != nil {
return nil, false, err
}
match, err := scanUntil(r, prefix, cmp)
if err != nil {
r.Close()
return nil, false, err
}
return r, match, err
}
// Read starting roughly at the middle file,
// until we find line that has prefix.
r, err := g.NewReader(curIndex)
if err != nil {
return nil, false, err
}
foundIndex, line, err := scanNext(r, prefix)
r.Close()
if err != nil {
return nil, false, err
}
// Compare this line to our search query.
val, err := cmp(line)
if err != nil {
return nil, false, err
}
if val < 0 {
// Line will come later
minIndex = foundIndex
} else if val == 0 {
// Stroke of luck, found the line
r, err := g.NewReader(foundIndex)
if err != nil {
return nil, false, err
}
match, err := scanUntil(r, prefix, cmp)
if !match {
panic("Expected match to be true")
}
if err != nil {
r.Close()
return nil, false, err
}
return r, true, err
} else {
// We passed it
maxIndex = curIndex - 1
}
}
}
// Scans and returns the first line that starts with 'prefix'
// Consumes line and returns it.
func scanNext(r *GroupReader, prefix string) (int, string, error) {
for {
line, err := r.ReadLine()
if err != nil {
return 0, "", err
}
if !strings.HasPrefix(line, prefix) {
continue
}
index := r.CurIndex()
return index, line, nil
}
}
// Returns true iff an exact match was found.
// Pushes line, does not consume it.
func scanUntil(r *GroupReader, prefix string, cmp SearchFunc) (bool, error) {
for {
line, err := r.ReadLine()
if err != nil {
return false, err
}
if !strings.HasPrefix(line, prefix) {
continue
}
val, err := cmp(line)
if err != nil {
return false, err
}
if val < 0 {
continue
} else if val == 0 {
r.PushLine(line)
return true, nil
} else {
r.PushLine(line)
return false, nil
}
}
}
// Searches backwards for the last line in Group with prefix.
// Scans each file forward until the end to find the last match.
func (g *Group) FindLast(prefix string) (match string, found bool, err error) {
g.mtx.Lock()
minIndex, maxIndex := g.minIndex, g.maxIndex
g.mtx.Unlock()
r, err := g.NewReader(maxIndex)
if err != nil {
return "", false, err
}
defer r.Close()
// Open files from the back and read
GROUP_LOOP:
for i := maxIndex; i >= minIndex; i-- {
err := r.SetIndex(i)
if err != nil {
return "", false, err
}
// Scan each line and test whether line matches
for {
line, err := r.ReadLine()
if err == io.EOF {
if found {
return match, found, nil
}
continue GROUP_LOOP
} else if err != nil {
return "", false, err
}
if strings.HasPrefix(line, prefix) {
match = line
found = true
}
if r.CurIndex() > i {
if found {
return match, found, nil
}
continue GROUP_LOOP
}
}
}
return
}
// GroupInfo holds information about the group.
type GroupInfo struct {
MinIndex int // index of the first file in the group, including head
@ -654,48 +488,6 @@ func (gr *GroupReader) Read(p []byte) (n int, err error) {
}
}
// ReadLine reads a line (without delimiter).
// just return io.EOF if no new lines found.
func (gr *GroupReader) ReadLine() (string, error) {
gr.mtx.Lock()
defer gr.mtx.Unlock()
// From PushLine
if gr.curLine != nil {
line := string(gr.curLine)
gr.curLine = nil
return line, nil
}
// Open file if not open yet
if gr.curReader == nil {
err := gr.openFile(gr.curIndex)
if err != nil {
return "", err
}
}
// Iterate over files until line is found
var linePrefix string
for {
bytesRead, err := gr.curReader.ReadBytes('\n')
if err == io.EOF {
// Open the next file
if err1 := gr.openFile(gr.curIndex + 1); err1 != nil {
return "", err1
}
if len(bytesRead) > 0 && bytesRead[len(bytesRead)-1] == byte('\n') {
return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil
}
linePrefix += string(bytesRead)
continue
} else if err != nil {
return "", err
}
return linePrefix + string(bytesRead[:len(bytesRead)-1]), nil
}
}
// IF index > gr.Group.maxIndex, returns io.EOF
// CONTRACT: caller should hold gr.mtx
func (gr *GroupReader) openFile(index int) error {
@ -725,20 +517,6 @@ func (gr *GroupReader) openFile(index int) error {
return nil
}
// PushLine makes the given line the current one, so the next time somebody
// calls ReadLine, this line will be returned.
// panics if called twice without calling ReadLine.
func (gr *GroupReader) PushLine(line string) {
gr.mtx.Lock()
defer gr.mtx.Unlock()
if gr.curLine == nil {
gr.curLine = []byte(line)
} else {
panic("PushLine failed, already have line")
}
}
// CurIndex returns cursor's file index.
func (gr *GroupReader) CurIndex() int {
gr.mtx.Lock()
@ -753,32 +531,3 @@ func (gr *GroupReader) SetIndex(index int) error {
defer gr.mtx.Unlock()
return gr.openFile(index)
}
//--------------------------------------------------------------------------------
// A simple SearchFunc that assumes that the marker is of form
// <prefix><number>.
// For example, if prefix is '#HEIGHT:', the markers of expected to be of the form:
//
// #HEIGHT:1
// ...
// #HEIGHT:2
// ...
func MakeSimpleSearchFunc(prefix string, target int) SearchFunc {
return func(line string) (int, error) {
if !strings.HasPrefix(line, prefix) {
return -1, fmt.Errorf("Marker line did not have prefix: %v", prefix)
}
i, err := strconv.Atoi(line[len(prefix):])
if err != nil {
return -1, fmt.Errorf("Failed to parse marker line: %v", err.Error())
}
if target < i {
return 1, nil
} else if target == i {
return 0, nil
} else {
return -1, nil
}
}
}

+ 0
- 199
libs/autofile/group_test.go View File

@ -1,13 +1,9 @@
package autofile
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"strconv"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@ -106,107 +102,6 @@ func TestCheckHeadSizeLimit(t *testing.T) {
destroyTestGroup(t, g)
}
func TestSearch(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 10*1000)
// Create some files in the group that have several INFO lines in them.
// Try to put the INFO lines in various spots.
for i := 0; i < 100; i++ {
// The random junk at the end ensures that this INFO linen
// is equally likely to show up at the end.
_, err := g.Head.Write([]byte(fmt.Sprintf("INFO %v %v\n", i, cmn.RandStr(123))))
require.NoError(t, err, "Failed to write to head")
g.checkHeadSizeLimit()
for j := 0; j < 10; j++ {
_, err1 := g.Head.Write([]byte(cmn.RandStr(123) + "\n"))
require.NoError(t, err1, "Failed to write to head")
g.checkHeadSizeLimit()
}
}
// Create a search func that searches for line
makeSearchFunc := func(target int) SearchFunc {
return func(line string) (int, error) {
parts := strings.Split(line, " ")
if len(parts) != 3 {
return -1, errors.New("Line did not have 3 parts")
}
i, err := strconv.Atoi(parts[1])
if err != nil {
return -1, errors.New("Failed to parse INFO: " + err.Error())
}
if target < i {
return 1, nil
} else if target == i {
return 0, nil
} else {
return -1, nil
}
}
}
// Now search for each number
for i := 0; i < 100; i++ {
gr, match, err := g.Search("INFO", makeSearchFunc(i))
require.NoError(t, err, "Failed to search for line, tc #%d", i)
assert.True(t, match, "Expected Search to return exact match, tc #%d", i)
line, err := gr.ReadLine()
require.NoError(t, err, "Failed to read line after search, tc #%d", i)
if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", i)) {
t.Fatalf("Failed to get correct line, tc #%d", i)
}
// Make sure we can continue to read from there.
cur := i + 1
for {
line, err := gr.ReadLine()
if err == io.EOF {
if cur == 99+1 {
// OK!
break
} else {
t.Fatalf("Got EOF after the wrong INFO #, tc #%d", i)
}
} else if err != nil {
t.Fatalf("Error reading line, tc #%d, err:\n%s", i, err)
}
if !strings.HasPrefix(line, "INFO ") {
continue
}
if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) {
t.Fatalf("Unexpected INFO #. Expected %v got:\n%v, tc #%d", cur, line, i)
}
cur++
}
gr.Close()
}
// Now search for something that is too small.
// We should get the first available line.
{
gr, match, err := g.Search("INFO", makeSearchFunc(-999))
require.NoError(t, err, "Failed to search for line")
assert.False(t, match, "Expected Search to not return exact match")
line, err := gr.ReadLine()
require.NoError(t, err, "Failed to read line after search")
if !strings.HasPrefix(line, "INFO 0 ") {
t.Error("Failed to fetch correct line, which is the earliest INFO")
}
err = gr.Close()
require.NoError(t, err, "Failed to close GroupReader")
}
// Now search for something that is too large.
// We should get an EOF error.
{
gr, _, err := g.Search("INFO", makeSearchFunc(999))
assert.Equal(t, io.EOF, err)
assert.Nil(t, gr)
}
// Cleanup
destroyTestGroup(t, g)
}
func TestRotateFile(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)
g.WriteLine("Line 1")
@ -237,100 +132,6 @@ func TestRotateFile(t *testing.T) {
destroyTestGroup(t, g)
}
func TestFindLast1(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("# a")
g.WriteLine("Line 3")
g.FlushAndSync()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.WriteLine("# b")
g.FlushAndSync()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast2(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("Line 3")
g.FlushAndSync()
g.RotateFile()
g.WriteLine("# a")
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("# b")
g.WriteLine("Line 6")
g.FlushAndSync()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast3(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)
g.WriteLine("Line 1")
g.WriteLine("# a")
g.WriteLine("Line 2")
g.WriteLine("# b")
g.WriteLine("Line 3")
g.FlushAndSync()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.FlushAndSync()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.True(t, found)
assert.Equal(t, "# b", match)
// Cleanup
destroyTestGroup(t, g)
}
func TestFindLast4(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)
g.WriteLine("Line 1")
g.WriteLine("Line 2")
g.WriteLine("Line 3")
g.FlushAndSync()
g.RotateFile()
g.WriteLine("Line 4")
g.WriteLine("Line 5")
g.WriteLine("Line 6")
g.FlushAndSync()
match, found, err := g.FindLast("#")
assert.NoError(t, err)
assert.False(t, found)
assert.Empty(t, match)
// Cleanup
destroyTestGroup(t, g)
}
func TestWrite(t *testing.T) {
g := createTestGroupWithHeadSizeLimit(t, 0)


Write Preview
Loading…
Cancel
Save