From 1261fca1608264cd14635585b6948ab359c88e37 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 30 Oct 2016 02:40:39 -0700 Subject: [PATCH] FindLast --- group.go | 96 ++++++++++++++++++++++++++---- group_test.go | 161 ++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 235 insertions(+), 22 deletions(-) diff --git a/group.go b/group.go index 48c770ca6..331b7e9e0 100644 --- a/group.go +++ b/group.go @@ -168,10 +168,16 @@ func (g *Group) RotateFile() { g.maxIndex += 1 } -func (g *Group) NewReader(index int) *GroupReader { +// NOTE: if error, returns no GroupReader. +// CONTRACT: Caller must close the returned GroupReader +func (g *Group) NewReader(index int) (*GroupReader, error) { r := newGroupReader(g) - r.SetIndex(index) - return r + err := r.SetIndex(index) + if err != nil { + return nil, err + } else { + return r, nil + } } // Returns -1 if line comes after, 0 if found, 1 if line comes before. @@ -181,7 +187,7 @@ type SearchFunc func(line string) (int, error) // then returns a GroupReader to start streaming lines // Returns true if an exact match was found, otherwise returns // the next greater line that starts with prefix. -// CONTRACT: caller is responsible for closing GroupReader. +// CONTRACT: Caller must close the returned GroupReader func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error) { g.mtx.Lock() minIndex, maxIndex := g.minIndex, g.maxIndex @@ -195,7 +201,10 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error // Base case, when there's only 1 choice left. if minIndex == maxIndex { - r := g.NewReader(maxIndex) + r, err := g.NewReader(maxIndex) + if err != nil { + return nil, false, err + } match, err := scanUntil(r, prefix, cmp) if err != nil { r.Close() @@ -207,8 +216,11 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error // Read starting roughly at the middle file, // until we find line that has prefix. - r := g.NewReader(curIndex) - foundIndex, line, err := scanFirst(r, prefix) + r, err := g.NewReader(curIndex) + if err != nil { + return nil, false, err + } + foundIndex, line, err := scanNext(r, prefix) r.Close() if err != nil { return nil, false, err @@ -224,7 +236,10 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error minIndex = foundIndex } else if val == 0 { // Stroke of luck, found the line - r := g.NewReader(foundIndex) + r, err := g.NewReader(foundIndex) + if err != nil { + return nil, false, err + } match, err := scanUntil(r, prefix, cmp) if !match { panic("Expected match to be true") @@ -244,7 +259,8 @@ func (g *Group) Search(prefix string, cmp SearchFunc) (*GroupReader, bool, error } // Scans and returns the first line that starts with 'prefix' -func scanFirst(r *GroupReader, prefix string) (int, string, error) { +// Consumes line and returns it. +func scanNext(r *GroupReader, prefix string) (int, string, error) { for { line, err := r.ReadLine() if err != nil { @@ -259,6 +275,7 @@ func scanFirst(r *GroupReader, prefix string) (int, string, error) { } // Returns true iff an exact match was found. +// Pushes line, does not consume it. func scanUntil(r *GroupReader, prefix string, cmp SearchFunc) (bool, error) { for { line, err := r.ReadLine() @@ -284,6 +301,47 @@ func scanUntil(r *GroupReader, prefix string, cmp SearchFunc) (bool, error) { } } +// Searches for the last line in Group with prefix. +func (g *Group) FindLast(prefix string) (match string, found bool, err error) { + g.mtx.Lock() + minIndex, maxIndex := g.minIndex, g.maxIndex + g.mtx.Unlock() + + r, err := g.NewReader(maxIndex) + if err != nil { + return "", false, err + } + defer r.Close() + + // Open files from the back and read +GROUP_LOOP: + for i := maxIndex; i >= minIndex; i-- { + err := r.SetIndex(i) + if err != nil { + return "", false, err + } + // Scan each line and test whether line matches + for { + line, err := r.ReadLineInCurrent() + if err == io.EOF { + if found { + return match, found, nil + } else { + continue GROUP_LOOP + } + } else if err != nil { + return "", false, err + } + if strings.HasPrefix(line, prefix) { + match = line + found = true + } + } + } + + return +} + type GroupInfo struct { MinIndex int MaxIndex int @@ -399,6 +457,18 @@ func (gr *GroupReader) Close() error { func (gr *GroupReader) ReadLine() (string, error) { gr.mtx.Lock() defer gr.mtx.Unlock() + return gr.readLineWithOptions(false) +} + +func (gr *GroupReader) ReadLineInCurrent() (string, error) { + gr.mtx.Lock() + defer gr.mtx.Unlock() + return gr.readLineWithOptions(true) +} + +// curFileOnly: if True, do not open new files, +// just return io.EOF if no new lines found. +func (gr *GroupReader) readLineWithOptions(curFileOnly bool) (string, error) { // From PushLine if gr.curLine != nil { @@ -420,7 +490,9 @@ func (gr *GroupReader) ReadLine() (string, error) { bytes, err := gr.curReader.ReadBytes('\n') if err != nil { if err != io.EOF { - return string(bytes), err + return "", err + } else if curFileOnly { + return "", err } else { // Open the next file err := gr.openFile(gr.curIndex + 1) @@ -483,8 +555,8 @@ func (gr *GroupReader) CurIndex() int { return gr.curIndex } -func (gr *GroupReader) SetIndex(index int) { +func (gr *GroupReader) SetIndex(index int) error { gr.mtx.Lock() defer gr.mtx.Unlock() - gr.openFile(index) + return gr.openFile(index) } diff --git a/group_test.go b/group_test.go index f7c70b709..672bd4d90 100644 --- a/group_test.go +++ b/group_test.go @@ -3,6 +3,7 @@ package autofile import ( "errors" "io" + "io/ioutil" "os" "strconv" "strings" @@ -30,6 +31,10 @@ func createTestGroup(t *testing.T, headSizeLimit int64) *Group { } g.SetHeadSizeLimit(headSizeLimit) g.stopTicker() + + if g == nil { + t.Fatal("Failed to create Group") + } return g } @@ -57,16 +62,13 @@ func assertGroupInfo(t *testing.T, gInfo GroupInfo, minIndex, maxIndex int, tota func TestCheckHeadSizeLimit(t *testing.T) { g := createTestGroup(t, 1000*1000) - if g == nil { - t.Error("Failed to create Group") - } // At first, there are no files. assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 0, 0) // Write 1000 bytes 999 times. for i := 0; i < 999; i++ { - _, err := g.Head.Write([]byte(RandStr(999) + "\n")) + err := g.WriteLine(RandStr(999)) if err != nil { t.Fatal("Error appending to head", err) } @@ -78,7 +80,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000) // Write 1000 more bytes. - _, err := g.Head.Write([]byte(RandStr(999) + "\n")) + err := g.WriteLine(RandStr(999)) if err != nil { t.Fatal("Error appending to head", err) } @@ -88,7 +90,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0) // Write 1000 more bytes. - _, err = g.Head.Write([]byte(RandStr(999) + "\n")) + err = g.WriteLine(RandStr(999)) if err != nil { t.Fatal("Error appending to head", err) } @@ -99,7 +101,7 @@ func TestCheckHeadSizeLimit(t *testing.T) { // Write 1000 bytes 999 times. for i := 0; i < 999; i++ { - _, err := g.Head.Write([]byte(RandStr(999) + "\n")) + err := g.WriteLine(RandStr(999)) if err != nil { t.Fatal("Error appending to head", err) } @@ -127,9 +129,6 @@ func TestCheckHeadSizeLimit(t *testing.T) { func TestSearch(t *testing.T) { g := createTestGroup(t, 10*1000) - if g == nil { - t.Error("Failed to create Group") - } // Create some files in the group that have several INFO lines in them. // Try to put the INFO lines in various spots. @@ -251,3 +250,145 @@ func TestSearch(t *testing.T) { // Cleanup destroyTestGroup(t, g) } + +func TestRotateFile(t *testing.T) { + g := createTestGroup(t, 0) + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + + // Read g.Head.Path+"000" + body1, err := ioutil.ReadFile(g.Head.Path + ".000") + if err != nil { + t.Error("Failed to read first rolled file") + } + if string(body1) != "Line 1\nLine 2\nLine 3\n" { + t.Errorf("Got unexpected contents: [%v]", string(body1)) + } + + // Read g.Head.Path + body2, err := ioutil.ReadFile(g.Head.Path) + if err != nil { + t.Error("Failed to read first rolled file") + } + if string(body2) != "Line 4\nLine 5\nLine 6\n" { + t.Errorf("Got unexpected contents: [%v]", string(body2)) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast1(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("# a") + g.WriteLine("Line 3") + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + g.WriteLine("# b") + + match, found, err := g.FindLast("#") + if err != nil { + t.Error("Unexpected error", err) + } + if !found { + t.Error("Expected found=True") + } + if match != "# b\n" { + t.Errorf("Unexpected match: [%v]", match) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast2(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.RotateFile() + g.WriteLine("# a") + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("# b") + g.WriteLine("Line 6") + + match, found, err := g.FindLast("#") + if err != nil { + t.Error("Unexpected error", err) + } + if !found { + t.Error("Expected found=True") + } + if match != "# b\n" { + t.Errorf("Unexpected match: [%v]", match) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast3(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("# a") + g.WriteLine("Line 2") + g.WriteLine("# b") + g.WriteLine("Line 3") + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + + match, found, err := g.FindLast("#") + if err != nil { + t.Error("Unexpected error", err) + } + if !found { + t.Error("Expected found=True") + } + if match != "# b\n" { + t.Errorf("Unexpected match: [%v]", match) + } + + // Cleanup + destroyTestGroup(t, g) +} + +func TestFindLast4(t *testing.T) { + g := createTestGroup(t, 0) + + g.WriteLine("Line 1") + g.WriteLine("Line 2") + g.WriteLine("Line 3") + g.RotateFile() + g.WriteLine("Line 4") + g.WriteLine("Line 5") + g.WriteLine("Line 6") + + match, found, err := g.FindLast("#") + if err != nil { + t.Error("Unexpected error", err) + } + if found { + t.Error("Expected found=False") + } + if match != "" { + t.Errorf("Unexpected match: [%v]", match) + } + + // Cleanup + destroyTestGroup(t, g) +}