You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

437 lines
11 KiB

8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
  1. package autofile
  2. import (
  3. "errors"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "os"
  8. "strconv"
  9. "strings"
  10. "testing"
  11. "github.com/stretchr/testify/assert"
  12. "github.com/stretchr/testify/require"
  13. cmn "github.com/tendermint/tmlibs/common"
  14. )
  15. // NOTE: Returned group has ticker stopped
  16. func createTestGroup(t *testing.T, headSizeLimit int64) *Group {
  17. testID := cmn.RandStr(12)
  18. testDir := "_test_" + testID
  19. err := cmn.EnsureDir(testDir, 0700)
  20. require.NoError(t, err, "Error creating dir")
  21. headPath := testDir + "/myfile"
  22. g, err := OpenGroup(headPath)
  23. require.NoError(t, err, "Error opening Group")
  24. g.SetHeadSizeLimit(headSizeLimit)
  25. g.stopTicker()
  26. require.NotEqual(t, nil, g, "Failed to create Group")
  27. return g
  28. }
  29. func destroyTestGroup(t *testing.T, g *Group) {
  30. err := os.RemoveAll(g.Dir)
  31. require.NoError(t, err, "Error removing test Group directory")
  32. }
  33. func assertGroupInfo(t *testing.T, gInfo GroupInfo, minIndex, maxIndex int, totalSize, headSize int64) {
  34. assert.Equal(t, minIndex, gInfo.MinIndex)
  35. assert.Equal(t, maxIndex, gInfo.MaxIndex)
  36. assert.Equal(t, totalSize, gInfo.TotalSize)
  37. assert.Equal(t, headSize, gInfo.HeadSize)
  38. }
  39. func TestCheckHeadSizeLimit(t *testing.T) {
  40. g := createTestGroup(t, 1000*1000)
  41. // At first, there are no files.
  42. assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 0, 0)
  43. // Write 1000 bytes 999 times.
  44. for i := 0; i < 999; i++ {
  45. err := g.WriteLine(cmn.RandStr(999))
  46. require.NoError(t, err, "Error appending to head")
  47. }
  48. g.Flush()
  49. assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000)
  50. // Even calling checkHeadSizeLimit manually won't rotate it.
  51. g.checkHeadSizeLimit()
  52. assertGroupInfo(t, g.ReadGroupInfo(), 0, 0, 999000, 999000)
  53. // Write 1000 more bytes.
  54. err := g.WriteLine(cmn.RandStr(999))
  55. require.NoError(t, err, "Error appending to head")
  56. g.Flush()
  57. // Calling checkHeadSizeLimit this time rolls it.
  58. g.checkHeadSizeLimit()
  59. assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1000000, 0)
  60. // Write 1000 more bytes.
  61. err = g.WriteLine(cmn.RandStr(999))
  62. require.NoError(t, err, "Error appending to head")
  63. g.Flush()
  64. // Calling checkHeadSizeLimit does nothing.
  65. g.checkHeadSizeLimit()
  66. assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 1001000, 1000)
  67. // Write 1000 bytes 999 times.
  68. for i := 0; i < 999; i++ {
  69. err = g.WriteLine(cmn.RandStr(999))
  70. require.NoError(t, err, "Error appending to head")
  71. }
  72. g.Flush()
  73. assertGroupInfo(t, g.ReadGroupInfo(), 0, 1, 2000000, 1000000)
  74. // Calling checkHeadSizeLimit rolls it again.
  75. g.checkHeadSizeLimit()
  76. assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2000000, 0)
  77. // Write 1000 more bytes.
  78. _, err = g.Head.Write([]byte(cmn.RandStr(999) + "\n"))
  79. require.NoError(t, err, "Error appending to head")
  80. g.Flush()
  81. assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000)
  82. // Calling checkHeadSizeLimit does nothing.
  83. g.checkHeadSizeLimit()
  84. assertGroupInfo(t, g.ReadGroupInfo(), 0, 2, 2001000, 1000)
  85. // Cleanup
  86. destroyTestGroup(t, g)
  87. }
  88. func TestSearch(t *testing.T) {
  89. g := createTestGroup(t, 10*1000)
  90. // Create some files in the group that have several INFO lines in them.
  91. // Try to put the INFO lines in various spots.
  92. for i := 0; i < 100; i++ {
  93. // The random junk at the end ensures that this INFO linen
  94. // is equally likely to show up at the end.
  95. _, err := g.Head.Write([]byte(fmt.Sprintf("INFO %v %v\n", i, cmn.RandStr(123))))
  96. require.NoError(t, err, "Failed to write to head")
  97. g.checkHeadSizeLimit()
  98. for j := 0; j < 10; j++ {
  99. _, err1 := g.Head.Write([]byte(cmn.RandStr(123) + "\n"))
  100. require.NoError(t, err1, "Failed to write to head")
  101. g.checkHeadSizeLimit()
  102. }
  103. }
  104. // Create a search func that searches for line
  105. makeSearchFunc := func(target int) SearchFunc {
  106. return func(line string) (int, error) {
  107. parts := strings.Split(line, " ")
  108. if len(parts) != 3 {
  109. return -1, errors.New("Line did not have 3 parts")
  110. }
  111. i, err := strconv.Atoi(parts[1])
  112. if err != nil {
  113. return -1, errors.New("Failed to parse INFO: " + err.Error())
  114. }
  115. if target < i {
  116. return 1, nil
  117. } else if target == i {
  118. return 0, nil
  119. } else {
  120. return -1, nil
  121. }
  122. }
  123. }
  124. // Now search for each number
  125. for i := 0; i < 100; i++ {
  126. t.Log("Testing for i", i)
  127. gr, match, err := g.Search("INFO", makeSearchFunc(i))
  128. require.NoError(t, err, "Failed to search for line")
  129. assert.True(t, match, "Expected Search to return exact match")
  130. line, err := gr.ReadLine()
  131. require.NoError(t, err, "Failed to read line after search")
  132. if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", i)) {
  133. t.Fatal("Failed to get correct line")
  134. }
  135. // Make sure we can continue to read from there.
  136. cur := i + 1
  137. for {
  138. line, err := gr.ReadLine()
  139. if err == io.EOF {
  140. if cur == 99+1 {
  141. // OK!
  142. break
  143. } else {
  144. t.Fatal("Got EOF after the wrong INFO #")
  145. }
  146. } else if err != nil {
  147. t.Fatal("Error reading line", err)
  148. }
  149. if !strings.HasPrefix(line, "INFO ") {
  150. continue
  151. }
  152. if !strings.HasPrefix(line, fmt.Sprintf("INFO %v ", cur)) {
  153. t.Fatalf("Unexpected INFO #. Expected %v got:\n%v", cur, line)
  154. }
  155. cur++
  156. }
  157. gr.Close()
  158. }
  159. // Now search for something that is too small.
  160. // We should get the first available line.
  161. {
  162. gr, match, err := g.Search("INFO", makeSearchFunc(-999))
  163. require.NoError(t, err, "Failed to search for line")
  164. assert.False(t, match, "Expected Search to not return exact match")
  165. line, err := gr.ReadLine()
  166. require.NoError(t, err, "Failed to read line after search")
  167. if !strings.HasPrefix(line, "INFO 0 ") {
  168. t.Error("Failed to fetch correct line, which is the earliest INFO")
  169. }
  170. err = gr.Close()
  171. require.NoError(t, err, "Failed to close GroupReader")
  172. }
  173. // Now search for something that is too large.
  174. // We should get an EOF error.
  175. {
  176. gr, _, err := g.Search("INFO", makeSearchFunc(999))
  177. assert.Equal(t, io.EOF, err)
  178. assert.Nil(t, gr)
  179. }
  180. // Cleanup
  181. destroyTestGroup(t, g)
  182. }
  183. func TestRotateFile(t *testing.T) {
  184. g := createTestGroup(t, 0)
  185. g.WriteLine("Line 1")
  186. g.WriteLine("Line 2")
  187. g.WriteLine("Line 3")
  188. g.Flush()
  189. g.RotateFile()
  190. g.WriteLine("Line 4")
  191. g.WriteLine("Line 5")
  192. g.WriteLine("Line 6")
  193. g.Flush()
  194. // Read g.Head.Path+"000"
  195. body1, err := ioutil.ReadFile(g.Head.Path + ".000")
  196. assert.NoError(t, err, "Failed to read first rolled file")
  197. if string(body1) != "Line 1\nLine 2\nLine 3\n" {
  198. t.Errorf("Got unexpected contents: [%v]", string(body1))
  199. }
  200. // Read g.Head.Path
  201. body2, err := ioutil.ReadFile(g.Head.Path)
  202. assert.NoError(t, err, "Failed to read first rolled file")
  203. if string(body2) != "Line 4\nLine 5\nLine 6\n" {
  204. t.Errorf("Got unexpected contents: [%v]", string(body2))
  205. }
  206. // Cleanup
  207. destroyTestGroup(t, g)
  208. }
  209. func TestFindLast1(t *testing.T) {
  210. g := createTestGroup(t, 0)
  211. g.WriteLine("Line 1")
  212. g.WriteLine("Line 2")
  213. g.WriteLine("# a")
  214. g.WriteLine("Line 3")
  215. g.Flush()
  216. g.RotateFile()
  217. g.WriteLine("Line 4")
  218. g.WriteLine("Line 5")
  219. g.WriteLine("Line 6")
  220. g.WriteLine("# b")
  221. g.Flush()
  222. match, found, err := g.FindLast("#")
  223. assert.NoError(t, err)
  224. assert.True(t, found)
  225. assert.Equal(t, "# b", match)
  226. // Cleanup
  227. destroyTestGroup(t, g)
  228. }
  229. func TestFindLast2(t *testing.T) {
  230. g := createTestGroup(t, 0)
  231. g.WriteLine("Line 1")
  232. g.WriteLine("Line 2")
  233. g.WriteLine("Line 3")
  234. g.Flush()
  235. g.RotateFile()
  236. g.WriteLine("# a")
  237. g.WriteLine("Line 4")
  238. g.WriteLine("Line 5")
  239. g.WriteLine("# b")
  240. g.WriteLine("Line 6")
  241. g.Flush()
  242. match, found, err := g.FindLast("#")
  243. assert.NoError(t, err)
  244. assert.True(t, found)
  245. assert.Equal(t, "# b", match)
  246. // Cleanup
  247. destroyTestGroup(t, g)
  248. }
  249. func TestFindLast3(t *testing.T) {
  250. g := createTestGroup(t, 0)
  251. g.WriteLine("Line 1")
  252. g.WriteLine("# a")
  253. g.WriteLine("Line 2")
  254. g.WriteLine("# b")
  255. g.WriteLine("Line 3")
  256. g.Flush()
  257. g.RotateFile()
  258. g.WriteLine("Line 4")
  259. g.WriteLine("Line 5")
  260. g.WriteLine("Line 6")
  261. g.Flush()
  262. match, found, err := g.FindLast("#")
  263. assert.NoError(t, err)
  264. assert.True(t, found)
  265. assert.Equal(t, "# b", match)
  266. // Cleanup
  267. destroyTestGroup(t, g)
  268. }
  269. func TestFindLast4(t *testing.T) {
  270. g := createTestGroup(t, 0)
  271. g.WriteLine("Line 1")
  272. g.WriteLine("Line 2")
  273. g.WriteLine("Line 3")
  274. g.Flush()
  275. g.RotateFile()
  276. g.WriteLine("Line 4")
  277. g.WriteLine("Line 5")
  278. g.WriteLine("Line 6")
  279. g.Flush()
  280. match, found, err := g.FindLast("#")
  281. assert.NoError(t, err)
  282. assert.False(t, found)
  283. assert.Empty(t, match)
  284. // Cleanup
  285. destroyTestGroup(t, g)
  286. }
  287. func TestWrite(t *testing.T) {
  288. g := createTestGroup(t, 0)
  289. written := []byte("Medusa")
  290. g.Write(written)
  291. g.Flush()
  292. read := make([]byte, len(written))
  293. gr, err := g.NewReader(0)
  294. require.NoError(t, err, "failed to create reader")
  295. _, err = gr.Read(read)
  296. assert.NoError(t, err, "failed to read data")
  297. assert.Equal(t, written, read)
  298. // Cleanup
  299. destroyTestGroup(t, g)
  300. }
  301. // test that Read reads the required amount of bytes from all the files in the
  302. // group and returns no error if n == size of the given slice.
  303. func TestGroupReaderRead(t *testing.T) {
  304. g := createTestGroup(t, 0)
  305. professor := []byte("Professor Monster")
  306. g.Write(professor)
  307. g.Flush()
  308. g.RotateFile()
  309. frankenstein := []byte("Frankenstein's Monster")
  310. g.Write(frankenstein)
  311. g.Flush()
  312. totalWrittenLength := len(professor) + len(frankenstein)
  313. read := make([]byte, totalWrittenLength)
  314. gr, err := g.NewReader(0)
  315. require.NoError(t, err, "failed to create reader")
  316. n, err := gr.Read(read)
  317. assert.NoError(t, err, "failed to read data")
  318. assert.Equal(t, totalWrittenLength, n, "not enough bytes read")
  319. professorPlusFrankenstein := professor
  320. professorPlusFrankenstein = append(professorPlusFrankenstein, frankenstein...)
  321. assert.Equal(t, professorPlusFrankenstein, read)
  322. // Cleanup
  323. destroyTestGroup(t, g)
  324. }
  325. // test that Read returns an error if number of bytes read < size of
  326. // the given slice. Subsequent call should return 0, io.EOF.
  327. func TestGroupReaderRead2(t *testing.T) {
  328. g := createTestGroup(t, 0)
  329. professor := []byte("Professor Monster")
  330. g.Write(professor)
  331. g.Flush()
  332. g.RotateFile()
  333. frankenstein := []byte("Frankenstein's Monster")
  334. frankensteinPart := []byte("Frankenstein")
  335. g.Write(frankensteinPart) // note writing only a part
  336. g.Flush()
  337. totalLength := len(professor) + len(frankenstein)
  338. read := make([]byte, totalLength)
  339. gr, err := g.NewReader(0)
  340. require.NoError(t, err, "failed to create reader")
  341. // 1) n < (size of the given slice), io.EOF
  342. n, err := gr.Read(read)
  343. assert.Equal(t, io.EOF, err)
  344. assert.Equal(t, len(professor)+len(frankensteinPart), n, "Read more/less bytes than it is in the group")
  345. // 2) 0, io.EOF
  346. n, err = gr.Read([]byte("0"))
  347. assert.Equal(t, io.EOF, err)
  348. assert.Equal(t, 0, n)
  349. // Cleanup
  350. destroyTestGroup(t, g)
  351. }
  352. func TestMinIndex(t *testing.T) {
  353. g := createTestGroup(t, 0)
  354. assert.Zero(t, g.MinIndex(), "MinIndex should be zero at the beginning")
  355. // Cleanup
  356. destroyTestGroup(t, g)
  357. }
  358. func TestMaxIndex(t *testing.T) {
  359. g := createTestGroup(t, 0)
  360. assert.Zero(t, g.MaxIndex(), "MaxIndex should be zero at the beginning")
  361. g.WriteLine("Line 1")
  362. g.Flush()
  363. g.RotateFile()
  364. assert.Equal(t, 1, g.MaxIndex(), "MaxIndex should point to the last file")
  365. // Cleanup
  366. destroyTestGroup(t, g)
  367. }