You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
303 lines
7.2 KiB
303 lines
7.2 KiB
package statesync
|
|
|
|
import (
|
|
"context"
|
|
"math/rand"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/tendermint/tendermint/internal/test/factory"
|
|
"github.com/tendermint/tendermint/types"
|
|
)
|
|
|
|
var (
|
|
startHeight int64 = 200
|
|
stopHeight int64 = 100
|
|
stopTime = time.Date(2019, 1, 1, 1, 0, 0, 0, time.UTC)
|
|
endTime = stopTime.Add(-1 * time.Second)
|
|
numWorkers = 1
|
|
)
|
|
|
|
func TestBlockQueueBasic(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
|
|
queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
// asynchronously fetch blocks and add it to the queue
|
|
for i := 0; i <= numWorkers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
queue.add(mockLBResp(ctx, t, peerID, height, endTime))
|
|
case <-queue.done():
|
|
wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
trackingHeight := startHeight
|
|
wg.Add(1)
|
|
|
|
loop:
|
|
for {
|
|
select {
|
|
case <-queue.done():
|
|
wg.Done()
|
|
break loop
|
|
|
|
case resp := <-queue.verifyNext():
|
|
// assert that the queue serializes the blocks
|
|
require.Equal(t, resp.block.Height, trackingHeight)
|
|
trackingHeight--
|
|
queue.success()
|
|
}
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
assert.Less(t, trackingHeight, stopHeight)
|
|
}
|
|
|
|
// Test with spurious failures and retries
|
|
func TestBlockQueueWithFailures(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
|
|
queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 200)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
failureRate := 4
|
|
for i := 0; i <= numWorkers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
if rand.Intn(failureRate) == 0 {
|
|
queue.retry(height)
|
|
} else {
|
|
queue.add(mockLBResp(ctx, t, peerID, height, endTime))
|
|
}
|
|
case <-queue.done():
|
|
wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
trackingHeight := startHeight
|
|
for {
|
|
select {
|
|
case resp := <-queue.verifyNext():
|
|
// assert that the queue serializes the blocks
|
|
assert.Equal(t, resp.block.Height, trackingHeight)
|
|
if rand.Intn(failureRate) == 0 {
|
|
queue.retry(resp.block.Height)
|
|
} else {
|
|
trackingHeight--
|
|
queue.success()
|
|
}
|
|
|
|
case <-queue.done():
|
|
wg.Wait()
|
|
assert.Less(t, trackingHeight, stopHeight)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Test that when all the blocks are retrieved that the queue still holds on to
|
|
// it's workers and in the event of failure can still fetch the failed block
|
|
func TestBlockQueueBlocks(t *testing.T) {
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 2)
|
|
expectedHeight := startHeight
|
|
retryHeight := stopHeight + 2
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
loop:
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
require.Equal(t, height, expectedHeight)
|
|
require.GreaterOrEqual(t, height, stopHeight)
|
|
expectedHeight--
|
|
queue.add(mockLBResp(ctx, t, peerID, height, endTime))
|
|
case <-time.After(1 * time.Second):
|
|
if expectedHeight >= stopHeight {
|
|
t.Fatalf("expected next height %d", expectedHeight)
|
|
}
|
|
break loop
|
|
}
|
|
}
|
|
|
|
// close any waiter channels that the previous worker left hanging
|
|
for _, ch := range queue.waiters {
|
|
close(ch)
|
|
}
|
|
queue.waiters = make([]chan int64, 0)
|
|
|
|
wg := &sync.WaitGroup{}
|
|
wg.Add(1)
|
|
// so far so good. The worker is waiting. Now we fail a previous
|
|
// block and check that the worker fetches them
|
|
go func(t *testing.T) {
|
|
defer wg.Done()
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
require.Equal(t, retryHeight, height)
|
|
case <-time.After(1 * time.Second):
|
|
require.Fail(t, "queue didn't ask worker to fetch failed height")
|
|
}
|
|
}(t)
|
|
queue.retry(retryHeight)
|
|
wg.Wait()
|
|
|
|
}
|
|
|
|
func TestBlockQueueAcceptsNoMoreBlocks(t *testing.T) {
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
|
|
defer queue.close()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
loop:
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
require.GreaterOrEqual(t, height, stopHeight)
|
|
queue.add(mockLBResp(ctx, t, peerID, height, endTime))
|
|
case <-time.After(1 * time.Second):
|
|
break loop
|
|
}
|
|
}
|
|
|
|
require.Len(t, queue.pending, int(startHeight-stopHeight)+1)
|
|
|
|
queue.add(mockLBResp(ctx, t, peerID, stopHeight-1, endTime))
|
|
require.Len(t, queue.pending, int(startHeight-stopHeight)+1)
|
|
}
|
|
|
|
// Test a scenario where more blocks are needed then just the stopheight because
|
|
// we haven't found a block with a small enough time.
|
|
func TestBlockQueueStopTime(t *testing.T) {
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
|
|
queue := newBlockQueue(startHeight, stopHeight, 1, stopTime, 1)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
baseTime := stopTime.Add(-50 * time.Second)
|
|
|
|
// asynchronously fetch blocks and add it to the queue
|
|
for i := 0; i <= numWorkers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
blockTime := baseTime.Add(time.Duration(height) * time.Second)
|
|
queue.add(mockLBResp(ctx, t, peerID, height, blockTime))
|
|
case <-queue.done():
|
|
wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
trackingHeight := startHeight
|
|
for {
|
|
select {
|
|
case resp := <-queue.verifyNext():
|
|
// assert that the queue serializes the blocks
|
|
assert.Equal(t, resp.block.Height, trackingHeight)
|
|
trackingHeight--
|
|
queue.success()
|
|
|
|
case <-queue.done():
|
|
wg.Wait()
|
|
assert.Less(t, trackingHeight, stopHeight-50)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBlockQueueInitialHeight(t *testing.T) {
|
|
peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
|
|
require.NoError(t, err)
|
|
const initialHeight int64 = 120
|
|
|
|
queue := newBlockQueue(startHeight, stopHeight, initialHeight, stopTime, 1)
|
|
wg := &sync.WaitGroup{}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
// asynchronously fetch blocks and add it to the queue
|
|
for i := 0; i <= numWorkers; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
for {
|
|
select {
|
|
case height := <-queue.nextHeight():
|
|
require.GreaterOrEqual(t, height, initialHeight)
|
|
queue.add(mockLBResp(ctx, t, peerID, height, endTime))
|
|
case <-queue.done():
|
|
wg.Done()
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
loop:
|
|
for {
|
|
select {
|
|
case <-queue.done():
|
|
wg.Wait()
|
|
require.NoError(t, queue.error())
|
|
break loop
|
|
|
|
case resp := <-queue.verifyNext():
|
|
require.GreaterOrEqual(t, resp.block.Height, initialHeight)
|
|
queue.success()
|
|
}
|
|
}
|
|
}
|
|
|
|
func mockLBResp(ctx context.Context, t *testing.T, peer types.NodeID, height int64, time time.Time) lightBlockResponse {
|
|
t.Helper()
|
|
vals, pv := factory.RandValidatorSet(ctx, 3, 10)
|
|
_, _, lb := mockLB(ctx, t, height, time, factory.MakeBlockID(), vals, pv)
|
|
return lightBlockResponse{
|
|
block: lb,
|
|
peer: peer,
|
|
}
|
|
}
|