You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

570 lines
17 KiB

  1. package evidence_test
  2. import (
  3. "context"
  4. "encoding/hex"
  5. "math/rand"
  6. "sync"
  7. "testing"
  8. "time"
  9. "github.com/fortytw2/leaktest"
  10. "github.com/stretchr/testify/assert"
  11. "github.com/stretchr/testify/mock"
  12. "github.com/stretchr/testify/require"
  13. dbm "github.com/tendermint/tm-db"
  14. "github.com/tendermint/tendermint/crypto"
  15. "github.com/tendermint/tendermint/crypto/tmhash"
  16. "github.com/tendermint/tendermint/internal/evidence"
  17. "github.com/tendermint/tendermint/internal/evidence/mocks"
  18. "github.com/tendermint/tendermint/internal/p2p"
  19. "github.com/tendermint/tendermint/internal/p2p/p2ptest"
  20. sm "github.com/tendermint/tendermint/internal/state"
  21. "github.com/tendermint/tendermint/libs/log"
  22. tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
  23. "github.com/tendermint/tendermint/types"
  24. )
  25. var (
  26. numEvidence = 10
  27. rng = rand.New(rand.NewSource(time.Now().UnixNano()))
  28. )
  29. type reactorTestSuite struct {
  30. network *p2ptest.Network
  31. logger log.Logger
  32. reactors map[types.NodeID]*evidence.Reactor
  33. pools map[types.NodeID]*evidence.Pool
  34. evidenceChannels map[types.NodeID]*p2p.Channel
  35. peerUpdates map[types.NodeID]*p2p.PeerUpdates
  36. peerChans map[types.NodeID]chan p2p.PeerUpdate
  37. nodes []*p2ptest.Node
  38. numStateStores int
  39. }
  40. func setup(ctx context.Context, t *testing.T, stateStores []sm.Store, chBuf uint) *reactorTestSuite {
  41. t.Helper()
  42. pID := make([]byte, 16)
  43. _, err := rng.Read(pID)
  44. require.NoError(t, err)
  45. numStateStores := len(stateStores)
  46. rts := &reactorTestSuite{
  47. numStateStores: numStateStores,
  48. logger: log.TestingLogger().With("testCase", t.Name()),
  49. network: p2ptest.MakeNetwork(ctx, t, p2ptest.NetworkOptions{NumNodes: numStateStores}),
  50. reactors: make(map[types.NodeID]*evidence.Reactor, numStateStores),
  51. pools: make(map[types.NodeID]*evidence.Pool, numStateStores),
  52. peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numStateStores),
  53. peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numStateStores),
  54. }
  55. chDesc := &p2p.ChannelDescriptor{ID: evidence.EvidenceChannel, MessageType: new(tmproto.Evidence)}
  56. rts.evidenceChannels = rts.network.MakeChannelsNoCleanup(ctx, t, chDesc)
  57. require.Len(t, rts.network.RandomNode().PeerManager.Peers(), 0)
  58. idx := 0
  59. evidenceTime := time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
  60. for nodeID := range rts.network.Nodes {
  61. logger := rts.logger.With("validator", idx)
  62. evidenceDB := dbm.NewMemDB()
  63. blockStore := &mocks.BlockStore{}
  64. state, _ := stateStores[idx].Load()
  65. blockStore.On("LoadBlockMeta", mock.AnythingOfType("int64")).Return(func(h int64) *types.BlockMeta {
  66. if h <= state.LastBlockHeight {
  67. return &types.BlockMeta{Header: types.Header{Time: evidenceTime}}
  68. }
  69. return nil
  70. })
  71. rts.pools[nodeID], err = evidence.NewPool(logger, evidenceDB, stateStores[idx], blockStore)
  72. require.NoError(t, err)
  73. rts.peerChans[nodeID] = make(chan p2p.PeerUpdate)
  74. rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
  75. rts.network.Nodes[nodeID].PeerManager.Register(ctx, rts.peerUpdates[nodeID])
  76. rts.nodes = append(rts.nodes, rts.network.Nodes[nodeID])
  77. chCreator := func(ctx context.Context, chdesc *p2p.ChannelDescriptor) (*p2p.Channel, error) {
  78. return rts.evidenceChannels[nodeID], nil
  79. }
  80. rts.reactors[nodeID], err = evidence.NewReactor(
  81. ctx,
  82. logger,
  83. chCreator,
  84. rts.peerUpdates[nodeID],
  85. rts.pools[nodeID])
  86. require.NoError(t, err)
  87. require.NoError(t, rts.reactors[nodeID].Start(ctx))
  88. require.True(t, rts.reactors[nodeID].IsRunning())
  89. idx++
  90. }
  91. t.Cleanup(func() {
  92. for _, r := range rts.reactors {
  93. if r.IsRunning() {
  94. require.NoError(t, r.Stop())
  95. require.False(t, r.IsRunning())
  96. }
  97. }
  98. })
  99. t.Cleanup(leaktest.Check(t))
  100. return rts
  101. }
  102. func (rts *reactorTestSuite) start(ctx context.Context, t *testing.T) {
  103. rts.network.Start(ctx, t)
  104. require.Len(t,
  105. rts.network.RandomNode().PeerManager.Peers(),
  106. rts.numStateStores-1,
  107. "network does not have expected number of nodes")
  108. }
  109. func (rts *reactorTestSuite) waitForEvidence(t *testing.T, evList types.EvidenceList, ids ...types.NodeID) {
  110. t.Helper()
  111. fn := func(pool *evidence.Pool) {
  112. var (
  113. localEvList []types.Evidence
  114. size int64
  115. loops int
  116. )
  117. // wait till we have at least the amount of evidence
  118. // that we expect. if there's more local evidence then
  119. // it doesn't make sense to wait longer and a
  120. // different assertion should catch the resulting error
  121. for len(localEvList) < len(evList) {
  122. // each evidence should not be more than 500 bytes
  123. localEvList, size = pool.PendingEvidence(int64(len(evList) * 500))
  124. if loops == 100 {
  125. t.Log("current wait status:", "|",
  126. "local", len(localEvList), "|",
  127. "waitlist", len(evList), "|",
  128. "size", size)
  129. }
  130. loops++
  131. }
  132. // put the reaped evidence in a map so we can quickly check we got everything
  133. evMap := make(map[string]types.Evidence)
  134. for _, e := range localEvList {
  135. evMap[string(e.Hash())] = e
  136. }
  137. for i, expectedEv := range evList {
  138. gotEv := evMap[string(expectedEv.Hash())]
  139. require.Equalf(
  140. t,
  141. expectedEv,
  142. gotEv,
  143. "evidence for pool %d in pool does not match; got: %v, expected: %v", i, gotEv, expectedEv,
  144. )
  145. }
  146. }
  147. if len(ids) == 1 {
  148. // special case waiting once, just to avoid the extra
  149. // goroutine, in the case that this hits a timeout,
  150. // the stack will be clearer.
  151. fn(rts.pools[ids[0]])
  152. return
  153. }
  154. wg := sync.WaitGroup{}
  155. for id := range rts.pools {
  156. if len(ids) > 0 && !p2ptest.NodeInSlice(id, ids) {
  157. // if an ID list is specified, then we only
  158. // want to wait for those pools that are
  159. // specified in the list, otherwise, wait for
  160. // all pools.
  161. continue
  162. }
  163. wg.Add(1)
  164. go func(id types.NodeID) { defer wg.Done(); fn(rts.pools[id]) }(id)
  165. }
  166. wg.Wait()
  167. }
  168. func createEvidenceList(
  169. ctx context.Context,
  170. t *testing.T,
  171. pool *evidence.Pool,
  172. val types.PrivValidator,
  173. numEvidence int,
  174. ) types.EvidenceList {
  175. t.Helper()
  176. evList := make([]types.Evidence, numEvidence)
  177. for i := 0; i < numEvidence; i++ {
  178. ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(
  179. ctx,
  180. int64(i+1),
  181. time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC),
  182. val,
  183. evidenceChainID,
  184. )
  185. require.NoError(t, err)
  186. require.NoError(t, pool.AddEvidence(ev),
  187. "adding evidence it#%d of %d to pool with height %d",
  188. i, numEvidence, pool.State().LastBlockHeight)
  189. evList[i] = ev
  190. }
  191. return evList
  192. }
  193. func TestReactorMultiDisconnect(t *testing.T) {
  194. ctx, cancel := context.WithCancel(context.Background())
  195. defer cancel()
  196. val := types.NewMockPV()
  197. height := int64(numEvidence) + 10
  198. stateDB1 := initializeValidatorState(ctx, t, val, height)
  199. stateDB2 := initializeValidatorState(ctx, t, val, height)
  200. rts := setup(ctx, t, []sm.Store{stateDB1, stateDB2}, 20)
  201. primary := rts.nodes[0]
  202. secondary := rts.nodes[1]
  203. _ = createEvidenceList(ctx, t, rts.pools[primary.NodeID], val, numEvidence)
  204. require.Equal(t, primary.PeerManager.Status(secondary.NodeID), p2p.PeerStatusDown)
  205. rts.start(ctx, t)
  206. require.Equal(t, primary.PeerManager.Status(secondary.NodeID), p2p.PeerStatusUp)
  207. // Ensure "disconnecting" the secondary peer from the primary more than once
  208. // is handled gracefully.
  209. primary.PeerManager.Disconnected(ctx, secondary.NodeID)
  210. require.Equal(t, primary.PeerManager.Status(secondary.NodeID), p2p.PeerStatusDown)
  211. _, err := primary.PeerManager.TryEvictNext()
  212. require.NoError(t, err)
  213. primary.PeerManager.Disconnected(ctx, secondary.NodeID)
  214. require.Equal(t, primary.PeerManager.Status(secondary.NodeID), p2p.PeerStatusDown)
  215. require.Equal(t, secondary.PeerManager.Status(primary.NodeID), p2p.PeerStatusUp)
  216. }
  217. // TestReactorBroadcastEvidence creates an environment of multiple peers that
  218. // are all at the same height. One peer, designated as a primary, gossips all
  219. // evidence to the remaining peers.
  220. func TestReactorBroadcastEvidence(t *testing.T) {
  221. numPeers := 7
  222. ctx, cancel := context.WithCancel(context.Background())
  223. defer cancel()
  224. // create a stateDB for all test suites (nodes)
  225. stateDBs := make([]sm.Store, numPeers)
  226. val := types.NewMockPV()
  227. // We need all validators saved for heights at least as high as we have
  228. // evidence for.
  229. height := int64(numEvidence) + 10
  230. for i := 0; i < numPeers; i++ {
  231. stateDBs[i] = initializeValidatorState(ctx, t, val, height)
  232. }
  233. rts := setup(ctx, t, stateDBs, 0)
  234. rts.start(ctx, t)
  235. // Create a series of fixtures where each suite contains a reactor and
  236. // evidence pool. In addition, we mark a primary suite and the rest are
  237. // secondaries where each secondary is added as a peer via a PeerUpdate to the
  238. // primary. As a result, the primary will gossip all evidence to each secondary.
  239. primary := rts.network.RandomNode()
  240. secondaries := make([]*p2ptest.Node, 0, len(rts.network.NodeIDs())-1)
  241. secondaryIDs := make([]types.NodeID, 0, cap(secondaries))
  242. for id := range rts.network.Nodes {
  243. if id == primary.NodeID {
  244. continue
  245. }
  246. secondaries = append(secondaries, rts.network.Nodes[id])
  247. secondaryIDs = append(secondaryIDs, id)
  248. }
  249. evList := createEvidenceList(ctx, t, rts.pools[primary.NodeID], val, numEvidence)
  250. // Add each secondary suite (node) as a peer to the primary suite (node). This
  251. // will cause the primary to gossip all evidence to the secondaries.
  252. for _, suite := range secondaries {
  253. rts.peerChans[primary.NodeID] <- p2p.PeerUpdate{
  254. Status: p2p.PeerStatusUp,
  255. NodeID: suite.NodeID,
  256. }
  257. }
  258. // Wait till all secondary suites (reactor) received all evidence from the
  259. // primary suite (node).
  260. rts.waitForEvidence(t, evList, secondaryIDs...)
  261. for _, pool := range rts.pools {
  262. require.Equal(t, numEvidence, int(pool.Size()))
  263. }
  264. }
  265. // TestReactorSelectiveBroadcast tests a context where we have two reactors
  266. // connected to one another but are at different heights. Reactor 1 which is
  267. // ahead receives a list of evidence.
  268. func TestReactorBroadcastEvidence_Lagging(t *testing.T) {
  269. val := types.NewMockPV()
  270. height1 := int64(numEvidence) + 10
  271. height2 := int64(numEvidence) / 2
  272. ctx, cancel := context.WithCancel(context.Background())
  273. defer cancel()
  274. // stateDB1 is ahead of stateDB2, where stateDB1 has all heights (1-20) and
  275. // stateDB2 only has heights 1-5.
  276. stateDB1 := initializeValidatorState(ctx, t, val, height1)
  277. stateDB2 := initializeValidatorState(ctx, t, val, height2)
  278. rts := setup(ctx, t, []sm.Store{stateDB1, stateDB2}, 100)
  279. rts.start(ctx, t)
  280. primary := rts.nodes[0]
  281. secondary := rts.nodes[1]
  282. // Send a list of valid evidence to the first reactor's, the one that is ahead,
  283. // evidence pool.
  284. evList := createEvidenceList(ctx, t, rts.pools[primary.NodeID], val, numEvidence)
  285. // Add each secondary suite (node) as a peer to the primary suite (node). This
  286. // will cause the primary to gossip all evidence to the secondaries.
  287. rts.peerChans[primary.NodeID] <- p2p.PeerUpdate{
  288. Status: p2p.PeerStatusUp,
  289. NodeID: secondary.NodeID,
  290. }
  291. // only ones less than the peers height should make it through
  292. rts.waitForEvidence(t, evList[:height2], secondary.NodeID)
  293. require.Equal(t, numEvidence, int(rts.pools[primary.NodeID].Size()))
  294. require.Equal(t, int(height2), int(rts.pools[secondary.NodeID].Size()))
  295. }
  296. func TestReactorBroadcastEvidence_Pending(t *testing.T) {
  297. val := types.NewMockPV()
  298. height := int64(10)
  299. ctx, cancel := context.WithCancel(context.Background())
  300. defer cancel()
  301. stateDB1 := initializeValidatorState(ctx, t, val, height)
  302. stateDB2 := initializeValidatorState(ctx, t, val, height)
  303. rts := setup(ctx, t, []sm.Store{stateDB1, stateDB2}, 100)
  304. primary := rts.nodes[0]
  305. secondary := rts.nodes[1]
  306. evList := createEvidenceList(ctx, t, rts.pools[primary.NodeID], val, numEvidence)
  307. // Manually add half the evidence to the secondary which will mark them as
  308. // pending.
  309. for i := 0; i < numEvidence/2; i++ {
  310. require.NoError(t, rts.pools[secondary.NodeID].AddEvidence(evList[i]))
  311. }
  312. // the secondary should have half the evidence as pending
  313. require.Equal(t, numEvidence/2, int(rts.pools[secondary.NodeID].Size()))
  314. rts.start(ctx, t)
  315. // The secondary reactor should have received all the evidence ignoring the
  316. // already pending evidence.
  317. rts.waitForEvidence(t, evList, secondary.NodeID)
  318. // check to make sure that all of the evidence has
  319. // propogated
  320. require.Len(t, rts.pools, 2)
  321. assert.EqualValues(t, numEvidence, rts.pools[primary.NodeID].Size(),
  322. "primary node should have all the evidence")
  323. assert.EqualValues(t, numEvidence, rts.pools[secondary.NodeID].Size(),
  324. "secondary nodes should have caught up")
  325. }
  326. func TestReactorBroadcastEvidence_Committed(t *testing.T) {
  327. val := types.NewMockPV()
  328. height := int64(10)
  329. ctx, cancel := context.WithCancel(context.Background())
  330. defer cancel()
  331. stateDB1 := initializeValidatorState(ctx, t, val, height)
  332. stateDB2 := initializeValidatorState(ctx, t, val, height)
  333. rts := setup(ctx, t, []sm.Store{stateDB1, stateDB2}, 0)
  334. primary := rts.nodes[0]
  335. secondary := rts.nodes[1]
  336. // add all evidence to the primary reactor
  337. evList := createEvidenceList(ctx, t, rts.pools[primary.NodeID], val, numEvidence)
  338. // Manually add half the evidence to the secondary which will mark them as
  339. // pending.
  340. for i := 0; i < numEvidence/2; i++ {
  341. require.NoError(t, rts.pools[secondary.NodeID].AddEvidence(evList[i]))
  342. }
  343. // the secondary should have half the evidence as pending
  344. require.Equal(t, numEvidence/2, int(rts.pools[secondary.NodeID].Size()))
  345. state, err := stateDB2.Load()
  346. require.NoError(t, err)
  347. // update the secondary's pool such that all pending evidence is committed
  348. state.LastBlockHeight++
  349. rts.pools[secondary.NodeID].Update(state, evList[:numEvidence/2])
  350. // the secondary should have half the evidence as committed
  351. require.Equal(t, 0, int(rts.pools[secondary.NodeID].Size()))
  352. // start the network and ensure it's configured
  353. rts.start(ctx, t)
  354. // The secondary reactor should have received all the evidence ignoring the
  355. // already committed evidence.
  356. rts.waitForEvidence(t, evList[numEvidence/2:], secondary.NodeID)
  357. require.Len(t, rts.pools, 2)
  358. assert.EqualValues(t, numEvidence, rts.pools[primary.NodeID].Size(),
  359. "primary node should have all the evidence")
  360. assert.EqualValues(t, numEvidence/2, rts.pools[secondary.NodeID].Size(),
  361. "secondary nodes should have caught up")
  362. }
  363. func TestReactorBroadcastEvidence_FullyConnected(t *testing.T) {
  364. numPeers := 7
  365. // create a stateDB for all test suites (nodes)
  366. stateDBs := make([]sm.Store, numPeers)
  367. val := types.NewMockPV()
  368. ctx, cancel := context.WithCancel(context.Background())
  369. defer cancel()
  370. // We need all validators saved for heights at least as high as we have
  371. // evidence for.
  372. height := int64(numEvidence) + 10
  373. for i := 0; i < numPeers; i++ {
  374. stateDBs[i] = initializeValidatorState(ctx, t, val, height)
  375. }
  376. rts := setup(ctx, t, stateDBs, 0)
  377. rts.start(ctx, t)
  378. evList := createEvidenceList(ctx, t, rts.pools[rts.network.RandomNode().NodeID], val, numEvidence)
  379. // every suite (reactor) connects to every other suite (reactor)
  380. for outerID, outerChan := range rts.peerChans {
  381. for innerID := range rts.peerChans {
  382. if outerID != innerID {
  383. outerChan <- p2p.PeerUpdate{
  384. Status: p2p.PeerStatusUp,
  385. NodeID: innerID,
  386. }
  387. }
  388. }
  389. }
  390. // wait till all suites (reactors) received all evidence from other suites (reactors)
  391. rts.waitForEvidence(t, evList)
  392. for _, pool := range rts.pools {
  393. require.Equal(t, numEvidence, int(pool.Size()))
  394. // commit state so we do not continue to repeat gossiping the same evidence
  395. state := pool.State()
  396. state.LastBlockHeight++
  397. pool.Update(state, evList)
  398. }
  399. }
  400. func TestEvidenceListSerialization(t *testing.T) {
  401. exampleVote := func(msgType byte) *types.Vote {
  402. var stamp, err = time.Parse(types.TimeFormat, "2017-12-25T03:00:01.234Z")
  403. require.NoError(t, err)
  404. return &types.Vote{
  405. Type: tmproto.SignedMsgType(msgType),
  406. Height: 3,
  407. Round: 2,
  408. Timestamp: stamp,
  409. BlockID: types.BlockID{
  410. Hash: tmhash.Sum([]byte("blockID_hash")),
  411. PartSetHeader: types.PartSetHeader{
  412. Total: 1000000,
  413. Hash: tmhash.Sum([]byte("blockID_part_set_header_hash")),
  414. },
  415. },
  416. ValidatorAddress: crypto.AddressHash([]byte("validator_address")),
  417. ValidatorIndex: 56789,
  418. }
  419. }
  420. val := &types.Validator{
  421. Address: crypto.AddressHash([]byte("validator_address")),
  422. VotingPower: 10,
  423. }
  424. valSet := types.NewValidatorSet([]*types.Validator{val})
  425. dupl, err := types.NewDuplicateVoteEvidence(
  426. exampleVote(1),
  427. exampleVote(2),
  428. defaultEvidenceTime,
  429. valSet,
  430. )
  431. require.NoError(t, err)
  432. testCases := map[string]struct {
  433. evidenceList []types.Evidence
  434. expBytes string
  435. }{
  436. "DuplicateVoteEvidence": {
  437. []types.Evidence{dupl},
  438. "0a85020a82020a79080210031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb031279080110031802224a0a208b01023386c371778ecb6368573e539afc3cc860ec3a2f614e54fe5652f4fc80122608c0843d122072db3d959635dff1bb567bedaa70573392c5159666a3f8caf11e413aac52207a2a0b08b1d381d20510809dca6f32146af1f4111082efb388211bc72c55bcd61e9ac3d538d5bb03180a200a2a060880dbaae105",
  439. },
  440. }
  441. for name, tc := range testCases {
  442. tc := tc
  443. t.Run(name, func(t *testing.T) {
  444. protoEv := make([]tmproto.Evidence, len(tc.evidenceList))
  445. for i := 0; i < len(tc.evidenceList); i++ {
  446. ev, err := types.EvidenceToProto(tc.evidenceList[i])
  447. require.NoError(t, err)
  448. protoEv[i] = *ev
  449. }
  450. epl := tmproto.EvidenceList{
  451. Evidence: protoEv,
  452. }
  453. bz, err := epl.Marshal()
  454. require.NoError(t, err)
  455. require.Equal(t, tc.expBytes, hex.EncodeToString(bz))
  456. })
  457. }
  458. }