You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

428 lines
10 KiB

  1. package mempool
  2. import (
  3. "context"
  4. "os"
  5. "strings"
  6. "sync"
  7. "testing"
  8. "time"
  9. "github.com/fortytw2/leaktest"
  10. "github.com/stretchr/testify/require"
  11. "github.com/tendermint/tendermint/abci/example/kvstore"
  12. abci "github.com/tendermint/tendermint/abci/types"
  13. "github.com/tendermint/tendermint/config"
  14. tmsync "github.com/tendermint/tendermint/internal/libs/sync"
  15. "github.com/tendermint/tendermint/internal/p2p"
  16. "github.com/tendermint/tendermint/internal/p2p/p2ptest"
  17. "github.com/tendermint/tendermint/libs/log"
  18. tmrand "github.com/tendermint/tendermint/libs/rand"
  19. protomem "github.com/tendermint/tendermint/proto/tendermint/mempool"
  20. "github.com/tendermint/tendermint/types"
  21. )
  22. type reactorTestSuite struct {
  23. network *p2ptest.Network
  24. logger log.Logger
  25. reactors map[types.NodeID]*Reactor
  26. mempoolChannels map[types.NodeID]*p2p.Channel
  27. mempools map[types.NodeID]*TxMempool
  28. kvstores map[types.NodeID]*kvstore.Application
  29. peerChans map[types.NodeID]chan p2p.PeerUpdate
  30. peerUpdates map[types.NodeID]*p2p.PeerUpdates
  31. nodes []types.NodeID
  32. }
  33. func setupReactors(ctx context.Context, t *testing.T, numNodes int, chBuf uint) *reactorTestSuite {
  34. t.Helper()
  35. cfg, err := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|"))
  36. require.NoError(t, err)
  37. t.Cleanup(func() { os.RemoveAll(cfg.RootDir) })
  38. rts := &reactorTestSuite{
  39. logger: log.TestingLogger().With("testCase", t.Name()),
  40. network: p2ptest.MakeNetwork(ctx, t, p2ptest.NetworkOptions{NumNodes: numNodes}),
  41. reactors: make(map[types.NodeID]*Reactor, numNodes),
  42. mempoolChannels: make(map[types.NodeID]*p2p.Channel, numNodes),
  43. mempools: make(map[types.NodeID]*TxMempool, numNodes),
  44. kvstores: make(map[types.NodeID]*kvstore.Application, numNodes),
  45. peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numNodes),
  46. peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
  47. }
  48. chDesc := GetChannelDescriptor(cfg.Mempool)
  49. rts.mempoolChannels = rts.network.MakeChannelsNoCleanup(ctx, t, chDesc)
  50. for nodeID := range rts.network.Nodes {
  51. rts.kvstores[nodeID] = kvstore.NewApplication()
  52. mempool := setup(ctx, t, 0)
  53. rts.mempools[nodeID] = mempool
  54. rts.peerChans[nodeID] = make(chan p2p.PeerUpdate)
  55. rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
  56. rts.network.Nodes[nodeID].PeerManager.Register(ctx, rts.peerUpdates[nodeID])
  57. rts.reactors[nodeID] = NewReactor(
  58. rts.logger.With("nodeID", nodeID),
  59. cfg.Mempool,
  60. rts.network.Nodes[nodeID].PeerManager,
  61. mempool,
  62. rts.mempoolChannels[nodeID],
  63. rts.peerUpdates[nodeID],
  64. )
  65. rts.nodes = append(rts.nodes, nodeID)
  66. require.NoError(t, rts.reactors[nodeID].Start(ctx))
  67. require.True(t, rts.reactors[nodeID].IsRunning())
  68. }
  69. require.Len(t, rts.reactors, numNodes)
  70. t.Cleanup(func() {
  71. for nodeID := range rts.reactors {
  72. if rts.reactors[nodeID].IsRunning() {
  73. require.NoError(t, rts.reactors[nodeID].Stop())
  74. require.False(t, rts.reactors[nodeID].IsRunning())
  75. }
  76. }
  77. })
  78. t.Cleanup(leaktest.Check(t))
  79. return rts
  80. }
  81. func (rts *reactorTestSuite) start(ctx context.Context, t *testing.T) {
  82. t.Helper()
  83. rts.network.Start(ctx, t)
  84. require.Len(t,
  85. rts.network.RandomNode().PeerManager.Peers(),
  86. len(rts.nodes)-1,
  87. "network does not have expected number of nodes")
  88. }
  89. func (rts *reactorTestSuite) assertMempoolChannelsDrained(t *testing.T) {
  90. t.Helper()
  91. for _, mch := range rts.mempoolChannels {
  92. require.Empty(t, mch.Out, "checking channel %q (len=%d)", mch.ID, len(mch.Out))
  93. }
  94. }
  95. func (rts *reactorTestSuite) waitForTxns(t *testing.T, txs []types.Tx, ids ...types.NodeID) {
  96. t.Helper()
  97. // ensure that the transactions get fully broadcast to the
  98. // rest of the network
  99. wg := &sync.WaitGroup{}
  100. for name, pool := range rts.mempools {
  101. if !p2ptest.NodeInSlice(name, ids) {
  102. continue
  103. }
  104. wg.Add(1)
  105. go func(pool *TxMempool) {
  106. defer wg.Done()
  107. require.Eventually(t, func() bool { return len(txs) == pool.Size() },
  108. time.Minute,
  109. 100*time.Millisecond,
  110. )
  111. }(pool)
  112. }
  113. wg.Wait()
  114. }
  115. func TestReactorBroadcastDoesNotPanic(t *testing.T) {
  116. ctx, cancel := context.WithCancel(context.Background())
  117. defer cancel()
  118. const numNodes = 2
  119. rts := setupReactors(ctx, t, numNodes, 0)
  120. observePanic := func(r interface{}) {
  121. t.Fatal("panic detected in reactor")
  122. }
  123. primary := rts.nodes[0]
  124. secondary := rts.nodes[1]
  125. primaryReactor := rts.reactors[primary]
  126. primaryMempool := primaryReactor.mempool
  127. secondaryReactor := rts.reactors[secondary]
  128. primaryReactor.observePanic = observePanic
  129. secondaryReactor.observePanic = observePanic
  130. firstTx := &WrappedTx{}
  131. primaryMempool.insertTx(firstTx)
  132. // run the router
  133. rts.start(ctx, t)
  134. closer := tmsync.NewCloser()
  135. primaryReactor.peerWG.Add(1)
  136. go primaryReactor.broadcastTxRoutine(ctx, secondary, closer)
  137. wg := &sync.WaitGroup{}
  138. for i := 0; i < 50; i++ {
  139. next := &WrappedTx{}
  140. wg.Add(1)
  141. go func() {
  142. defer wg.Done()
  143. primaryMempool.insertTx(next)
  144. }()
  145. }
  146. err := primaryReactor.Stop()
  147. require.NoError(t, err)
  148. primaryReactor.peerWG.Wait()
  149. wg.Wait()
  150. }
  151. func TestReactorBroadcastTxs(t *testing.T) {
  152. numTxs := 1000
  153. numNodes := 10
  154. ctx, cancel := context.WithCancel(context.Background())
  155. defer cancel()
  156. rts := setupReactors(ctx, t, numNodes, 0)
  157. primary := rts.nodes[0]
  158. secondaries := rts.nodes[1:]
  159. txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, UnknownPeerID)
  160. // run the router
  161. rts.start(ctx, t)
  162. // Wait till all secondary suites (reactor) received all mempool txs from the
  163. // primary suite (node).
  164. rts.waitForTxns(t, convertTex(txs), secondaries...)
  165. }
  166. // regression test for https://github.com/tendermint/tendermint/issues/5408
  167. func TestReactorConcurrency(t *testing.T) {
  168. numTxs := 5
  169. numNodes := 2
  170. ctx, cancel := context.WithCancel(context.Background())
  171. defer cancel()
  172. rts := setupReactors(ctx, t, numNodes, 0)
  173. primary := rts.nodes[0]
  174. secondary := rts.nodes[1]
  175. rts.start(ctx, t)
  176. var wg sync.WaitGroup
  177. for i := 0; i < 1000; i++ {
  178. wg.Add(2)
  179. // 1. submit a bunch of txs
  180. // 2. update the whole mempool
  181. txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, UnknownPeerID)
  182. go func() {
  183. defer wg.Done()
  184. mempool := rts.mempools[primary]
  185. mempool.Lock()
  186. defer mempool.Unlock()
  187. deliverTxResponses := make([]*abci.ResponseDeliverTx, len(txs))
  188. for i := range txs {
  189. deliverTxResponses[i] = &abci.ResponseDeliverTx{Code: 0}
  190. }
  191. require.NoError(t, mempool.Update(1, convertTex(txs), deliverTxResponses, nil, nil))
  192. }()
  193. // 1. submit a bunch of txs
  194. // 2. update none
  195. _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs, UnknownPeerID)
  196. go func() {
  197. defer wg.Done()
  198. mempool := rts.mempools[secondary]
  199. mempool.Lock()
  200. defer mempool.Unlock()
  201. err := mempool.Update(1, []types.Tx{}, make([]*abci.ResponseDeliverTx, 0), nil, nil)
  202. require.NoError(t, err)
  203. }()
  204. // flush the mempool
  205. rts.mempools[secondary].Flush()
  206. }
  207. wg.Wait()
  208. }
  209. func TestReactorNoBroadcastToSender(t *testing.T) {
  210. numTxs := 1000
  211. numNodes := 2
  212. ctx, cancel := context.WithCancel(context.Background())
  213. defer cancel()
  214. rts := setupReactors(ctx, t, numNodes, uint(numTxs))
  215. primary := rts.nodes[0]
  216. secondary := rts.nodes[1]
  217. peerID := uint16(1)
  218. _ = checkTxs(ctx, t, rts.mempools[primary], numTxs, peerID)
  219. rts.start(ctx, t)
  220. time.Sleep(100 * time.Millisecond)
  221. require.Eventually(t, func() bool {
  222. return rts.mempools[secondary].Size() == 0
  223. }, time.Minute, 100*time.Millisecond)
  224. rts.assertMempoolChannelsDrained(t)
  225. }
  226. func TestReactor_MaxTxBytes(t *testing.T) {
  227. numNodes := 2
  228. cfg := config.TestConfig()
  229. ctx, cancel := context.WithCancel(context.Background())
  230. defer cancel()
  231. rts := setupReactors(ctx, t, numNodes, 0)
  232. primary := rts.nodes[0]
  233. secondary := rts.nodes[1]
  234. // Broadcast a tx, which has the max size and ensure it's received by the
  235. // second reactor.
  236. tx1 := tmrand.Bytes(cfg.Mempool.MaxTxBytes)
  237. err := rts.reactors[primary].mempool.CheckTx(
  238. ctx,
  239. tx1,
  240. nil,
  241. TxInfo{
  242. SenderID: UnknownPeerID,
  243. },
  244. )
  245. require.NoError(t, err)
  246. rts.start(ctx, t)
  247. rts.reactors[primary].mempool.Flush()
  248. rts.reactors[secondary].mempool.Flush()
  249. // broadcast a tx, which is beyond the max size and ensure it's not sent
  250. tx2 := tmrand.Bytes(cfg.Mempool.MaxTxBytes + 1)
  251. err = rts.mempools[primary].CheckTx(ctx, tx2, nil, TxInfo{SenderID: UnknownPeerID})
  252. require.Error(t, err)
  253. rts.assertMempoolChannelsDrained(t)
  254. }
  255. func TestDontExhaustMaxActiveIDs(t *testing.T) {
  256. // we're creating a single node network, but not starting the
  257. // network.
  258. ctx, cancel := context.WithCancel(context.Background())
  259. defer cancel()
  260. rts := setupReactors(ctx, t, 1, MaxActiveIDs+1)
  261. nodeID := rts.nodes[0]
  262. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  263. require.NoError(t, err)
  264. // ensure the reactor does not panic (i.e. exhaust active IDs)
  265. for i := 0; i < MaxActiveIDs+1; i++ {
  266. rts.peerChans[nodeID] <- p2p.PeerUpdate{
  267. Status: p2p.PeerStatusUp,
  268. NodeID: peerID,
  269. }
  270. rts.mempoolChannels[nodeID].Out <- p2p.Envelope{
  271. To: peerID,
  272. Message: &protomem.Txs{
  273. Txs: [][]byte{},
  274. },
  275. }
  276. }
  277. require.Eventually(
  278. t,
  279. func() bool {
  280. for _, mch := range rts.mempoolChannels {
  281. if len(mch.Out) > 0 {
  282. return false
  283. }
  284. }
  285. return true
  286. },
  287. time.Minute,
  288. 10*time.Millisecond,
  289. )
  290. rts.assertMempoolChannelsDrained(t)
  291. }
  292. func TestMempoolIDsPanicsIfNodeRequestsOvermaxActiveIDs(t *testing.T) {
  293. if testing.Short() {
  294. t.Skip("skipping test in short mode")
  295. }
  296. // 0 is already reserved for UnknownPeerID
  297. ids := NewMempoolIDs()
  298. peerID, err := types.NewNodeID("0011223344556677889900112233445566778899")
  299. require.NoError(t, err)
  300. for i := 0; i < MaxActiveIDs-1; i++ {
  301. ids.ReserveForPeer(peerID)
  302. }
  303. require.Panics(t, func() {
  304. ids.ReserveForPeer(peerID)
  305. })
  306. }
  307. func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) {
  308. if testing.Short() {
  309. t.Skip("skipping test in short mode")
  310. }
  311. ctx, cancel := context.WithCancel(context.Background())
  312. defer cancel()
  313. rts := setupReactors(ctx, t, 2, 0)
  314. primary := rts.nodes[0]
  315. secondary := rts.nodes[1]
  316. rts.start(ctx, t)
  317. // disconnect peer
  318. rts.peerChans[primary] <- p2p.PeerUpdate{
  319. Status: p2p.PeerStatusDown,
  320. NodeID: secondary,
  321. }
  322. time.Sleep(500 * time.Millisecond)
  323. txs := checkTxs(ctx, t, rts.reactors[primary].mempool, 4, UnknownPeerID)
  324. require.Equal(t, 4, len(txs))
  325. require.Equal(t, 4, rts.mempools[primary].Size())
  326. require.Equal(t, 0, rts.mempools[secondary].Size())
  327. }