You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

606 lines
17 KiB

9 years ago
9 years ago
8 years ago
8 years ago
9 years ago
8 years ago
8 years ago
9 years ago
9 years ago
8 years ago
8 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
8 years ago
8 years ago
8 years ago
8 years ago
8 years ago
9 years ago
8 years ago
8 years ago
9 years ago
8 years ago
9 years ago
9 years ago
8 years ago
8 years ago
8 years ago
8 years ago
9 years ago
9 years ago
9 years ago
9 years ago
8 years ago
7 years ago
9 years ago
7 years ago
9 years ago
9 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
7 years ago
9 years ago
7 years ago
9 years ago
  1. package mempool
  2. import (
  3. "bytes"
  4. "container/list"
  5. "crypto/sha256"
  6. "encoding/binary"
  7. "fmt"
  8. "sync"
  9. "sync/atomic"
  10. "time"
  11. "github.com/pkg/errors"
  12. abci "github.com/tendermint/tendermint/abci/types"
  13. auto "github.com/tendermint/tendermint/libs/autofile"
  14. "github.com/tendermint/tendermint/libs/clist"
  15. cmn "github.com/tendermint/tendermint/libs/common"
  16. "github.com/tendermint/tendermint/libs/log"
  17. cfg "github.com/tendermint/tendermint/config"
  18. "github.com/tendermint/tendermint/proxy"
  19. "github.com/tendermint/tendermint/types"
  20. )
  21. /*
  22. The mempool pushes new txs onto the proxyAppConn.
  23. It gets a stream of (req, res) tuples from the proxy.
  24. The mempool stores good txs in a concurrent linked-list.
  25. Multiple concurrent go-routines can traverse this linked-list
  26. safely by calling .NextWait() on each element.
  27. So we have several go-routines:
  28. 1. Consensus calling Update() and Reap() synchronously
  29. 2. Many mempool reactor's peer routines calling CheckTx()
  30. 3. Many mempool reactor's peer routines traversing the txs linked list
  31. 4. Another goroutine calling GarbageCollectTxs() periodically
  32. To manage these goroutines, there are three methods of locking.
  33. 1. Mutations to the linked-list is protected by an internal mtx (CList is goroutine-safe)
  34. 2. Mutations to the linked-list elements are atomic
  35. 3. CheckTx() calls can be paused upon Update() and Reap(), protected by .proxyMtx
  36. Garbage collection of old elements from mempool.txs is handlde via
  37. the DetachPrev() call, which makes old elements not reachable by
  38. peer broadcastTxRoutine() automatically garbage collected.
  39. TODO: Better handle abci client errors. (make it automatically handle connection errors)
  40. */
  41. var (
  42. // ErrTxInCache is returned to the client if we saw tx earlier
  43. ErrTxInCache = errors.New("Tx already exists in cache")
  44. // ErrMempoolIsFull means Tendermint & an application can't handle that much load
  45. ErrMempoolIsFull = errors.New("Mempool is full")
  46. )
  47. // TxID is the hex encoded hash of the bytes as a types.Tx.
  48. func TxID(tx []byte) string {
  49. return fmt.Sprintf("%X", types.Tx(tx).Hash())
  50. }
  51. // Mempool is an ordered in-memory pool for transactions before they are proposed in a consensus
  52. // round. Transaction validity is checked using the CheckTx abci message before the transaction is
  53. // added to the pool. The Mempool uses a concurrent list structure for storing transactions that
  54. // can be efficiently accessed by multiple concurrent readers.
  55. type Mempool struct {
  56. config *cfg.MempoolConfig
  57. proxyMtx sync.Mutex
  58. proxyAppConn proxy.AppConnMempool
  59. txs *clist.CList // concurrent linked-list of good txs
  60. counter int64 // simple incrementing counter
  61. height int64 // the last block Update()'d to
  62. rechecking int32 // for re-checking filtered txs on Update()
  63. recheckCursor *clist.CElement // next expected response
  64. recheckEnd *clist.CElement // re-checking stops here
  65. notifiedTxsAvailable bool
  66. txsAvailable chan struct{} // fires once for each height, when the mempool is not empty
  67. // Filter mempool to only accept txs for which filter(tx) returns true.
  68. filter func(types.Tx) bool
  69. // Keep a cache of already-seen txs.
  70. // This reduces the pressure on the proxyApp.
  71. cache txCache
  72. // A log of mempool txs
  73. wal *auto.AutoFile
  74. logger log.Logger
  75. metrics *Metrics
  76. }
  77. // MempoolOption sets an optional parameter on the Mempool.
  78. type MempoolOption func(*Mempool)
  79. // NewMempool returns a new Mempool with the given configuration and connection to an application.
  80. func NewMempool(
  81. config *cfg.MempoolConfig,
  82. proxyAppConn proxy.AppConnMempool,
  83. height int64,
  84. options ...MempoolOption,
  85. ) *Mempool {
  86. mempool := &Mempool{
  87. config: config,
  88. proxyAppConn: proxyAppConn,
  89. txs: clist.New(),
  90. counter: 0,
  91. height: height,
  92. rechecking: 0,
  93. recheckCursor: nil,
  94. recheckEnd: nil,
  95. logger: log.NewNopLogger(),
  96. metrics: NopMetrics(),
  97. }
  98. if config.CacheSize > 0 {
  99. mempool.cache = newMapTxCache(config.CacheSize)
  100. } else {
  101. mempool.cache = nopTxCache{}
  102. }
  103. proxyAppConn.SetResponseCallback(mempool.resCb)
  104. for _, option := range options {
  105. option(mempool)
  106. }
  107. return mempool
  108. }
  109. // EnableTxsAvailable initializes the TxsAvailable channel,
  110. // ensuring it will trigger once every height when transactions are available.
  111. // NOTE: not thread safe - should only be called once, on startup
  112. func (mem *Mempool) EnableTxsAvailable() {
  113. mem.txsAvailable = make(chan struct{}, 1)
  114. }
  115. // SetLogger sets the Logger.
  116. func (mem *Mempool) SetLogger(l log.Logger) {
  117. mem.logger = l
  118. }
  119. // WithFilter sets a filter for mempool to only accept txs for which f(tx)
  120. // returns true.
  121. func WithFilter(f func(types.Tx) bool) MempoolOption {
  122. return func(mem *Mempool) { mem.filter = f }
  123. }
  124. // WithMetrics sets the metrics.
  125. func WithMetrics(metrics *Metrics) MempoolOption {
  126. return func(mem *Mempool) { mem.metrics = metrics }
  127. }
  128. // CloseWAL closes and discards the underlying WAL file.
  129. // Any further writes will not be relayed to disk.
  130. func (mem *Mempool) CloseWAL() bool {
  131. if mem == nil {
  132. return false
  133. }
  134. mem.proxyMtx.Lock()
  135. defer mem.proxyMtx.Unlock()
  136. if mem.wal == nil {
  137. return false
  138. }
  139. if err := mem.wal.Close(); err != nil && mem.logger != nil {
  140. mem.logger.Error("Mempool.CloseWAL", "err", err)
  141. }
  142. mem.wal = nil
  143. return true
  144. }
  145. func (mem *Mempool) InitWAL() {
  146. walDir := mem.config.WalDir()
  147. if walDir != "" {
  148. err := cmn.EnsureDir(walDir, 0700)
  149. if err != nil {
  150. cmn.PanicSanity(errors.Wrap(err, "Error ensuring Mempool wal dir"))
  151. }
  152. af, err := auto.OpenAutoFile(walDir + "/wal")
  153. if err != nil {
  154. cmn.PanicSanity(errors.Wrap(err, "Error opening Mempool wal file"))
  155. }
  156. mem.wal = af
  157. }
  158. }
  159. // Lock locks the mempool. The consensus must be able to hold lock to safely update.
  160. func (mem *Mempool) Lock() {
  161. mem.proxyMtx.Lock()
  162. }
  163. // Unlock unlocks the mempool.
  164. func (mem *Mempool) Unlock() {
  165. mem.proxyMtx.Unlock()
  166. }
  167. // Size returns the number of transactions in the mempool.
  168. func (mem *Mempool) Size() int {
  169. return mem.txs.Len()
  170. }
  171. // Flushes the mempool connection to ensure async resCb calls are done e.g.
  172. // from CheckTx.
  173. func (mem *Mempool) FlushAppConn() error {
  174. return mem.proxyAppConn.FlushSync()
  175. }
  176. // Flush removes all transactions from the mempool and cache
  177. func (mem *Mempool) Flush() {
  178. mem.proxyMtx.Lock()
  179. defer mem.proxyMtx.Unlock()
  180. mem.cache.Reset()
  181. for e := mem.txs.Front(); e != nil; e = e.Next() {
  182. mem.txs.Remove(e)
  183. e.DetachPrev()
  184. }
  185. }
  186. // TxsFront returns the first transaction in the ordered list for peer
  187. // goroutines to call .NextWait() on.
  188. func (mem *Mempool) TxsFront() *clist.CElement {
  189. return mem.txs.Front()
  190. }
  191. // TxsWaitChan returns a channel to wait on transactions. It will be closed
  192. // once the mempool is not empty (ie. the internal `mem.txs` has at least one
  193. // element)
  194. func (mem *Mempool) TxsWaitChan() <-chan struct{} {
  195. return mem.txs.WaitChan()
  196. }
  197. // CheckTx executes a new transaction against the application to determine its validity
  198. // and whether it should be added to the mempool.
  199. // It blocks if we're waiting on Update() or Reap().
  200. // cb: A callback from the CheckTx command.
  201. // It gets called from another goroutine.
  202. // CONTRACT: Either cb will get called, or err returned.
  203. func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) {
  204. mem.proxyMtx.Lock()
  205. defer mem.proxyMtx.Unlock()
  206. if mem.Size() >= mem.config.Size {
  207. return ErrMempoolIsFull
  208. }
  209. if mem.filter != nil && !mem.filter(tx) {
  210. return
  211. }
  212. // CACHE
  213. if !mem.cache.Push(tx) {
  214. return ErrTxInCache
  215. }
  216. // END CACHE
  217. // WAL
  218. if mem.wal != nil {
  219. // TODO: Notify administrators when WAL fails
  220. _, err := mem.wal.Write([]byte(tx))
  221. if err != nil {
  222. mem.logger.Error("Error writing to WAL", "err", err)
  223. }
  224. _, err = mem.wal.Write([]byte("\n"))
  225. if err != nil {
  226. mem.logger.Error("Error writing to WAL", "err", err)
  227. }
  228. }
  229. // END WAL
  230. // NOTE: proxyAppConn may error if tx buffer is full
  231. if err = mem.proxyAppConn.Error(); err != nil {
  232. return err
  233. }
  234. reqRes := mem.proxyAppConn.CheckTxAsync(tx)
  235. if cb != nil {
  236. reqRes.SetCallback(cb)
  237. }
  238. return nil
  239. }
  240. // ABCI callback function
  241. func (mem *Mempool) resCb(req *abci.Request, res *abci.Response) {
  242. if mem.recheckCursor == nil {
  243. mem.resCbNormal(req, res)
  244. } else {
  245. mem.resCbRecheck(req, res)
  246. }
  247. mem.metrics.Size.Set(float64(mem.Size()))
  248. }
  249. func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) {
  250. switch r := res.Value.(type) {
  251. case *abci.Response_CheckTx:
  252. tx := req.GetCheckTx().Tx
  253. if r.CheckTx.Code == abci.CodeTypeOK {
  254. mem.counter++
  255. memTx := &mempoolTx{
  256. counter: mem.counter,
  257. height: mem.height,
  258. tx: tx,
  259. }
  260. mem.txs.PushBack(memTx)
  261. mem.logger.Info("Added good transaction", "tx", TxID(tx), "res", r, "total", mem.Size())
  262. mem.notifyTxsAvailable()
  263. } else {
  264. // ignore bad transaction
  265. mem.logger.Info("Rejected bad transaction", "tx", TxID(tx), "res", r)
  266. // remove from cache (it might be good later)
  267. mem.cache.Remove(tx)
  268. }
  269. default:
  270. // ignore other messages
  271. }
  272. }
  273. func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
  274. switch r := res.Value.(type) {
  275. case *abci.Response_CheckTx:
  276. memTx := mem.recheckCursor.Value.(*mempoolTx)
  277. if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) {
  278. cmn.PanicSanity(fmt.Sprintf("Unexpected tx response from proxy during recheck\n"+
  279. "Expected %X, got %X", r.CheckTx.Data, memTx.tx))
  280. }
  281. if r.CheckTx.Code == abci.CodeTypeOK {
  282. // Good, nothing to do.
  283. } else {
  284. // Tx became invalidated due to newly committed block.
  285. mem.txs.Remove(mem.recheckCursor)
  286. mem.recheckCursor.DetachPrev()
  287. // remove from cache (it might be good later)
  288. mem.cache.Remove(req.GetCheckTx().Tx)
  289. }
  290. if mem.recheckCursor == mem.recheckEnd {
  291. mem.recheckCursor = nil
  292. } else {
  293. mem.recheckCursor = mem.recheckCursor.Next()
  294. }
  295. if mem.recheckCursor == nil {
  296. // Done!
  297. atomic.StoreInt32(&mem.rechecking, 0)
  298. mem.logger.Info("Done rechecking txs")
  299. // incase the recheck removed all txs
  300. if mem.Size() > 0 {
  301. mem.notifyTxsAvailable()
  302. }
  303. }
  304. default:
  305. // ignore other messages
  306. }
  307. }
  308. // TxsAvailable returns a channel which fires once for every height,
  309. // and only when transactions are available in the mempool.
  310. // NOTE: the returned channel may be nil if EnableTxsAvailable was not called.
  311. func (mem *Mempool) TxsAvailable() <-chan struct{} {
  312. return mem.txsAvailable
  313. }
  314. func (mem *Mempool) notifyTxsAvailable() {
  315. if mem.Size() == 0 {
  316. panic("notified txs available but mempool is empty!")
  317. }
  318. if mem.txsAvailable != nil && !mem.notifiedTxsAvailable {
  319. // channel cap is 1, so this will send once
  320. mem.notifiedTxsAvailable = true
  321. select {
  322. case mem.txsAvailable <- struct{}{}:
  323. default:
  324. }
  325. }
  326. }
  327. // ReapMaxBytes reaps transactions from the mempool up to n bytes total.
  328. // If max is negative, there is no cap on the size of all returned
  329. // transactions (~ all available transactions).
  330. func (mem *Mempool) ReapMaxBytes(max int) types.Txs {
  331. var buf [binary.MaxVarintLen64]byte
  332. mem.proxyMtx.Lock()
  333. defer mem.proxyMtx.Unlock()
  334. for atomic.LoadInt32(&mem.rechecking) > 0 {
  335. // TODO: Something better?
  336. time.Sleep(time.Millisecond * 10)
  337. }
  338. var cur int
  339. // TODO: we will get a performance boost if we have a good estimate of avg
  340. // size per tx, and set the initial capacity based off of that.
  341. // txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), max/mem.avgTxSize))
  342. txs := make([]types.Tx, 0, mem.txs.Len())
  343. for e := mem.txs.Front(); e != nil; e = e.Next() {
  344. memTx := e.Value.(*mempoolTx)
  345. // amino.UvarintSize is not used here because it won't be possible to reuse buf
  346. aminoOverhead := binary.PutUvarint(buf[:], uint64(len(memTx.tx)))
  347. if max > 0 && cur+len(memTx.tx)+aminoOverhead > max {
  348. return txs
  349. }
  350. cur += len(memTx.tx) + aminoOverhead
  351. txs = append(txs, memTx.tx)
  352. }
  353. return txs
  354. }
  355. // ReapMaxTxs reaps up to max transactions from the mempool.
  356. // If max is negative, there is no cap on the size of all returned
  357. // transactions (~ all available transactions).
  358. func (mem *Mempool) ReapMaxTxs(max int) types.Txs {
  359. mem.proxyMtx.Lock()
  360. defer mem.proxyMtx.Unlock()
  361. if max < 0 {
  362. max = mem.txs.Len()
  363. }
  364. for atomic.LoadInt32(&mem.rechecking) > 0 {
  365. // TODO: Something better?
  366. time.Sleep(time.Millisecond * 10)
  367. }
  368. txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), max))
  369. for e := mem.txs.Front(); e != nil && len(txs) <= max; e = e.Next() {
  370. memTx := e.Value.(*mempoolTx)
  371. txs = append(txs, memTx.tx)
  372. }
  373. return txs
  374. }
  375. // Update informs the mempool that the given txs were committed and can be discarded.
  376. // NOTE: this should be called *after* block is committed by consensus.
  377. // NOTE: unsafe; Lock/Unlock must be managed by caller
  378. func (mem *Mempool) Update(height int64, txs types.Txs, filter func(types.Tx) bool) error {
  379. // First, create a lookup map of txns in new txs.
  380. txsMap := make(map[string]struct{}, len(txs))
  381. for _, tx := range txs {
  382. txsMap[string(tx)] = struct{}{}
  383. }
  384. // Set height
  385. mem.height = height
  386. mem.notifiedTxsAvailable = false
  387. if filter != nil {
  388. mem.filter = filter
  389. }
  390. // Remove transactions that are already in txs.
  391. goodTxs := mem.filterTxs(txsMap)
  392. // Recheck mempool txs if any txs were committed in the block
  393. // NOTE/XXX: in some apps a tx could be invalidated due to EndBlock,
  394. // so we really still do need to recheck, but this is for debugging
  395. if mem.config.Recheck && (mem.config.RecheckEmpty || len(goodTxs) > 0) {
  396. mem.logger.Info("Recheck txs", "numtxs", len(goodTxs), "height", height)
  397. mem.recheckTxs(goodTxs)
  398. // At this point, mem.txs are being rechecked.
  399. // mem.recheckCursor re-scans mem.txs and possibly removes some txs.
  400. // Before mem.Reap(), we should wait for mem.recheckCursor to be nil.
  401. }
  402. // Update metrics
  403. mem.metrics.Size.Set(float64(mem.Size()))
  404. return nil
  405. }
  406. func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx {
  407. goodTxs := make([]types.Tx, 0, mem.txs.Len())
  408. for e := mem.txs.Front(); e != nil; e = e.Next() {
  409. memTx := e.Value.(*mempoolTx)
  410. // Remove the tx if it's alredy in a block.
  411. if _, ok := blockTxsMap[string(memTx.tx)]; ok {
  412. // remove from clist
  413. mem.txs.Remove(e)
  414. e.DetachPrev()
  415. // NOTE: we don't remove committed txs from the cache.
  416. continue
  417. }
  418. // Good tx!
  419. goodTxs = append(goodTxs, memTx.tx)
  420. }
  421. return goodTxs
  422. }
  423. // NOTE: pass in goodTxs because mem.txs can mutate concurrently.
  424. func (mem *Mempool) recheckTxs(goodTxs []types.Tx) {
  425. if len(goodTxs) == 0 {
  426. return
  427. }
  428. atomic.StoreInt32(&mem.rechecking, 1)
  429. mem.recheckCursor = mem.txs.Front()
  430. mem.recheckEnd = mem.txs.Back()
  431. // Push txs to proxyAppConn
  432. // NOTE: resCb() may be called concurrently.
  433. for _, tx := range goodTxs {
  434. mem.proxyAppConn.CheckTxAsync(tx)
  435. }
  436. mem.proxyAppConn.FlushAsync()
  437. }
  438. //--------------------------------------------------------------------------------
  439. // mempoolTx is a transaction that successfully ran
  440. type mempoolTx struct {
  441. counter int64 // a simple incrementing counter
  442. height int64 // height that this tx had been validated in
  443. tx types.Tx //
  444. }
  445. // Height returns the height for this transaction
  446. func (memTx *mempoolTx) Height() int64 {
  447. return atomic.LoadInt64(&memTx.height)
  448. }
  449. //--------------------------------------------------------------------------------
  450. type txCache interface {
  451. Reset()
  452. Push(tx types.Tx) bool
  453. Remove(tx types.Tx)
  454. }
  455. // mapTxCache maintains a cache of transactions. This only stores
  456. // the hash of the tx, due to memory concerns.
  457. type mapTxCache struct {
  458. mtx sync.Mutex
  459. size int
  460. map_ map[[sha256.Size]byte]*list.Element
  461. list *list.List // to remove oldest tx when cache gets too big
  462. }
  463. var _ txCache = (*mapTxCache)(nil)
  464. // newMapTxCache returns a new mapTxCache.
  465. func newMapTxCache(cacheSize int) *mapTxCache {
  466. return &mapTxCache{
  467. size: cacheSize,
  468. map_: make(map[[sha256.Size]byte]*list.Element, cacheSize),
  469. list: list.New(),
  470. }
  471. }
  472. // Reset resets the cache to an empty state.
  473. func (cache *mapTxCache) Reset() {
  474. cache.mtx.Lock()
  475. cache.map_ = make(map[[sha256.Size]byte]*list.Element, cache.size)
  476. cache.list.Init()
  477. cache.mtx.Unlock()
  478. }
  479. // Push adds the given tx to the cache and returns true. It returns false if tx
  480. // is already in the cache.
  481. func (cache *mapTxCache) Push(tx types.Tx) bool {
  482. cache.mtx.Lock()
  483. defer cache.mtx.Unlock()
  484. // Use the tx hash in the cache
  485. txHash := sha256.Sum256(tx)
  486. if _, exists := cache.map_[txHash]; exists {
  487. return false
  488. }
  489. if cache.list.Len() >= cache.size {
  490. popped := cache.list.Front()
  491. poppedTxHash := popped.Value.([sha256.Size]byte)
  492. delete(cache.map_, poppedTxHash)
  493. if popped != nil {
  494. cache.list.Remove(popped)
  495. }
  496. }
  497. cache.list.PushBack(txHash)
  498. cache.map_[txHash] = cache.list.Back()
  499. return true
  500. }
  501. // Remove removes the given tx from the cache.
  502. func (cache *mapTxCache) Remove(tx types.Tx) {
  503. cache.mtx.Lock()
  504. txHash := sha256.Sum256(tx)
  505. popped := cache.map_[txHash]
  506. delete(cache.map_, txHash)
  507. if popped != nil {
  508. cache.list.Remove(popped)
  509. }
  510. cache.mtx.Unlock()
  511. }
  512. type nopTxCache struct{}
  513. var _ txCache = (*nopTxCache)(nil)
  514. func (nopTxCache) Reset() {}
  515. func (nopTxCache) Push(types.Tx) bool { return true }
  516. func (nopTxCache) Remove(types.Tx) {}