You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

702 lines
20 KiB

9 years ago
9 years ago
8 years ago
8 years ago
9 years ago
8 years ago
8 years ago
9 years ago
9 years ago
8 years ago
8 years ago
8 years ago
8 years ago
9 years ago
8 years ago
8 years ago
9 years ago
8 years ago
9 years ago
9 years ago
9 years ago
8 years ago
8 years ago
8 years ago
8 years ago
9 years ago
9 years ago
9 years ago
9 years ago
9 years ago
8 years ago
8 years ago
9 years ago
8 years ago
9 years ago
9 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
max-bytes PR follow-up (#2318) * ReapMaxTxs: return all txs if max is negative this mirrors ReapMaxBytes behavior See https://github.com/tendermint/tendermint/pull/2184#discussion_r214439950 * increase MaxAminoOverheadForBlock tested with: ``` func TestMaxAminoOverheadForBlock(t *testing.T) { maxChainID := "" for i := 0; i < MaxChainIDLen; i++ { maxChainID += "𠜎" } h := Header{ ChainID: maxChainID, Height: 10, Time: time.Now().UTC(), NumTxs: 100, TotalTxs: 200, LastBlockID: makeBlockID(make([]byte, 20), 300, make([]byte, 20)), LastCommitHash: tmhash.Sum([]byte("last_commit_hash")), DataHash: tmhash.Sum([]byte("data_hash")), ValidatorsHash: tmhash.Sum([]byte("validators_hash")), NextValidatorsHash: tmhash.Sum([]byte("next_validators_hash")), ConsensusHash: tmhash.Sum([]byte("consensus_hash")), AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), ProposerAddress: tmhash.Sum([]byte("proposer_address")), } b := Block{ Header: h, Data: Data{Txs: makeTxs(10000, 100)}, Evidence: EvidenceData{}, LastCommit: &Commit{}, } bz, err := cdc.MarshalBinary(b) require.NoError(t, err) assert.Equal(t, MaxHeaderBytes+MaxAminoOverheadForBlock-2, len(bz)-1000000-20000-1) } ``` * fix MaxYYY constants calculation by using math.MaxInt64 See https://github.com/tendermint/tendermint/pull/2184#discussion_r214444244 * pass mempool filter as an option See https://github.com/tendermint/tendermint/pull/2184#discussion_r214445869 * fixes after Dev's comments
6 years ago
8 years ago
9 years ago
8 years ago
9 years ago
  1. package mempool
  2. import (
  3. "bytes"
  4. "container/list"
  5. "crypto/sha256"
  6. "fmt"
  7. "sync"
  8. "sync/atomic"
  9. "time"
  10. "github.com/pkg/errors"
  11. abci "github.com/tendermint/tendermint/abci/types"
  12. cfg "github.com/tendermint/tendermint/config"
  13. auto "github.com/tendermint/tendermint/libs/autofile"
  14. "github.com/tendermint/tendermint/libs/clist"
  15. cmn "github.com/tendermint/tendermint/libs/common"
  16. "github.com/tendermint/tendermint/libs/log"
  17. "github.com/tendermint/tendermint/proxy"
  18. "github.com/tendermint/tendermint/types"
  19. )
  20. // PreCheckFunc is an optional filter executed before CheckTx and rejects
  21. // transaction if false is returned. An example would be to ensure that a
  22. // transaction doesn't exceeded the block size.
  23. type PreCheckFunc func(types.Tx) error
  24. // PostCheckFunc is an optional filter executed after CheckTx and rejects
  25. // transaction if false is returned. An example would be to ensure a
  26. // transaction doesn't require more gas than available for the block.
  27. type PostCheckFunc func(types.Tx, *abci.ResponseCheckTx) error
  28. /*
  29. The mempool pushes new txs onto the proxyAppConn.
  30. It gets a stream of (req, res) tuples from the proxy.
  31. The mempool stores good txs in a concurrent linked-list.
  32. Multiple concurrent go-routines can traverse this linked-list
  33. safely by calling .NextWait() on each element.
  34. So we have several go-routines:
  35. 1. Consensus calling Update() and Reap() synchronously
  36. 2. Many mempool reactor's peer routines calling CheckTx()
  37. 3. Many mempool reactor's peer routines traversing the txs linked list
  38. 4. Another goroutine calling GarbageCollectTxs() periodically
  39. To manage these goroutines, there are three methods of locking.
  40. 1. Mutations to the linked-list is protected by an internal mtx (CList is goroutine-safe)
  41. 2. Mutations to the linked-list elements are atomic
  42. 3. CheckTx() calls can be paused upon Update() and Reap(), protected by .proxyMtx
  43. Garbage collection of old elements from mempool.txs is handlde via
  44. the DetachPrev() call, which makes old elements not reachable by
  45. peer broadcastTxRoutine() automatically garbage collected.
  46. TODO: Better handle abci client errors. (make it automatically handle connection errors)
  47. */
  48. var (
  49. // ErrTxInCache is returned to the client if we saw tx earlier
  50. ErrTxInCache = errors.New("Tx already exists in cache")
  51. // ErrMempoolIsFull means Tendermint & an application can't handle that much load
  52. ErrMempoolIsFull = errors.New("Mempool is full")
  53. )
  54. // ErrPreCheck is returned when tx is too big
  55. type ErrPreCheck struct {
  56. Reason error
  57. }
  58. func (e ErrPreCheck) Error() string {
  59. return e.Reason.Error()
  60. }
  61. // IsPreCheckError returns true if err is due to pre check failure.
  62. func IsPreCheckError(err error) bool {
  63. _, ok := err.(ErrPreCheck)
  64. return ok
  65. }
  66. // PreCheckAminoMaxBytes checks that the size of the transaction plus the amino
  67. // overhead is smaller or equal to the expected maxBytes.
  68. func PreCheckAminoMaxBytes(maxBytes int64) PreCheckFunc {
  69. return func(tx types.Tx) error {
  70. // We have to account for the amino overhead in the tx size as well
  71. // NOTE: fieldNum = 1 as types.Block.Data contains Txs []Tx as first field.
  72. // If this field order ever changes this needs to updated here accordingly.
  73. // NOTE: if some []Tx are encoded without a parenting struct, the
  74. // fieldNum is also equal to 1.
  75. aminoOverhead := types.ComputeAminoOverhead(tx, 1)
  76. txSize := int64(len(tx)) + aminoOverhead
  77. if txSize > maxBytes {
  78. return fmt.Errorf("Tx size (including amino overhead) is too big: %d, max: %d",
  79. txSize, maxBytes)
  80. }
  81. return nil
  82. }
  83. }
  84. // PostCheckMaxGas checks that the wanted gas is smaller or equal to the passed
  85. // maxGas. Returns nil if maxGas is -1.
  86. func PostCheckMaxGas(maxGas int64) PostCheckFunc {
  87. return func(tx types.Tx, res *abci.ResponseCheckTx) error {
  88. if maxGas == -1 {
  89. return nil
  90. }
  91. if res.GasWanted > maxGas {
  92. return fmt.Errorf("gas wanted %d is greater than max gas %d",
  93. res.GasWanted, maxGas)
  94. }
  95. return nil
  96. }
  97. }
  98. // TxID is the hex encoded hash of the bytes as a types.Tx.
  99. func TxID(tx []byte) string {
  100. return fmt.Sprintf("%X", types.Tx(tx).Hash())
  101. }
  102. // Mempool is an ordered in-memory pool for transactions before they are proposed in a consensus
  103. // round. Transaction validity is checked using the CheckTx abci message before the transaction is
  104. // added to the pool. The Mempool uses a concurrent list structure for storing transactions that
  105. // can be efficiently accessed by multiple concurrent readers.
  106. type Mempool struct {
  107. config *cfg.MempoolConfig
  108. proxyMtx sync.Mutex
  109. proxyAppConn proxy.AppConnMempool
  110. txs *clist.CList // concurrent linked-list of good txs
  111. counter int64 // simple incrementing counter
  112. height int64 // the last block Update()'d to
  113. rechecking int32 // for re-checking filtered txs on Update()
  114. recheckCursor *clist.CElement // next expected response
  115. recheckEnd *clist.CElement // re-checking stops here
  116. notifiedTxsAvailable bool
  117. txsAvailable chan struct{} // fires once for each height, when the mempool is not empty
  118. preCheck PreCheckFunc
  119. postCheck PostCheckFunc
  120. // Keep a cache of already-seen txs.
  121. // This reduces the pressure on the proxyApp.
  122. cache txCache
  123. // A log of mempool txs
  124. wal *auto.AutoFile
  125. logger log.Logger
  126. metrics *Metrics
  127. }
  128. // MempoolOption sets an optional parameter on the Mempool.
  129. type MempoolOption func(*Mempool)
  130. // NewMempool returns a new Mempool with the given configuration and connection to an application.
  131. func NewMempool(
  132. config *cfg.MempoolConfig,
  133. proxyAppConn proxy.AppConnMempool,
  134. height int64,
  135. options ...MempoolOption,
  136. ) *Mempool {
  137. mempool := &Mempool{
  138. config: config,
  139. proxyAppConn: proxyAppConn,
  140. txs: clist.New(),
  141. counter: 0,
  142. height: height,
  143. rechecking: 0,
  144. recheckCursor: nil,
  145. recheckEnd: nil,
  146. logger: log.NewNopLogger(),
  147. metrics: NopMetrics(),
  148. }
  149. if config.CacheSize > 0 {
  150. mempool.cache = newMapTxCache(config.CacheSize)
  151. } else {
  152. mempool.cache = nopTxCache{}
  153. }
  154. proxyAppConn.SetResponseCallback(mempool.resCb)
  155. for _, option := range options {
  156. option(mempool)
  157. }
  158. return mempool
  159. }
  160. // EnableTxsAvailable initializes the TxsAvailable channel,
  161. // ensuring it will trigger once every height when transactions are available.
  162. // NOTE: not thread safe - should only be called once, on startup
  163. func (mem *Mempool) EnableTxsAvailable() {
  164. mem.txsAvailable = make(chan struct{}, 1)
  165. }
  166. // SetLogger sets the Logger.
  167. func (mem *Mempool) SetLogger(l log.Logger) {
  168. mem.logger = l
  169. }
  170. // WithPreCheck sets a filter for the mempool to reject a tx if f(tx) returns
  171. // false. This is ran before CheckTx.
  172. func WithPreCheck(f PreCheckFunc) MempoolOption {
  173. return func(mem *Mempool) { mem.preCheck = f }
  174. }
  175. // WithPostCheck sets a filter for the mempool to reject a tx if f(tx) returns
  176. // false. This is ran after CheckTx.
  177. func WithPostCheck(f PostCheckFunc) MempoolOption {
  178. return func(mem *Mempool) { mem.postCheck = f }
  179. }
  180. // WithMetrics sets the metrics.
  181. func WithMetrics(metrics *Metrics) MempoolOption {
  182. return func(mem *Mempool) { mem.metrics = metrics }
  183. }
  184. // InitWAL creates a directory for the WAL file and opens a file itself.
  185. //
  186. // *panics* if can't create directory or open file.
  187. // *not thread safe*
  188. func (mem *Mempool) InitWAL() {
  189. walDir := mem.config.WalDir()
  190. err := cmn.EnsureDir(walDir, 0700)
  191. if err != nil {
  192. panic(errors.Wrap(err, "Error ensuring Mempool WAL dir"))
  193. }
  194. af, err := auto.OpenAutoFile(walDir + "/wal")
  195. if err != nil {
  196. panic(errors.Wrap(err, "Error opening Mempool WAL file"))
  197. }
  198. mem.wal = af
  199. }
  200. // CloseWAL closes and discards the underlying WAL file.
  201. // Any further writes will not be relayed to disk.
  202. func (mem *Mempool) CloseWAL() {
  203. mem.proxyMtx.Lock()
  204. defer mem.proxyMtx.Unlock()
  205. if err := mem.wal.Close(); err != nil {
  206. mem.logger.Error("Error closing WAL", "err", err)
  207. }
  208. mem.wal = nil
  209. }
  210. // Lock locks the mempool. The consensus must be able to hold lock to safely update.
  211. func (mem *Mempool) Lock() {
  212. mem.proxyMtx.Lock()
  213. }
  214. // Unlock unlocks the mempool.
  215. func (mem *Mempool) Unlock() {
  216. mem.proxyMtx.Unlock()
  217. }
  218. // Size returns the number of transactions in the mempool.
  219. func (mem *Mempool) Size() int {
  220. return mem.txs.Len()
  221. }
  222. // Flushes the mempool connection to ensure async resCb calls are done e.g.
  223. // from CheckTx.
  224. func (mem *Mempool) FlushAppConn() error {
  225. return mem.proxyAppConn.FlushSync()
  226. }
  227. // Flush removes all transactions from the mempool and cache
  228. func (mem *Mempool) Flush() {
  229. mem.proxyMtx.Lock()
  230. defer mem.proxyMtx.Unlock()
  231. mem.cache.Reset()
  232. for e := mem.txs.Front(); e != nil; e = e.Next() {
  233. mem.txs.Remove(e)
  234. e.DetachPrev()
  235. }
  236. }
  237. // TxsFront returns the first transaction in the ordered list for peer
  238. // goroutines to call .NextWait() on.
  239. func (mem *Mempool) TxsFront() *clist.CElement {
  240. return mem.txs.Front()
  241. }
  242. // TxsWaitChan returns a channel to wait on transactions. It will be closed
  243. // once the mempool is not empty (ie. the internal `mem.txs` has at least one
  244. // element)
  245. func (mem *Mempool) TxsWaitChan() <-chan struct{} {
  246. return mem.txs.WaitChan()
  247. }
  248. // CheckTx executes a new transaction against the application to determine its validity
  249. // and whether it should be added to the mempool.
  250. // It blocks if we're waiting on Update() or Reap().
  251. // cb: A callback from the CheckTx command.
  252. // It gets called from another goroutine.
  253. // CONTRACT: Either cb will get called, or err returned.
  254. func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) {
  255. mem.proxyMtx.Lock()
  256. defer mem.proxyMtx.Unlock()
  257. if mem.Size() >= mem.config.Size {
  258. return ErrMempoolIsFull
  259. }
  260. if mem.preCheck != nil {
  261. if err := mem.preCheck(tx); err != nil {
  262. return ErrPreCheck{err}
  263. }
  264. }
  265. // CACHE
  266. if !mem.cache.Push(tx) {
  267. return ErrTxInCache
  268. }
  269. // END CACHE
  270. // WAL
  271. if mem.wal != nil {
  272. // TODO: Notify administrators when WAL fails
  273. _, err := mem.wal.Write([]byte(tx))
  274. if err != nil {
  275. mem.logger.Error("Error writing to WAL", "err", err)
  276. }
  277. _, err = mem.wal.Write([]byte("\n"))
  278. if err != nil {
  279. mem.logger.Error("Error writing to WAL", "err", err)
  280. }
  281. }
  282. // END WAL
  283. // NOTE: proxyAppConn may error if tx buffer is full
  284. if err = mem.proxyAppConn.Error(); err != nil {
  285. return err
  286. }
  287. reqRes := mem.proxyAppConn.CheckTxAsync(tx)
  288. if cb != nil {
  289. reqRes.SetCallback(cb)
  290. }
  291. return nil
  292. }
  293. // ABCI callback function
  294. func (mem *Mempool) resCb(req *abci.Request, res *abci.Response) {
  295. if mem.recheckCursor == nil {
  296. mem.resCbNormal(req, res)
  297. } else {
  298. mem.metrics.RecheckTimes.Add(1)
  299. mem.resCbRecheck(req, res)
  300. }
  301. mem.metrics.Size.Set(float64(mem.Size()))
  302. }
  303. func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) {
  304. switch r := res.Value.(type) {
  305. case *abci.Response_CheckTx:
  306. tx := req.GetCheckTx().Tx
  307. var postCheckErr error
  308. if mem.postCheck != nil {
  309. postCheckErr = mem.postCheck(tx, r.CheckTx)
  310. }
  311. if (r.CheckTx.Code == abci.CodeTypeOK) && postCheckErr == nil {
  312. mem.counter++
  313. memTx := &mempoolTx{
  314. counter: mem.counter,
  315. height: mem.height,
  316. gasWanted: r.CheckTx.GasWanted,
  317. tx: tx,
  318. }
  319. mem.txs.PushBack(memTx)
  320. mem.logger.Info("Added good transaction",
  321. "tx", TxID(tx),
  322. "res", r,
  323. "height", memTx.height,
  324. "total", mem.Size(),
  325. "counter", memTx.counter,
  326. )
  327. mem.metrics.TxSizeBytes.Observe(float64(len(tx)))
  328. mem.notifyTxsAvailable()
  329. } else {
  330. // ignore bad transaction
  331. mem.logger.Info("Rejected bad transaction", "tx", TxID(tx), "res", r, "err", postCheckErr)
  332. mem.metrics.FailedTxs.Add(1)
  333. // remove from cache (it might be good later)
  334. mem.cache.Remove(tx)
  335. }
  336. default:
  337. // ignore other messages
  338. }
  339. }
  340. func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) {
  341. switch r := res.Value.(type) {
  342. case *abci.Response_CheckTx:
  343. tx := req.GetCheckTx().Tx
  344. memTx := mem.recheckCursor.Value.(*mempoolTx)
  345. if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) {
  346. cmn.PanicSanity(
  347. fmt.Sprintf(
  348. "Unexpected tx response from proxy during recheck\nExpected %X, got %X",
  349. r.CheckTx.Data,
  350. memTx.tx,
  351. ),
  352. )
  353. }
  354. var postCheckErr error
  355. if mem.postCheck != nil {
  356. postCheckErr = mem.postCheck(tx, r.CheckTx)
  357. }
  358. if (r.CheckTx.Code == abci.CodeTypeOK) && postCheckErr == nil {
  359. // Good, nothing to do.
  360. } else {
  361. // Tx became invalidated due to newly committed block.
  362. mem.logger.Info("Tx is no longer valid", "tx", TxID(tx), "res", r, "err", postCheckErr)
  363. mem.txs.Remove(mem.recheckCursor)
  364. mem.recheckCursor.DetachPrev()
  365. // remove from cache (it might be good later)
  366. mem.cache.Remove(tx)
  367. }
  368. if mem.recheckCursor == mem.recheckEnd {
  369. mem.recheckCursor = nil
  370. } else {
  371. mem.recheckCursor = mem.recheckCursor.Next()
  372. }
  373. if mem.recheckCursor == nil {
  374. // Done!
  375. atomic.StoreInt32(&mem.rechecking, 0)
  376. mem.logger.Info("Done rechecking txs")
  377. // incase the recheck removed all txs
  378. if mem.Size() > 0 {
  379. mem.notifyTxsAvailable()
  380. }
  381. }
  382. default:
  383. // ignore other messages
  384. }
  385. }
  386. // TxsAvailable returns a channel which fires once for every height,
  387. // and only when transactions are available in the mempool.
  388. // NOTE: the returned channel may be nil if EnableTxsAvailable was not called.
  389. func (mem *Mempool) TxsAvailable() <-chan struct{} {
  390. return mem.txsAvailable
  391. }
  392. func (mem *Mempool) notifyTxsAvailable() {
  393. if mem.Size() == 0 {
  394. panic("notified txs available but mempool is empty!")
  395. }
  396. if mem.txsAvailable != nil && !mem.notifiedTxsAvailable {
  397. // channel cap is 1, so this will send once
  398. mem.notifiedTxsAvailable = true
  399. select {
  400. case mem.txsAvailable <- struct{}{}:
  401. default:
  402. }
  403. }
  404. }
  405. // ReapMaxBytesMaxGas reaps transactions from the mempool up to maxBytes bytes total
  406. // with the condition that the total gasWanted must be less than maxGas.
  407. // If both maxes are negative, there is no cap on the size of all returned
  408. // transactions (~ all available transactions).
  409. func (mem *Mempool) ReapMaxBytesMaxGas(maxBytes, maxGas int64) types.Txs {
  410. mem.proxyMtx.Lock()
  411. defer mem.proxyMtx.Unlock()
  412. for atomic.LoadInt32(&mem.rechecking) > 0 {
  413. // TODO: Something better?
  414. time.Sleep(time.Millisecond * 10)
  415. }
  416. var totalBytes int64
  417. var totalGas int64
  418. // TODO: we will get a performance boost if we have a good estimate of avg
  419. // size per tx, and set the initial capacity based off of that.
  420. // txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), max/mem.avgTxSize))
  421. txs := make([]types.Tx, 0, mem.txs.Len())
  422. for e := mem.txs.Front(); e != nil; e = e.Next() {
  423. memTx := e.Value.(*mempoolTx)
  424. // Check total size requirement
  425. aminoOverhead := types.ComputeAminoOverhead(memTx.tx, 1)
  426. if maxBytes > -1 && totalBytes+int64(len(memTx.tx))+aminoOverhead > maxBytes {
  427. return txs
  428. }
  429. totalBytes += int64(len(memTx.tx)) + aminoOverhead
  430. // Check total gas requirement
  431. if maxGas > -1 && totalGas+memTx.gasWanted > maxGas {
  432. return txs
  433. }
  434. totalGas += memTx.gasWanted
  435. txs = append(txs, memTx.tx)
  436. }
  437. return txs
  438. }
  439. // ReapMaxTxs reaps up to max transactions from the mempool.
  440. // If max is negative, there is no cap on the size of all returned
  441. // transactions (~ all available transactions).
  442. func (mem *Mempool) ReapMaxTxs(max int) types.Txs {
  443. mem.proxyMtx.Lock()
  444. defer mem.proxyMtx.Unlock()
  445. if max < 0 {
  446. max = mem.txs.Len()
  447. }
  448. for atomic.LoadInt32(&mem.rechecking) > 0 {
  449. // TODO: Something better?
  450. time.Sleep(time.Millisecond * 10)
  451. }
  452. txs := make([]types.Tx, 0, cmn.MinInt(mem.txs.Len(), max))
  453. for e := mem.txs.Front(); e != nil && len(txs) <= max; e = e.Next() {
  454. memTx := e.Value.(*mempoolTx)
  455. txs = append(txs, memTx.tx)
  456. }
  457. return txs
  458. }
  459. // Update informs the mempool that the given txs were committed and can be discarded.
  460. // NOTE: this should be called *after* block is committed by consensus.
  461. // NOTE: unsafe; Lock/Unlock must be managed by caller
  462. func (mem *Mempool) Update(
  463. height int64,
  464. txs types.Txs,
  465. preCheck PreCheckFunc,
  466. postCheck PostCheckFunc,
  467. ) error {
  468. // First, create a lookup map of txns in new txs.
  469. txsMap := make(map[string]struct{}, len(txs))
  470. for _, tx := range txs {
  471. txsMap[string(tx)] = struct{}{}
  472. }
  473. // Set height
  474. mem.height = height
  475. mem.notifiedTxsAvailable = false
  476. if preCheck != nil {
  477. mem.preCheck = preCheck
  478. }
  479. if postCheck != nil {
  480. mem.postCheck = postCheck
  481. }
  482. // Remove transactions that are already in txs.
  483. goodTxs := mem.filterTxs(txsMap)
  484. // Recheck mempool txs if any txs were committed in the block
  485. if mem.config.Recheck && len(goodTxs) > 0 {
  486. mem.logger.Info("Recheck txs", "numtxs", len(goodTxs), "height", height)
  487. mem.recheckTxs(goodTxs)
  488. // At this point, mem.txs are being rechecked.
  489. // mem.recheckCursor re-scans mem.txs and possibly removes some txs.
  490. // Before mem.Reap(), we should wait for mem.recheckCursor to be nil.
  491. }
  492. // Update metrics
  493. mem.metrics.Size.Set(float64(mem.Size()))
  494. return nil
  495. }
  496. func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx {
  497. goodTxs := make([]types.Tx, 0, mem.txs.Len())
  498. for e := mem.txs.Front(); e != nil; e = e.Next() {
  499. memTx := e.Value.(*mempoolTx)
  500. // Remove the tx if it's alredy in a block.
  501. if _, ok := blockTxsMap[string(memTx.tx)]; ok {
  502. // remove from clist
  503. mem.txs.Remove(e)
  504. e.DetachPrev()
  505. // NOTE: we don't remove committed txs from the cache.
  506. continue
  507. }
  508. // Good tx!
  509. goodTxs = append(goodTxs, memTx.tx)
  510. }
  511. return goodTxs
  512. }
  513. // NOTE: pass in goodTxs because mem.txs can mutate concurrently.
  514. func (mem *Mempool) recheckTxs(goodTxs []types.Tx) {
  515. if len(goodTxs) == 0 {
  516. return
  517. }
  518. atomic.StoreInt32(&mem.rechecking, 1)
  519. mem.recheckCursor = mem.txs.Front()
  520. mem.recheckEnd = mem.txs.Back()
  521. // Push txs to proxyAppConn
  522. // NOTE: resCb() may be called concurrently.
  523. for _, tx := range goodTxs {
  524. mem.proxyAppConn.CheckTxAsync(tx)
  525. }
  526. mem.proxyAppConn.FlushAsync()
  527. }
  528. //--------------------------------------------------------------------------------
  529. // mempoolTx is a transaction that successfully ran
  530. type mempoolTx struct {
  531. counter int64 // a simple incrementing counter
  532. height int64 // height that this tx had been validated in
  533. gasWanted int64 // amount of gas this tx states it will require
  534. tx types.Tx //
  535. }
  536. // Height returns the height for this transaction
  537. func (memTx *mempoolTx) Height() int64 {
  538. return atomic.LoadInt64(&memTx.height)
  539. }
  540. //--------------------------------------------------------------------------------
  541. type txCache interface {
  542. Reset()
  543. Push(tx types.Tx) bool
  544. Remove(tx types.Tx)
  545. }
  546. // mapTxCache maintains a cache of transactions. This only stores
  547. // the hash of the tx, due to memory concerns.
  548. type mapTxCache struct {
  549. mtx sync.Mutex
  550. size int
  551. map_ map[[sha256.Size]byte]*list.Element
  552. list *list.List // to remove oldest tx when cache gets too big
  553. }
  554. var _ txCache = (*mapTxCache)(nil)
  555. // newMapTxCache returns a new mapTxCache.
  556. func newMapTxCache(cacheSize int) *mapTxCache {
  557. return &mapTxCache{
  558. size: cacheSize,
  559. map_: make(map[[sha256.Size]byte]*list.Element, cacheSize),
  560. list: list.New(),
  561. }
  562. }
  563. // Reset resets the cache to an empty state.
  564. func (cache *mapTxCache) Reset() {
  565. cache.mtx.Lock()
  566. cache.map_ = make(map[[sha256.Size]byte]*list.Element, cache.size)
  567. cache.list.Init()
  568. cache.mtx.Unlock()
  569. }
  570. // Push adds the given tx to the cache and returns true. It returns false if tx
  571. // is already in the cache.
  572. func (cache *mapTxCache) Push(tx types.Tx) bool {
  573. cache.mtx.Lock()
  574. defer cache.mtx.Unlock()
  575. // Use the tx hash in the cache
  576. txHash := sha256.Sum256(tx)
  577. if moved, exists := cache.map_[txHash]; exists {
  578. cache.list.MoveToFront(moved)
  579. return false
  580. }
  581. if cache.list.Len() >= cache.size {
  582. popped := cache.list.Front()
  583. poppedTxHash := popped.Value.([sha256.Size]byte)
  584. delete(cache.map_, poppedTxHash)
  585. if popped != nil {
  586. cache.list.Remove(popped)
  587. }
  588. }
  589. cache.list.PushBack(txHash)
  590. cache.map_[txHash] = cache.list.Back()
  591. return true
  592. }
  593. // Remove removes the given tx from the cache.
  594. func (cache *mapTxCache) Remove(tx types.Tx) {
  595. cache.mtx.Lock()
  596. txHash := sha256.Sum256(tx)
  597. popped := cache.map_[txHash]
  598. delete(cache.map_, txHash)
  599. if popped != nil {
  600. cache.list.Remove(popped)
  601. }
  602. cache.mtx.Unlock()
  603. }
  604. type nopTxCache struct{}
  605. var _ txCache = (*nopTxCache)(nil)
  606. func (nopTxCache) Reset() {}
  607. func (nopTxCache) Push(types.Tx) bool { return true }
  608. func (nopTxCache) Remove(types.Tx) {}