You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

215 lines
5.5 KiB

limit number of /subscribe clients and queries per client (#3269) * limit number of /subscribe clients and queries per client Add the following config variables (under [rpc] section): * max_subscription_clients * max_subscriptions_per_client * timeout_broadcast_tx_commit Fixes #2826 new HTTPClient interface for subscriptions finalize HTTPClient events interface remove EventSubscriber fix data race ``` WARNING: DATA RACE Read at 0x00c000a36060 by goroutine 129: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe.func1() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:168 +0x1f0 Previous write at 0x00c000a36060 by goroutine 132: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:191 +0x4e0 github.com/tendermint/tendermint/rpc/client.WaitForOneEvent() /go/src/github.com/tendermint/tendermint/rpc/client/helpers.go:64 +0x178 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync.func1() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:139 +0x298 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 Goroutine 129 (running) created at: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:164 +0x4b7 github.com/tendermint/tendermint/rpc/client.WaitForOneEvent() /go/src/github.com/tendermint/tendermint/rpc/client/helpers.go:64 +0x178 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync.func1() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:139 +0x298 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 Goroutine 132 (running) created at: testing.(*T).Run() /usr/local/go/src/testing/testing.go:878 +0x659 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:119 +0x186 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 ================== ``` lite client works (tested manually) godoc comments httpclient: do not close the out channel use TimeoutBroadcastTxCommit no timeout for unsubscribe but 1s Local (5s HTTP) timeout for resubscribe format code change Subscribe#out cap to 1 and replace config vars with RPCConfig TimeoutBroadcastTxCommit can't be greater than rpcserver.WriteTimeout rpc: Context as first parameter to all functions reformat code fixes after my own review fixes after Ethan's review add test stubs fix config.toml * fixes after manual testing - rpc: do not recommend to use BroadcastTxCommit because it's slow and wastes Tendermint resources (pubsub) - rpc: better error in Subscribe and BroadcastTxCommit - HTTPClient: do not resubscribe if err = ErrAlreadySubscribed * fixes after Ismail's review * Update rpc/grpc/grpc_test.go Co-Authored-By: melekes <anton.kalyaev@gmail.com>
6 years ago
limit number of /subscribe clients and queries per client (#3269) * limit number of /subscribe clients and queries per client Add the following config variables (under [rpc] section): * max_subscription_clients * max_subscriptions_per_client * timeout_broadcast_tx_commit Fixes #2826 new HTTPClient interface for subscriptions finalize HTTPClient events interface remove EventSubscriber fix data race ``` WARNING: DATA RACE Read at 0x00c000a36060 by goroutine 129: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe.func1() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:168 +0x1f0 Previous write at 0x00c000a36060 by goroutine 132: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:191 +0x4e0 github.com/tendermint/tendermint/rpc/client.WaitForOneEvent() /go/src/github.com/tendermint/tendermint/rpc/client/helpers.go:64 +0x178 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync.func1() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:139 +0x298 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 Goroutine 129 (running) created at: github.com/tendermint/tendermint/rpc/client.(*Local).Subscribe() /go/src/github.com/tendermint/tendermint/rpc/client/localclient.go:164 +0x4b7 github.com/tendermint/tendermint/rpc/client.WaitForOneEvent() /go/src/github.com/tendermint/tendermint/rpc/client/helpers.go:64 +0x178 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync.func1() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:139 +0x298 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 Goroutine 132 (running) created at: testing.(*T).Run() /usr/local/go/src/testing/testing.go:878 +0x659 github.com/tendermint/tendermint/rpc/client_test.TestTxEventsSentWithBroadcastTxSync() /go/src/github.com/tendermint/tendermint/rpc/client/event_test.go:119 +0x186 testing.tRunner() /usr/local/go/src/testing/testing.go:827 +0x162 ================== ``` lite client works (tested manually) godoc comments httpclient: do not close the out channel use TimeoutBroadcastTxCommit no timeout for unsubscribe but 1s Local (5s HTTP) timeout for resubscribe format code change Subscribe#out cap to 1 and replace config vars with RPCConfig TimeoutBroadcastTxCommit can't be greater than rpcserver.WriteTimeout rpc: Context as first parameter to all functions reformat code fixes after my own review fixes after Ethan's review add test stubs fix config.toml * fixes after manual testing - rpc: do not recommend to use BroadcastTxCommit because it's slow and wastes Tendermint resources (pubsub) - rpc: better error in Subscribe and BroadcastTxCommit - HTTPClient: do not resubscribe if err = ErrAlreadySubscribed * fixes after Ismail's review * Update rpc/grpc/grpc_test.go Co-Authored-By: melekes <anton.kalyaev@gmail.com>
6 years ago
  1. package core
  2. import (
  3. "encoding/base64"
  4. "fmt"
  5. "time"
  6. cfg "github.com/tendermint/tendermint/config"
  7. "github.com/tendermint/tendermint/crypto"
  8. "github.com/tendermint/tendermint/internal/consensus"
  9. mempl "github.com/tendermint/tendermint/internal/mempool"
  10. "github.com/tendermint/tendermint/internal/p2p"
  11. "github.com/tendermint/tendermint/internal/proxy"
  12. "github.com/tendermint/tendermint/internal/statesync"
  13. tmjson "github.com/tendermint/tendermint/libs/json"
  14. "github.com/tendermint/tendermint/libs/log"
  15. ctypes "github.com/tendermint/tendermint/rpc/core/types"
  16. sm "github.com/tendermint/tendermint/state"
  17. "github.com/tendermint/tendermint/state/indexer"
  18. "github.com/tendermint/tendermint/types"
  19. )
  20. const (
  21. // see README
  22. defaultPerPage = 30
  23. maxPerPage = 100
  24. // SubscribeTimeout is the maximum time we wait to subscribe for an event.
  25. // must be less than the server's write timeout (see rpcserver.DefaultConfig)
  26. SubscribeTimeout = 5 * time.Second
  27. // genesisChunkSize is the maximum size, in bytes, of each
  28. // chunk in the genesis structure for the chunked API
  29. genesisChunkSize = 16 * 1024 * 1024 // 16
  30. )
  31. //----------------------------------------------
  32. // These interfaces are used by RPC and must be thread safe
  33. type consensusState interface {
  34. GetState() sm.State
  35. GetValidators() (int64, []*types.Validator)
  36. GetLastHeight() int64
  37. GetRoundStateJSON() ([]byte, error)
  38. GetRoundStateSimpleJSON() ([]byte, error)
  39. }
  40. type transport interface {
  41. Listeners() []string
  42. IsListening() bool
  43. NodeInfo() types.NodeInfo
  44. }
  45. type peers interface {
  46. AddPersistentPeers([]string) error
  47. AddUnconditionalPeerIDs([]string) error
  48. AddPrivatePeerIDs([]string) error
  49. DialPeersAsync([]string) error
  50. Peers() p2p.IPeerSet
  51. }
  52. type consensusReactor interface {
  53. WaitSync() bool
  54. GetPeerState(peerID types.NodeID) (*consensus.PeerState, bool)
  55. }
  56. type peerManager interface {
  57. Peers() []types.NodeID
  58. Addresses(types.NodeID) []p2p.NodeAddress
  59. }
  60. //----------------------------------------------
  61. // Environment contains objects and interfaces used by the RPC. It is expected
  62. // to be setup once during startup.
  63. type Environment struct {
  64. // external, thread safe interfaces
  65. ProxyAppQuery proxy.AppConnQuery
  66. ProxyAppMempool proxy.AppConnMempool
  67. // interfaces defined in types and above
  68. StateStore sm.Store
  69. BlockStore sm.BlockStore
  70. EvidencePool sm.EvidencePool
  71. ConsensusState consensusState
  72. ConsensusReactor consensusReactor
  73. P2PPeers peers
  74. // Legacy p2p stack
  75. P2PTransport transport
  76. // interfaces for new p2p interfaces
  77. PeerManager peerManager
  78. // objects
  79. PubKey crypto.PubKey
  80. GenDoc *types.GenesisDoc // cache the genesis structure
  81. EventSinks []indexer.EventSink
  82. EventBus *types.EventBus // thread safe
  83. Mempool mempl.Mempool
  84. BlockSyncReactor consensus.BlockSyncReactor
  85. StateSyncMetricer statesync.Metricer
  86. Logger log.Logger
  87. Config cfg.RPCConfig
  88. // cache of chunked genesis data.
  89. genChunks []string
  90. }
  91. //----------------------------------------------
  92. func validatePage(pagePtr *int, perPage, totalCount int) (int, error) {
  93. // this can only happen if we haven't first run validatePerPage
  94. if perPage < 1 {
  95. panic(fmt.Errorf("%w (%d)", ctypes.ErrZeroOrNegativePerPage, perPage))
  96. }
  97. if pagePtr == nil { // no page parameter
  98. return 1, nil
  99. }
  100. pages := ((totalCount - 1) / perPage) + 1
  101. if pages == 0 {
  102. pages = 1 // one page (even if it's empty)
  103. }
  104. page := *pagePtr
  105. if page <= 0 || page > pages {
  106. return 1, fmt.Errorf("%w expected range: [1, %d], given %d", ctypes.ErrPageOutOfRange, pages, page)
  107. }
  108. return page, nil
  109. }
  110. func (env *Environment) validatePerPage(perPagePtr *int) int {
  111. if perPagePtr == nil { // no per_page parameter
  112. return defaultPerPage
  113. }
  114. perPage := *perPagePtr
  115. if perPage < 1 {
  116. return defaultPerPage
  117. // in unsafe mode there is no max on the page size but in safe mode
  118. // we cap it to maxPerPage
  119. } else if perPage > maxPerPage && !env.Config.Unsafe {
  120. return maxPerPage
  121. }
  122. return perPage
  123. }
  124. // InitGenesisChunks configures the environment and should be called on service
  125. // startup.
  126. func (env *Environment) InitGenesisChunks() error {
  127. if env.genChunks != nil {
  128. return nil
  129. }
  130. if env.GenDoc == nil {
  131. return nil
  132. }
  133. data, err := tmjson.Marshal(env.GenDoc)
  134. if err != nil {
  135. return err
  136. }
  137. for i := 0; i < len(data); i += genesisChunkSize {
  138. end := i + genesisChunkSize
  139. if end > len(data) {
  140. end = len(data)
  141. }
  142. env.genChunks = append(env.genChunks, base64.StdEncoding.EncodeToString(data[i:end]))
  143. }
  144. return nil
  145. }
  146. func validateSkipCount(page, perPage int) int {
  147. skipCount := (page - 1) * perPage
  148. if skipCount < 0 {
  149. return 0
  150. }
  151. return skipCount
  152. }
  153. // latestHeight can be either latest committed or uncommitted (+1) height.
  154. func (env *Environment) getHeight(latestHeight int64, heightPtr *int64) (int64, error) {
  155. if heightPtr != nil {
  156. height := *heightPtr
  157. if height <= 0 {
  158. return 0, fmt.Errorf("%w (requested height: %d)", ctypes.ErrZeroOrNegativeHeight, height)
  159. }
  160. if height > latestHeight {
  161. return 0, fmt.Errorf("%w (requested height: %d, blockchain height: %d)",
  162. ctypes.ErrHeightExceedsChainHead, height, latestHeight)
  163. }
  164. base := env.BlockStore.Base()
  165. if height < base {
  166. return 0, fmt.Errorf("%w (requested height: %d, base height: %d)", ctypes.ErrHeightNotAvailable, height, base)
  167. }
  168. return height, nil
  169. }
  170. return latestHeight, nil
  171. }
  172. func (env *Environment) latestUncommittedHeight() int64 {
  173. nodeIsSyncing := env.ConsensusReactor.WaitSync()
  174. if nodeIsSyncing {
  175. return env.BlockStore.Height()
  176. }
  177. return env.BlockStore.Height() + 1
  178. }