You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

363 lines
11 KiB

7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
lint: Enable Golint (#4212) * Fix many golint errors * Fix golint errors in the 'lite' package * Don't export Pool.store * Fix typo * Revert unwanted changes * Fix errors in counter package * Fix linter errors in kvstore package * Fix linter error in example package * Fix error in tests package * Fix linter errors in v2 package * Fix linter errors in consensus package * Fix linter errors in evidence package * Fix linter error in fail package * Fix linter errors in query package * Fix linter errors in core package * Fix linter errors in node package * Fix linter errors in mempool package * Fix linter error in conn package * Fix linter errors in pex package * Rename PEXReactor export to Reactor * Fix linter errors in trust package * Fix linter errors in upnp package * Fix linter errors in p2p package * Fix linter errors in proxy package * Fix linter errors in mock_test package * Fix linter error in client_test package * Fix linter errors in coretypes package * Fix linter errors in coregrpc package * Fix linter errors in rpcserver package * Fix linter errors in rpctypes package * Fix linter errors in rpctest package * Fix linter error in json2wal script * Fix linter error in wal2json script * Fix linter errors in kv package * Fix linter error in state package * Fix linter error in grpc_client * Fix linter errors in types package * Fix linter error in version package * Fix remaining errors * Address review comments * Fix broken tests * Reconcile package coregrpc * Fix golangci bot error * Fix new golint errors * Fix broken reference * Enable golint linter * minor changes to bring golint into line * fix failing test * fix pex reactor naming * address PR comments
5 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
7 years ago
  1. package evidence
  2. import (
  3. "fmt"
  4. "sync"
  5. "time"
  6. clist "github.com/tendermint/tendermint/libs/clist"
  7. "github.com/tendermint/tendermint/libs/log"
  8. "github.com/tendermint/tendermint/libs/service"
  9. tmsync "github.com/tendermint/tendermint/libs/sync"
  10. "github.com/tendermint/tendermint/p2p"
  11. tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
  12. "github.com/tendermint/tendermint/types"
  13. )
  14. var (
  15. _ service.Service = (*Reactor)(nil)
  16. // ChannelShims contains a map of ChannelDescriptorShim objects, where each
  17. // object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
  18. // p2p proto.Message the new p2p Channel is responsible for handling.
  19. //
  20. //
  21. // TODO: Remove once p2p refactor is complete.
  22. // ref: https://github.com/tendermint/tendermint/issues/5670
  23. ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
  24. EvidenceChannel: {
  25. MsgType: new(tmproto.EvidenceList),
  26. Descriptor: &p2p.ChannelDescriptor{
  27. ID: byte(EvidenceChannel),
  28. Priority: 6,
  29. RecvMessageCapacity: maxMsgSize,
  30. MaxSendBytes: 400,
  31. },
  32. },
  33. }
  34. )
  35. const (
  36. EvidenceChannel = p2p.ChannelID(0x38)
  37. maxMsgSize = 1048576 // 1MB TODO make it configurable
  38. // broadcast all uncommitted evidence this often. This sets when the reactor
  39. // goes back to the start of the list and begins sending the evidence again.
  40. // Most evidence should be committed in the very next block that is why we wait
  41. // just over the block production rate before sending evidence again.
  42. broadcastEvidenceIntervalS = 10
  43. )
  44. // Reactor handles evpool evidence broadcasting amongst peers.
  45. type Reactor struct {
  46. service.BaseService
  47. evpool *Pool
  48. evidenceCh *p2p.Channel
  49. peerUpdates *p2p.PeerUpdates
  50. closeCh chan struct{}
  51. peerWG sync.WaitGroup
  52. mtx tmsync.Mutex
  53. peerRoutines map[p2p.NodeID]*tmsync.Closer
  54. }
  55. // NewReactor returns a reference to a new evidence reactor, which implements the
  56. // service.Service interface. It accepts a p2p Channel dedicated for handling
  57. // envelopes with EvidenceList messages.
  58. func NewReactor(
  59. logger log.Logger,
  60. evidenceCh *p2p.Channel,
  61. peerUpdates *p2p.PeerUpdates,
  62. evpool *Pool,
  63. ) *Reactor {
  64. r := &Reactor{
  65. evpool: evpool,
  66. evidenceCh: evidenceCh,
  67. peerUpdates: peerUpdates,
  68. closeCh: make(chan struct{}),
  69. peerRoutines: make(map[p2p.NodeID]*tmsync.Closer),
  70. }
  71. r.BaseService = *service.NewBaseService(logger, "Evidence", r)
  72. return r
  73. }
  74. // OnStart starts separate go routines for each p2p Channel and listens for
  75. // envelopes on each. In addition, it also listens for peer updates and handles
  76. // messages on that p2p channel accordingly. The caller must be sure to execute
  77. // OnStop to ensure the outbound p2p Channels are closed. No error is returned.
  78. func (r *Reactor) OnStart() error {
  79. go r.processEvidenceCh()
  80. go r.processPeerUpdates()
  81. return nil
  82. }
  83. // OnStop stops the reactor by signaling to all spawned goroutines to exit and
  84. // blocking until they all exit.
  85. func (r *Reactor) OnStop() {
  86. r.mtx.Lock()
  87. for _, c := range r.peerRoutines {
  88. c.Close()
  89. }
  90. r.mtx.Unlock()
  91. // Wait for all spawned peer evidence broadcasting goroutines to gracefully
  92. // exit.
  93. r.peerWG.Wait()
  94. // Close closeCh to signal to all spawned goroutines to gracefully exit. All
  95. // p2p Channels should execute Close().
  96. close(r.closeCh)
  97. // Wait for all p2p Channels to be closed before returning. This ensures we
  98. // can easily reason about synchronization of all p2p Channels and ensure no
  99. // panics will occur.
  100. <-r.evidenceCh.Done()
  101. <-r.peerUpdates.Done()
  102. }
  103. // handleEvidenceMessage handles envelopes sent from peers on the EvidenceChannel.
  104. // It returns an error only if the Envelope.Message is unknown for this channel
  105. // or if the given evidence is invalid. This should never be called outside of
  106. // handleMessage.
  107. func (r *Reactor) handleEvidenceMessage(envelope p2p.Envelope) error {
  108. logger := r.Logger.With("peer", envelope.From)
  109. switch msg := envelope.Message.(type) {
  110. case *tmproto.EvidenceList:
  111. // TODO: Refactor the Evidence type to not contain a list since we only ever
  112. // send and receive one piece of evidence at a time. Or potentially consider
  113. // batching evidence.
  114. //
  115. // see: https://github.com/tendermint/tendermint/issues/4729
  116. for i := 0; i < len(msg.Evidence); i++ {
  117. ev, err := types.EvidenceFromProto(&msg.Evidence[i])
  118. if err != nil {
  119. logger.Error("failed to convert evidence", "err", err)
  120. continue
  121. }
  122. if err := r.evpool.AddEvidence(ev); err != nil {
  123. // If we're given invalid evidence by the peer, notify the router that
  124. // we should remove this peer by returning an error.
  125. if _, ok := err.(*types.ErrInvalidEvidence); ok {
  126. return err
  127. }
  128. }
  129. }
  130. default:
  131. return fmt.Errorf("received unknown message: %T", msg)
  132. }
  133. return nil
  134. }
  135. // handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
  136. // It will handle errors and any possible panics gracefully. A caller can handle
  137. // any error returned by sending a PeerError on the respective channel.
  138. func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
  139. defer func() {
  140. if e := recover(); e != nil {
  141. err = fmt.Errorf("panic in processing message: %v", e)
  142. }
  143. }()
  144. r.Logger.Debug("received message", "message", envelope.Message, "peer", envelope.From)
  145. switch chID {
  146. case EvidenceChannel:
  147. err = r.handleEvidenceMessage(envelope)
  148. default:
  149. err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
  150. }
  151. return err
  152. }
  153. // processEvidenceCh implements a blocking event loop where we listen for p2p
  154. // Envelope messages from the evidenceCh.
  155. func (r *Reactor) processEvidenceCh() {
  156. defer r.evidenceCh.Close()
  157. for {
  158. select {
  159. case envelope := <-r.evidenceCh.In:
  160. if err := r.handleMessage(r.evidenceCh.ID, envelope); err != nil {
  161. r.Logger.Error("failed to process message", "ch_id", r.evidenceCh.ID, "envelope", envelope, "err", err)
  162. r.evidenceCh.Error <- p2p.PeerError{
  163. NodeID: envelope.From,
  164. Err: err,
  165. }
  166. }
  167. case <-r.closeCh:
  168. r.Logger.Debug("stopped listening on evidence channel; closing...")
  169. return
  170. }
  171. }
  172. }
  173. // processPeerUpdate processes a PeerUpdate. For new or live peers it will check
  174. // if an evidence broadcasting goroutine needs to be started. For down or
  175. // removed peers, it will check if an evidence broadcasting goroutine
  176. // exists and signal that it should exit.
  177. //
  178. // FIXME: The peer may be behind in which case it would simply ignore the
  179. // evidence and treat it as invalid. This would cause the peer to disconnect.
  180. // The peer may also receive the same piece of evidence multiple times if it
  181. // connects/disconnects frequently from the broadcasting peer(s).
  182. //
  183. // REF: https://github.com/tendermint/tendermint/issues/4727
  184. func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
  185. r.Logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
  186. r.mtx.Lock()
  187. defer r.mtx.Unlock()
  188. switch peerUpdate.Status {
  189. case p2p.PeerStatusUp:
  190. // Do not allow starting new evidence broadcast loops after reactor shutdown
  191. // has been initiated. This can happen after we've manually closed all
  192. // peer broadcast loops and closed r.closeCh, but the router still sends
  193. // in-flight peer updates.
  194. if !r.IsRunning() {
  195. return
  196. }
  197. // Check if we've already started a goroutine for this peer, if not we create
  198. // a new done channel so we can explicitly close the goroutine if the peer
  199. // is later removed, we increment the waitgroup so the reactor can stop
  200. // safely, and finally start the goroutine to broadcast evidence to that peer.
  201. _, ok := r.peerRoutines[peerUpdate.NodeID]
  202. if !ok {
  203. closer := tmsync.NewCloser()
  204. r.peerRoutines[peerUpdate.NodeID] = closer
  205. r.peerWG.Add(1)
  206. go r.broadcastEvidenceLoop(peerUpdate.NodeID, closer)
  207. }
  208. case p2p.PeerStatusDown:
  209. // Check if we've started an evidence broadcasting goroutine for this peer.
  210. // If we have, we signal to terminate the goroutine via the channel's closure.
  211. // This will internally decrement the peer waitgroup and remove the peer
  212. // from the map of peer evidence broadcasting goroutines.
  213. closer, ok := r.peerRoutines[peerUpdate.NodeID]
  214. if ok {
  215. closer.Close()
  216. }
  217. }
  218. }
  219. // processPeerUpdates initiates a blocking process where we listen for and handle
  220. // PeerUpdate messages. When the reactor is stopped, we will catch the signal and
  221. // close the p2p PeerUpdatesCh gracefully.
  222. func (r *Reactor) processPeerUpdates() {
  223. defer r.peerUpdates.Close()
  224. for {
  225. select {
  226. case peerUpdate := <-r.peerUpdates.Updates():
  227. r.processPeerUpdate(peerUpdate)
  228. case <-r.closeCh:
  229. r.Logger.Debug("stopped listening on peer updates channel; closing...")
  230. return
  231. }
  232. }
  233. }
  234. // broadcastEvidenceLoop starts a blocking process that continuously reads pieces
  235. // of evidence off of a linked-list and sends the evidence in a p2p Envelope to
  236. // the given peer by ID. This should be invoked in a goroutine per unique peer
  237. // ID via an appropriate PeerUpdate. The goroutine can be signaled to gracefully
  238. // exit by either explicitly closing the provided doneCh or by the reactor
  239. // signaling to stop.
  240. //
  241. // TODO: This should be refactored so that we do not blindly gossip evidence
  242. // that the peer has already received or may not be ready for.
  243. //
  244. // REF: https://github.com/tendermint/tendermint/issues/4727
  245. func (r *Reactor) broadcastEvidenceLoop(peerID p2p.NodeID, closer *tmsync.Closer) {
  246. var next *clist.CElement
  247. defer func() {
  248. r.mtx.Lock()
  249. delete(r.peerRoutines, peerID)
  250. r.mtx.Unlock()
  251. r.peerWG.Done()
  252. if e := recover(); e != nil {
  253. r.Logger.Error("recovering from broadcasting evidence loop", "err", e)
  254. }
  255. }()
  256. for {
  257. // This happens because the CElement we were looking at got garbage
  258. // collected (removed). That is, .NextWaitChan() returned nil. So we can go
  259. // ahead and start from the beginning.
  260. if next == nil {
  261. select {
  262. case <-r.evpool.EvidenceWaitChan(): // wait until next evidence is available
  263. if next = r.evpool.EvidenceFront(); next == nil {
  264. continue
  265. }
  266. case <-closer.Done():
  267. // The peer is marked for removal via a PeerUpdate as the doneCh was
  268. // explicitly closed to signal we should exit.
  269. return
  270. case <-r.closeCh:
  271. // The reactor has signaled that we are stopped and thus we should
  272. // implicitly exit this peer's goroutine.
  273. return
  274. }
  275. }
  276. ev := next.Value.(types.Evidence)
  277. evProto, err := types.EvidenceToProto(ev)
  278. if err != nil {
  279. panic(fmt.Errorf("failed to convert evidence: %w", err))
  280. }
  281. // Send the evidence to the corresponding peer. Note, the peer may be behind
  282. // and thus would not be able to process the evidence correctly. Also, the
  283. // peer may receive this piece of evidence multiple times if it added and
  284. // removed frequently from the broadcasting peer.
  285. r.evidenceCh.Out <- p2p.Envelope{
  286. To: peerID,
  287. Message: &tmproto.EvidenceList{
  288. Evidence: []tmproto.Evidence{*evProto},
  289. },
  290. }
  291. r.Logger.Debug("gossiped evidence to peer", "evidence", ev, "peer", peerID)
  292. select {
  293. case <-time.After(time.Second * broadcastEvidenceIntervalS):
  294. // start from the beginning after broadcastEvidenceIntervalS seconds
  295. next = nil
  296. case <-next.NextWaitChan():
  297. next = next.Next()
  298. case <-closer.Done():
  299. // The peer is marked for removal via a PeerUpdate as the doneCh was
  300. // explicitly closed to signal we should exit.
  301. return
  302. case <-r.closeCh:
  303. // The reactor has signaled that we are stopped and thus we should
  304. // implicitly exit this peer's goroutine.
  305. return
  306. }
  307. }
  308. }