You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

420 lines
12 KiB

8 years ago
7 years ago
7 years ago
7 years ago
8 years ago
10 years ago
8 years ago
8 years ago
10 years ago
10 years ago
  1. package blockchain
  2. import (
  3. "fmt"
  4. "reflect"
  5. "time"
  6. amino "github.com/tendermint/go-amino"
  7. cmn "github.com/tendermint/tendermint/libs/common"
  8. "github.com/tendermint/tendermint/libs/log"
  9. "github.com/tendermint/tendermint/p2p"
  10. sm "github.com/tendermint/tendermint/state"
  11. "github.com/tendermint/tendermint/types"
  12. )
  13. const (
  14. // BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
  15. BlockchainChannel = byte(0x40)
  16. trySyncIntervalMS = 10
  17. // stop syncing when last block's time is
  18. // within this much of the system time.
  19. // stopSyncingDurationMinutes = 10
  20. // ask for best height every 10s
  21. statusUpdateIntervalSeconds = 10
  22. // check if we should switch to consensus reactor
  23. switchToConsensusIntervalSeconds = 1
  24. // NOTE: keep up to date with bcBlockResponseMessage
  25. bcBlockResponseMessagePrefixSize = 4
  26. bcBlockResponseMessageFieldKeySize = 1
  27. maxMsgSize = types.MaxBlockSizeBytes +
  28. bcBlockResponseMessagePrefixSize +
  29. bcBlockResponseMessageFieldKeySize
  30. )
  31. type consensusReactor interface {
  32. // for when we switch from blockchain reactor and fast sync to
  33. // the consensus machine
  34. SwitchToConsensus(sm.State, int)
  35. }
  36. type peerError struct {
  37. err error
  38. peerID p2p.ID
  39. }
  40. func (e peerError) Error() string {
  41. return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error())
  42. }
  43. // BlockchainReactor handles long-term catchup syncing.
  44. type BlockchainReactor struct {
  45. p2p.BaseReactor
  46. // immutable
  47. initialState sm.State
  48. blockExec *sm.BlockExecutor
  49. store *BlockStore
  50. pool *BlockPool
  51. fastSync bool
  52. requestsCh <-chan BlockRequest
  53. errorsCh <-chan peerError
  54. }
  55. // NewBlockchainReactor returns new reactor instance.
  56. func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *BlockStore,
  57. fastSync bool) *BlockchainReactor {
  58. if state.LastBlockHeight != store.Height() {
  59. panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
  60. store.Height()))
  61. }
  62. requestsCh := make(chan BlockRequest, maxTotalRequesters)
  63. const capacity = 1000 // must be bigger than peers count
  64. errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
  65. pool := NewBlockPool(
  66. store.Height()+1,
  67. requestsCh,
  68. errorsCh,
  69. )
  70. bcR := &BlockchainReactor{
  71. initialState: state,
  72. blockExec: blockExec,
  73. store: store,
  74. pool: pool,
  75. fastSync: fastSync,
  76. requestsCh: requestsCh,
  77. errorsCh: errorsCh,
  78. }
  79. bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
  80. return bcR
  81. }
  82. // SetLogger implements cmn.Service by setting the logger on reactor and pool.
  83. func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
  84. bcR.BaseService.Logger = l
  85. bcR.pool.Logger = l
  86. }
  87. // OnStart implements cmn.Service.
  88. func (bcR *BlockchainReactor) OnStart() error {
  89. if bcR.fastSync {
  90. err := bcR.pool.Start()
  91. if err != nil {
  92. return err
  93. }
  94. go bcR.poolRoutine()
  95. }
  96. return nil
  97. }
  98. // OnStop implements cmn.Service.
  99. func (bcR *BlockchainReactor) OnStop() {
  100. bcR.pool.Stop()
  101. }
  102. // GetChannels implements Reactor
  103. func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
  104. return []*p2p.ChannelDescriptor{
  105. {
  106. ID: BlockchainChannel,
  107. Priority: 10,
  108. SendQueueCapacity: 1000,
  109. RecvBufferCapacity: 50 * 4096,
  110. RecvMessageCapacity: maxMsgSize,
  111. },
  112. }
  113. }
  114. // AddPeer implements Reactor by sending our state to peer.
  115. func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
  116. msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
  117. if !peer.Send(BlockchainChannel, msgBytes) {
  118. // doing nothing, will try later in `poolRoutine`
  119. }
  120. // peer is added to the pool once we receive the first
  121. // bcStatusResponseMessage from the peer and call pool.SetPeerHeight
  122. }
  123. // RemovePeer implements Reactor by removing peer from the pool.
  124. func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
  125. bcR.pool.RemovePeer(peer.ID())
  126. }
  127. // respondToPeer loads a block and sends it to the requesting peer,
  128. // if we have it. Otherwise, we'll respond saying we don't have it.
  129. // According to the Tendermint spec, if all nodes are honest,
  130. // no node should be requesting for a block that's non-existent.
  131. func (bcR *BlockchainReactor) respondToPeer(msg *bcBlockRequestMessage,
  132. src p2p.Peer) (queued bool) {
  133. block := bcR.store.LoadBlock(msg.Height)
  134. if block != nil {
  135. msgBytes := cdc.MustMarshalBinaryBare(&bcBlockResponseMessage{Block: block})
  136. return src.TrySend(BlockchainChannel, msgBytes)
  137. }
  138. bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height)
  139. msgBytes := cdc.MustMarshalBinaryBare(&bcNoBlockResponseMessage{Height: msg.Height})
  140. return src.TrySend(BlockchainChannel, msgBytes)
  141. }
  142. // Receive implements Reactor by handling 4 types of messages (look below).
  143. func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
  144. msg, err := decodeMsg(msgBytes)
  145. if err != nil {
  146. bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "msg", msg, "err", err, "bytes", msgBytes)
  147. bcR.Switch.StopPeerForError(src, err)
  148. return
  149. }
  150. bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
  151. switch msg := msg.(type) {
  152. case *bcBlockRequestMessage:
  153. if queued := bcR.respondToPeer(msg, src); !queued {
  154. // Unfortunately not queued since the queue is full.
  155. }
  156. case *bcBlockResponseMessage:
  157. // Got a block.
  158. bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes))
  159. case *bcStatusRequestMessage:
  160. // Send peer our state.
  161. msgBytes := cdc.MustMarshalBinaryBare(&bcStatusResponseMessage{bcR.store.Height()})
  162. queued := src.TrySend(BlockchainChannel, msgBytes)
  163. if !queued {
  164. // sorry
  165. }
  166. case *bcStatusResponseMessage:
  167. // Got a peer status. Unverified.
  168. bcR.pool.SetPeerHeight(src.ID(), msg.Height)
  169. default:
  170. bcR.Logger.Error(cmn.Fmt("Unknown message type %v", reflect.TypeOf(msg)))
  171. }
  172. }
  173. // Handle messages from the poolReactor telling the reactor what to do.
  174. // NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
  175. func (bcR *BlockchainReactor) poolRoutine() {
  176. trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
  177. statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
  178. switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
  179. blocksSynced := 0
  180. chainID := bcR.initialState.ChainID
  181. state := bcR.initialState
  182. lastHundred := time.Now()
  183. lastRate := 0.0
  184. didProcessCh := make(chan struct{}, 1)
  185. FOR_LOOP:
  186. for {
  187. select {
  188. case request := <-bcR.requestsCh:
  189. peer := bcR.Switch.Peers().Get(request.PeerID)
  190. if peer == nil {
  191. continue FOR_LOOP // Peer has since been disconnected.
  192. }
  193. msgBytes := cdc.MustMarshalBinaryBare(&bcBlockRequestMessage{request.Height})
  194. queued := peer.TrySend(BlockchainChannel, msgBytes)
  195. if !queued {
  196. // We couldn't make the request, send-queue full.
  197. // The pool handles timeouts, just let it go.
  198. continue FOR_LOOP
  199. }
  200. case err := <-bcR.errorsCh:
  201. peer := bcR.Switch.Peers().Get(err.peerID)
  202. if peer != nil {
  203. bcR.Switch.StopPeerForError(peer, err)
  204. }
  205. case <-statusUpdateTicker.C:
  206. // ask for status updates
  207. go bcR.BroadcastStatusRequest() // nolint: errcheck
  208. case <-switchToConsensusTicker.C:
  209. height, numPending, lenRequesters := bcR.pool.GetStatus()
  210. outbound, inbound, _ := bcR.Switch.NumPeers()
  211. bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters,
  212. "outbound", outbound, "inbound", inbound)
  213. if bcR.pool.IsCaughtUp() {
  214. bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
  215. bcR.pool.Stop()
  216. conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
  217. conR.SwitchToConsensus(state, blocksSynced)
  218. break FOR_LOOP
  219. }
  220. case <-trySyncTicker.C: // chan time
  221. select {
  222. case didProcessCh <- struct{}{}:
  223. default:
  224. }
  225. case <-didProcessCh:
  226. // NOTE: It is a subtle mistake to process more than a single block
  227. // at a time (e.g. 10) here, because we only TrySend 1 request per
  228. // loop. The ratio mismatch can result in starving of blocks, a
  229. // sudden burst of requests and responses, and repeat.
  230. // Consequently, it is better to split these routines rather than
  231. // coupling them as it's written here. TODO uncouple from request
  232. // routine.
  233. // See if there are any blocks to sync.
  234. first, second := bcR.pool.PeekTwoBlocks()
  235. //bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
  236. if first == nil || second == nil {
  237. // We need both to sync the first block.
  238. continue FOR_LOOP
  239. } else {
  240. // Try again quickly next loop.
  241. didProcessCh <- struct{}{}
  242. }
  243. firstParts := first.MakePartSet(state.ConsensusParams.BlockPartSizeBytes)
  244. firstPartsHeader := firstParts.Header()
  245. firstID := types.BlockID{first.Hash(), firstPartsHeader}
  246. // Finally, verify the first block using the second's commit
  247. // NOTE: we can probably make this more efficient, but note that calling
  248. // first.Hash() doesn't verify the tx contents, so MakePartSet() is
  249. // currently necessary.
  250. err := state.Validators.VerifyCommit(
  251. chainID, firstID, first.Height, second.LastCommit)
  252. if err != nil {
  253. bcR.Logger.Error("Error in validation", "err", err)
  254. peerID := bcR.pool.RedoRequest(first.Height)
  255. peer := bcR.Switch.Peers().Get(peerID)
  256. if peer != nil {
  257. // NOTE: we've already removed the peer's request, but we
  258. // still need to clean up the rest.
  259. bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err))
  260. }
  261. continue FOR_LOOP
  262. } else {
  263. bcR.pool.PopRequest()
  264. // TODO: batch saves so we dont persist to disk every block
  265. bcR.store.SaveBlock(first, firstParts, second.LastCommit)
  266. // TODO: same thing for app - but we would need a way to
  267. // get the hash without persisting the state
  268. var err error
  269. state, err = bcR.blockExec.ApplyBlock(state, firstID, first)
  270. if err != nil {
  271. // TODO This is bad, are we zombie?
  272. cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v",
  273. first.Height, first.Hash(), err))
  274. }
  275. blocksSynced++
  276. if blocksSynced%100 == 0 {
  277. lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
  278. bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
  279. "max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
  280. lastHundred = time.Now()
  281. }
  282. }
  283. continue FOR_LOOP
  284. case <-bcR.Quit():
  285. break FOR_LOOP
  286. }
  287. }
  288. }
  289. // BroadcastStatusRequest broadcasts `BlockStore` height.
  290. func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
  291. msgBytes := cdc.MustMarshalBinaryBare(&bcStatusRequestMessage{bcR.store.Height()})
  292. bcR.Switch.Broadcast(BlockchainChannel, msgBytes)
  293. return nil
  294. }
  295. //-----------------------------------------------------------------------------
  296. // Messages
  297. // BlockchainMessage is a generic message for this reactor.
  298. type BlockchainMessage interface{}
  299. func RegisterBlockchainMessages(cdc *amino.Codec) {
  300. cdc.RegisterInterface((*BlockchainMessage)(nil), nil)
  301. cdc.RegisterConcrete(&bcBlockRequestMessage{}, "tendermint/mempool/BlockRequest", nil)
  302. cdc.RegisterConcrete(&bcBlockResponseMessage{}, "tendermint/mempool/BlockResponse", nil)
  303. cdc.RegisterConcrete(&bcNoBlockResponseMessage{}, "tendermint/mempool/NoBlockResponse", nil)
  304. cdc.RegisterConcrete(&bcStatusResponseMessage{}, "tendermint/mempool/StatusResponse", nil)
  305. cdc.RegisterConcrete(&bcStatusRequestMessage{}, "tendermint/mempool/StatusRequest", nil)
  306. }
  307. func decodeMsg(bz []byte) (msg BlockchainMessage, err error) {
  308. if len(bz) > maxMsgSize {
  309. return msg, fmt.Errorf("Msg exceeds max size (%d > %d)", len(bz), maxMsgSize)
  310. }
  311. err = cdc.UnmarshalBinaryBare(bz, &msg)
  312. return
  313. }
  314. //-------------------------------------
  315. type bcBlockRequestMessage struct {
  316. Height int64
  317. }
  318. func (m *bcBlockRequestMessage) String() string {
  319. return cmn.Fmt("[bcBlockRequestMessage %v]", m.Height)
  320. }
  321. type bcNoBlockResponseMessage struct {
  322. Height int64
  323. }
  324. func (brm *bcNoBlockResponseMessage) String() string {
  325. return cmn.Fmt("[bcNoBlockResponseMessage %d]", brm.Height)
  326. }
  327. //-------------------------------------
  328. type bcBlockResponseMessage struct {
  329. Block *types.Block
  330. }
  331. func (m *bcBlockResponseMessage) String() string {
  332. return cmn.Fmt("[bcBlockResponseMessage %v]", m.Block.Height)
  333. }
  334. //-------------------------------------
  335. type bcStatusRequestMessage struct {
  336. Height int64
  337. }
  338. func (m *bcStatusRequestMessage) String() string {
  339. return cmn.Fmt("[bcStatusRequestMessage %v]", m.Height)
  340. }
  341. //-------------------------------------
  342. type bcStatusResponseMessage struct {
  343. Height int64
  344. }
  345. func (m *bcStatusResponseMessage) String() string {
  346. return cmn.Fmt("[bcStatusResponseMessage %v]", m.Height)
  347. }