package p2p import ( "fmt" "math" "sync" "time" "github.com/tendermint/tendermint/config" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p/conn" ) const ( // wait a random amount of time from this interval // before dialing peers or reconnecting to help prevent DoS dialRandomizerIntervalMilliseconds = 3000 // repeatedly try to reconnect for a few minutes // ie. 5 * 20 = 100s reconnectAttempts = 20 reconnectInterval = 5 * time.Second // then move into exponential backoff mode for ~1day // ie. 3**10 = 16hrs reconnectBackOffAttempts = 10 reconnectBackOffBaseSeconds = 3 ) // MConnConfig returns an MConnConfig with fields updated // from the P2PConfig. func MConnConfig(cfg *config.P2PConfig) conn.MConnConfig { mConfig := conn.DefaultMConnConfig() mConfig.FlushThrottle = cfg.FlushThrottleTimeout mConfig.SendRate = cfg.SendRate mConfig.RecvRate = cfg.RecvRate mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize return mConfig } //----------------------------------------------------------------------------- // An AddrBook represents an address book from the pex package, which is used // to store peer addresses. type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error AddOurAddress(*NetAddress) OurAddress(*NetAddress) bool MarkGood(*NetAddress) RemoveAddress(*NetAddress) HasAddress(*NetAddress) bool Save() } // PeerFilterFunc to be implemented by filter hooks after a new Peer has been // fully setup. type PeerFilterFunc func(IPeerSet, Peer) error //----------------------------------------------------------------------------- // Switch handles peer connections and exposes an API to receive incoming messages // on `Reactors`. Each `Reactor` is responsible for handling incoming messages of one // or more `Channels`. So while sending outgoing messages is typically performed on the peer, // incoming messages are received on the reactor. type Switch struct { cmn.BaseService config *config.P2PConfig reactors map[string]Reactor chDescs []*conn.ChannelDescriptor reactorsByCh map[byte]Reactor peers *PeerSet dialing *cmn.CMap reconnecting *cmn.CMap nodeInfo NodeInfo // our node info nodeKey *NodeKey // our node privkey addrBook AddrBook transport Transport filterTimeout time.Duration peerFilters []PeerFilterFunc rng *cmn.Rand // seed for randomizing dial times and orders metrics *Metrics } // SwitchOption sets an optional parameter on the Switch. type SwitchOption func(*Switch) // NewSwitch creates a new Switch with the given config. func NewSwitch( cfg *config.P2PConfig, transport Transport, options ...SwitchOption, ) *Switch { sw := &Switch{ config: cfg, reactors: make(map[string]Reactor), chDescs: make([]*conn.ChannelDescriptor, 0), reactorsByCh: make(map[byte]Reactor), peers: NewPeerSet(), dialing: cmn.NewCMap(), reconnecting: cmn.NewCMap(), metrics: NopMetrics(), transport: transport, filterTimeout: defaultFilterTimeout, } // Ensure we have a completely undeterministic PRNG. sw.rng = cmn.NewRand() sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw) for _, option := range options { option(sw) } return sw } // SwitchFilterTimeout sets the timeout used for peer filters. func SwitchFilterTimeout(timeout time.Duration) SwitchOption { return func(sw *Switch) { sw.filterTimeout = timeout } } // SwitchPeerFilters sets the filters for rejection of new peers. func SwitchPeerFilters(filters ...PeerFilterFunc) SwitchOption { return func(sw *Switch) { sw.peerFilters = filters } } // WithMetrics sets the metrics. func WithMetrics(metrics *Metrics) SwitchOption { return func(sw *Switch) { sw.metrics = metrics } } //--------------------------------------------------------------------- // Switch setup // AddReactor adds the given reactor to the switch. // NOTE: Not goroutine safe. func (sw *Switch) AddReactor(name string, reactor Reactor) Reactor { // Validate the reactor. // No two reactors can share the same channel. reactorChannels := reactor.GetChannels() for _, chDesc := range reactorChannels { chID := chDesc.ID if sw.reactorsByCh[chID] != nil { cmn.PanicSanity(fmt.Sprintf("Channel %X has multiple reactors %v & %v", chID, sw.reactorsByCh[chID], reactor)) } sw.chDescs = append(sw.chDescs, chDesc) sw.reactorsByCh[chID] = reactor } sw.reactors[name] = reactor reactor.SetSwitch(sw) return reactor } // Reactors returns a map of reactors registered on the switch. // NOTE: Not goroutine safe. func (sw *Switch) Reactors() map[string]Reactor { return sw.reactors } // Reactor returns the reactor with the given name. // NOTE: Not goroutine safe. func (sw *Switch) Reactor(name string) Reactor { return sw.reactors[name] } // SetNodeInfo sets the switch's NodeInfo for checking compatibility and handshaking with other nodes. // NOTE: Not goroutine safe. func (sw *Switch) SetNodeInfo(nodeInfo NodeInfo) { sw.nodeInfo = nodeInfo } // NodeInfo returns the switch's NodeInfo. // NOTE: Not goroutine safe. func (sw *Switch) NodeInfo() NodeInfo { return sw.nodeInfo } // SetNodeKey sets the switch's private key for authenticated encryption. // NOTE: Not goroutine safe. func (sw *Switch) SetNodeKey(nodeKey *NodeKey) { sw.nodeKey = nodeKey } //--------------------------------------------------------------------- // Service start/stop // OnStart implements BaseService. It starts all the reactors and peers. func (sw *Switch) OnStart() error { // Start reactors for _, reactor := range sw.reactors { err := reactor.Start() if err != nil { return cmn.ErrorWrap(err, "failed to start %v", reactor) } } // Start accepting Peers. go sw.acceptRoutine() return nil } // OnStop implements BaseService. It stops all peers and reactors. func (sw *Switch) OnStop() { // Stop peers for _, p := range sw.peers.List() { sw.transport.Cleanup(p) p.Stop() if sw.peers.Remove(p) { sw.metrics.Peers.Add(float64(-1)) } } // Stop reactors sw.Logger.Debug("Switch: Stopping reactors") for _, reactor := range sw.reactors { reactor.Stop() } } //--------------------------------------------------------------------- // Peers // Broadcast runs a go routine for each attempted send, which will block trying // to send for defaultSendTimeoutSeconds. Returns a channel which receives // success values for each attempted send (false if times out). Channel will be // closed once msg bytes are sent to all peers (or time out). // // NOTE: Broadcast uses goroutines, so order of broadcast may not be preserved. func (sw *Switch) Broadcast(chID byte, msgBytes []byte) chan bool { successChan := make(chan bool, len(sw.peers.List())) sw.Logger.Debug("Broadcast", "channel", chID, "msgBytes", fmt.Sprintf("%X", msgBytes)) var wg sync.WaitGroup for _, peer := range sw.peers.List() { wg.Add(1) go func(peer Peer) { defer wg.Done() success := peer.Send(chID, msgBytes) successChan <- success }(peer) } go func() { wg.Wait() close(successChan) }() return successChan } // NumPeers returns the count of outbound/inbound and outbound-dialing peers. func (sw *Switch) NumPeers() (outbound, inbound, dialing int) { peers := sw.peers.List() for _, peer := range peers { if peer.IsOutbound() { outbound++ } else { inbound++ } } dialing = sw.dialing.Size() return } // MaxNumOutboundPeers returns a maximum number of outbound peers. func (sw *Switch) MaxNumOutboundPeers() int { return sw.config.MaxNumOutboundPeers } // Peers returns the set of peers that are connected to the switch. func (sw *Switch) Peers() IPeerSet { return sw.peers } // StopPeerForError disconnects from a peer due to external error. // If the peer is persistent, it will attempt to reconnect. // TODO: make record depending on reason. func (sw *Switch) StopPeerForError(peer Peer, reason interface{}) { sw.Logger.Error("Stopping peer for error", "peer", peer, "err", reason) sw.stopAndRemovePeer(peer, reason) if peer.IsPersistent() { addr := peer.OriginalAddr() if addr == nil { // FIXME: persistent peers can't be inbound right now. // self-reported address for inbound persistent peers addr = peer.NodeInfo().NetAddress() } go sw.reconnectToPeer(addr) } } // StopPeerGracefully disconnects from a peer gracefully. // TODO: handle graceful disconnects. func (sw *Switch) StopPeerGracefully(peer Peer) { sw.Logger.Info("Stopping peer gracefully") sw.stopAndRemovePeer(peer, nil) } func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) } sw.transport.Cleanup(peer) peer.Stop() for _, reactor := range sw.reactors { reactor.RemovePeer(peer, reason) } } // reconnectToPeer tries to reconnect to the addr, first repeatedly // with a fixed interval, then with exponential backoff. // If no success after all that, it stops trying, and leaves it // to the PEX/Addrbook to find the peer with the addr again // NOTE: this will keep trying even if the handshake or auth fails. // TODO: be more explicit with error types so we only retry on certain failures // - ie. if we're getting ErrDuplicatePeer we can stop // because the addrbook got us the peer back already func (sw *Switch) reconnectToPeer(addr *NetAddress) { if sw.reconnecting.Has(string(addr.ID)) { return } sw.reconnecting.Set(string(addr.ID), addr) defer sw.reconnecting.Delete(string(addr.ID)) start := time.Now() sw.Logger.Info("Reconnecting to peer", "addr", addr) for i := 0; i < reconnectAttempts; i++ { if !sw.IsRunning() { return } if sw.IsDialingOrExistingAddress(addr) { sw.Logger.Debug("Peer connection has been established or dialed while we waiting next try", "addr", addr) return } err := sw.DialPeerWithAddress(addr, true) if err == nil { return // success } sw.Logger.Info("Error reconnecting to peer. Trying again", "tries", i, "err", err, "addr", addr) // sleep a set amount sw.randomSleep(reconnectInterval) continue } sw.Logger.Error("Failed to reconnect to peer. Beginning exponential backoff", "addr", addr, "elapsed", time.Since(start)) for i := 0; i < reconnectBackOffAttempts; i++ { if !sw.IsRunning() { return } // sleep an exponentially increasing amount sleepIntervalSeconds := math.Pow(reconnectBackOffBaseSeconds, float64(i)) sw.randomSleep(time.Duration(sleepIntervalSeconds) * time.Second) err := sw.DialPeerWithAddress(addr, true) if err == nil { return // success } sw.Logger.Info("Error reconnecting to peer. Trying again", "tries", i, "err", err, "addr", addr) } sw.Logger.Error("Failed to reconnect to peer. Giving up", "addr", addr, "elapsed", time.Since(start)) } // SetAddrBook allows to set address book on Switch. func (sw *Switch) SetAddrBook(addrBook AddrBook) { sw.addrBook = addrBook } // MarkPeerAsGood marks the given peer as good when it did something useful // like contributed to consensus. func (sw *Switch) MarkPeerAsGood(peer Peer) { if sw.addrBook != nil { sw.addrBook.MarkGood(peer.NodeInfo().NetAddress()) } } //--------------------------------------------------------------------- // Dialing // DialPeersAsync dials a list of peers asynchronously in random order (optionally, making them persistent). // Used to dial peers from config on startup or from unsafe-RPC (trusted sources). // TODO: remove addrBook arg since it's now set on the switch func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent bool) error { netAddrs, errs := NewNetAddressStrings(peers) // only log errors, dial correct addresses for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } ourAddr := sw.nodeInfo.NetAddress() // TODO: this code feels like it's in the wrong place. // The integration tests depend on the addrBook being saved // right away but maybe we can change that. Recall that // the addrBook is only written to disk every 2min if addrBook != nil { // add peers to `addrBook` for _, netAddr := range netAddrs { // do not add our address or ID if !netAddr.Same(ourAddr) { if err := addrBook.AddAddress(netAddr, ourAddr); err != nil { sw.Logger.Error("Can't add peer's address to addrbook", "err", err) } } } // Persist some peers to disk right away. // NOTE: integration tests depend on this addrBook.Save() } // permute the list, dial them in random order. perm := sw.rng.Perm(len(netAddrs)) for i := 0; i < len(perm); i++ { go func(i int) { j := perm[i] addr := netAddrs[j] if addr.Same(ourAddr) { sw.Logger.Debug("Ignore attempt to connect to ourselves", "addr", addr, "ourAddr", ourAddr) return } sw.randomSleep(0) if sw.IsDialingOrExistingAddress(addr) { sw.Logger.Debug("Ignore attempt to connect to an existing peer", "addr", addr) return } err := sw.DialPeerWithAddress(addr, persistent) if err != nil { switch err.(type) { case ErrSwitchConnectToSelf, ErrSwitchDuplicatePeerID: sw.Logger.Debug("Error dialing peer", "err", err) default: sw.Logger.Error("Error dialing peer", "err", err) } } }(i) } return nil } // DialPeerWithAddress dials the given peer and runs sw.addPeer if it connects and authenticates successfully. // If `persistent == true`, the switch will always try to reconnect to this peer if the connection ever fails. func (sw *Switch) DialPeerWithAddress(addr *NetAddress, persistent bool) error { sw.dialing.Set(string(addr.ID), addr) defer sw.dialing.Delete(string(addr.ID)) return sw.addOutboundPeerWithConfig(addr, sw.config, persistent) } // sleep for interval plus some random amount of ms on [0, dialRandomizerIntervalMilliseconds] func (sw *Switch) randomSleep(interval time.Duration) { r := time.Duration(sw.rng.Int63n(dialRandomizerIntervalMilliseconds)) * time.Millisecond time.Sleep(r + interval) } // IsDialingOrExistingAddress returns true if switch has a peer with the given // address or dialing it at the moment. func (sw *Switch) IsDialingOrExistingAddress(addr *NetAddress) bool { return sw.dialing.Has(string(addr.ID)) || sw.peers.Has(addr.ID) || (!sw.config.AllowDuplicateIP && sw.peers.HasIP(addr.IP)) } func (sw *Switch) acceptRoutine() { for { p, err := sw.transport.Accept(peerConfig{ chDescs: sw.chDescs, onPeerError: sw.StopPeerForError, reactorsByCh: sw.reactorsByCh, metrics: sw.metrics, }) if err != nil { switch err.(type) { case ErrRejected: rErr := err.(ErrRejected) if rErr.IsSelf() { // Remove the given address from the address book and add to our addresses // to avoid dialing in the future. addr := rErr.Addr() sw.addrBook.RemoveAddress(&addr) sw.addrBook.AddOurAddress(&addr) } sw.Logger.Info( "Inbound Peer rejected", "err", err, "numPeers", sw.peers.Size(), ) continue case *ErrTransportClosed: sw.Logger.Error( "Stopped accept routine, as transport is closed", "numPeers", sw.peers.Size(), ) default: sw.Logger.Error( "Accept on transport errored", "err", err, "numPeers", sw.peers.Size(), ) // We could instead have a retry loop around the acceptRoutine, // but that would need to stop and let the node shutdown eventually. // So might as well panic and let process managers restart the node. // There's no point in letting the node run without the acceptRoutine, // since it won't be able to accept new connections. panic(fmt.Errorf("accept routine exited: %v", err)) } break } // Ignore connection if we already have enough peers. _, in, _ := sw.NumPeers() if in >= sw.config.MaxNumInboundPeers { sw.Logger.Info( "Ignoring inbound connection: already have enough inbound peers", "address", p.NodeInfo().NetAddress().String(), "have", in, "max", sw.config.MaxNumInboundPeers, ) sw.transport.Cleanup(p) continue } if err := sw.addPeer(p); err != nil { sw.transport.Cleanup(p) if p.IsRunning() { _ = p.Stop() } sw.Logger.Info( "Ignoring inbound connection: error while adding peer", "err", err, "id", p.ID(), ) } } } // dial the peer; make secret connection; authenticate against the dialed ID; // add the peer. // if dialing fails, start the reconnect loop. If handhsake fails, its over. // If peer is started succesffuly, reconnectLoop will start when // StopPeerForError is called func (sw *Switch) addOutboundPeerWithConfig( addr *NetAddress, cfg *config.P2PConfig, persistent bool, ) error { sw.Logger.Info("Dialing peer", "address", addr) // XXX(xla): Remove the leakage of test concerns in implementation. if cfg.TestDialFail { go sw.reconnectToPeer(addr) return fmt.Errorf("dial err (peerConfig.DialFail == true)") } p, err := sw.transport.Dial(*addr, peerConfig{ chDescs: sw.chDescs, onPeerError: sw.StopPeerForError, persistent: persistent, reactorsByCh: sw.reactorsByCh, metrics: sw.metrics, }) if err != nil { switch e := err.(type) { case ErrRejected: if e.IsSelf() { // Remove the given address from the address book and add to our addresses // to avoid dialing in the future. sw.addrBook.RemoveAddress(addr) sw.addrBook.AddOurAddress(addr) return err } } // retry persistent peers after // any dial error besides IsSelf() if persistent { go sw.reconnectToPeer(addr) } return err } if err := sw.addPeer(p); err != nil { sw.transport.Cleanup(p) if p.IsRunning() { _ = p.Stop() } return err } return nil } func (sw *Switch) filterPeer(p Peer) error { // Avoid duplicate if sw.peers.Has(p.ID()) { return ErrRejected{id: p.ID(), isDuplicate: true} } errc := make(chan error, len(sw.peerFilters)) for _, f := range sw.peerFilters { go func(f PeerFilterFunc, p Peer, errc chan<- error) { errc <- f(sw.peers, p) }(f, p, errc) } for i := 0; i < cap(errc); i++ { select { case err := <-errc: if err != nil { return ErrRejected{id: p.ID(), err: err, isFiltered: true} } case <-time.After(sw.filterTimeout): return ErrFilterTimeout{} } } return nil } // addPeer starts up the Peer and adds it to the Switch. Error is returned if // the peer is filtered out or failed to start or can't be added. func (sw *Switch) addPeer(p Peer) error { if err := sw.filterPeer(p); err != nil { return err } p.SetLogger(sw.Logger.With("peer", p.NodeInfo().NetAddress())) // Handle the shut down case where the switch has stopped but we're // concurrently trying to add a peer. if sw.IsRunning() { // All good. Start peer if err := sw.startInitPeer(p); err != nil { return err } } else { sw.Logger.Error("Won't start a peer - switch is not running", "peer", p) } // Add the peer to .peers. // We start it first so that a peer in the list is safe to Stop. // It should not err since we already checked peers.Has(). if err := sw.peers.Add(p); err != nil { return err } sw.Logger.Info("Added peer", "peer", p) sw.metrics.Peers.Add(float64(1)) return nil } func (sw *Switch) startInitPeer(p Peer) error { err := p.Start() // spawn send/recv routines if err != nil { // Should never happen sw.Logger.Error( "Error starting peer", "err", err, "peer", p, ) return err } for _, reactor := range sw.reactors { reactor.AddPeer(p) } return nil }