You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
540 lines
17 KiB
540 lines
17 KiB
package pex
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"runtime/debug"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/tendermint/tendermint/internal/p2p"
|
|
"github.com/tendermint/tendermint/internal/p2p/conn"
|
|
"github.com/tendermint/tendermint/libs/log"
|
|
tmmath "github.com/tendermint/tendermint/libs/math"
|
|
"github.com/tendermint/tendermint/libs/service"
|
|
protop2p "github.com/tendermint/tendermint/proto/tendermint/p2p"
|
|
"github.com/tendermint/tendermint/types"
|
|
)
|
|
|
|
var (
|
|
_ service.Service = (*ReactorV2)(nil)
|
|
_ p2p.Wrapper = (*protop2p.PexMessage)(nil)
|
|
)
|
|
|
|
// TODO: Consolidate with params file.
|
|
// See https://github.com/tendermint/tendermint/issues/6371
|
|
const (
|
|
// the minimum time one peer can send another request to the same peer
|
|
minReceiveRequestInterval = 100 * time.Millisecond
|
|
|
|
// the maximum amount of addresses that can be included in a response
|
|
maxAddresses uint16 = 100
|
|
|
|
// allocated time to resolve a node address into a set of endpoints
|
|
resolveTimeout = 3 * time.Second
|
|
|
|
// How long to wait when there are no peers available before trying again
|
|
noAvailablePeersWaitPeriod = 1 * time.Second
|
|
|
|
// indicates the ping rate of the pex reactor when the peer store is full.
|
|
// The reactor should still look to add new peers in order to flush out low
|
|
// scoring peers that are still in the peer store
|
|
fullCapacityInterval = 10 * time.Minute
|
|
)
|
|
|
|
// TODO: We should decide whether we want channel descriptors to be housed
|
|
// within each reactor (as they are now) or, considering that the reactor doesn't
|
|
// really need to care about the channel descriptors, if they should be housed
|
|
// in the node module.
|
|
func ChannelDescriptor() conn.ChannelDescriptor {
|
|
return conn.ChannelDescriptor{
|
|
ID: PexChannel,
|
|
Priority: 1,
|
|
SendQueueCapacity: 10,
|
|
RecvMessageCapacity: maxMsgSize,
|
|
RecvBufferCapacity: 32,
|
|
MaxSendBytes: 200,
|
|
}
|
|
}
|
|
|
|
// ReactorV2 is a PEX reactor for the new P2P stack. The legacy reactor
|
|
// is Reactor.
|
|
//
|
|
// FIXME: Rename this when Reactor is removed, and consider moving to p2p/.
|
|
//
|
|
// The peer exchange or PEX reactor supports the peer manager by sending
|
|
// requests to other peers for addresses that can be given to the peer manager
|
|
// and at the same time advertises addresses to peers that need more.
|
|
//
|
|
// The reactor is able to tweak the intensity of it's search by decreasing or
|
|
// increasing the interval between each request. It tracks connected peers via
|
|
// a linked list, sending a request to the node at the front of the list and
|
|
// adding it to the back of the list once a response is received.
|
|
type ReactorV2 struct {
|
|
service.BaseService
|
|
|
|
peerManager *p2p.PeerManager
|
|
pexCh *p2p.Channel
|
|
peerUpdates *p2p.PeerUpdates
|
|
closeCh chan struct{}
|
|
|
|
// list of available peers to loop through and send peer requests to
|
|
availablePeers map[types.NodeID]struct{}
|
|
|
|
mtx sync.RWMutex
|
|
|
|
// requestsSent keeps track of which peers the PEX reactor has sent requests
|
|
// to. This prevents the sending of spurious responses.
|
|
// NOTE: If a node never responds, they will remain in this map until a
|
|
// peer down status update is sent
|
|
requestsSent map[types.NodeID]struct{}
|
|
|
|
// lastReceivedRequests keeps track of when peers send a request to prevent
|
|
// peers from sending requests too often (as defined by
|
|
// minReceiveRequestInterval).
|
|
lastReceivedRequests map[types.NodeID]time.Time
|
|
|
|
// the time when another request will be sent
|
|
nextRequestTime time.Time
|
|
|
|
// keep track of how many new peers to existing peers we have received to
|
|
// extrapolate the size of the network
|
|
newPeers uint32
|
|
totalPeers uint32
|
|
|
|
// discoveryRatio is the inverse ratio of new peers to old peers squared.
|
|
// This is multiplied by the minimum duration to calculate how long to wait
|
|
// between each request.
|
|
discoveryRatio float32
|
|
}
|
|
|
|
// NewReactor returns a reference to a new reactor.
|
|
func NewReactorV2(
|
|
logger log.Logger,
|
|
peerManager *p2p.PeerManager,
|
|
pexCh *p2p.Channel,
|
|
peerUpdates *p2p.PeerUpdates,
|
|
) *ReactorV2 {
|
|
|
|
r := &ReactorV2{
|
|
peerManager: peerManager,
|
|
pexCh: pexCh,
|
|
peerUpdates: peerUpdates,
|
|
closeCh: make(chan struct{}),
|
|
availablePeers: make(map[types.NodeID]struct{}),
|
|
requestsSent: make(map[types.NodeID]struct{}),
|
|
lastReceivedRequests: make(map[types.NodeID]time.Time),
|
|
}
|
|
|
|
r.BaseService = *service.NewBaseService(logger, "PEX", r)
|
|
return r
|
|
}
|
|
|
|
// OnStart starts separate go routines for each p2p Channel and listens for
|
|
// envelopes on each. In addition, it also listens for peer updates and handles
|
|
// messages on that p2p channel accordingly. The caller must be sure to execute
|
|
// OnStop to ensure the outbound p2p Channels are closed.
|
|
func (r *ReactorV2) OnStart() error {
|
|
go r.processPexCh()
|
|
go r.processPeerUpdates()
|
|
return nil
|
|
}
|
|
|
|
// OnStop stops the reactor by signaling to all spawned goroutines to exit and
|
|
// blocking until they all exit.
|
|
func (r *ReactorV2) OnStop() {
|
|
// Close closeCh to signal to all spawned goroutines to gracefully exit. All
|
|
// p2p Channels should execute Close().
|
|
close(r.closeCh)
|
|
|
|
// Wait for all p2p Channels to be closed before returning. This ensures we
|
|
// can easily reason about synchronization of all p2p Channels and ensure no
|
|
// panics will occur.
|
|
<-r.pexCh.Done()
|
|
<-r.peerUpdates.Done()
|
|
}
|
|
|
|
// processPexCh implements a blocking event loop where we listen for p2p
|
|
// Envelope messages from the pexCh.
|
|
func (r *ReactorV2) processPexCh() {
|
|
defer r.pexCh.Close()
|
|
|
|
for {
|
|
select {
|
|
case <-r.closeCh:
|
|
r.Logger.Debug("stopped listening on PEX channel; closing...")
|
|
return
|
|
|
|
// outbound requests for new peers
|
|
case <-r.waitUntilNextRequest():
|
|
r.sendRequestForPeers()
|
|
|
|
// inbound requests for new peers or responses to requests sent by this
|
|
// reactor
|
|
case envelope := <-r.pexCh.In:
|
|
if err := r.handleMessage(r.pexCh.ID, envelope); err != nil {
|
|
r.Logger.Error("failed to process message", "ch_id", r.pexCh.ID, "envelope", envelope, "err", err)
|
|
r.pexCh.Error <- p2p.PeerError{
|
|
NodeID: envelope.From,
|
|
Err: err,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// processPeerUpdates initiates a blocking process where we listen for and handle
|
|
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
|
|
// close the p2p PeerUpdatesCh gracefully.
|
|
func (r *ReactorV2) processPeerUpdates() {
|
|
defer r.peerUpdates.Close()
|
|
|
|
for {
|
|
select {
|
|
case peerUpdate := <-r.peerUpdates.Updates():
|
|
r.processPeerUpdate(peerUpdate)
|
|
|
|
case <-r.closeCh:
|
|
r.Logger.Debug("stopped listening on peer updates channel; closing...")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// handlePexMessage handles envelopes sent from peers on the PexChannel.
|
|
func (r *ReactorV2) handlePexMessage(envelope p2p.Envelope) error {
|
|
logger := r.Logger.With("peer", envelope.From)
|
|
|
|
switch msg := envelope.Message.(type) {
|
|
|
|
case *protop2p.PexRequest:
|
|
// Check if the peer hasn't sent a prior request too close to this one
|
|
// in time.
|
|
if err := r.markPeerRequest(envelope.From); err != nil {
|
|
return err
|
|
}
|
|
|
|
// parse and send the legacy PEX addresses
|
|
pexAddresses := r.resolve(r.peerManager.Advertise(envelope.From, maxAddresses))
|
|
r.pexCh.Out <- p2p.Envelope{
|
|
To: envelope.From,
|
|
Message: &protop2p.PexResponse{Addresses: pexAddresses},
|
|
}
|
|
|
|
case *protop2p.PexResponse:
|
|
// check if the response matches a request that was made to that peer
|
|
if err := r.markPeerResponse(envelope.From); err != nil {
|
|
return err
|
|
}
|
|
|
|
// check the size of the response
|
|
if len(msg.Addresses) > int(maxAddresses) {
|
|
return fmt.Errorf("peer sent too many addresses (max: %d, got: %d)",
|
|
maxAddresses,
|
|
len(msg.Addresses),
|
|
)
|
|
}
|
|
|
|
for _, pexAddress := range msg.Addresses {
|
|
// no protocol is prefixed so we assume the default (mconn)
|
|
peerAddress, err := p2p.ParseNodeAddress(
|
|
fmt.Sprintf("%s@%s:%d", pexAddress.ID, pexAddress.IP, pexAddress.Port))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
added, err := r.peerManager.Add(peerAddress)
|
|
if err != nil {
|
|
logger.Error("failed to add PEX address", "address", peerAddress, "err", err)
|
|
}
|
|
if added {
|
|
r.newPeers++
|
|
logger.Debug("added PEX address", "address", peerAddress)
|
|
}
|
|
r.totalPeers++
|
|
}
|
|
|
|
// V2 PEX MESSAGES
|
|
case *protop2p.PexRequestV2:
|
|
// check if the peer hasn't sent a prior request too close to this one
|
|
// in time
|
|
if err := r.markPeerRequest(envelope.From); err != nil {
|
|
return err
|
|
}
|
|
|
|
// request peers from the peer manager and parse the NodeAddresses into
|
|
// URL strings
|
|
nodeAddresses := r.peerManager.Advertise(envelope.From, maxAddresses)
|
|
pexAddressesV2 := make([]protop2p.PexAddressV2, len(nodeAddresses))
|
|
for idx, addr := range nodeAddresses {
|
|
pexAddressesV2[idx] = protop2p.PexAddressV2{
|
|
URL: addr.String(),
|
|
}
|
|
}
|
|
r.pexCh.Out <- p2p.Envelope{
|
|
To: envelope.From,
|
|
Message: &protop2p.PexResponseV2{Addresses: pexAddressesV2},
|
|
}
|
|
|
|
case *protop2p.PexResponseV2:
|
|
// check if the response matches a request that was made to that peer
|
|
if err := r.markPeerResponse(envelope.From); err != nil {
|
|
return err
|
|
}
|
|
|
|
// check the size of the response
|
|
if len(msg.Addresses) > int(maxAddresses) {
|
|
return fmt.Errorf("peer sent too many addresses (max: %d, got: %d)",
|
|
maxAddresses,
|
|
len(msg.Addresses),
|
|
)
|
|
}
|
|
|
|
for _, pexAddress := range msg.Addresses {
|
|
peerAddress, err := p2p.ParseNodeAddress(pexAddress.URL)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
added, err := r.peerManager.Add(peerAddress)
|
|
if err != nil {
|
|
logger.Error("failed to add V2 PEX address", "address", peerAddress, "err", err)
|
|
}
|
|
if added {
|
|
r.newPeers++
|
|
logger.Debug("added V2 PEX address", "address", peerAddress)
|
|
}
|
|
r.totalPeers++
|
|
}
|
|
|
|
default:
|
|
return fmt.Errorf("received unknown message: %T", msg)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolve resolves a set of peer addresses into PEX addresses.
|
|
//
|
|
// FIXME: This is necessary because the current PEX protocol only supports
|
|
// IP/port pairs, while the P2P stack uses NodeAddress URLs. The PEX protocol
|
|
// should really use URLs too, to exchange DNS names instead of IPs and allow
|
|
// different transport protocols (e.g. QUIC and MemoryTransport).
|
|
//
|
|
// FIXME: We may want to cache and parallelize this, but for now we'll just rely
|
|
// on the operating system to cache it for us.
|
|
func (r *ReactorV2) resolve(addresses []p2p.NodeAddress) []protop2p.PexAddress {
|
|
limit := len(addresses)
|
|
pexAddresses := make([]protop2p.PexAddress, 0, limit)
|
|
|
|
for _, address := range addresses {
|
|
ctx, cancel := context.WithTimeout(context.Background(), resolveTimeout)
|
|
endpoints, err := address.Resolve(ctx)
|
|
r.Logger.Debug("resolved node address", "endpoints", endpoints)
|
|
cancel()
|
|
|
|
if err != nil {
|
|
r.Logger.Debug("failed to resolve address", "address", address, "err", err)
|
|
continue
|
|
}
|
|
|
|
for _, endpoint := range endpoints {
|
|
r.Logger.Debug("checking endpint", "IP", endpoint.IP, "Port", endpoint.Port)
|
|
if len(pexAddresses) >= limit {
|
|
return pexAddresses
|
|
|
|
} else if endpoint.IP != nil {
|
|
r.Logger.Debug("appending pex address")
|
|
// PEX currently only supports IP-networked transports (as
|
|
// opposed to e.g. p2p.MemoryTransport).
|
|
//
|
|
// FIXME: as the PEX address contains no information about the
|
|
// protocol, we jam this into the ID. We won't need to this once
|
|
// we support URLs
|
|
pexAddresses = append(pexAddresses, protop2p.PexAddress{
|
|
ID: string(address.NodeID),
|
|
IP: endpoint.IP.String(),
|
|
Port: uint32(endpoint.Port),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
return pexAddresses
|
|
}
|
|
|
|
// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
|
|
// It will handle errors and any possible panics gracefully. A caller can handle
|
|
// any error returned by sending a PeerError on the respective channel.
|
|
func (r *ReactorV2) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
|
|
defer func() {
|
|
if e := recover(); e != nil {
|
|
err = fmt.Errorf("panic in processing message: %v", e)
|
|
r.Logger.Error(
|
|
"recovering from processing message panic",
|
|
"err", err,
|
|
"stack", string(debug.Stack()),
|
|
)
|
|
}
|
|
}()
|
|
|
|
r.Logger.Debug("received PEX message", "peer", envelope.From)
|
|
|
|
switch chID {
|
|
case p2p.ChannelID(PexChannel):
|
|
err = r.handlePexMessage(envelope)
|
|
|
|
default:
|
|
err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
// processPeerUpdate processes a PeerUpdate. For added peers, PeerStatusUp, we
|
|
// send a request for addresses.
|
|
func (r *ReactorV2) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
|
|
r.Logger.Debug("received PEX peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
|
|
|
|
r.mtx.Lock()
|
|
defer r.mtx.Unlock()
|
|
|
|
switch peerUpdate.Status {
|
|
case p2p.PeerStatusUp:
|
|
r.availablePeers[peerUpdate.NodeID] = struct{}{}
|
|
case p2p.PeerStatusDown:
|
|
delete(r.availablePeers, peerUpdate.NodeID)
|
|
delete(r.requestsSent, peerUpdate.NodeID)
|
|
delete(r.lastReceivedRequests, peerUpdate.NodeID)
|
|
default:
|
|
}
|
|
}
|
|
|
|
func (r *ReactorV2) waitUntilNextRequest() <-chan time.Time {
|
|
return time.After(time.Until(r.nextRequestTime))
|
|
}
|
|
|
|
// sendRequestForPeers pops the first peerID off the list and sends the
|
|
// peer a request for more peer addresses. The function then moves the
|
|
// peer into the requestsSent bucket and calculates when the next request
|
|
// time should be
|
|
func (r *ReactorV2) sendRequestForPeers() {
|
|
r.mtx.Lock()
|
|
defer r.mtx.Unlock()
|
|
if len(r.availablePeers) == 0 {
|
|
// no peers are available
|
|
r.Logger.Debug("no available peers to send request to, waiting...")
|
|
r.nextRequestTime = time.Now().Add(noAvailablePeersWaitPeriod)
|
|
|
|
return
|
|
}
|
|
var peerID types.NodeID
|
|
|
|
// use range to get a random peer.
|
|
for peerID = range r.availablePeers {
|
|
break
|
|
}
|
|
|
|
// The node accommodates for both pex systems
|
|
if r.isLegacyPeer(peerID) {
|
|
r.pexCh.Out <- p2p.Envelope{
|
|
To: peerID,
|
|
Message: &protop2p.PexRequest{},
|
|
}
|
|
} else {
|
|
r.pexCh.Out <- p2p.Envelope{
|
|
To: peerID,
|
|
Message: &protop2p.PexRequestV2{},
|
|
}
|
|
}
|
|
|
|
// remove the peer from the abvailable peers list and mark it in the requestsSent map
|
|
delete(r.availablePeers, peerID)
|
|
r.requestsSent[peerID] = struct{}{}
|
|
|
|
r.calculateNextRequestTime()
|
|
r.Logger.Debug("peer request sent", "next_request_time", r.nextRequestTime)
|
|
}
|
|
|
|
// calculateNextRequestTime implements something of a proportional controller
|
|
// to estimate how often the reactor should be requesting new peer addresses.
|
|
// The dependent variable in this calculation is the ratio of new peers to
|
|
// all peers that the reactor receives. The interval is thus calculated as the
|
|
// inverse squared. In the beginning, all peers should be new peers.
|
|
// We expect this ratio to be near 1 and thus the interval to be as short
|
|
// as possible. As the node becomes more familiar with the network the ratio of
|
|
// new nodes will plummet to a very small number, meaning the interval expands
|
|
// to its upper bound.
|
|
// CONTRACT: Must use a write lock as nextRequestTime is updated
|
|
func (r *ReactorV2) calculateNextRequestTime() {
|
|
// check if the peer store is full. If so then there is no need
|
|
// to send peer requests too often
|
|
if ratio := r.peerManager.PeerRatio(); ratio >= 0.95 {
|
|
r.Logger.Debug("peer manager near full ratio, sleeping...",
|
|
"sleep_period", fullCapacityInterval, "ratio", ratio)
|
|
r.nextRequestTime = time.Now().Add(fullCapacityInterval)
|
|
return
|
|
}
|
|
|
|
// baseTime represents the shortest interval that we can send peer requests
|
|
// in. For example if we have 10 peers and we can't send a message to the
|
|
// same peer every 500ms, then we can send a request every 50ms. In practice
|
|
// we use a safety margin of 2, ergo 100ms
|
|
peers := tmmath.MinInt(len(r.availablePeers), 50)
|
|
baseTime := minReceiveRequestInterval
|
|
if peers > 0 {
|
|
baseTime = minReceiveRequestInterval * 2 / time.Duration(peers)
|
|
}
|
|
|
|
if r.totalPeers > 0 || r.discoveryRatio == 0 {
|
|
// find the ratio of new peers. NOTE: We add 1 to both sides to avoid
|
|
// divide by zero problems
|
|
ratio := float32(r.totalPeers+1) / float32(r.newPeers+1)
|
|
// square the ratio in order to get non linear time intervals
|
|
// NOTE: The longest possible interval for a network with 100 or more peers
|
|
// where a node is connected to 50 of them is 2 minutes.
|
|
r.discoveryRatio = ratio * ratio
|
|
r.newPeers = 0
|
|
r.totalPeers = 0
|
|
}
|
|
// NOTE: As ratio is always >= 1, discovery ratio is >= 1. Therefore we don't need to worry
|
|
// about the next request time being less than the minimum time
|
|
r.nextRequestTime = time.Now().Add(baseTime * time.Duration(r.discoveryRatio))
|
|
}
|
|
|
|
func (r *ReactorV2) markPeerRequest(peer types.NodeID) error {
|
|
r.mtx.Lock()
|
|
defer r.mtx.Unlock()
|
|
if lastRequestTime, ok := r.lastReceivedRequests[peer]; ok {
|
|
if time.Now().Before(lastRequestTime.Add(minReceiveRequestInterval)) {
|
|
return fmt.Errorf("peer sent a request too close after a prior one. Minimum interval: %v",
|
|
minReceiveRequestInterval)
|
|
}
|
|
}
|
|
r.lastReceivedRequests[peer] = time.Now()
|
|
return nil
|
|
}
|
|
|
|
func (r *ReactorV2) markPeerResponse(peer types.NodeID) error {
|
|
r.mtx.Lock()
|
|
defer r.mtx.Unlock()
|
|
// check if a request to this peer was sent
|
|
if _, ok := r.requestsSent[peer]; !ok {
|
|
return fmt.Errorf("peer sent a PEX response when none was requested (%v)", peer)
|
|
}
|
|
delete(r.requestsSent, peer)
|
|
// attach to the back of the list so that the peer can be used again for
|
|
// future requests
|
|
|
|
r.availablePeers[peer] = struct{}{}
|
|
return nil
|
|
}
|
|
|
|
// all addresses must use a MCONN protocol for the peer to be considered part of the
|
|
// legacy p2p pex system
|
|
func (r *ReactorV2) isLegacyPeer(peer types.NodeID) bool {
|
|
for _, addr := range r.peerManager.Addresses(peer) {
|
|
if addr.Protocol != p2p.MConnProtocol {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|