Aleksandr Bezobchuk
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
22 changed files with 1277 additions and 88 deletions
Split View
Diff Options
-
+2 -0blockchain/v0/reactor.go
-
+8 -0consensus/reactor.go
-
+2 -0evidence/reactor.go
-
+2 -0mempool/reactor.go
-
+18 -3mempool/reactor_test.go
-
+37 -9node/node.go
-
+12 -3p2p/conn/connection.go
-
+66 -5p2p/metrics.go
-
+9 -2p2p/p2ptest/network.go
-
+2 -0p2p/pex/pex_reactor.go
-
+277 -0p2p/pqueue.go
-
+9 -12p2p/queue.go
-
+163 -28p2p/router.go
-
+62 -0p2p/router_init_test.go
-
+72 -15p2p/router_test.go
-
+283 -0p2p/wdrr_queue.go
-
+208 -0p2p/wdrr_queue_test.go
-
+4 -0statesync/reactor.go
-
+2 -0test/e2e/docker/Dockerfile
-
+11 -6test/e2e/runner/load.go
-
+16 -5test/e2e/runner/main.go
-
+12 -0test/e2e/runner/setup.go
+ 2
- 0
blockchain/v0/reactor.go
View File
+ 8
- 0
consensus/reactor.go
View File
+ 2
- 0
evidence/reactor.go
View File
+ 2
- 0
mempool/reactor.go
View File
+ 18
- 3
mempool/reactor_test.go
View File
+ 37
- 9
node/node.go
View File
+ 12
- 3
p2p/conn/connection.go
View File
+ 66
- 5
p2p/metrics.go
View File
+ 9
- 2
p2p/p2ptest/network.go
View File
+ 2
- 0
p2p/pex/pex_reactor.go
View File
+ 277
- 0
p2p/pqueue.go
View File
| @ -0,0 +1,277 @@ | |||
| package p2p | |||
| import ( | |||
| "container/heap" | |||
| "sort" | |||
| "strconv" | |||
| "time" | |||
| "github.com/gogo/protobuf/proto" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| tmsync "github.com/tendermint/tendermint/libs/sync" | |||
| ) | |||
| // pqEnvelope defines a wrapper around an Envelope with priority to be inserted | |||
| // into a priority queue used for Envelope scheduling. | |||
| type pqEnvelope struct { | |||
| envelope Envelope | |||
| priority uint | |||
| size uint | |||
| timestamp time.Time | |||
| index int | |||
| } | |||
| // priorityQueue defines a type alias for a priority queue implementation. | |||
| type priorityQueue []*pqEnvelope | |||
| func (pq priorityQueue) get(i int) *pqEnvelope { return pq[i] } | |||
| func (pq priorityQueue) Len() int { return len(pq) } | |||
| func (pq priorityQueue) Less(i, j int) bool { | |||
| // if both elements have the same priority, prioritize based on most recent | |||
| if pq[i].priority == pq[j].priority { | |||
| return pq[i].timestamp.After(pq[j].timestamp) | |||
| } | |||
| // otherwise, pick the pqEnvelope with the higher priority | |||
| return pq[i].priority > pq[j].priority | |||
| } | |||
| func (pq priorityQueue) Swap(i, j int) { | |||
| pq[i], pq[j] = pq[j], pq[i] | |||
| pq[i].index = i | |||
| pq[j].index = j | |||
| } | |||
| func (pq *priorityQueue) Push(x interface{}) { | |||
| n := len(*pq) | |||
| pqEnv := x.(*pqEnvelope) | |||
| pqEnv.index = n | |||
| *pq = append(*pq, pqEnv) | |||
| } | |||
| func (pq *priorityQueue) Pop() interface{} { | |||
| old := *pq | |||
| n := len(old) | |||
| pqEnv := old[n-1] | |||
| old[n-1] = nil | |||
| pqEnv.index = -1 | |||
| *pq = old[:n-1] | |||
| return pqEnv | |||
| } | |||
| // Assert the priority queue scheduler implements the queue interface at | |||
| // compile-time. | |||
| var _ queue = (*pqScheduler)(nil) | |||
| type pqScheduler struct { | |||
| logger log.Logger | |||
| metrics *Metrics | |||
| size uint | |||
| sizes map[uint]uint // cumulative priority sizes | |||
| pq *priorityQueue | |||
| chDescs []ChannelDescriptor | |||
| capacity uint | |||
| chPriorities map[ChannelID]uint | |||
| enqueueCh chan Envelope | |||
| dequeueCh chan Envelope | |||
| closer *tmsync.Closer | |||
| done *tmsync.Closer | |||
| } | |||
| func newPQScheduler( | |||
| logger log.Logger, | |||
| m *Metrics, | |||
| chDescs []ChannelDescriptor, | |||
| enqueueBuf, dequeueBuf, capacity uint, | |||
| ) *pqScheduler { | |||
| // copy each ChannelDescriptor and sort them by ascending channel priority | |||
| chDescsCopy := make([]ChannelDescriptor, len(chDescs)) | |||
| copy(chDescsCopy, chDescs) | |||
| sort.Slice(chDescsCopy, func(i, j int) bool { return chDescsCopy[i].Priority < chDescsCopy[j].Priority }) | |||
| var ( | |||
| chPriorities = make(map[ChannelID]uint) | |||
| sizes = make(map[uint]uint) | |||
| ) | |||
| for _, chDesc := range chDescsCopy { | |||
| chID := ChannelID(chDesc.ID) | |||
| chPriorities[chID] = uint(chDesc.Priority) | |||
| sizes[uint(chDesc.Priority)] = 0 | |||
| } | |||
| pq := make(priorityQueue, 0) | |||
| heap.Init(&pq) | |||
| return &pqScheduler{ | |||
| logger: logger.With("router", "scheduler"), | |||
| metrics: m, | |||
| chDescs: chDescsCopy, | |||
| capacity: capacity, | |||
| chPriorities: chPriorities, | |||
| pq: &pq, | |||
| sizes: sizes, | |||
| enqueueCh: make(chan Envelope, enqueueBuf), | |||
| dequeueCh: make(chan Envelope, dequeueBuf), | |||
| closer: tmsync.NewCloser(), | |||
| done: tmsync.NewCloser(), | |||
| } | |||
| } | |||
| func (s *pqScheduler) enqueue() chan<- Envelope { | |||
| return s.enqueueCh | |||
| } | |||
| func (s *pqScheduler) dequeue() <-chan Envelope { | |||
| return s.dequeueCh | |||
| } | |||
| func (s *pqScheduler) close() { | |||
| s.closer.Close() | |||
| <-s.done.Done() | |||
| } | |||
| func (s *pqScheduler) closed() <-chan struct{} { | |||
| return s.closer.Done() | |||
| } | |||
| // start starts non-blocking process that starts the priority queue scheduler. | |||
| func (s *pqScheduler) start() { | |||
| go s.process() | |||
| } | |||
| // process starts a block process where we listen for Envelopes to enqueue. If | |||
| // there is sufficient capacity, it will be enqueued into the priority queue, | |||
| // otherwise, we attempt to dequeue enough elements from the priority queue to | |||
| // make room for the incoming Envelope by dropping lower priority elements. If | |||
| // there isn't sufficient capacity at lower priorities for the incoming Envelope, | |||
| // it is dropped. | |||
| // | |||
| // After we attempt to enqueue the incoming Envelope, if the priority queue is | |||
| // non-empty, we pop the top Envelope and send it on the dequeueCh. | |||
| func (s *pqScheduler) process() { | |||
| defer s.done.Close() | |||
| for { | |||
| select { | |||
| case e := <-s.enqueueCh: | |||
| chIDStr := strconv.Itoa(int(e.channelID)) | |||
| pqEnv := &pqEnvelope{ | |||
| envelope: e, | |||
| size: uint(proto.Size(e.Message)), | |||
| priority: s.chPriorities[e.channelID], | |||
| timestamp: time.Now().UTC(), | |||
| } | |||
| // enqueue | |||
| // Check if we have sufficient capacity to simply enqueue the incoming | |||
| // Envelope. | |||
| if s.size+pqEnv.size <= s.capacity { | |||
| // enqueue the incoming Envelope | |||
| s.push(pqEnv) | |||
| } else { | |||
| // There is not sufficient capacity to simply enqueue the incoming | |||
| // Envelope. So we have to attempt to make room for it by dropping lower | |||
| // priority Envelopes or drop the incoming Envelope otherwise. | |||
| // The cumulative size of all enqueue envelopes at the incoming envelope's | |||
| // priority or lower. | |||
| total := s.sizes[pqEnv.priority] | |||
| if total >= pqEnv.size { | |||
| // There is room for the incoming Envelope, so we drop as many lower | |||
| // priority Envelopes as we need to. | |||
| var ( | |||
| canEnqueue bool | |||
| tmpSize = s.size | |||
| i = s.pq.Len() - 1 | |||
| ) | |||
| // Drop lower priority Envelopes until sufficient capacity exists for | |||
| // the incoming Envelope | |||
| for i >= 0 && !canEnqueue { | |||
| pqEnvTmp := s.pq.get(i) | |||
| if pqEnvTmp.priority < pqEnv.priority { | |||
| if tmpSize+pqEnv.size <= s.capacity { | |||
| canEnqueue = true | |||
| } else { | |||
| pqEnvTmpChIDStr := strconv.Itoa(int(pqEnvTmp.envelope.channelID)) | |||
| s.metrics.PeerQueueDroppedMsgs.With("ch_id", pqEnvTmpChIDStr).Add(1) | |||
| s.logger.Debug( | |||
| "dropped envelope", | |||
| "ch_id", pqEnvTmpChIDStr, | |||
| "priority", pqEnvTmp.priority, | |||
| "msg_size", pqEnvTmp.size, | |||
| "capacity", s.capacity, | |||
| ) | |||
| // dequeue/drop from the priority queue | |||
| heap.Remove(s.pq, pqEnvTmp.index) | |||
| // update the size tracker | |||
| tmpSize -= pqEnvTmp.size | |||
| // start from the end again | |||
| i = s.pq.Len() - 1 | |||
| } | |||
| } else { | |||
| i-- | |||
| } | |||
| } | |||
| // enqueue the incoming Envelope | |||
| s.push(pqEnv) | |||
| } else { | |||
| // There is not sufficient capacity to drop lower priority Envelopes, | |||
| // so we drop the incoming Envelope. | |||
| s.metrics.PeerQueueDroppedMsgs.With("ch_id", chIDStr).Add(1) | |||
| s.logger.Debug( | |||
| "dropped envelope", | |||
| "ch_id", chIDStr, | |||
| "priority", pqEnv.priority, | |||
| "msg_size", pqEnv.size, | |||
| "capacity", s.capacity, | |||
| ) | |||
| } | |||
| } | |||
| // dequeue | |||
| for s.pq.Len() > 0 { | |||
| pqEnv = heap.Pop(s.pq).(*pqEnvelope) | |||
| s.size -= pqEnv.size | |||
| // deduct the Envelope size from all the relevant cumulative sizes | |||
| for i := 0; i < len(s.chDescs) && pqEnv.priority <= uint(s.chDescs[i].Priority); i++ { | |||
| s.sizes[uint(s.chDescs[i].Priority)] -= pqEnv.size | |||
| } | |||
| s.dequeueCh <- pqEnv.envelope | |||
| } | |||
| case <-s.closer.Done(): | |||
| return | |||
| } | |||
| } | |||
| } | |||
| func (s *pqScheduler) push(pqEnv *pqEnvelope) { | |||
| chIDStr := strconv.Itoa(int(pqEnv.envelope.channelID)) | |||
| // enqueue the incoming Envelope | |||
| heap.Push(s.pq, pqEnv) | |||
| s.size += pqEnv.size | |||
| s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Set(float64(pqEnv.size)) | |||
| // Update the cumulative sizes by adding the Envelope's size to every | |||
| // priority less than or equal to it. | |||
| for i := 0; i < len(s.chDescs) && pqEnv.priority <= uint(s.chDescs[i].Priority); i++ { | |||
| s.sizes[uint(s.chDescs[i].Priority)] += pqEnv.size | |||
| } | |||
| } | |||
+ 9
- 12
p2p/queue.go
View File
+ 163
- 28
p2p/router.go
View File
+ 62
- 0
p2p/router_init_test.go
View File
| @ -0,0 +1,62 @@ | |||
| package p2p | |||
| import ( | |||
| "os" | |||
| "testing" | |||
| "github.com/stretchr/testify/require" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| ) | |||
| func TestRouter_ConstructQueueFactory(t *testing.T) { | |||
| t.Run("ValidateOptionsPopulatesDefaultQueue", func(t *testing.T) { | |||
| opts := RouterOptions{} | |||
| require.NoError(t, opts.Validate()) | |||
| require.Equal(t, "fifo", opts.QueueType) | |||
| }) | |||
| t.Run("Default", func(t *testing.T) { | |||
| require.Zero(t, os.Getenv("TM_P2P_QUEUE")) | |||
| opts := RouterOptions{} | |||
| r, err := NewRouter(log.NewNopLogger(), nil, NodeInfo{}, nil, nil, nil, opts) | |||
| require.NoError(t, err) | |||
| _, ok := r.queueFactory(1).(*fifoQueue) | |||
| require.True(t, ok) | |||
| }) | |||
| t.Run("Fifo", func(t *testing.T) { | |||
| opts := RouterOptions{QueueType: queueTypeFifo} | |||
| r, err := NewRouter(log.NewNopLogger(), nil, NodeInfo{}, nil, nil, nil, opts) | |||
| require.NoError(t, err) | |||
| _, ok := r.queueFactory(1).(*fifoQueue) | |||
| require.True(t, ok) | |||
| }) | |||
| t.Run("Priority", func(t *testing.T) { | |||
| opts := RouterOptions{QueueType: queueTypePriority} | |||
| r, err := NewRouter(log.NewNopLogger(), nil, NodeInfo{}, nil, nil, nil, opts) | |||
| require.NoError(t, err) | |||
| q, ok := r.queueFactory(1).(*pqScheduler) | |||
| require.True(t, ok) | |||
| defer q.close() | |||
| }) | |||
| t.Run("WDRR", func(t *testing.T) { | |||
| opts := RouterOptions{QueueType: queueTypeWDRR} | |||
| r, err := NewRouter(log.NewNopLogger(), nil, NodeInfo{}, nil, nil, nil, opts) | |||
| require.NoError(t, err) | |||
| q, ok := r.queueFactory(1).(*wdrrScheduler) | |||
| require.True(t, ok) | |||
| defer q.close() | |||
| }) | |||
| t.Run("NonExistant", func(t *testing.T) { | |||
| opts := RouterOptions{QueueType: "fast"} | |||
| _, err := NewRouter(log.NewNopLogger(), nil, NodeInfo{}, nil, nil, nil, opts) | |||
| require.Error(t, err) | |||
| require.Contains(t, err.Error(), "fast") | |||
| }) | |||
| t.Run("InternalsSafeWhenUnspecified", func(t *testing.T) { | |||
| r := &Router{} | |||
| require.Zero(t, r.options.QueueType) | |||
| fn, err := r.createQueueFactory() | |||
| require.Error(t, err) | |||
| require.Nil(t, fn) | |||
| }) | |||
| } | |||
+ 72
- 15
p2p/router_test.go
View File
+ 283
- 0
p2p/wdrr_queue.go
View File
| @ -0,0 +1,283 @@ | |||
| package p2p | |||
| import ( | |||
| "sort" | |||
| "strconv" | |||
| "github.com/gogo/protobuf/proto" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| tmsync "github.com/tendermint/tendermint/libs/sync" | |||
| ) | |||
| const defaultCapacity uint = 1048576 // 1MB | |||
| // wrappedEnvelope wraps a p2p Envelope with its precomputed size. | |||
| type wrappedEnvelope struct { | |||
| envelope Envelope | |||
| size uint | |||
| } | |||
| // assert the WDDR scheduler implements the queue interface at compile-time | |||
| var _ queue = (*wdrrScheduler)(nil) | |||
| // wdrrQueue implements a Weighted Deficit Round Robin (WDRR) scheduling | |||
| // algorithm via the queue interface. A WDRR queue is created per peer, where | |||
| // the queue will have N number of flows. Each flow corresponds to a p2p Channel, | |||
| // so there are n input flows and a single output source, the peer's connection. | |||
| // | |||
| // The WDRR scheduler contains a shared buffer with a fixed capacity. | |||
| // | |||
| // Each flow has the following: | |||
| // - quantum: The number of bytes that is added to the deficit counter of the | |||
| // flow in each round. The flow can send at most quantum bytes at a time. Each | |||
| // flow has its own unique quantum, which gives the queue its weighted nature. | |||
| // A higher quantum corresponds to a higher weight/priority. The quantum is | |||
| // computed as MaxSendBytes * Priority. | |||
| // - deficit counter: The number of bytes that the flow is allowed to transmit | |||
| // when it is its turn. | |||
| // | |||
| // See: https://en.wikipedia.org/wiki/Deficit_round_robin | |||
| type wdrrScheduler struct { | |||
| logger log.Logger | |||
| metrics *Metrics | |||
| chDescs []ChannelDescriptor | |||
| capacity uint | |||
| size uint | |||
| chPriorities map[ChannelID]uint | |||
| buffer map[ChannelID][]wrappedEnvelope | |||
| quanta map[ChannelID]uint | |||
| deficits map[ChannelID]uint | |||
| closer *tmsync.Closer | |||
| doneCh *tmsync.Closer | |||
| enqueueCh chan Envelope | |||
| dequeueCh chan Envelope | |||
| } | |||
| func newWDRRScheduler( | |||
| logger log.Logger, | |||
| m *Metrics, | |||
| chDescs []ChannelDescriptor, | |||
| enqueueBuf, dequeueBuf, capacity uint, | |||
| ) *wdrrScheduler { | |||
| // copy each ChannelDescriptor and sort them by channel priority | |||
| chDescsCopy := make([]ChannelDescriptor, len(chDescs)) | |||
| copy(chDescsCopy, chDescs) | |||
| sort.Slice(chDescsCopy, func(i, j int) bool { return chDescsCopy[i].Priority > chDescsCopy[j].Priority }) | |||
| var ( | |||
| buffer = make(map[ChannelID][]wrappedEnvelope) | |||
| chPriorities = make(map[ChannelID]uint) | |||
| quanta = make(map[ChannelID]uint) | |||
| deficits = make(map[ChannelID]uint) | |||
| ) | |||
| for _, chDesc := range chDescsCopy { | |||
| chID := ChannelID(chDesc.ID) | |||
| chPriorities[chID] = uint(chDesc.Priority) | |||
| buffer[chID] = make([]wrappedEnvelope, 0) | |||
| quanta[chID] = chDesc.MaxSendBytes * uint(chDesc.Priority) | |||
| } | |||
| return &wdrrScheduler{ | |||
| logger: logger.With("queue", "wdrr"), | |||
| metrics: m, | |||
| capacity: capacity, | |||
| chPriorities: chPriorities, | |||
| chDescs: chDescsCopy, | |||
| buffer: buffer, | |||
| quanta: quanta, | |||
| deficits: deficits, | |||
| closer: tmsync.NewCloser(), | |||
| doneCh: tmsync.NewCloser(), | |||
| enqueueCh: make(chan Envelope, enqueueBuf), | |||
| dequeueCh: make(chan Envelope, dequeueBuf), | |||
| } | |||
| } | |||
| // enqueue returns an unbuffered write-only channel which a producer can send on. | |||
| func (s *wdrrScheduler) enqueue() chan<- Envelope { | |||
| return s.enqueueCh | |||
| } | |||
| // dequeue returns an unbuffered read-only channel which a consumer can read from. | |||
| func (s *wdrrScheduler) dequeue() <-chan Envelope { | |||
| return s.dequeueCh | |||
| } | |||
| func (s *wdrrScheduler) closed() <-chan struct{} { | |||
| return s.closer.Done() | |||
| } | |||
| // close closes the WDRR queue. After this call enqueue() will block, so the | |||
| // caller must select on closed() as well to avoid blocking forever. The | |||
| // enqueue() and dequeue() along with the internal channels will NOT be closed. | |||
| // Note, close() will block until all externally spawned goroutines have exited. | |||
| func (s *wdrrScheduler) close() { | |||
| s.closer.Close() | |||
| <-s.doneCh.Done() | |||
| } | |||
| // start starts the WDRR queue process in a blocking goroutine. This must be | |||
| // called before the queue can start to process and accept Envelopes. | |||
| func (s *wdrrScheduler) start() { | |||
| go s.process() | |||
| } | |||
| // process starts a blocking WDRR scheduler process, where we continuously | |||
| // evaluate if we need to attempt to enqueue an Envelope or schedule Envelopes | |||
| // to be dequeued and subsequently read and sent on the source connection. | |||
| // Internally, each p2p Channel maps to a flow, where each flow has a deficit | |||
| // and a quantum. | |||
| // | |||
| // For each Envelope requested to be enqueued, we evaluate if there is sufficient | |||
| // capacity in the shared buffer to add the Envelope. If so, it is added. | |||
| // Otherwise, we evaluate all flows of lower priority where we attempt find an | |||
| // existing Envelope in the shared buffer of sufficient size that can be dropped | |||
| // in place of the incoming Envelope. If there is no such Envelope that can be | |||
| // dropped, then the incoming Envelope is dropped. | |||
| // | |||
| // When there is nothing to be enqueued, we perform the WDRR algorithm and | |||
| // determine which Envelopes can be dequeued. For each Envelope that can be | |||
| // dequeued, it is sent on the dequeueCh. Specifically, for each flow, if it is | |||
| // non-empty, its deficit counter is incremented by its quantum value. Then, the | |||
| // value of the deficit counter is a maximal amount of bytes that can be sent at | |||
| // this round. If the deficit counter is greater than the Envelopes's message | |||
| // size at the head of the queue (HoQ), this envelope can be sent and the value | |||
| // of the counter is decremented by the message's size. Then, the size of the | |||
| // next Envelopes's message is compared to the counter value, etc. Once the flow | |||
| // is empty or the value of the counter is insufficient, the scheduler will skip | |||
| // to the next flow. If the flow is empty, the value of the deficit counter is | |||
| // reset to 0. | |||
| // | |||
| // XXX/TODO: Evaluate the single goroutine scheduler mechanism. In other words, | |||
| // evaluate the effectiveness and performance of having a single goroutine | |||
| // perform handling both enqueueing and dequeueing logic. Specifically, there | |||
| // is potentially contention between reading off of enqueueCh and trying to | |||
| // enqueue while also attempting to perform the WDRR algorithm and find the next | |||
| // set of Envelope(s) to send on the dequeueCh. Alternatively, we could consider | |||
| // separate scheduling goroutines, but then that requires the use of mutexes and | |||
| // possibly a degrading performance. | |||
| func (s *wdrrScheduler) process() { | |||
| defer s.doneCh.Close() | |||
| for { | |||
| select { | |||
| case <-s.closer.Done(): | |||
| return | |||
| case e := <-s.enqueueCh: | |||
| // attempt to enqueue the incoming Envelope | |||
| chIDStr := strconv.Itoa(int(e.channelID)) | |||
| wEnv := wrappedEnvelope{envelope: e, size: uint(proto.Size(e.Message))} | |||
| msgSize := wEnv.size | |||
| // If we're at capacity, we need to either drop the incoming Envelope or | |||
| // an Envelope from a lower priority flow. Otherwise, we add the (wrapped) | |||
| // envelope to the flow's queue. | |||
| if s.size+wEnv.size > s.capacity { | |||
| chPriority := s.chPriorities[e.channelID] | |||
| var ( | |||
| canDrop bool | |||
| dropIdx int | |||
| dropChID ChannelID | |||
| ) | |||
| // Evaluate all lower priority flows and determine if there exists an | |||
| // Envelope that is of equal or greater size that we can drop in favor | |||
| // of the incoming Envelope. | |||
| for i := len(s.chDescs) - 1; i >= 0 && uint(s.chDescs[i].Priority) < chPriority && !canDrop; i-- { | |||
| currChID := ChannelID(s.chDescs[i].ID) | |||
| flow := s.buffer[currChID] | |||
| for j := 0; j < len(flow) && !canDrop; j++ { | |||
| if flow[j].size >= wEnv.size { | |||
| canDrop = true | |||
| dropIdx = j | |||
| dropChID = currChID | |||
| break | |||
| } | |||
| } | |||
| } | |||
| // If we can drop an existing Envelope, drop it and enqueue the incoming | |||
| // Envelope. | |||
| if canDrop { | |||
| chIDStr = strconv.Itoa(int(dropChID)) | |||
| chPriority = s.chPriorities[dropChID] | |||
| msgSize = s.buffer[dropChID][dropIdx].size | |||
| // Drop Envelope for the lower priority flow and update the queue's | |||
| // buffer size | |||
| s.size -= msgSize | |||
| s.buffer[dropChID] = append(s.buffer[dropChID][:dropIdx], s.buffer[dropChID][dropIdx+1:]...) | |||
| // add the incoming Envelope and update queue's buffer size | |||
| s.size += wEnv.size | |||
| s.buffer[e.channelID] = append(s.buffer[e.channelID], wEnv) | |||
| s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Set(float64(wEnv.size)) | |||
| } | |||
| // We either dropped the incoming Enevelope or one from an existing | |||
| // lower priority flow. | |||
| s.metrics.PeerQueueDroppedMsgs.With("ch_id", chIDStr).Add(1) | |||
| s.logger.Debug( | |||
| "dropped envelope", | |||
| "ch_id", chIDStr, | |||
| "priority", chPriority, | |||
| "capacity", s.capacity, | |||
| "msg_size", msgSize, | |||
| ) | |||
| } else { | |||
| // we have sufficient capacity to enqueue the incoming Envelope | |||
| s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Set(float64(wEnv.size)) | |||
| s.buffer[e.channelID] = append(s.buffer[e.channelID], wEnv) | |||
| s.size += wEnv.size | |||
| } | |||
| default: | |||
| // perform the WDRR algorithm | |||
| for _, chDesc := range s.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| // only consider non-empty flows | |||
| if len(s.buffer[chID]) > 0 { | |||
| // bump flow's quantum | |||
| s.deficits[chID] += s.quanta[chID] | |||
| // grab the flow's current deficit counter and HoQ (wrapped) Envelope | |||
| d := s.deficits[chID] | |||
| we := s.buffer[chID][0] | |||
| // While the flow is non-empty and we can send the current Envelope | |||
| // on the dequeueCh: | |||
| // | |||
| // 1. send the Envelope | |||
| // 2. update the scheduler's shared buffer's size | |||
| // 3. update the flow's deficit | |||
| // 4. remove from the flow's queue | |||
| // 5. grab the next HoQ Envelope and flow's deficit | |||
| for len(s.buffer[chID]) > 0 && d >= we.size { | |||
| s.dequeueCh <- we.envelope | |||
| s.size -= we.size | |||
| s.deficits[chID] -= we.size | |||
| s.buffer[chID] = s.buffer[chID][1:] | |||
| if len(s.buffer[chID]) > 0 { | |||
| d = s.deficits[chID] | |||
| we = s.buffer[chID][0] | |||
| } | |||
| } | |||
| } | |||
| // reset the flow's deficit to zero if it is empty | |||
| if len(s.buffer[chID]) == 0 { | |||
| s.deficits[chID] = 0 | |||
| } | |||
| } | |||
| } | |||
| } | |||
| } | |||
+ 208
- 0
p2p/wdrr_queue_test.go
View File
| @ -0,0 +1,208 @@ | |||
| package p2p | |||
| import ( | |||
| "math" | |||
| "math/rand" | |||
| "testing" | |||
| "time" | |||
| gogotypes "github.com/gogo/protobuf/types" | |||
| "github.com/stretchr/testify/require" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| tmsync "github.com/tendermint/tendermint/libs/sync" | |||
| ) | |||
| type testMessage = gogotypes.StringValue | |||
| func TestWDRRQueue_EqualWeights(t *testing.T) { | |||
| chDescs := []ChannelDescriptor{ | |||
| {ID: 0x01, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x02, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x03, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x04, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x05, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x06, Priority: 1, MaxSendBytes: 4}, | |||
| } | |||
| peerQueue := newWDRRScheduler(log.NewNopLogger(), NopMetrics(), chDescs, 1000, 1000, 120) | |||
| peerQueue.start() | |||
| totalMsgs := make(map[ChannelID]int) | |||
| deliveredMsgs := make(map[ChannelID]int) | |||
| successRates := make(map[ChannelID]float64) | |||
| closer := tmsync.NewCloser() | |||
| go func() { | |||
| timout := 10 * time.Second | |||
| ticker := time.NewTicker(timout) | |||
| defer ticker.Stop() | |||
| for { | |||
| select { | |||
| case e := <-peerQueue.dequeue(): | |||
| deliveredMsgs[e.channelID]++ | |||
| ticker.Reset(timout) | |||
| case <-ticker.C: | |||
| closer.Close() | |||
| } | |||
| } | |||
| }() | |||
| rng := rand.New(rand.NewSource(time.Now().UnixNano())) | |||
| maxMsgs := 5000 | |||
| minMsgs := 1000 | |||
| for _, chDesc := range chDescs { | |||
| total := rng.Intn(maxMsgs-minMsgs) + minMsgs // total = rand[minMsgs, maxMsgs) | |||
| totalMsgs[ChannelID(chDesc.ID)] = total | |||
| go func(cID ChannelID, n int) { | |||
| for i := 0; i < n; i++ { | |||
| peerQueue.enqueue() <- Envelope{ | |||
| channelID: cID, | |||
| Message: &testMessage{Value: "foo"}, // 5 bytes | |||
| } | |||
| } | |||
| }(ChannelID(chDesc.ID), total) | |||
| } | |||
| // wait for dequeueing to complete | |||
| <-closer.Done() | |||
| // close queue and wait for cleanup | |||
| peerQueue.close() | |||
| <-peerQueue.closed() | |||
| var ( | |||
| sum float64 | |||
| stdDev float64 | |||
| ) | |||
| for _, chDesc := range peerQueue.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| require.Zero(t, peerQueue.deficits[chID], "expected flow deficit to be zero") | |||
| require.Len(t, peerQueue.buffer[chID], 0, "expected flow queue to be empty") | |||
| total := totalMsgs[chID] | |||
| delivered := deliveredMsgs[chID] | |||
| successRate := float64(delivered) / float64(total) | |||
| sum += successRate | |||
| successRates[chID] = successRate | |||
| // require some messages dropped | |||
| require.Less(t, delivered, total, "expected some messages to be dropped") | |||
| require.Less(t, successRate, 1.0, "expected a success rate below 100%") | |||
| } | |||
| require.Zero(t, peerQueue.size, "expected scheduler size to be zero") | |||
| numFlows := float64(len(peerQueue.buffer)) | |||
| mean := sum / numFlows | |||
| for _, successRate := range successRates { | |||
| stdDev += math.Pow(successRate-mean, 2) | |||
| } | |||
| stdDev = math.Sqrt(stdDev / numFlows) | |||
| require.Less(t, stdDev, 0.02, "expected success rate standard deviation to be less than 2%") | |||
| } | |||
| func TestWDRRQueue_DecreasingWeights(t *testing.T) { | |||
| chDescs := []ChannelDescriptor{ | |||
| {ID: 0x01, Priority: 18, MaxSendBytes: 4}, | |||
| {ID: 0x02, Priority: 10, MaxSendBytes: 4}, | |||
| {ID: 0x03, Priority: 2, MaxSendBytes: 4}, | |||
| {ID: 0x04, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x05, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x06, Priority: 1, MaxSendBytes: 4}, | |||
| } | |||
| peerQueue := newWDRRScheduler(log.NewNopLogger(), NopMetrics(), chDescs, 0, 0, 500) | |||
| peerQueue.start() | |||
| totalMsgs := make(map[ChannelID]int) | |||
| deliveredMsgs := make(map[ChannelID]int) | |||
| successRates := make(map[ChannelID]float64) | |||
| for _, chDesc := range chDescs { | |||
| total := 1000 | |||
| totalMsgs[ChannelID(chDesc.ID)] = total | |||
| go func(cID ChannelID, n int) { | |||
| for i := 0; i < n; i++ { | |||
| peerQueue.enqueue() <- Envelope{ | |||
| channelID: cID, | |||
| Message: &testMessage{Value: "foo"}, // 5 bytes | |||
| } | |||
| } | |||
| }(ChannelID(chDesc.ID), total) | |||
| } | |||
| closer := tmsync.NewCloser() | |||
| go func() { | |||
| timout := 20 * time.Second | |||
| ticker := time.NewTicker(timout) | |||
| defer ticker.Stop() | |||
| for { | |||
| select { | |||
| case e := <-peerQueue.dequeue(): | |||
| deliveredMsgs[e.channelID]++ | |||
| ticker.Reset(timout) | |||
| case <-ticker.C: | |||
| closer.Close() | |||
| } | |||
| } | |||
| }() | |||
| // wait for dequeueing to complete | |||
| <-closer.Done() | |||
| // close queue and wait for cleanup | |||
| peerQueue.close() | |||
| <-peerQueue.closed() | |||
| for i, chDesc := range peerQueue.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| require.Zero(t, peerQueue.deficits[chID], "expected flow deficit to be zero") | |||
| require.Len(t, peerQueue.buffer[chID], 0, "expected flow queue to be empty") | |||
| total := totalMsgs[chID] | |||
| delivered := deliveredMsgs[chID] | |||
| successRate := float64(delivered) / float64(total) | |||
| successRates[chID] = successRate | |||
| // Require some messages dropped. Note, the top weighted flows may not have | |||
| // any dropped if lower priority non-empty queues always exist. | |||
| if i > 2 { | |||
| require.Less(t, delivered, total, "expected some messages to be dropped") | |||
| require.Less(t, successRate, 1.0, "expected a success rate below 100%") | |||
| } | |||
| } | |||
| require.Zero(t, peerQueue.size, "expected scheduler size to be zero") | |||
| // require channel 0x01 to have the highest success rate due to its weight | |||
| ch01Rate := successRates[ChannelID(chDescs[0].ID)] | |||
| for i := 1; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch01Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| // require channel 0x02 to have the 2nd highest success rate due to its weight | |||
| ch02Rate := successRates[ChannelID(chDescs[1].ID)] | |||
| for i := 2; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch02Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| // require channel 0x03 to have the 3rd highest success rate due to its weight | |||
| ch03Rate := successRates[ChannelID(chDescs[2].ID)] | |||
| for i := 3; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch03Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| } | |||