Browse Source
p2p: remove wdrr queue (#7064)
This code hasn't been battle tested, and seems to have grown increasingly flaky int tests. Given our general direction of reducing queue complexity over the next couple of releases I think it makes sense to remove it.pull/7062/head
Sam Kleinman
3 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 11 additions and 522 deletions
Split View
Diff Options
-
+1 -0CHANGELOG_PENDING.md
-
+1 -1config/config.go
-
+3 -0internal/p2p/pqueue_test.go
-
+4 -16internal/p2p/router.go
-
+0 -8internal/p2p/router_init_test.go
-
+0 -287internal/p2p/wdrr_queue.go
-
+0 -208internal/p2p/wdrr_queue_test.go
-
+1 -1test/e2e/generator/generate.go
-
+1 -1test/e2e/pkg/testnet.go
+ 1
- 0
CHANGELOG_PENDING.md
View File
+ 1
- 1
config/config.go
View File
+ 3
- 0
internal/p2p/pqueue_test.go
View File
+ 4
- 16
internal/p2p/router.go
View File
+ 0
- 8
internal/p2p/router_init_test.go
View File
+ 0
- 287
internal/p2p/wdrr_queue.go
View File
| @ -1,287 +0,0 @@ | |||
| package p2p | |||
| import ( | |||
| "fmt" | |||
| "sort" | |||
| "strconv" | |||
| "github.com/gogo/protobuf/proto" | |||
| tmsync "github.com/tendermint/tendermint/internal/libs/sync" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| ) | |||
| // wrappedEnvelope wraps a p2p Envelope with its precomputed size. | |||
| type wrappedEnvelope struct { | |||
| envelope Envelope | |||
| size uint | |||
| } | |||
| // assert the WDRR scheduler implements the queue interface at compile-time | |||
| var _ queue = (*wdrrScheduler)(nil) | |||
| // wdrrQueue implements a Weighted Deficit Round Robin (WDRR) scheduling | |||
| // algorithm via the queue interface. A WDRR queue is created per peer, where | |||
| // the queue will have N number of flows. Each flow corresponds to a p2p Channel, | |||
| // so there are n input flows and a single output source, the peer's connection. | |||
| // | |||
| // The WDRR scheduler contains a shared buffer with a fixed capacity. | |||
| // | |||
| // Each flow has the following: | |||
| // - quantum: The number of bytes that is added to the deficit counter of the | |||
| // flow in each round. The flow can send at most quantum bytes at a time. Each | |||
| // flow has its own unique quantum, which gives the queue its weighted nature. | |||
| // A higher quantum corresponds to a higher weight/priority. The quantum is | |||
| // computed as MaxSendBytes * Priority. | |||
| // - deficit counter: The number of bytes that the flow is allowed to transmit | |||
| // when it is its turn. | |||
| // | |||
| // See: https://en.wikipedia.org/wiki/Deficit_round_robin | |||
| type wdrrScheduler struct { | |||
| logger log.Logger | |||
| metrics *Metrics | |||
| chDescs []ChannelDescriptor | |||
| capacity uint | |||
| size uint | |||
| chPriorities map[ChannelID]uint | |||
| buffer map[ChannelID][]wrappedEnvelope | |||
| quanta map[ChannelID]uint | |||
| deficits map[ChannelID]uint | |||
| closer *tmsync.Closer | |||
| doneCh *tmsync.Closer | |||
| enqueueCh chan Envelope | |||
| dequeueCh chan Envelope | |||
| } | |||
| func newWDRRScheduler( | |||
| logger log.Logger, | |||
| m *Metrics, | |||
| chDescs []ChannelDescriptor, | |||
| enqueueBuf, dequeueBuf, capacity uint, | |||
| ) *wdrrScheduler { | |||
| // copy each ChannelDescriptor and sort them by channel priority | |||
| chDescsCopy := make([]ChannelDescriptor, len(chDescs)) | |||
| copy(chDescsCopy, chDescs) | |||
| sort.Slice(chDescsCopy, func(i, j int) bool { return chDescsCopy[i].Priority > chDescsCopy[j].Priority }) | |||
| var ( | |||
| buffer = make(map[ChannelID][]wrappedEnvelope) | |||
| chPriorities = make(map[ChannelID]uint) | |||
| quanta = make(map[ChannelID]uint) | |||
| deficits = make(map[ChannelID]uint) | |||
| ) | |||
| for _, chDesc := range chDescsCopy { | |||
| chID := ChannelID(chDesc.ID) | |||
| chPriorities[chID] = uint(chDesc.Priority) | |||
| buffer[chID] = make([]wrappedEnvelope, 0) | |||
| quanta[chID] = chDesc.MaxSendBytes * uint(chDesc.Priority) | |||
| } | |||
| return &wdrrScheduler{ | |||
| logger: logger.With("queue", "wdrr"), | |||
| metrics: m, | |||
| capacity: capacity, | |||
| chPriorities: chPriorities, | |||
| chDescs: chDescsCopy, | |||
| buffer: buffer, | |||
| quanta: quanta, | |||
| deficits: deficits, | |||
| closer: tmsync.NewCloser(), | |||
| doneCh: tmsync.NewCloser(), | |||
| enqueueCh: make(chan Envelope, enqueueBuf), | |||
| dequeueCh: make(chan Envelope, dequeueBuf), | |||
| } | |||
| } | |||
| // enqueue returns an unbuffered write-only channel which a producer can send on. | |||
| func (s *wdrrScheduler) enqueue() chan<- Envelope { | |||
| return s.enqueueCh | |||
| } | |||
| // dequeue returns an unbuffered read-only channel which a consumer can read from. | |||
| func (s *wdrrScheduler) dequeue() <-chan Envelope { | |||
| return s.dequeueCh | |||
| } | |||
| func (s *wdrrScheduler) closed() <-chan struct{} { | |||
| return s.closer.Done() | |||
| } | |||
| // close closes the WDRR queue. After this call enqueue() will block, so the | |||
| // caller must select on closed() as well to avoid blocking forever. The | |||
| // enqueue() and dequeue() along with the internal channels will NOT be closed. | |||
| // Note, close() will block until all externally spawned goroutines have exited. | |||
| func (s *wdrrScheduler) close() { | |||
| s.closer.Close() | |||
| <-s.doneCh.Done() | |||
| } | |||
| // start starts the WDRR queue process in a blocking goroutine. This must be | |||
| // called before the queue can start to process and accept Envelopes. | |||
| func (s *wdrrScheduler) start() { | |||
| go s.process() | |||
| } | |||
| // process starts a blocking WDRR scheduler process, where we continuously | |||
| // evaluate if we need to attempt to enqueue an Envelope or schedule Envelopes | |||
| // to be dequeued and subsequently read and sent on the source connection. | |||
| // Internally, each p2p Channel maps to a flow, where each flow has a deficit | |||
| // and a quantum. | |||
| // | |||
| // For each Envelope requested to be enqueued, we evaluate if there is sufficient | |||
| // capacity in the shared buffer to add the Envelope. If so, it is added. | |||
| // Otherwise, we evaluate all flows of lower priority where we attempt find an | |||
| // existing Envelope in the shared buffer of sufficient size that can be dropped | |||
| // in place of the incoming Envelope. If there is no such Envelope that can be | |||
| // dropped, then the incoming Envelope is dropped. | |||
| // | |||
| // When there is nothing to be enqueued, we perform the WDRR algorithm and | |||
| // determine which Envelopes can be dequeued. For each Envelope that can be | |||
| // dequeued, it is sent on the dequeueCh. Specifically, for each flow, if it is | |||
| // non-empty, its deficit counter is incremented by its quantum value. Then, the | |||
| // value of the deficit counter is a maximal amount of bytes that can be sent at | |||
| // this round. If the deficit counter is greater than the Envelopes's message | |||
| // size at the head of the queue (HoQ), this envelope can be sent and the value | |||
| // of the counter is decremented by the message's size. Then, the size of the | |||
| // next Envelopes's message is compared to the counter value, etc. Once the flow | |||
| // is empty or the value of the counter is insufficient, the scheduler will skip | |||
| // to the next flow. If the flow is empty, the value of the deficit counter is | |||
| // reset to 0. | |||
| // | |||
| // XXX/TODO: Evaluate the single goroutine scheduler mechanism. In other words, | |||
| // evaluate the effectiveness and performance of having a single goroutine | |||
| // perform handling both enqueueing and dequeueing logic. Specifically, there | |||
| // is potentially contention between reading off of enqueueCh and trying to | |||
| // enqueue while also attempting to perform the WDRR algorithm and find the next | |||
| // set of Envelope(s) to send on the dequeueCh. Alternatively, we could consider | |||
| // separate scheduling goroutines, but then that requires the use of mutexes and | |||
| // possibly a degrading performance. | |||
| func (s *wdrrScheduler) process() { | |||
| defer s.doneCh.Close() | |||
| for { | |||
| select { | |||
| case <-s.closer.Done(): | |||
| return | |||
| case e := <-s.enqueueCh: | |||
| // attempt to enqueue the incoming Envelope | |||
| chIDStr := strconv.Itoa(int(e.channelID)) | |||
| wEnv := wrappedEnvelope{envelope: e, size: uint(proto.Size(e.Message))} | |||
| msgSize := wEnv.size | |||
| s.metrics.PeerPendingSendBytes.With("peer_id", string(e.To)).Add(float64(msgSize)) | |||
| // If we're at capacity, we need to either drop the incoming Envelope or | |||
| // an Envelope from a lower priority flow. Otherwise, we add the (wrapped) | |||
| // envelope to the flow's queue. | |||
| if s.size+wEnv.size > s.capacity { | |||
| chPriority := s.chPriorities[e.channelID] | |||
| var ( | |||
| canDrop bool | |||
| dropIdx int | |||
| dropChID ChannelID | |||
| ) | |||
| // Evaluate all lower priority flows and determine if there exists an | |||
| // Envelope that is of equal or greater size that we can drop in favor | |||
| // of the incoming Envelope. | |||
| for i := len(s.chDescs) - 1; i >= 0 && uint(s.chDescs[i].Priority) < chPriority && !canDrop; i-- { | |||
| currChID := ChannelID(s.chDescs[i].ID) | |||
| flow := s.buffer[currChID] | |||
| for j := 0; j < len(flow) && !canDrop; j++ { | |||
| if flow[j].size >= wEnv.size { | |||
| canDrop = true | |||
| dropIdx = j | |||
| dropChID = currChID | |||
| break | |||
| } | |||
| } | |||
| } | |||
| // If we can drop an existing Envelope, drop it and enqueue the incoming | |||
| // Envelope. | |||
| if canDrop { | |||
| chIDStr = strconv.Itoa(int(dropChID)) | |||
| chPriority = s.chPriorities[dropChID] | |||
| msgSize = s.buffer[dropChID][dropIdx].size | |||
| // Drop Envelope for the lower priority flow and update the queue's | |||
| // buffer size | |||
| s.size -= msgSize | |||
| s.buffer[dropChID] = append(s.buffer[dropChID][:dropIdx], s.buffer[dropChID][dropIdx+1:]...) | |||
| // add the incoming Envelope and update queue's buffer size | |||
| s.size += wEnv.size | |||
| s.buffer[e.channelID] = append(s.buffer[e.channelID], wEnv) | |||
| s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Set(float64(wEnv.size)) | |||
| } | |||
| // We either dropped the incoming Enevelope or one from an existing | |||
| // lower priority flow. | |||
| s.metrics.PeerQueueDroppedMsgs.With("ch_id", chIDStr).Add(1) | |||
| s.logger.Debug( | |||
| "dropped envelope", | |||
| "ch_id", chIDStr, | |||
| "priority", chPriority, | |||
| "capacity", s.capacity, | |||
| "msg_size", msgSize, | |||
| ) | |||
| } else { | |||
| // we have sufficient capacity to enqueue the incoming Envelope | |||
| s.metrics.PeerQueueMsgSize.With("ch_id", chIDStr).Set(float64(wEnv.size)) | |||
| s.buffer[e.channelID] = append(s.buffer[e.channelID], wEnv) | |||
| s.size += wEnv.size | |||
| } | |||
| default: | |||
| // perform the WDRR algorithm | |||
| for _, chDesc := range s.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| // only consider non-empty flows | |||
| if len(s.buffer[chID]) > 0 { | |||
| // bump flow's quantum | |||
| s.deficits[chID] += s.quanta[chID] | |||
| // grab the flow's current deficit counter and HoQ (wrapped) Envelope | |||
| d := s.deficits[chID] | |||
| we := s.buffer[chID][0] | |||
| // While the flow is non-empty and we can send the current Envelope | |||
| // on the dequeueCh: | |||
| // | |||
| // 1. send the Envelope | |||
| // 2. update the scheduler's shared buffer's size | |||
| // 3. update the flow's deficit | |||
| // 4. remove from the flow's queue | |||
| // 5. grab the next HoQ Envelope and flow's deficit | |||
| for len(s.buffer[chID]) > 0 && d >= we.size { | |||
| s.metrics.PeerSendBytesTotal.With( | |||
| "chID", fmt.Sprint(chID), | |||
| "peer_id", string(we.envelope.To)).Add(float64(we.size)) | |||
| s.dequeueCh <- we.envelope | |||
| s.size -= we.size | |||
| s.deficits[chID] -= we.size | |||
| s.buffer[chID] = s.buffer[chID][1:] | |||
| if len(s.buffer[chID]) > 0 { | |||
| d = s.deficits[chID] | |||
| we = s.buffer[chID][0] | |||
| } | |||
| } | |||
| } | |||
| // reset the flow's deficit to zero if it is empty | |||
| if len(s.buffer[chID]) == 0 { | |||
| s.deficits[chID] = 0 | |||
| } | |||
| } | |||
| } | |||
| } | |||
| } | |||
+ 0
- 208
internal/p2p/wdrr_queue_test.go
View File
| @ -1,208 +0,0 @@ | |||
| package p2p | |||
| import ( | |||
| "math" | |||
| "math/rand" | |||
| "testing" | |||
| "time" | |||
| gogotypes "github.com/gogo/protobuf/types" | |||
| "github.com/stretchr/testify/require" | |||
| tmsync "github.com/tendermint/tendermint/internal/libs/sync" | |||
| "github.com/tendermint/tendermint/libs/log" | |||
| ) | |||
| type testMessage = gogotypes.StringValue | |||
| func TestWDRRQueue_EqualWeights(t *testing.T) { | |||
| chDescs := []ChannelDescriptor{ | |||
| {ID: 0x01, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x02, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x03, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x04, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x05, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x06, Priority: 1, MaxSendBytes: 4}, | |||
| } | |||
| peerQueue := newWDRRScheduler(log.NewNopLogger(), NopMetrics(), chDescs, 1000, 1000, 120) | |||
| peerQueue.start() | |||
| totalMsgs := make(map[ChannelID]int) | |||
| deliveredMsgs := make(map[ChannelID]int) | |||
| successRates := make(map[ChannelID]float64) | |||
| closer := tmsync.NewCloser() | |||
| go func() { | |||
| timout := 10 * time.Second | |||
| ticker := time.NewTicker(timout) | |||
| defer ticker.Stop() | |||
| for { | |||
| select { | |||
| case e := <-peerQueue.dequeue(): | |||
| deliveredMsgs[e.channelID]++ | |||
| ticker.Reset(timout) | |||
| case <-ticker.C: | |||
| closer.Close() | |||
| } | |||
| } | |||
| }() | |||
| rng := rand.New(rand.NewSource(time.Now().UnixNano())) | |||
| maxMsgs := 5000 | |||
| minMsgs := 1000 | |||
| for _, chDesc := range chDescs { | |||
| total := rng.Intn(maxMsgs-minMsgs) + minMsgs // total = rand[minMsgs, maxMsgs) | |||
| totalMsgs[ChannelID(chDesc.ID)] = total | |||
| go func(cID ChannelID, n int) { | |||
| for i := 0; i < n; i++ { | |||
| peerQueue.enqueue() <- Envelope{ | |||
| channelID: cID, | |||
| Message: &testMessage{Value: "foo"}, // 5 bytes | |||
| } | |||
| } | |||
| }(ChannelID(chDesc.ID), total) | |||
| } | |||
| // wait for dequeueing to complete | |||
| <-closer.Done() | |||
| // close queue and wait for cleanup | |||
| peerQueue.close() | |||
| <-peerQueue.closed() | |||
| var ( | |||
| sum float64 | |||
| stdDev float64 | |||
| ) | |||
| for _, chDesc := range peerQueue.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| require.Zero(t, peerQueue.deficits[chID], "expected flow deficit to be zero") | |||
| require.Len(t, peerQueue.buffer[chID], 0, "expected flow queue to be empty") | |||
| total := totalMsgs[chID] | |||
| delivered := deliveredMsgs[chID] | |||
| successRate := float64(delivered) / float64(total) | |||
| sum += successRate | |||
| successRates[chID] = successRate | |||
| // require some messages dropped | |||
| require.Less(t, delivered, total, "expected some messages to be dropped") | |||
| require.Less(t, successRate, 1.0, "expected a success rate below 100%") | |||
| } | |||
| require.Zero(t, peerQueue.size, "expected scheduler size to be zero") | |||
| numFlows := float64(len(peerQueue.buffer)) | |||
| mean := sum / numFlows | |||
| for _, successRate := range successRates { | |||
| stdDev += math.Pow(successRate-mean, 2) | |||
| } | |||
| stdDev = math.Sqrt(stdDev / numFlows) | |||
| require.Less(t, stdDev, 0.02, "expected success rate standard deviation to be less than 2%") | |||
| } | |||
| func TestWDRRQueue_DecreasingWeights(t *testing.T) { | |||
| chDescs := []ChannelDescriptor{ | |||
| {ID: 0x01, Priority: 18, MaxSendBytes: 4}, | |||
| {ID: 0x02, Priority: 10, MaxSendBytes: 4}, | |||
| {ID: 0x03, Priority: 2, MaxSendBytes: 4}, | |||
| {ID: 0x04, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x05, Priority: 1, MaxSendBytes: 4}, | |||
| {ID: 0x06, Priority: 1, MaxSendBytes: 4}, | |||
| } | |||
| peerQueue := newWDRRScheduler(log.NewNopLogger(), NopMetrics(), chDescs, 0, 0, 500) | |||
| peerQueue.start() | |||
| totalMsgs := make(map[ChannelID]int) | |||
| deliveredMsgs := make(map[ChannelID]int) | |||
| successRates := make(map[ChannelID]float64) | |||
| for _, chDesc := range chDescs { | |||
| total := 1000 | |||
| totalMsgs[ChannelID(chDesc.ID)] = total | |||
| go func(cID ChannelID, n int) { | |||
| for i := 0; i < n; i++ { | |||
| peerQueue.enqueue() <- Envelope{ | |||
| channelID: cID, | |||
| Message: &testMessage{Value: "foo"}, // 5 bytes | |||
| } | |||
| } | |||
| }(ChannelID(chDesc.ID), total) | |||
| } | |||
| closer := tmsync.NewCloser() | |||
| go func() { | |||
| timout := 20 * time.Second | |||
| ticker := time.NewTicker(timout) | |||
| defer ticker.Stop() | |||
| for { | |||
| select { | |||
| case e := <-peerQueue.dequeue(): | |||
| deliveredMsgs[e.channelID]++ | |||
| ticker.Reset(timout) | |||
| case <-ticker.C: | |||
| closer.Close() | |||
| } | |||
| } | |||
| }() | |||
| // wait for dequeueing to complete | |||
| <-closer.Done() | |||
| // close queue and wait for cleanup | |||
| peerQueue.close() | |||
| <-peerQueue.closed() | |||
| for i, chDesc := range peerQueue.chDescs { | |||
| chID := ChannelID(chDesc.ID) | |||
| require.Zero(t, peerQueue.deficits[chID], "expected flow deficit to be zero") | |||
| require.Len(t, peerQueue.buffer[chID], 0, "expected flow queue to be empty") | |||
| total := totalMsgs[chID] | |||
| delivered := deliveredMsgs[chID] | |||
| successRate := float64(delivered) / float64(total) | |||
| successRates[chID] = successRate | |||
| // Require some messages dropped. Note, the top weighted flows may not have | |||
| // any dropped if lower priority non-empty queues always exist. | |||
| if i > 2 { | |||
| require.Less(t, delivered, total, "expected some messages to be dropped") | |||
| require.Less(t, successRate, 1.0, "expected a success rate below 100%") | |||
| } | |||
| } | |||
| require.Zero(t, peerQueue.size, "expected scheduler size to be zero") | |||
| // require channel 0x01 to have the highest success rate due to its weight | |||
| ch01Rate := successRates[ChannelID(chDescs[0].ID)] | |||
| for i := 1; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch01Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| // require channel 0x02 to have the 2nd highest success rate due to its weight | |||
| ch02Rate := successRates[ChannelID(chDescs[1].ID)] | |||
| for i := 2; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch02Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| // require channel 0x03 to have the 3rd highest success rate due to its weight | |||
| ch03Rate := successRates[ChannelID(chDescs[2].ID)] | |||
| for i := 3; i < len(chDescs); i++ { | |||
| require.GreaterOrEqual(t, ch03Rate, successRates[ChannelID(chDescs[i].ID)]) | |||
| } | |||
| } | |||