|
@ -0,0 +1,225 @@ |
|
|
|
|
|
package main |
|
|
|
|
|
|
|
|
|
|
|
import ( |
|
|
|
|
|
"bytes" |
|
|
|
|
|
"encoding/json" |
|
|
|
|
|
"fmt" |
|
|
|
|
|
"os" |
|
|
|
|
|
"reflect" |
|
|
|
|
|
"sync" |
|
|
|
|
|
"time" |
|
|
|
|
|
|
|
|
|
|
|
. "github.com/tendermint/go-common" |
|
|
|
|
|
"github.com/tendermint/go-crypto" |
|
|
|
|
|
|
|
|
|
|
|
// register rpc and event types with go-
|
|
|
|
|
|
ctypes "github.com/tendermint/tendermint/rpc/core/types" |
|
|
|
|
|
// "github.com/tendermint/tendermint/types"
|
|
|
|
|
|
client "github.com/tendermint/tendermint/rpc/client" |
|
|
|
|
|
|
|
|
|
|
|
"github.com/gorilla/websocket" |
|
|
|
|
|
"github.com/rcrowley/go-metrics" |
|
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
//------------------------------------------------------
|
|
|
|
|
|
// Connect to all validators for a blockchain
|
|
|
|
|
|
|
|
|
|
|
|
type Blockchain struct { |
|
|
|
|
|
ID string |
|
|
|
|
|
Validators []Validator |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
type Validator struct { |
|
|
|
|
|
ID string |
|
|
|
|
|
PubKey crypto.PubKey |
|
|
|
|
|
IP string |
|
|
|
|
|
Port int |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
//------------------------------------------------------
|
|
|
|
|
|
// Generic system to subscribe to events and record their frequency
|
|
|
|
|
|
|
|
|
|
|
|
// Metrics for a given event
|
|
|
|
|
|
type EventMetric struct { |
|
|
|
|
|
ID string `json:"id"` |
|
|
|
|
|
Started time.Time `json:"start_time"` |
|
|
|
|
|
LastHeard time.Time `json:"last_heard"` |
|
|
|
|
|
MinDuration int64 `json:"min_duration"` |
|
|
|
|
|
MaxDuration int64 `json:"max_duration"` |
|
|
|
|
|
|
|
|
|
|
|
// tracks event count and rate
|
|
|
|
|
|
meter metrics.Meter |
|
|
|
|
|
|
|
|
|
|
|
// filled in from the Meter
|
|
|
|
|
|
Count int64 `json:"count"` |
|
|
|
|
|
Rate1 float64 `json:"rate_1"` |
|
|
|
|
|
Rate5 float64 `json:"rate_5"` |
|
|
|
|
|
Rate15 float64 `json:"rate_15"` |
|
|
|
|
|
RateMean float64 `json:"rate_mean"` |
|
|
|
|
|
|
|
|
|
|
|
// XXX: move this
|
|
|
|
|
|
// latency for node itself (not related to event)
|
|
|
|
|
|
Latency float64 `json:"latency_mean"` |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// Each node gets an event meter to track events for that node
|
|
|
|
|
|
type EventMeter struct { |
|
|
|
|
|
QuitService |
|
|
|
|
|
|
|
|
|
|
|
wsc *client.WSClient |
|
|
|
|
|
|
|
|
|
|
|
mtx sync.Mutex |
|
|
|
|
|
events map[string]*EventMetric |
|
|
|
|
|
|
|
|
|
|
|
// to record latency
|
|
|
|
|
|
timer metrics.Timer |
|
|
|
|
|
lastPing time.Time |
|
|
|
|
|
receivedPong bool |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func NewEventMeter(addr string) *EventMeter { |
|
|
|
|
|
em := &EventMeter{ |
|
|
|
|
|
wsc: client.NewWSClient(addr), |
|
|
|
|
|
events: make(map[string]*EventMetric), |
|
|
|
|
|
timer: metrics.NewTimer(), |
|
|
|
|
|
receivedPong: true, |
|
|
|
|
|
} |
|
|
|
|
|
em.QuitService = *NewQuitService(nil, "EventMeter", em) |
|
|
|
|
|
return em |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func (em *EventMeter) OnStart() error { |
|
|
|
|
|
em.QuitService.OnStart() |
|
|
|
|
|
if err := em.wsc.OnStart(); err != nil { |
|
|
|
|
|
return err |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
em.wsc.Conn.SetPongHandler(func(m string) error { |
|
|
|
|
|
// NOTE: https://github.com/gorilla/websocket/issues/97
|
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
defer em.mtx.Unlock() |
|
|
|
|
|
em.receivedPong = true |
|
|
|
|
|
em.timer.UpdateSince(em.lastPing) |
|
|
|
|
|
return nil |
|
|
|
|
|
}) |
|
|
|
|
|
go em.receiveRoutine() |
|
|
|
|
|
return nil |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func (em *EventMeter) OnStop() { |
|
|
|
|
|
em.wsc.OnStop() |
|
|
|
|
|
em.QuitService.OnStop() |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func (em *EventMeter) Subscribe(eventid string) error { |
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
defer em.mtx.Unlock() |
|
|
|
|
|
|
|
|
|
|
|
if _, ok := em.events[eventid]; ok { |
|
|
|
|
|
return fmt.Errorf("Subscription already exists") |
|
|
|
|
|
} |
|
|
|
|
|
if err := em.wsc.Subscribe(eventid); err != nil { |
|
|
|
|
|
return err |
|
|
|
|
|
} |
|
|
|
|
|
em.events[eventid] = &EventMetric{ |
|
|
|
|
|
Started: time.Now(), |
|
|
|
|
|
MinDuration: 1 << 62, |
|
|
|
|
|
meter: metrics.NewMeter(), |
|
|
|
|
|
} |
|
|
|
|
|
return nil |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
func (em *EventMeter) Unsubscribe(eventid string) error { |
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
defer em.mtx.Unlock() |
|
|
|
|
|
|
|
|
|
|
|
if err := em.wsc.Unsubscribe(eventid); err != nil { |
|
|
|
|
|
return err |
|
|
|
|
|
} |
|
|
|
|
|
// XXX: should we persist or save this info first?
|
|
|
|
|
|
delete(em.events, eventid) |
|
|
|
|
|
return nil |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
//------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
func (em *EventMeter) receiveRoutine() { |
|
|
|
|
|
logTicker := time.NewTicker(time.Second * 3) |
|
|
|
|
|
pingTicker := time.NewTicker(time.Second * 1) |
|
|
|
|
|
for { |
|
|
|
|
|
select { |
|
|
|
|
|
case <-logTicker.C: |
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
for _, metric := range em.events { |
|
|
|
|
|
metric.Count = metric.meter.Count() |
|
|
|
|
|
metric.Rate1 = metric.meter.Rate1() |
|
|
|
|
|
metric.Rate5 = metric.meter.Rate5() |
|
|
|
|
|
metric.Rate15 = metric.meter.Rate15() |
|
|
|
|
|
metric.RateMean = metric.meter.RateMean() |
|
|
|
|
|
|
|
|
|
|
|
metric.Latency = em.timer.Mean() |
|
|
|
|
|
|
|
|
|
|
|
b, err := json.Marshal(metric) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
// TODO
|
|
|
|
|
|
log.Error(err.Error()) |
|
|
|
|
|
continue |
|
|
|
|
|
} |
|
|
|
|
|
var out bytes.Buffer |
|
|
|
|
|
json.Indent(&out, b, "", "\t") |
|
|
|
|
|
out.WriteTo(os.Stdout) |
|
|
|
|
|
} |
|
|
|
|
|
em.mtx.Unlock() |
|
|
|
|
|
case <-pingTicker.C: |
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
|
|
|
|
|
|
// ping to record latency
|
|
|
|
|
|
if !em.receivedPong { |
|
|
|
|
|
// XXX: why is the pong taking so long? should we stop the conn?
|
|
|
|
|
|
em.mtx.Unlock() |
|
|
|
|
|
continue |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
em.lastPing = time.Now() |
|
|
|
|
|
em.receivedPong = false |
|
|
|
|
|
err := em.wsc.Conn.WriteMessage(websocket.PingMessage, []byte{}) |
|
|
|
|
|
if err != nil { |
|
|
|
|
|
log.Error("Failed to write ping message on websocket", "error", err) |
|
|
|
|
|
em.wsc.Stop() |
|
|
|
|
|
return |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
em.mtx.Unlock() |
|
|
|
|
|
|
|
|
|
|
|
case r := <-em.wsc.ResultsCh: |
|
|
|
|
|
em.mtx.Lock() |
|
|
|
|
|
switch r := r.(type) { |
|
|
|
|
|
case *ctypes.ResultEvent: |
|
|
|
|
|
id, _ := r.Event, r.Data |
|
|
|
|
|
metric, ok := em.events[id] |
|
|
|
|
|
if !ok { |
|
|
|
|
|
// we already unsubscribed, or got an unexpected event
|
|
|
|
|
|
continue |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
last := metric.LastHeard |
|
|
|
|
|
metric.LastHeard = time.Now() |
|
|
|
|
|
metric.meter.Mark(1) |
|
|
|
|
|
dur := int64(metric.LastHeard.Sub(last)) |
|
|
|
|
|
if dur < metric.MinDuration { |
|
|
|
|
|
metric.MinDuration = dur |
|
|
|
|
|
} |
|
|
|
|
|
if !last.IsZero() && dur > metric.MaxDuration { |
|
|
|
|
|
metric.MaxDuration = dur |
|
|
|
|
|
} |
|
|
|
|
|
default: |
|
|
|
|
|
log.Error("Unknown result event type", "type", reflect.TypeOf(r)) |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
em.mtx.Unlock() |
|
|
|
|
|
case <-em.Quit: |
|
|
|
|
|
break |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
} |