From 1dec3e139aae913bb26c52e1e875401db842dd0f Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Tue, 6 Jul 2021 14:26:18 -0400 Subject: [PATCH] add stacktrace to panic logs (#6662) --- internal/blockchain/v0/reactor.go | 7 ++++++- internal/consensus/reactor.go | 7 ++++++- internal/evidence/reactor.go | 12 +++++++++++- internal/mempool/v0/reactor.go | 12 +++++++++++- internal/mempool/v1/reactor.go | 12 +++++++++++- internal/p2p/pex/reactor.go | 6 ++++++ internal/statesync/reactor.go | 6 ++++++ 7 files changed, 57 insertions(+), 5 deletions(-) diff --git a/internal/blockchain/v0/reactor.go b/internal/blockchain/v0/reactor.go index 444dfff56..9a9584857 100644 --- a/internal/blockchain/v0/reactor.go +++ b/internal/blockchain/v0/reactor.go @@ -2,6 +2,7 @@ package v0 import ( "fmt" + "runtime/debug" "sync" "time" @@ -265,7 +266,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) - r.Logger.Error("recovering from processing message panic", "err", err) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 59a1b899d..ef3494e32 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -2,6 +2,7 @@ package consensus import ( "fmt" + "runtime/debug" "time" cstypes "github.com/tendermint/tendermint/internal/consensus/types" @@ -1207,7 +1208,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) - r.Logger.Error("recovering from processing message panic", "err", err) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/evidence/reactor.go b/internal/evidence/reactor.go index 57202d5e8..a454038fd 100644 --- a/internal/evidence/reactor.go +++ b/internal/evidence/reactor.go @@ -2,6 +2,7 @@ package evidence import ( "fmt" + "runtime/debug" "sync" "time" @@ -165,6 +166,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() @@ -296,7 +302,11 @@ func (r *Reactor) broadcastEvidenceLoop(peerID types.NodeID, closer *tmsync.Clos r.peerWG.Done() if e := recover(); e != nil { - r.Logger.Error("recovering from broadcasting evidence loop", "err", e) + r.Logger.Error( + "recovering from broadcasting evidence loop", + "err", e, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/mempool/v0/reactor.go b/internal/mempool/v0/reactor.go index 5bfff0efe..29dec5833 100644 --- a/internal/mempool/v0/reactor.go +++ b/internal/mempool/v0/reactor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "runtime/debug" "sync" "time" @@ -188,6 +189,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() @@ -312,7 +318,11 @@ func (r *Reactor) broadcastTxRoutine(peerID types.NodeID, closer *tmsync.Closer) r.peerWG.Done() if e := recover(); e != nil { - r.Logger.Error("recovering from broadcasting mempool loop", "err", e) + r.Logger.Error( + "recovering from broadcasting mempool loop", + "err", e, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/mempool/v1/reactor.go b/internal/mempool/v1/reactor.go index 6fdfe2d06..436dd9d27 100644 --- a/internal/mempool/v1/reactor.go +++ b/internal/mempool/v1/reactor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "runtime/debug" "sync" "time" @@ -187,6 +188,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() @@ -311,7 +317,11 @@ func (r *Reactor) broadcastTxRoutine(peerID types.NodeID, closer *tmsync.Closer) r.peerWG.Done() if e := recover(); e != nil { - r.Logger.Error("recovering from broadcasting mempool loop", "err", e) + r.Logger.Error( + "recovering from broadcasting mempool loop", + "err", e, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/p2p/pex/reactor.go b/internal/p2p/pex/reactor.go index 4a8738fd8..8cff2f95b 100644 --- a/internal/p2p/pex/reactor.go +++ b/internal/p2p/pex/reactor.go @@ -3,6 +3,7 @@ package pex import ( "context" "fmt" + "runtime/debug" "sync" "time" @@ -367,6 +368,11 @@ func (r *ReactorV2) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (er defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }() diff --git a/internal/statesync/reactor.go b/internal/statesync/reactor.go index 6608fd763..4e7ba91d3 100644 --- a/internal/statesync/reactor.go +++ b/internal/statesync/reactor.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "runtime/debug" "sort" "time" @@ -644,6 +645,11 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err defer func() { if e := recover(); e != nil { err = fmt.Errorf("panic in processing message: %v", e) + r.Logger.Error( + "recovering from processing message panic", + "err", err, + "stack", string(debug.Stack()), + ) } }()