You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

750 lines
23 KiB

package evidence
import (
"fmt"
"sync"
"time"
"github.com/gogo/protobuf/proto"
gogotypes "github.com/gogo/protobuf/types"
dbm "github.com/tendermint/tm-db"
clist "github.com/tendermint/tendermint/libs/clist"
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
sm "github.com/tendermint/tendermint/state"
"github.com/tendermint/tendermint/store"
"github.com/tendermint/tendermint/types"
)
const (
baseKeyCommitted = byte(0x00)
baseKeyPending = byte(0x01)
baseKeyPOLC = byte(0x02)
baseKeyAwaiting = byte(0x03)
)
// Pool maintains a pool of valid evidence to be broadcasted and committed
type Pool struct {
logger log.Logger
evidenceStore dbm.DB
evidenceList *clist.CList // concurrent linked-list of evidence
// needed to load validators to verify evidence
stateDB dbm.DB
// needed to load headers to verify evidence
blockStore *store.BlockStore
mtx sync.Mutex
// latest state
state sm.State
// a map of active validators and respective last heights validator is active
// if it was in validator set after EvidenceParams.MaxAgeNumBlocks or
// currently is (ie. [MaxAgeNumBlocks, CurrentHeight])
// In simple words, it means it's still bonded -> therefore slashable.
valToLastHeight valToLastHeightMap
nextEvidenceTrialEndedHeight int64
}
// Validator.Address -> Last height it was in validator set
type valToLastHeightMap map[string]int64
func NewPool(stateDB, evidenceDB dbm.DB, blockStore *store.BlockStore) (*Pool, error) {
var (
state = sm.LoadState(stateDB)
)
valToLastHeight, err := buildValToLastHeightMap(state, stateDB, blockStore)
if err != nil {
return nil, err
}
pool := &Pool{
stateDB: stateDB,
blockStore: blockStore,
state: state,
logger: log.NewNopLogger(),
evidenceStore: evidenceDB,
evidenceList: clist.New(),
valToLastHeight: valToLastHeight,
nextEvidenceTrialEndedHeight: -1,
}
// if pending evidence already in db, in event of prior failure, then load it back to the evidenceList
evList := pool.AllPendingEvidence()
for _, ev := range evList {
pool.evidenceList.PushBack(ev)
}
return pool, nil
}
// PendingEvidence is used primarily as part of block proposal and returns up to maxNum of uncommitted evidence.
// If maxNum is -1, all evidence is returned. Pending evidence is prioritised based on time.
func (evpool *Pool) PendingEvidence(maxNum uint32) []types.Evidence {
evpool.removeExpiredPendingEvidence()
evidence, err := evpool.listEvidence(baseKeyPending, int64(maxNum))
if err != nil {
evpool.logger.Error("Unable to retrieve pending evidence", "err", err)
}
return evidence
}
func (evpool *Pool) AllPendingEvidence() []types.Evidence {
evpool.removeExpiredPendingEvidence()
evidence, err := evpool.listEvidence(baseKeyPending, -1)
if err != nil {
evpool.logger.Error("Unable to retrieve pending evidence", "err", err)
}
return evidence
}
// Update uses the latest block & state to update its copy of the state,
// validator to last height map and calls MarkEvidenceAsCommitted.
func (evpool *Pool) Update(block *types.Block, state sm.State) {
// sanity check
if state.LastBlockHeight != block.Height {
panic(fmt.Sprintf("Failed EvidencePool.Update sanity check: got state.Height=%d with block.Height=%d",
state.LastBlockHeight,
block.Height,
),
)
}
// remove evidence from pending and mark committed
evpool.MarkEvidenceAsCommitted(block.Height, block.Evidence.Evidence)
// prune pending, committed and potential evidence and polc's periodically
if block.Height%state.ConsensusParams.Evidence.MaxAgeNumBlocks == 0 {
evpool.pruneExpiredPOLC()
evpool.removeExpiredPendingEvidence()
}
if evpool.nextEvidenceTrialEndedHeight > 0 && block.Height < evpool.nextEvidenceTrialEndedHeight {
evpool.upgradePotentialAmnesiaEvidence()
}
// update the state
evpool.mtx.Lock()
defer evpool.mtx.Unlock()
evpool.state = state
evpool.updateValToLastHeight(block.Height, state)
}
// AddPOLC adds a proof of lock change to the evidence database
// that may be needed in the future to verify votes
func (evpool *Pool) AddPOLC(polc types.ProofOfLockChange) error {
key := keyPOLC(polc)
pbplc, err := polc.ToProto()
if err != nil {
return err
}
polcBytes, err := proto.Marshal(pbplc)
if err != nil {
return fmt.Errorf("addPOLC: unable to marshal ProofOfLockChange: %w", err)
}
return evpool.evidenceStore.Set(key, polcBytes)
}
// AddEvidence checks the evidence is valid and adds it to the pool. If
// evidence is composite (ConflictingHeadersEvidence), it will be broken up
// into smaller pieces.
func (evpool *Pool) AddEvidence(evidence types.Evidence) error {
var (
state = evpool.State()
evList = []types.Evidence{evidence}
)
valSet, err := sm.LoadValidators(evpool.stateDB, evidence.Height())
if err != nil {
return fmt.Errorf("can't load validators at height #%d: %w", evidence.Height(), err)
}
// Break composite evidence into smaller pieces.
if ce, ok := evidence.(types.CompositeEvidence); ok {
evpool.logger.Info("Breaking up composite evidence", "ev", evidence)
blockMeta := evpool.blockStore.LoadBlockMeta(evidence.Height())
if blockMeta == nil {
return fmt.Errorf("don't have block meta at height #%d", evidence.Height())
}
if err := ce.VerifyComposite(&blockMeta.Header, valSet); err != nil {
return err
}
// XXX: Copy here since this should be a rare case.
evpool.mtx.Lock()
valToLastHeightCopy := make(valToLastHeightMap, len(evpool.valToLastHeight))
for k, v := range evpool.valToLastHeight {
valToLastHeightCopy[k] = v
}
evpool.mtx.Unlock()
evList = ce.Split(&blockMeta.Header, valSet, valToLastHeightCopy)
}
for _, ev := range evList {
if evpool.Has(ev) {
continue
}
// For lunatic validator evidence, a header needs to be fetched.
var header *types.Header
if _, ok := ev.(*types.LunaticValidatorEvidence); ok {
header = evpool.Header(ev.Height())
if header == nil {
return fmt.Errorf("don't have block meta at height #%d", ev.Height())
}
}
// 1) Verify against state.
if err := sm.VerifyEvidence(evpool.stateDB, state, ev, header); err != nil {
return fmt.Errorf("failed to verify %v: %w", ev, err)
}
// For potential amnesia evidence, if this node is indicted it shall retrieve a polc
// to form AmensiaEvidence
if pe, ok := ev.(types.PotentialAmnesiaEvidence); ok {
var (
height = pe.Height()
exists = false
polc types.ProofOfLockChange
)
pe.HeightStamp = evpool.State().LastBlockHeight
// a) first try to find a corresponding polc
for round := pe.VoteB.Round; round > pe.VoteA.Round; round-- {
polc, err = evpool.RetrievePOLC(height, round)
if err != nil {
evpool.logger.Error("Failed to retrieve polc for potential amnesia evidence", "err", err, "pae", pe.String())
continue
}
if err == nil && !polc.IsAbsent() {
// we should not need to verify it if both the polc and potential amnesia evidence have already
// been verified. We replace the potential amnesia evidence.
ae := types.MakeAmnesiaEvidence(pe, polc)
err := evpool.AddEvidence(ae)
if err != nil {
evpool.logger.Error("Failed to create amnesia evidence from potential amnesia evidence", "err", err)
// revert back to processing potential amnesia evidence
exists = false
} else {
evpool.logger.Info("Formed amnesia evidence from own polc", "amnesiaEvidence", ae)
}
break
}
}
// b) check if amnesia evidence can be made now or if we need to enact the trial period
if !exists && pe.Primed(1, pe.HeightStamp) {
err := evpool.AddEvidence(types.MakeAmnesiaEvidence(pe, types.EmptyPOLC()))
if err != nil {
return err
}
} else if !exists && evpool.State().LastBlockHeight+evpool.State().ConsensusParams.Evidence.ProofTrialPeriod <
pe.Height()+evpool.State().ConsensusParams.Evidence.MaxAgeNumBlocks {
// if we can't find a proof of lock change and we know that the trial period will finish before the
// evidence has expired, then we commence the trial period by saving it in the awaiting bucket
pbe, err := types.EvidenceToProto(pe)
if err != nil {
return err
}
evBytes, err := pbe.Marshal()
if err != nil {
return err
}
key := keyAwaiting(pe)
err = evpool.evidenceStore.Set(key, evBytes)
if err != nil {
return err
}
// keep track of when the next pe has finished the trial period
if evpool.nextEvidenceTrialEndedHeight == -1 {
evpool.nextEvidenceTrialEndedHeight = ev.Height() + evpool.State().ConsensusParams.Evidence.ProofTrialPeriod
}
}
// we don't need to do anymore processing so we can move on to the next piece of evidence
continue
}
// 2) Save to store.
if err := evpool.addPendingEvidence(ev); err != nil {
return fmt.Errorf("database error when adding evidence: %v", err)
}
// 3) Add evidence to clist.
evpool.evidenceList.PushBack(ev)
evpool.logger.Info("Verified new evidence of byzantine behaviour", "evidence", ev)
}
return nil
}
// MarkEvidenceAsCommitted marks all the evidence as committed and removes it
// from the queue.
func (evpool *Pool) MarkEvidenceAsCommitted(height int64, evidence []types.Evidence) {
// make a map of committed evidence to remove from the clist
blockEvidenceMap := make(map[string]struct{})
for _, ev := range evidence {
// As the evidence is stored in the block store we only need to record the height that it was saved at.
key := keyCommitted(ev)
h := gogotypes.Int64Value{Value: height}
evBytes, err := proto.Marshal(&h)
if err != nil {
panic(err)
}
if err := evpool.evidenceStore.Set(key, evBytes); err != nil {
evpool.logger.Error("Unable to add committed evidence", "err", err)
// if we can't move evidence to committed then don't remove the evidence from pending
continue
}
// if pending, remove from that bucket, remember not all evidence has been seen before
if evpool.IsPending(ev) {
evpool.removePendingEvidence(ev)
blockEvidenceMap[evMapKey(ev)] = struct{}{}
}
}
// remove committed evidence from the clist
if len(blockEvidenceMap) != 0 {
evpool.removeEvidenceFromList(blockEvidenceMap)
}
}
// Has checks whether the evidence exists either pending or already committed
func (evpool *Pool) Has(evidence types.Evidence) bool {
return evpool.IsPending(evidence) || evpool.IsCommitted(evidence)
}
// IsEvidenceExpired checks whether evidence is past the maximum age where it can be used
func (evpool *Pool) IsEvidenceExpired(evidence types.Evidence) bool {
return evpool.IsExpired(evidence.Height(), evidence.Time())
}
// IsExpired checks whether evidence or a polc is expired by checking whether a height and time is older
// than set by the evidence consensus parameters
func (evpool *Pool) IsExpired(height int64, time time.Time) bool {
var (
params = evpool.State().ConsensusParams.Evidence
ageDuration = evpool.State().LastBlockTime.Sub(time)
ageNumBlocks = evpool.State().LastBlockHeight - height
)
return ageNumBlocks > params.MaxAgeNumBlocks &&
ageDuration > params.MaxAgeDuration
}
// IsCommitted returns true if we have already seen this exact evidence and it is already marked as committed.
func (evpool *Pool) IsCommitted(evidence types.Evidence) bool {
key := keyCommitted(evidence)
ok, err := evpool.evidenceStore.Has(key)
if err != nil {
evpool.logger.Error("Unable to find committed evidence", "err", err)
}
return ok
}
// IsPending checks whether the evidence is already pending. DB errors are passed to the logger.
func (evpool *Pool) IsPending(evidence types.Evidence) bool {
key := keyPending(evidence)
ok, err := evpool.evidenceStore.Has(key)
if err != nil {
evpool.logger.Error("Unable to find pending evidence", "err", err)
}
return ok
}
// RetrievePOLC attempts to find a polc at the given height and round, if not there than exist returns false, all
// database errors are automatically logged
func (evpool *Pool) RetrievePOLC(height int64, round int32) (polc types.ProofOfLockChange, err error) {
var pbpolc tmproto.ProofOfLockChange
key := keyPOLCFromHeightAndRound(height, round)
polcBytes, err := evpool.evidenceStore.Get(key)
if err != nil {
evpool.logger.Error("Unable to retrieve polc", "err", err)
return polc, err
}
if polcBytes == nil {
return polc, fmt.Errorf("nil value in database for key: %s", key)
}
err = proto.Unmarshal(polcBytes, &pbpolc)
if err != nil {
return polc, err
}
plc, err := types.ProofOfLockChangeFromProto(&pbpolc)
if err != nil {
return polc, err
}
return *plc, err
}
// EvidenceFront goes to the first evidence in the clist
func (evpool *Pool) EvidenceFront() *clist.CElement {
return evpool.evidenceList.Front()
}
// EvidenceWaitChan is a channel that closes once the first evidence in the list is there. i.e Front is not nil
func (evpool *Pool) EvidenceWaitChan() <-chan struct{} {
return evpool.evidenceList.WaitChan()
}
// SetLogger sets the Logger.
func (evpool *Pool) SetLogger(l log.Logger) {
evpool.logger = l
}
// Header gets the header from the block store at a specified height.
// Is used for validation of LunaticValidatorEvidence
func (evpool *Pool) Header(height int64) *types.Header {
blockMeta := evpool.blockStore.LoadBlockMeta(height)
if blockMeta == nil {
return nil
}
return &blockMeta.Header
}
// ValidatorLastHeight returns the last height of the validator w/ the
// given address. 0 - if address never was a validator or was such a
// long time ago (> ConsensusParams.Evidence.MaxAgeDuration && >
// ConsensusParams.Evidence.MaxAgeNumBlocks).
func (evpool *Pool) ValidatorLastHeight(address []byte) int64 {
evpool.mtx.Lock()
defer evpool.mtx.Unlock()
h, ok := evpool.valToLastHeight[string(address)]
if !ok {
return 0
}
return h
}
// State returns the current state of the evpool.
func (evpool *Pool) State() sm.State {
evpool.mtx.Lock()
defer evpool.mtx.Unlock()
return evpool.state
}
func (evpool *Pool) addPendingEvidence(evidence types.Evidence) error {
evi, err := types.EvidenceToProto(evidence)
if err != nil {
return fmt.Errorf("unable to convert to proto, err: %w", err)
}
evBytes, err := proto.Marshal(evi)
if err != nil {
return fmt.Errorf("unable to marshal evidence: %w", err)
}
key := keyPending(evidence)
return evpool.evidenceStore.Set(key, evBytes)
}
func (evpool *Pool) removePendingEvidence(evidence types.Evidence) {
key := keyPending(evidence)
if err := evpool.evidenceStore.Delete(key); err != nil {
evpool.logger.Error("Unable to delete pending evidence", "err", err)
} else {
evpool.logger.Info("Deleted pending evidence", "evidence", evidence)
}
}
// listEvidence lists up to maxNum pieces of evidence for the given prefix key.
// If maxNum is -1, there's no cap on the size of returned evidence.
func (evpool *Pool) listEvidence(prefixKey byte, maxNum int64) ([]types.Evidence, error) {
var count int64
var evidence []types.Evidence
iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{prefixKey})
if err != nil {
return nil, fmt.Errorf("database error: %v", err)
}
defer iter.Close()
for ; iter.Valid(); iter.Next() {
if count == maxNum {
return evidence, nil
}
count++
val := iter.Value()
var (
ev types.Evidence
evpb tmproto.Evidence
)
err := proto.Unmarshal(val, &evpb)
if err != nil {
return nil, err
}
ev, err = types.EvidenceFromProto(&evpb)
if err != nil {
return nil, err
}
evidence = append(evidence, ev)
}
return evidence, nil
}
func (evpool *Pool) removeExpiredPendingEvidence() {
iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{baseKeyPending})
if err != nil {
evpool.logger.Error("Unable to iterate over pending evidence", "err", err)
return
}
defer iter.Close()
blockEvidenceMap := make(map[string]struct{})
for ; iter.Valid(); iter.Next() {
evBytes := iter.Value()
var (
ev types.Evidence
evpb tmproto.Evidence
)
err := proto.Unmarshal(evBytes, &evpb)
if err != nil {
evpool.logger.Error("Unable to unmarshal Evidence", "err", err)
continue
}
ev, err = types.EvidenceFromProto(&evpb)
if err != nil {
evpool.logger.Error("Error in transition evidence from protobuf", "err", err)
continue
}
if !evpool.IsExpired(ev.Height()-1, ev.Time()) {
if len(blockEvidenceMap) != 0 {
evpool.removeEvidenceFromList(blockEvidenceMap)
}
return
}
evpool.removePendingEvidence(ev)
blockEvidenceMap[evMapKey(ev)] = struct{}{}
}
}
func (evpool *Pool) removeEvidenceFromList(
blockEvidenceMap map[string]struct{}) {
for e := evpool.evidenceList.Front(); e != nil; e = e.Next() {
// Remove from clist
ev := e.Value.(types.Evidence)
if _, ok := blockEvidenceMap[evMapKey(ev)]; ok {
evpool.evidenceList.Remove(e)
e.DetachPrev()
}
}
}
func (evpool *Pool) pruneExpiredPOLC() {
evpool.logger.Debug("Pruning expired POLC's")
iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{baseKeyPOLC})
if err != nil {
evpool.logger.Error("Unable to iterate over POLC's", "err", err)
return
}
defer iter.Close()
for ; iter.Valid(); iter.Next() {
proofBytes := iter.Value()
var (
pbproof tmproto.ProofOfLockChange
)
err := proto.Unmarshal(proofBytes, &pbproof)
if err != nil {
evpool.logger.Error("Unable to unmarshal POLC", "err", err)
continue
}
proof, err := types.ProofOfLockChangeFromProto(&pbproof)
if err != nil {
evpool.logger.Error("Unable to transition POLC from protobuf", "err", err)
continue
}
if !evpool.IsExpired(proof.Height()-1, proof.Time()) {
return
}
err = evpool.evidenceStore.Delete(iter.Key())
if err != nil {
evpool.logger.Error("Unable to delete expired POLC", "err", err)
continue
}
evpool.logger.Info("Deleted expired POLC", "polc", proof)
}
}
// upgrades any potential evidence that has undergone the trial period and is primed to be made into
// amnesia evidence
func (evpool *Pool) upgradePotentialAmnesiaEvidence() int64 {
iter, err := dbm.IteratePrefix(evpool.evidenceStore, []byte{baseKeyAwaiting})
if err != nil {
evpool.logger.Error("Unable to iterate over POLC's", "err", err)
return -1
}
defer iter.Close()
trialPeriod := evpool.State().ConsensusParams.Evidence.ProofTrialPeriod
// 1) Iterate through all potential amnesia evidence in order of height
for ; iter.Valid(); iter.Next() {
paeBytes := iter.Value()
// 2) Retrieve the evidence
var evpb tmproto.Evidence
err := evpb.Unmarshal(paeBytes)
if err != nil {
evpool.logger.Error("Unable to unmarshal potential amnesia evidence", "err", err)
continue
}
ev, err := types.EvidenceFromProto(&evpb)
if err != nil {
evpool.logger.Error("coverting to evidence from proto", "err", err)
continue
}
// 3) Check if the trial period has lapsed and amnesia evidence can be formed
if pe, ok := ev.(*types.PotentialAmnesiaEvidence); ok {
if pe.Primed(trialPeriod, evpool.State().LastBlockHeight) {
ae := types.MakeAmnesiaEvidence(*pe, types.EmptyPOLC())
err := evpool.AddEvidence(ae)
if err != nil {
evpool.logger.Error("Unable to add amnesia evidence", "err", err)
continue
}
err = evpool.evidenceStore.Delete(iter.Key())
if err != nil {
evpool.logger.Error("Unable to delete potential amnesia evidence", "err", err)
continue
}
} else {
evpool.logger.Debug("Potential amnesia evidence not ready to be upgraded. Ready at height", "height",
pe.HeightStamp+trialPeriod)
// once we reach a piece of evidence that isn't ready send back the height with which it will be ready
return pe.HeightStamp + trialPeriod
}
}
}
// if we have no evidence left to process we want to reset nextEvidenceTrialEndedHeight
return -1
}
func evMapKey(ev types.Evidence) string {
return string(ev.Hash())
}
func (evpool *Pool) updateValToLastHeight(blockHeight int64, state sm.State) {
// Update current validators & add new ones.
for _, val := range state.Validators.Validators {
evpool.valToLastHeight[string(val.Address)] = blockHeight
}
// Remove validators outside of MaxAgeNumBlocks & MaxAgeDuration.
removeHeight := blockHeight - state.ConsensusParams.Evidence.MaxAgeNumBlocks
if removeHeight >= 1 {
for val, height := range evpool.valToLastHeight {
if height <= removeHeight {
delete(evpool.valToLastHeight, val)
}
}
}
}
func buildValToLastHeightMap(state sm.State, stateDB dbm.DB, blockStore *store.BlockStore) (valToLastHeightMap, error) {
var (
valToLastHeight = make(map[string]int64)
params = state.ConsensusParams.Evidence
numBlocks = int64(0)
minAgeTime = time.Now().Add(-params.MaxAgeDuration)
height = state.LastBlockHeight
)
if height == 0 {
return valToLastHeight, nil
}
meta := blockStore.LoadBlockMeta(height)
if meta == nil {
return nil, fmt.Errorf("block meta for height %d not found", height)
}
blockTime := meta.Header.Time
// From state.LastBlockHeight, build a map of "active" validators until
// MaxAgeNumBlocks is passed and block time is less than now() -
// MaxAgeDuration.
for height >= 1 && (numBlocks <= params.MaxAgeNumBlocks || !blockTime.Before(minAgeTime)) {
valSet, err := sm.LoadValidators(stateDB, height)
if err != nil {
// last stored height -> return
if _, ok := err.(sm.ErrNoValSetForHeight); ok {
return valToLastHeight, nil
}
return nil, fmt.Errorf("validator set for height %d not found", height)
}
for _, val := range valSet.Validators {
key := string(val.Address)
if _, ok := valToLastHeight[key]; !ok {
valToLastHeight[key] = height
}
}
height--
if height > 0 {
// NOTE: we assume here blockStore and state.Validators are in sync. I.e if
// block N is stored, then validators for height N are also stored in
// state.
meta := blockStore.LoadBlockMeta(height)
if meta == nil {
return nil, fmt.Errorf("block meta for height %d not found", height)
}
blockTime = meta.Header.Time
}
numBlocks++
}
return valToLastHeight, nil
}
// big endian padded hex
func bE(h int64) string {
return fmt.Sprintf("%0.16X", h)
}
func keyCommitted(evidence types.Evidence) []byte {
return append([]byte{baseKeyCommitted}, keySuffix(evidence)...)
}
func keyPending(evidence types.Evidence) []byte {
return append([]byte{baseKeyPending}, keySuffix(evidence)...)
}
func keyAwaiting(evidence types.Evidence) []byte {
return append([]byte{baseKeyAwaiting}, keySuffix(evidence)...)
}
func keyPOLC(polc types.ProofOfLockChange) []byte {
return keyPOLCFromHeightAndRound(polc.Height(), polc.Round())
}
func keyPOLCFromHeightAndRound(height int64, round int32) []byte {
return append([]byte{baseKeyPOLC}, []byte(fmt.Sprintf("%s/%s", bE(height), bE(int64(round))))...)
}
func keySuffix(evidence types.Evidence) []byte {
return []byte(fmt.Sprintf("%s/%X", bE(evidence.Height()), evidence.Hash()))
}
// ErrInvalidEvidence returns when evidence failed to validate
type ErrInvalidEvidence struct {
Reason error
}
func (e ErrInvalidEvidence) Error() string {
return fmt.Sprintf("evidence is not valid: %v ", e.Reason)
}