tendermint/consensus/replay.go

package consensus

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"strconv"
	"strings"
	"time"

	. "github.com/tendermint/go-common"
	"github.com/tendermint/go-wire"

	sm "github.com/tendermint/tendermint/state"
	"github.com/tendermint/tendermint/tailseek"
	"github.com/tendermint/tendermint/types"
)

//--------------------------------------------------------
// types and functions for savings consensus messages

type ConsensusLogMessage struct {
	Time time.Time                    `json:"time"`
	Msg  ConsensusLogMessageInterface `json:"msg"`
}

type ConsensusLogMessageInterface interface{}

var _ = wire.RegisterInterface(
	struct{ ConsensusLogMessageInterface }{},
	wire.ConcreteType{&types.EventDataRoundState{}, 0x01},
	wire.ConcreteType{msgInfo{}, 0x02},
	wire.ConcreteType{timeoutInfo{}, 0x03},
)

// called in newStep and for each pass in receiveRoutine
func (cs *ConsensusState) saveMsg(msg ConsensusLogMessageInterface) {
	if cs.msgLogFP != nil {
		var n int
		var err error
		wire.WriteJSON(ConsensusLogMessage{time.Now(), msg}, cs.msgLogFP, &n, &err)
		wire.WriteTo([]byte("\n"), cs.msgLogFP, &n, &err) // one message per line
		if err != nil {
			log.Error("Error writing to consensus message log file", "err", err, "msg", msg)
		}
	}
}

//--------------------------------------------------------
// replay the messages

// Interactive playback
func (cs ConsensusState) ReplayConsole(file string) error {
	return cs.replay(file, true)
}

// Full playback, with tests
func (cs ConsensusState) ReplayMessages(file string) error {
	return cs.replay(file, false)
}

func (cs *ConsensusState) catchupReplay(height int) error {
	if !cs.msgLogExists {
		return nil
	}

	if cs.msgLogFP == nil {
		log.Warn("consensus msg log is nil")
		return nil
	}

	log.Notice("Catchup by replaying consensus messages")
	f := cs.msgLogFP

	n, err := seek.SeekFromEndOfFile(f, func(lineBytes []byte) bool {
		var err error
		var msg ConsensusLogMessage
		wire.ReadJSON(&msg, lineBytes, &err)
		if err != nil {
			panic(Fmt("Failed to read cs_msg_log json: %v", err))
		}
		m, ok := msg.Msg.(*types.EventDataRoundState)
		if ok && m.Step == RoundStepNewHeight.String() {
			r, err := f.Seek(0, 1)
			// TODO: ensure the height matches
			return true
		}
		return false
	})

	if err != nil {
		return err
	}

	// we found it, now we can replay everything
	pb := newPlayback("", cs.msgLogFP, cs, nil, cs.state.Copy())

	reader := bufio.NewReader(cs.msgLogFP)
	i := 0
	for {
		i += 1
		msgBytes, err := reader.ReadBytes('\n')
		if err == io.EOF {
			break
		} else if err != nil {
			return err
		} else if len(msgBytes) == 0 {
			continue
		}
		// the first msg is the NewHeight event, so we can ignore it
		if i == 1 {
			continue
		}

		// NOTE: since the priv key is set when the msgs are received
		// it will attempt to eg double sign but we can just ignore it
		// since the votes will be replayed and we'll get to the next step
		if err := pb.readReplayMessage(msgBytes); err != nil {
			return err

		}
		if i >= n {
			break
		}
	}
	return nil

}

// replay all msgs or start the console
func (cs *ConsensusState) replay(file string, console bool) error {
	if cs.IsRunning() {
		return errors.New("cs is already running, cannot replay")
	}

	cs.startForReplay()

	// set the FP to nil so we don't overwrite
	if cs.msgLogFP != nil {
		cs.msgLogFP.Close()
		cs.msgLogFP = nil
	}

	// ensure all new step events are regenerated as expected
	newStepCh := cs.evsw.SubscribeToEvent("replay-test", types.EventStringNewRoundStep(), 1)

	fp, err := os.OpenFile(file, os.O_RDONLY, 0666)
	if err != nil {
		return err
	}

	pb := newPlayback(file, fp, cs, newStepCh, cs.state.Copy())
	defer pb.fp.Close()

	var nextN int // apply N msgs in a row
	for pb.scanner.Scan() {
		if nextN == 0 && console {
			nextN = pb.replayConsoleLoop()
		}

		if err := pb.readReplayMessage(pb.scanner.Bytes()); err != nil {
			return err
		}

		if nextN > 0 {
			nextN -= 1
		}
		pb.count += 1
	}
	return nil
}

//------------------------------------------------
// playback manager

type playback struct {
	cs           *ConsensusState
	file         string
	fp           *os.File
	scanner      *bufio.Scanner
	newStepCh    chan interface{}
	genesisState *sm.State
	count        int
}

func newPlayback(file string, fp *os.File, cs *ConsensusState, ch chan interface{}, genState *sm.State) *playback {
	return &playback{
		cs:           cs,
		file:         file,
		newStepCh:    ch,
		genesisState: genState,
		fp:           fp,
		scanner:      bufio.NewScanner(fp),
	}
}

// go back count steps by resetting the state and running (pb.count - count) steps
func (pb *playback) replayReset(count int) error {

	pb.cs.Stop()

	newCs := NewConsensusState(pb.genesisState.Copy(), pb.cs.proxyAppCtx, pb.cs.blockStore, pb.cs.mempool)
	newCs.SetEventSwitch(pb.cs.evsw)

	// ensure all new step events are regenerated as expected
	pb.newStepCh = newCs.evsw.SubscribeToEvent("replay-test", types.EventStringNewRoundStep(), 1)

	newCs.startForReplay()

	pb.fp.Close()
	fp, err := os.OpenFile(pb.file, os.O_RDONLY, 0666)
	if err != nil {
		return err
	}
	pb.fp = fp
	pb.scanner = bufio.NewScanner(fp)
	count = pb.count - count
	log.Notice(Fmt("Reseting from %d to %d", pb.count, count))
	pb.count = 0
	pb.cs = newCs
	for i := 0; pb.scanner.Scan() && i < count; i++ {
		if err := pb.readReplayMessage(pb.scanner.Bytes()); err != nil {
			return err
		}
		pb.count += 1
	}
	return nil
}

func (cs *ConsensusState) startForReplay() {
	cs.BaseService.OnStart()
	go cs.receiveRoutine(0)
	// since we replay tocks we just ignore ticks
	go func() {
		for {
			select {
			case <-cs.tickChan:
			case <-cs.Quit:
				return
			}
		}
	}()
}

// console function for parsing input and running commands
func (pb *playback) replayConsoleLoop() int {
	for {
		fmt.Printf("> ")
		bufReader := bufio.NewReader(os.Stdin)
		line, more, err := bufReader.ReadLine()
		if more {
			Exit("input is too long")
		} else if err != nil {
			Exit(err.Error())
		}

		tokens := strings.Split(string(line), " ")
		if len(tokens) == 0 {
			continue
		}

		switch tokens[0] {
		case "next":
			// "next" -> replay next message
			// "next N" -> replay next N messages

			if len(tokens) == 1 {
				return 0
			} else {
				i, err := strconv.Atoi(tokens[1])
				if err != nil {
					fmt.Println("next takes an integer argument")
				} else {
					return i
				}
			}

		case "back":
			// "back" -> go back one message
			// "back N" -> go back N messages

			// NOTE: "back" is not supported in the state machine design,
			// so we restart and replay up to

			if len(tokens) == 1 {
				pb.replayReset(1)
			} else {
				i, err := strconv.Atoi(tokens[1])
				if err != nil {
					fmt.Println("back takes an integer argument")
				} else if i > pb.count {
					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
				} else {
					pb.replayReset(i)
				}
			}

		case "rs":
			// "rs" -> print entire round state
			// "rs short" -> print height/round/step
			// "rs <field>" -> print another field of the round state

			rs := pb.cs.RoundState
			if len(tokens) == 1 {
				fmt.Println(rs)
			} else {
				switch tokens[1] {
				case "short":
					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
				case "validators":
					fmt.Println(rs.Validators)
				case "proposal":
					fmt.Println(rs.Proposal)
				case "proposal_block":
					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
				case "locked_round":
					fmt.Println(rs.LockedRound)
				case "locked_block":
					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
				case "votes":
					fmt.Println(rs.Votes.StringIndented("    "))

				default:
					fmt.Println("Unknown option", tokens[1])
				}
			}
		case "n":
			fmt.Println(pb.count)
		}
	}
	return 0
}

func (pb *playback) readReplayMessage(msgBytes []byte) error {
	var err error
	var msg ConsensusLogMessage
	wire.ReadJSON(&msg, msgBytes, &err)
	if err != nil {
		return fmt.Errorf("Error reading json data: %v", err)
	}

	// for logging
	switch m := msg.Msg.(type) {
	case *types.EventDataRoundState:
		log.Notice("New Step", "height", m.Height, "round", m.Round, "step", m.Step)
		// these are playback checks
		ticker := time.After(time.Second * 2)
		if pb.newStepCh != nil {
			select {
			case mi := <-pb.newStepCh:
				m2 := mi.(*types.EventDataRoundState)
				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
					return fmt.Errorf("RoundState mismatch. Got %v; Expected %v", m2, m)
				}
			case <-ticker:
				return fmt.Errorf("Failed to read off newStepCh")
			}
		}
	case msgInfo:
		peerKey := m.PeerKey
		if peerKey == "" {
			peerKey = "local"
		}
		switch msg := m.Msg.(type) {
		case *ProposalMessage:
			p := msg.Proposal
			log.Notice("Proposal", "height", p.Height, "round", p.Round, "header",
				p.BlockPartsHeader, "pol", p.POLRound, "peer", peerKey)
		case *BlockPartMessage:
			log.Notice("BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerKey)
		case *VoteMessage:
			v := msg.Vote
			log.Notice("Vote", "height", v.Height, "round", v.Round, "type", v.Type,
				"hash", v.BlockHash, "header", v.BlockPartsHeader, "peer", peerKey)
		}
		// internal or from peer
		if m.PeerKey == "" {
			pb.cs.internalMsgQueue <- m
		} else {
			pb.cs.peerMsgQueue <- m
		}
	case timeoutInfo:
		log.Notice("Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
		pb.cs.tockChan <- m
	default:
		return fmt.Errorf("Unknown ConsensusLogMessage type: %v", reflect.TypeOf(msg.Msg))
	}
	return nil
}

// Read lines starting from the end of the file until we read a line that causes found to return true
func SeekFromEndOfFile(f *os.File, found func([]byte) bool) (nLines int, err error) {
	var current int64
	// start at the end
	current, err = f.Seek(0, 2)
	if err != nil {
		fmt.Println("1")
		return
	}

	// backup until we find the the right line
	for {
		current -= 1
		if current < 0 {
			return
		}
		// backup one and read a new byte
		if _, err = f.Seek(current, 0); err != nil {
			fmt.Println("2", current)
			return
		}
		b := make([]byte, 1)
		if _, err = f.Read(b); err != nil {
			return
		}
		if b[0] == '\n' || len(b) == 0 {
			nLines += 1

			// read a full line
			reader := bufio.NewReader(f)
			lineBytes, _ := reader.ReadBytes('\n')
			if len(lineBytes) == 0 {
				continue
			}

			if found(lineBytes) {
				f.Seek(current, 0)
				return
			}
		}
	}
}