state.State and wal.writeHeight after handshake

consensus/replay.go

@@ -101,6 +101,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 	// Search for height marker
 	gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight))
 	if err == io.EOF {
+		log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight)
 		return nil
 	} else if err != nil {
 		return err

consensus/state.go

@@ -362,6 +362,14 @@ func (cs *ConsensusState) OnStart() error {
 		// let's go for it anyways, maybe we're fine
 	}
 
+	// If the latest block was applied in the tmsp handshake,
+	// we may not have written the current height to the wal,
+	// so write it here in case
+	if cs.Step == RoundStepNewHeight {
+		log.Warn("wal.writeHeight", "height", cs.Height)
+		cs.wal.writeHeight(cs.Height)
+	}
+
 	// now start the receiveRoutine
 	go cs.receiveRoutine(0)
 

state/execution.go

@@ -344,6 +344,9 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
 		return errors.New(Fmt("Error on replay: %v", err))
 	}
 
+	// Save the state
+	h.state.Save()
+
 	// TODO: (on restart) replay mempool
 
 	return nil
@@ -378,7 +381,6 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, appConnCon
 			// so load the intermediate state and update the hash
 			h.state.LoadIntermediate()
 			h.state.AppHash = appHash
-			h.state.Save()
 			log.Debug("TMSP RelpayBlocks: Loaded intermediate state and updated state.AppHash")
 		} else {
 			PanicSanity(Fmt("Unexpected state.AppHash: state.AppHash %X; app.AppHash %X, lastBlock.AppHash %X", stateAppHash, appHash, lastBlockAppHash))

test/docker/Dockerfile

@@ -21,5 +21,8 @@ COPY . $REPO
 RUN go install ./cmd/tendermint
 RUN bash scripts/install_tmsp_apps.sh
 
+# expose the volume for debugging
+VOLUME $REPO
+
 EXPOSE 46656
 EXPOSE 46657

test/persist/test_failure_indices.sh

@@ -18,7 +18,7 @@ function start_procs(){
 		PID_TENDERMINT=$!
 	else
 		# run in foreground, fail
-		FAIL_TEST_INDEX=$indexToFail tendermint node &> tendermint_${name}.log
+		FAIL_TEST_INDEX=$indexToFail tendermint node --log_level=debug &> tendermint_${name}.log
 		PID_TENDERMINT=$!
 	fi
 }