Rebased to master the existing `ProcessProposal` PR (#7752)

* Rebased and git-squashed the commits in PR #7091

  - add processproposal proto/boilerplate/logic

  - mockery

  - gofmt

  - fix test

  - gofmt

  - move UNKNOWN response behaviour to reject

* Fixed build of some UTs

* Addressed William's comment on context

* Adapted TestProcessProposal

* BaseApp needs to ACCEPT vote extensions by default

* Added missing ProcessProposal to socket_server.go

* Re-renamed TwoThirdPrevote... to Valid...

* Addressed William's comment on ProcessProposal error

* Addressed Callum's comments

* fmt

Co-authored-by: mconcat <monoidconcat@gmail.com>

abci/client/client.go

@@ -46,6 +46,7 @@ type Client interface {
 	Commit(context.Context) (*types.ResponseCommit, error)
 	InitChain(context.Context, types.RequestInitChain) (*types.ResponseInitChain, error)
 	PrepareProposal(context.Context, types.RequestPrepareProposal) (*types.ResponsePrepareProposal, error)
+	ProcessProposal(context.Context, types.RequestProcessProposal) (*types.ResponseProcessProposal, error)
 	ExtendVote(context.Context, types.RequestExtendVote) (*types.ResponseExtendVote, error)
 	VerifyVoteExtension(context.Context, types.RequestVerifyVoteExtension) (*types.ResponseVerifyVoteExtension, error)
 	BeginBlock(context.Context, types.RequestBeginBlock) (*types.ResponseBeginBlock, error)

abci/client/grpc_client.go

@@ -377,6 +377,14 @@ func (cli *grpcClient) PrepareProposal(
 	return cli.client.PrepareProposal(ctx, req.GetPrepareProposal(), grpc.WaitForReady(true))
 }
 
+func (cli *grpcClient) ProcessProposal(
+	ctx context.Context,
+	params types.RequestProcessProposal) (*types.ResponseProcessProposal, error) {
+
+	req := types.ToRequestProcessProposal(params)
+	return cli.client.ProcessProposal(ctx, req.GetProcessProposal(), grpc.WaitForReady(true))
+}
+
 func (cli *grpcClient) ExtendVote(
 	ctx context.Context,
 	params types.RequestExtendVote) (*types.ResponseExtendVote, error) {

abci/client/local_client.go

@@ -233,6 +233,17 @@ func (app *localClient) PrepareProposal(
 	return &res, nil
 }
 
+func (app *localClient) ProcessProposal(
+	ctx context.Context,
+	req types.RequestProcessProposal) (*types.ResponseProcessProposal, error) {
+
+	app.mtx.Lock()
+	defer app.mtx.Unlock()
+
+	res := app.Application.ProcessProposal(req)
+	return &res, nil
+}
+
 func (app *localClient) ExtendVote(
 	ctx context.Context,
 	req types.RequestExtendVote) (*types.ResponseExtendVote, error) {

abci/client/mocks/client.go

@@ -450,6 +450,29 @@ func (_m *Client) PrepareProposal(_a0 context.Context, _a1 types.RequestPrepareP
 	return r0, r1
 }
 
+// ProcessProposal provides a mock function with given fields: _a0, _a1
+func (_m *Client) ProcessProposal(_a0 context.Context, _a1 types.RequestProcessProposal) (*types.ResponseProcessProposal, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *types.ResponseProcessProposal
+	if rf, ok := ret.Get(0).(func(context.Context, types.RequestProcessProposal) *types.ResponseProcessProposal); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseProcessProposal)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.RequestProcessProposal) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // Query provides a mock function with given fields: _a0, _a1
 func (_m *Client) Query(_a0 context.Context, _a1 types.RequestQuery) (*types.ResponseQuery, error) {
 	ret := _m.Called(_a0, _a1)

abci/client/socket_client.go

@@ -415,6 +415,18 @@ func (cli *socketClient) PrepareProposal(
 	return reqres.Response.GetPrepareProposal(), nil
 }
 
+func (cli *socketClient) ProcessProposal(
+	ctx context.Context,
+	req types.RequestProcessProposal,
+) (*types.ResponseProcessProposal, error) {
+
+	reqres, err := cli.queueRequestAndFlush(ctx, types.ToRequestProcessProposal(req))
+	if err != nil {
+		return nil, err
+	}
+	return reqres.Response.GetProcessProposal(), nil
+}
+
 func (cli *socketClient) ExtendVote(
 	ctx context.Context,
 	req types.RequestExtendVote) (*types.ResponseExtendVote, error) {

abci/example/kvstore/persistent_kvstore.go

@@ -189,6 +189,16 @@ func (app *PersistentKVStoreApplication) PrepareProposal(
 	return types.ResponsePrepareProposal{BlockData: app.substPrepareTx(req.BlockData)}
 }
 
+func (app *PersistentKVStoreApplication) ProcessProposal(
+	req types.RequestProcessProposal) types.ResponseProcessProposal {
+	for _, tx := range req.Txs {
+		if len(tx) == 0 {
+			return types.ResponseProcessProposal{Result: types.ResponseProcessProposal_REJECT}
+		}
+	}
+	return types.ResponseProcessProposal{Result: types.ResponseProcessProposal_ACCEPT}
+}
+
 //---------------------------------------------
 // update validators
 

abci/server/socket_server.go

@@ -243,6 +243,9 @@ func (s *SocketServer) handleRequest(req *types.Request, responses chan<- *types
 	case *types.Request_PrepareProposal:
 		res := s.app.PrepareProposal(*r.PrepareProposal)
 		responses <- types.ToResponsePrepareProposal(res)
+	case *types.Request_ProcessProposal:
+		res := s.app.ProcessProposal(*r.ProcessProposal)
+		responses <- types.ToResponseProcessProposal(res)
 	case *types.Request_LoadSnapshotChunk:
 		res := s.app.LoadSnapshotChunk(*r.LoadSnapshotChunk)
 		responses <- types.ToResponseLoadSnapshotChunk(res)

abci/types/application.go

@@ -19,6 +19,7 @@ type Application interface {
 	// Consensus Connection
 	InitChain(RequestInitChain) ResponseInitChain // Initialize blockchain w validators/other info from TendermintCore
 	PrepareProposal(RequestPrepareProposal) ResponsePrepareProposal
+	ProcessProposal(RequestProcessProposal) ResponseProcessProposal
 	// Signals the beginning of a block
 	BeginBlock(RequestBeginBlock) ResponseBeginBlock
 	// Deliver a tx for full processing
@@ -72,7 +73,9 @@ func (BaseApplication) ExtendVote(req RequestExtendVote) ResponseExtendVote {
 }
 
 func (BaseApplication) VerifyVoteExtension(req RequestVerifyVoteExtension) ResponseVerifyVoteExtension {
-	return ResponseVerifyVoteExtension{}
+	return ResponseVerifyVoteExtension{
+		Result: ResponseVerifyVoteExtension_ACCEPT,
+	}
 }
 
 func (BaseApplication) Query(req RequestQuery) ResponseQuery {
@@ -111,6 +114,10 @@ func (BaseApplication) PrepareProposal(req RequestPrepareProposal) ResponsePrepa
 	return ResponsePrepareProposal{}
 }
 
+func (BaseApplication) ProcessProposal(req RequestProcessProposal) ResponseProcessProposal {
+	return ResponseProcessProposal{}
+}
+
 //-------------------------------------------------------
 
 // GRPCApplication is a GRPC wrapper for Application
@@ -211,3 +218,9 @@ func (app *GRPCApplication) PrepareProposal(
 	res := app.app.PrepareProposal(*req)
 	return &res, nil
 }
+
+func (app *GRPCApplication) ProcessProposal(
+	ctx context.Context, req *RequestProcessProposal) (*ResponseProcessProposal, error) {
+	res := app.app.ProcessProposal(*req)
+	return &res, nil
+}

abci/types/messages.go

@@ -128,6 +128,12 @@ func ToRequestPrepareProposal(req RequestPrepareProposal) *Request {
 	}
 }
 
+func ToRequestProcessProposal(req RequestProcessProposal) *Request {
+	return &Request{
+		Value: &Request_ProcessProposal{&req},
+	}
+}
+
 //----------------------------------------
 
 func ToResponseException(errStr string) *Response {
@@ -236,3 +242,9 @@ func ToResponsePrepareProposal(res ResponsePrepareProposal) *Response {
 		Value: &Response_PrepareProposal{&res},
 	}
 }
+
+func ToResponseProcessProposal(res ResponseProcessProposal) *Response {
+	return &Response{
+		Value: &Response_ProcessProposal{&res},
+	}
+}

abci/types/result.go

@@ -43,14 +43,24 @@ func (r ResponseQuery) IsErr() bool {
 	return r.Code != CodeTypeOK
 }
 
+// IsUnknown returns true if Code is Unknown
+func (r ResponseVerifyVoteExtension) IsUnknown() bool {
+	return r.Result == ResponseVerifyVoteExtension_UNKNOWN
+}
+
 // IsOK returns true if Code is OK
 func (r ResponseVerifyVoteExtension) IsOK() bool {
-	return r.Result <= ResponseVerifyVoteExtension_ACCEPT
+	return r.Result == ResponseVerifyVoteExtension_ACCEPT
 }
 
 // IsErr returns true if Code is something other than OK.
 func (r ResponseVerifyVoteExtension) IsErr() bool {
-	return r.Result > ResponseVerifyVoteExtension_ACCEPT
+	return r.Result != ResponseVerifyVoteExtension_ACCEPT
+}
+
+// IsOK returns true if Code is OK
+func (r ResponseProcessProposal) IsOK() bool {
+	return r.Result == ResponseProcessProposal_ACCEPT
 }
 
 //---------------------------------------------------------------------------

internal/consensus/state.go

@@ -1439,11 +1439,12 @@ func (cs *State) defaultDoPrevote(ctx context.Context, height int64, round int32
 		return
 	}
 
-	// Validate proposal block
+	// Validate proposal block, from Tendermint's perspective
 	err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock)
 	if err != nil {
 		// ProposalBlock is invalid, prevote nil.
-		logger.Error("prevote step: ProposalBlock is invalid; prevoting nil", "err", err)
+		logger.Error("prevote step: consensus deems this block invalid; prevoting nil",
+			"err", err)
 		cs.signAddVote(ctx, tmproto.PrevoteType, nil, types.PartSetHeader{})
 		return
 	}
@@ -1506,6 +1507,30 @@ func (cs *State) defaultDoPrevote(ctx context.Context, height int64, round int32
 		}
 	}
 
+	/*
+		Before prevoting on the block received from the proposer for the current round and height,
+		we request the Application, via `ProcessProposal` ABCI call, to confirm that the block is
+		valid. If the Application does not accept the block, Tendermint prevotes `nil`.
+
+		WARNING: misuse of block rejection by the Application can seriously compromise Tendermint's
+		liveness properties. Please see `PrepareProosal`-`ProcessProposal` coherence and determinism
+		properties in the ABCI++ specification.
+	*/
+	stateMachineValidBlock, err := cs.blockExec.ProcessProposal(ctx, cs.ProposalBlock)
+	if err != nil {
+		panic(fmt.Sprintf(
+			"state machine returned an error (%v) when calling ProcessProposal", err,
+		))
+	}
+
+	// Vote nil if the Application rejected the block
+	if !stateMachineValidBlock {
+		logger.Error("prevote step: state machine rejected a proposed block; this should not happen:"+
+			"the proposer may be misbehaving; prevoting nil", "err", err)
+		cs.signAddVote(ctx, tmproto.PrevoteType, nil, types.PartSetHeader{})
+		return
+	}
+
 	logger.Debug("prevote step: ProposalBlock is valid but was not our locked block or " +
 		"did not receive a more recent majority; prevoting nil")
 	cs.signAddVote(ctx, tmproto.PrevoteType, nil, types.PartSetHeader{})

internal/consensus/types/round_state.go

@@ -81,6 +81,15 @@ type RoundState struct {
 	LockedBlock         *types.Block        `json:"locked_block"`
 	LockedBlockParts    *types.PartSet      `json:"locked_block_parts"`
 
+	// The variables below starting with "Valid..." derive their name from
+	// the algorithm presented in this paper:
+	// [The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938).
+	// Therefore, "Valid...":
+	//   * means that the block or round that the variable refers to has
+	//     received 2/3+ non-`nil` prevotes (a.k.a. a *polka*)
+	//   * has nothing to do with whether the Application returned "Accept" in its
+	//     response to `ProcessProposal`, or "Reject"
+
 	// Last known round with POL for non-nil valid block.
 	ValidRound int32        `json:"valid_round"`
 	ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above.
@@ -187,8 +196,8 @@ func (rs *RoundState) StringIndented(indent string) string {
 %s  ProposalBlock: %v %v
 %s  LockedRound:   %v
 %s  LockedBlock:   %v %v
-%s  ValidRound:   %v
-%s  ValidBlock:   %v %v
+%s  ValidRound:    %v
+%s  ValidBlock:    %v %v
 %s  Votes:         %v
 %s  LastCommit:    %v
 %s  LastValidators:%v

internal/proxy/app_conn.go

@@ -21,6 +21,7 @@ type AppConnConsensus interface {
 	InitChain(context.Context, types.RequestInitChain) (*types.ResponseInitChain, error)
 
 	PrepareProposal(context.Context, types.RequestPrepareProposal) (*types.ResponsePrepareProposal, error)
+	ProcessProposal(context.Context, types.RequestProcessProposal) (*types.ResponseProcessProposal, error)
 	ExtendVote(context.Context, types.RequestExtendVote) (*types.ResponseExtendVote, error)
 	VerifyVoteExtension(context.Context, types.RequestVerifyVoteExtension) (*types.ResponseVerifyVoteExtension, error)
 	BeginBlock(context.Context, types.RequestBeginBlock) (*types.ResponseBeginBlock, error)
@@ -98,6 +99,14 @@ func (app *appConnConsensus) PrepareProposal(
 	return app.appConn.PrepareProposal(ctx, req)
 }
 
+func (app *appConnConsensus) ProcessProposal(
+	ctx context.Context,
+	req types.RequestProcessProposal,
+) (*types.ResponseProcessProposal, error) {
+	defer addTimeSample(app.metrics.MethodTiming.With("method", "process_proposal", "type", "sync"))()
+	return app.appConn.ProcessProposal(ctx, req)
+}
+
 func (app *appConnConsensus) ExtendVote(
 	ctx context.Context,
 	req types.RequestExtendVote,

internal/proxy/mocks/app_conn_consensus.go

@@ -192,6 +192,29 @@ func (_m *AppConnConsensus) PrepareProposal(_a0 context.Context, _a1 types.Reque
 	return r0, r1
 }
 
+// ProcessProposal provides a mock function with given fields: _a0, _a1
+func (_m *AppConnConsensus) ProcessProposal(_a0 context.Context, _a1 types.RequestProcessProposal) (*types.ResponseProcessProposal, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *types.ResponseProcessProposal
+	if rf, ok := ret.Get(0).(func(context.Context, types.RequestProcessProposal) *types.ResponseProcessProposal); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*types.ResponseProcessProposal)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.RequestProcessProposal) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // SetResponseCallback provides a mock function with given fields: _a0
 func (_m *AppConnConsensus) SetResponseCallback(_a0 abciclient.Callback) {
 	_m.Called(_a0)

internal/state/execution.go

@@ -148,6 +148,23 @@ func (blockExec *BlockExecutor) CreateProposalBlock(
 	return state.MakeBlock(height, modifiedTxs, commit, evidence, proposerAddr)
 }
 
+func (blockExec *BlockExecutor) ProcessProposal(
+	ctx context.Context,
+	block *types.Block,
+) (bool, error) {
+	req := abci.RequestProcessProposal{
+		Txs:    block.Data.Txs.ToSliceOfBytes(),
+		Header: *block.Header.ToProto(),
+	}
+
+	resp, err := blockExec.proxyApp.ProcessProposal(ctx, req)
+	if err != nil {
+		return false, ErrInvalidBlock(err)
+	}
+
+	return resp.IsOK(), nil
+}
+
 // ValidateBlock validates the given block against the given state.
 // If the block is invalid, it returns an error.
 // Validation does not mutate state, but does require historical information from the stateDB,

internal/state/execution_test.go

@@ -24,6 +24,7 @@ import (
 	"github.com/tendermint/tendermint/internal/state/mocks"
 	sf "github.com/tendermint/tendermint/internal/state/test/factory"
 	"github.com/tendermint/tendermint/internal/store"
+	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	tmtime "github.com/tendermint/tendermint/libs/time"
 	"github.com/tendermint/tendermint/types"
@@ -239,6 +240,47 @@ func TestBeginBlockByzantineValidators(t *testing.T) {
 	assert.Equal(t, abciEv, app.ByzantineValidators)
 }
 
+func TestProcessProposal(t *testing.T) {
+	height := 1
+	runTest := func(txs types.Txs, expectAccept bool) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		app := &testApp{}
+		cc := abciclient.NewLocalCreator(app)
+		logger := log.TestingLogger()
+		proxyApp := proxy.NewAppConns(cc, logger, proxy.NopMetrics())
+		err := proxyApp.Start(ctx)
+		require.NoError(t, err)
+
+		state, stateDB, _ := makeState(t, 1, height)
+		stateStore := sm.NewStore(stateDB)
+		blockStore := store.NewBlockStore(dbm.NewMemDB())
+
+		blockExec := sm.NewBlockExecutor(
+			stateStore,
+			logger,
+			proxyApp.Consensus(),
+			mmock.Mempool{},
+			sm.EmptyEvidencePool{},
+			blockStore,
+		)
+
+		block, err := sf.MakeBlock(state, int64(height), new(types.Commit))
+		require.NoError(t, err)
+		block.Txs = txs
+		acceptBlock, err := blockExec.ProcessProposal(ctx, block)
+		require.NoError(t, err)
+		require.Equal(t, expectAccept, acceptBlock)
+	}
+	goodTxs := factory.MakeTenTxs(int64(height))
+	runTest(goodTxs, true)
+	// testApp has process proposal fail if any tx is 0-len
+	badTxs := factory.MakeTenTxs(int64(height))
+	badTxs[0] = types.Tx{}
+	runTest(badTxs, false)
+}
+
 func TestValidateValidatorUpdates(t *testing.T) {
 	pubkey1 := ed25519.GenPrivKey().PubKey()
 	pubkey2 := ed25519.GenPrivKey().PubKey()

internal/state/helpers_test.go

@@ -327,3 +327,12 @@ func (app *testApp) Commit() abci.ResponseCommit {
 func (app *testApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
 	return
 }
+
+func (app *testApp) ProcessProposal(req abci.RequestProcessProposal) abci.ResponseProcessProposal {
+	for _, tx := range req.Txs {
+		if len(tx) == 0 {
+			return abci.ResponseProcessProposal{Result: abci.ResponseProcessProposal_REJECT}
+		}
+	}
+	return abci.ResponseProcessProposal{Result: abci.ResponseProcessProposal_ACCEPT}
+}

types/tx.go

@@ -93,6 +93,7 @@ func (txs Txs) ToSliceOfBytes() [][]byte {
 }
 
 // ToTxs converts a raw slice of byte slices into a Txs type.
+// TODO This function is to disappear when TxRecord is introduced
 func ToTxs(txs [][]byte) Txs {
 	txBzs := make(Txs, len(txs))
 	for i := 0; i < len(txs); i++ {