rpc: add pagination to /validators (#3993)

* Add pagination to /validators

- closes #3472

Signed-off-by: Marko Baricevic <marbar3778@yahoo.com>

* add swagger params, default returns all

* address pr comments

* golint fix

* swagger default change, change to default in comment

* swagger.yaml: replace x-example with example

https://swagger.io/docs/specification/adding-examples/

* Revert "swagger.yaml: replace x-example with example"

This reverts commit 9df1b006de.

* update changelog and remove extra body close

CHANGELOG_PENDING.md

@@ -11,6 +11,7 @@ program](https://hackerone.com/tendermint).
 ### BREAKING CHANGES:
 
 - CLI/RPC/Config
+  - [rpc] \#3471 Paginate `/validators` response (default: 30 vals per page)
   - [rpc] \#3188 Remove `BlockMeta` in `ResultBlock` in favor of `BlockId` for `/block`
   - [rpc] `/block_results` response format updated (see RPC docs for details)
     ```
@@ -32,6 +33,7 @@ program](https://hackerone.com/tendermint).
 
 - Go API
   - [libs/pubsub] [\#4070](https://github.com/tendermint/tendermint/pull/4070) `Query#(Matches|Conditions)` returns an error.
+  - [rpc/client] \#3471 `Validators` now requires two more args: `page` and `perPage`
 
 - Blockchain Protocol
   - [abci] \#2521 Remove `TotalTxs` and `NumTxs` from `Header`

lite/client/provider.go

@@ -106,7 +106,7 @@ func (p *provider) getValidatorSet(chainID string, height int64) (valset *types.
 		err = fmt.Errorf("expected height >= 1, got height %v", height)
 		return
 	}
-	res, err := p.client.Validators(&height)
+	res, err := p.client.Validators(&height, 0, 0)
 	if err != nil {
 		// TODO pass through other types of errors.
 		return nil, lerr.ErrUnknownValidators(chainID, height)

lite/proxy/proxy.go

@@ -134,7 +134,7 @@ func makeValidatorsFunc(c rpcclient.Client) func(
 	height *int64,
 ) (*ctypes.ResultValidators, error) {
 	return func(ctx *rpctypes.Context, height *int64) (*ctypes.ResultValidators, error) {
-		return c.Validators(height)
+		return c.Validators(height, 0, 0)
 	}
 }
 

p2p/switch_test.go

@@ -368,9 +368,10 @@ func TestSwitchStopPeerForError(t *testing.T) {
 	defer s.Close()
 
 	scrapeMetrics := func() string {
-		resp, _ := http.Get(s.URL)
+		resp, err := http.Get(s.URL)
+		assert.NoError(t, err)
+		defer resp.Body.Close()
 		buf, _ := ioutil.ReadAll(resp.Body)
-		resp.Body.Close()
 		return string(buf)
 	}
 

rpc/client/httpclient.go

@@ -339,9 +339,13 @@ func (c *baseRPCClient) TxSearch(query string, prove bool, page, perPage int) (*
 	return result, nil
 }
 
-func (c *baseRPCClient) Validators(height *int64) (*ctypes.ResultValidators, error) {
+func (c *baseRPCClient) Validators(height *int64, page, perPage int) (*ctypes.ResultValidators, error) {
 	result := new(ctypes.ResultValidators)
-	_, err := c.caller.Call("validators", map[string]interface{}{"height": height}, result)
+	_, err := c.caller.Call("validators", map[string]interface{}{
+		"height":   height,
+		"page":     page,
+		"per_page": perPage,
+	}, result)
 	if err != nil {
 		return nil, errors.Wrap(err, "Validators")
 	}

rpc/client/interface.go

@@ -66,7 +66,7 @@ type SignClient interface {
 	Block(height *int64) (*ctypes.ResultBlock, error)
 	BlockResults(height *int64) (*ctypes.ResultBlockResults, error)
 	Commit(height *int64) (*ctypes.ResultCommit, error)
-	Validators(height *int64) (*ctypes.ResultValidators, error)
+	Validators(height *int64, page, perPage int) (*ctypes.ResultValidators, error)
 	Tx(hash []byte, prove bool) (*ctypes.ResultTx, error)
 	TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error)
 }

rpc/client/localclient.go

@@ -148,8 +148,8 @@ func (c *Local) Commit(height *int64) (*ctypes.ResultCommit, error) {
 	return core.Commit(c.ctx, height)
 }
 
-func (c *Local) Validators(height *int64) (*ctypes.ResultValidators, error) {
-	return core.Validators(c.ctx, height)
+func (c *Local) Validators(height *int64, page, perPage int) (*ctypes.ResultValidators, error) {
+	return core.Validators(c.ctx, height, page, perPage)
 }
 
 func (c *Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {

rpc/client/mock/client.go

@@ -149,8 +149,8 @@ func (c Client) Commit(height *int64) (*ctypes.ResultCommit, error) {
 	return core.Commit(&rpctypes.Context{}, height)
 }
 
-func (c Client) Validators(height *int64) (*ctypes.ResultValidators, error) {
-	return core.Validators(&rpctypes.Context{}, height)
+func (c Client) Validators(height *int64, page, perPage int) (*ctypes.ResultValidators, error) {
+	return core.Validators(&rpctypes.Context{}, height, page, perPage)
 }
 
 func (c Client) BroadcastEvidence(ev types.Evidence) (*ctypes.ResultBroadcastEvidence, error) {

rpc/client/rpc_test.go

@@ -152,7 +152,7 @@ func TestGenesisAndValidators(t *testing.T) {
 		gval := gen.Genesis.Validators[0]
 
 		// get the current validators
-		vals, err := c.Validators(nil)
+		vals, err := c.Validators(nil, 0, 0)
 		require.Nil(t, err, "%d: %+v", i, err)
 		require.Equal(t, 1, len(vals.Validators))
 		val := vals.Validators[0]

rpc/core/consensus.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	cm "github.com/tendermint/tendermint/consensus"
+	cmn "github.com/tendermint/tendermint/libs/common"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	rpctypes "github.com/tendermint/tendermint/rpc/lib/types"
 	sm "github.com/tendermint/tendermint/state"
@@ -13,7 +14,7 @@ import (
 // Note the validators are sorted by their address - this is the canonical
 // order for the validators in the set as used in computing their Merkle root.
 // More: https://tendermint.com/rpc/#/Info/validators
-func Validators(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultValidators, error) {
+func Validators(ctx *rpctypes.Context, heightPtr *int64, page, perPage int) (*ctypes.ResultValidators, error) {
 	// The latest validator that we know is the
 	// NextValidator of the last block.
 	height := consensusState.GetState().LastBlockHeight + 1
@@ -26,9 +27,21 @@ func Validators(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultValidato
 	if err != nil {
 		return nil, err
 	}
+
+	totalCount := len(validators.Validators)
+	perPage = validatePerPage(perPage)
+	page, err = validatePage(page, perPage, totalCount)
+	if err != nil {
+		return nil, err
+	}
+
+	skipCount := validateSkipCount(page, perPage)
+
+	v := validators.Validators[skipCount : skipCount+cmn.MinInt(perPage, totalCount-skipCount)]
+
 	return &ctypes.ResultValidators{
 		BlockHeight: height,
-		Validators:  validators.Validators}, nil
+		Validators:  v}, nil
 }
 
 // DumpConsensusState dumps consensus state.

rpc/core/routes.go

@@ -23,7 +23,7 @@ var Routes = map[string]*rpc.RPCFunc{
 	"commit":               rpc.NewRPCFunc(Commit, "height"),
 	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove"),
 	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page"),
-	"validators":           rpc.NewRPCFunc(Validators, "height"),
+	"validators":           rpc.NewRPCFunc(Validators, "height,page,per_page"),
 	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""),
 	"consensus_state":      rpc.NewRPCFunc(ConsensusState, ""),
 	"consensus_params":     rpc.NewRPCFunc(ConsensusParams, "height"),

rpc/lib/server/handlers_test.go

@@ -71,6 +71,7 @@ func TestRPCParams(t *testing.T) {
 		rec := httptest.NewRecorder()
 		mux.ServeHTTP(rec, req)
 		res := rec.Result()
+		defer res.Body.Close()
 		// Always expecting back a JSONRPCResponse
 		assert.True(t, statusOK(res.StatusCode), "#%d: should always return 2XX", i)
 		blob, err := ioutil.ReadAll(res.Body)
@@ -78,7 +79,6 @@ func TestRPCParams(t *testing.T) {
 			t.Errorf("#%d: err reading body: %v", i, err)
 			continue
 		}
-		res.Body.Close()
 
 		recv := new(types.RPCResponse)
 		assert.Nil(t, json.Unmarshal(blob, recv), "#%d: expecting successful parsing of an RPCResponse:\nblob: %s", i, blob)

rpc/swagger/swagger.yaml

@@ -584,6 +584,20 @@ paths:
           description: height to return. If no height is provided, it will fetch validato set at the latest block. 0 means latest
           default: 0
           x-example: 1
+        - in: query
+          name: page
+          type: number
+          description: "Page number (1-based)"
+          required: false
+          x-example: 1
+          default: 0
+        - in: query
+          name: per_page
+          type: number
+          description: "Number of entries per page (max: 100)"
+          required: false
+          x-example: 30
+          default: 30
       tags:
         - Info
       description: |