Merge pull request #1567 from tendermint/909-limit-tx-search-output

limit /tx_search output

CHANGELOG.md

@@ -15,6 +15,10 @@ BUG FIXES
 
 - [consensus] Fix issue #1575 where a late proposer can get stuck
 
+BREAKING:
+- [rpc] `/tx_search` now outputs maximum `?per_page` txs (you can provide custom `?per_page` up to 100; defaults to 30). You can set the `?page` (starts at 1).
+- [rpc] `/unconfirmed_txs` now outputs maximum `?limit` txs (you can provide custom `?limit` up to 100; defaults to 30)
+
 ## 0.19.3 (May 14th, 2018)
 
 FEATURES

rpc/client/httpclient.go

@@ -204,17 +204,19 @@ func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 	return result, nil
 }
 
-func (c *HTTP) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) {
-	results := new([]*ctypes.ResultTx)
+func (c *HTTP) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) {
+	result := new(ctypes.ResultTxSearch)
 	params := map[string]interface{}{
-		"query": query,
-		"prove": prove,
+		"query":    query,
+		"prove":    prove,
+		"page":     page,
+		"per_page": perPage,
 	}
-	_, err := c.rpc.Call("tx_search", params, results)
+	_, err := c.rpc.Call("tx_search", params, result)
 	if err != nil {
 		return nil, errors.Wrap(err, "TxSearch")
 	}
-	return *results, nil
+	return result, nil
 }
 
 func (c *HTTP) Validators(height *int64) (*ctypes.ResultValidators, error) {

rpc/client/interface.go

@@ -50,7 +50,7 @@ type SignClient interface {
 	Commit(height *int64) (*ctypes.ResultCommit, error)
 	Validators(height *int64) (*ctypes.ResultValidators, error)
 	Tx(hash []byte, prove bool) (*ctypes.ResultTx, error)
-	TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error)
+	TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error)
 }
 
 // HistoryClient shows us data from genesis to now in large chunks.

rpc/client/localclient.go

@@ -128,8 +128,8 @@ func (Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 	return core.Tx(hash, prove)
 }
 
-func (Local) TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) {
-	return core.TxSearch(query, prove)
+func (Local) TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) {
+	return core.TxSearch(query, prove, page, perPage)
 }
 
 func (c *Local) Subscribe(ctx context.Context, subscriber string, query tmpubsub.Query, out chan<- interface{}) error {

rpc/client/rpc_test.go

@@ -334,11 +334,11 @@ func TestTxSearch(t *testing.T) {
 
 		// now we query for the tx.
 		// since there's only one tx, we know index=0.
-		results, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true)
+		result, err := c.TxSearch(fmt.Sprintf("tx.hash='%v'", txHash), true, 1, 30)
 		require.Nil(t, err, "%+v", err)
-		require.Len(t, results, 1)
+		require.Len(t, result.Txs, 1)
 
-		ptx := results[0]
+		ptx := result.Txs[0]
 		assert.EqualValues(t, txHeight, ptx.Height)
 		assert.EqualValues(t, tx, ptx.Tx)
 		assert.Zero(t, ptx.Index)
@@ -352,14 +352,14 @@ func TestTxSearch(t *testing.T) {
 		}
 
 		// we query for non existing tx
-		results, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false)
+		result, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false, 1, 30)
 		require.Nil(t, err, "%+v", err)
-		require.Len(t, results, 0)
+		require.Len(t, result.Txs, 0)
 
 		// we query using a tag (see kvstore application)
-		results, err = c.TxSearch("app.creator='jae'", false)
+		result, err = c.TxSearch("app.creator='jae'", false, 1, 30)
 		require.Nil(t, err, "%+v", err)
-		if len(results) == 0 {
+		if len(result.Txs) == 0 {
 			t.Fatal("expected a lot of transactions")
 		}
 	}

rpc/core/README.md

@@ -13,3 +13,9 @@ go get github.com/melekes/godoc2md
 
 godoc2md -template rpc/core/doc_template.txt github.com/tendermint/tendermint/rpc/core | grep -v -e "pipe.go" -e "routes.go" -e "dev.go" | sed 's$/src/target$https://github.com/tendermint/tendermint/tree/master/rpc/core$'
 ```
+
+## Pagination
+
+Requests that return multiple items will be paginated to 30 items by default.
+You can specify further pages with the ?page parameter. You can also set a
+custom page size up to 100 with the ?per_page parameter.

rpc/core/mempool.go

@@ -209,7 +209,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 	}
 }
 
-// Get unconfirmed transactions including their number.
+// Get unconfirmed transactions (maximum ?limit entries) including their number.
 //
 // ```shell
 // curl 'localhost:46657/unconfirmed_txs'
@@ -232,9 +232,18 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
 //   "id": "",
 //   "jsonrpc": "2.0"
 // }
+//
+// ### Query Parameters
+//
+// | Parameter | Type | Default | Required | Description                          |
+// |-----------+------+---------+----------+--------------------------------------|
+// | limit     | int  | 30      | false    | Maximum number of entries (max: 100) |
 // ```
-func UnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) {
-	txs := mempool.Reap(-1)
+func UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) {
+	// reuse per_page validator
+	limit = validatePerPage(limit)
+
+	txs := mempool.Reap(limit)
 	return &ctypes.ResultUnconfirmedTxs{len(txs), txs}, nil
 }
 

rpc/core/pipe.go

@@ -14,6 +14,12 @@ import (
 	"github.com/tendermint/tmlibs/log"
 )
 
+const (
+	// see README
+	defaultPerPage = 30
+	maxPerPage     = 100
+)
+
 var subscribeTimeout = 5 * time.Second
 
 //----------------------------------------------
@@ -117,3 +123,21 @@ func SetLogger(l log.Logger) {
 func SetEventBus(b *types.EventBus) {
 	eventBus = b
 }
+
+func validatePage(page, perPage, totalCount int) int {
+	pages := ((totalCount - 1) / perPage) + 1
+	if page < 1 {
+		page = 1
+	} else if page > pages {
+		page = pages
+	}
+
+	return page
+}
+
+func validatePerPage(perPage int) int {
+	if perPage < 1 || perPage > maxPerPage {
+		return defaultPerPage
+	}
+	return perPage
+}

rpc/core/pipe_test.go

@@ -0,0 +1,67 @@
+package core
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPaginationPage(t *testing.T) {
+
+	cases := []struct {
+		totalCount int
+		perPage    int
+		page       int
+		newPage    int
+	}{
+		{0, 10, 0, 1},
+		{0, 10, 1, 1},
+		{0, 10, 2, 1},
+
+		{5, 10, -1, 1},
+		{5, 10, 0, 1},
+		{5, 10, 1, 1},
+		{5, 10, 2, 1},
+		{5, 10, 2, 1},
+
+		{5, 5, 1, 1},
+		{5, 5, 2, 1},
+		{5, 5, 3, 1},
+
+		{5, 3, 2, 2},
+		{5, 3, 3, 2},
+
+		{5, 2, 2, 2},
+		{5, 2, 3, 3},
+		{5, 2, 4, 3},
+	}
+
+	for _, c := range cases {
+		p := validatePage(c.page, c.perPage, c.totalCount)
+		assert.Equal(t, c.newPage, p, fmt.Sprintf("%v", c))
+	}
+
+}
+
+func TestPaginationPerPage(t *testing.T) {
+
+	cases := []struct {
+		totalCount int
+		perPage    int
+		newPerPage int
+	}{
+		{5, 0, defaultPerPage},
+		{5, 1, 1},
+		{5, 2, 2},
+		{5, defaultPerPage, defaultPerPage},
+		{5, maxPerPage - 1, maxPerPage - 1},
+		{5, maxPerPage, maxPerPage},
+		{5, maxPerPage + 1, defaultPerPage},
+	}
+
+	for _, c := range cases {
+		p := validatePerPage(c.perPage)
+		assert.Equal(t, c.newPerPage, p, fmt.Sprintf("%v", c))
+	}
+}

rpc/core/routes.go

@@ -22,11 +22,11 @@ var Routes = map[string]*rpc.RPCFunc{
 	"block_results":        rpc.NewRPCFunc(BlockResults, "height"),
 	"commit":               rpc.NewRPCFunc(Commit, "height"),
 	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove"),
-	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove"),
+	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page"),
 	"validators":           rpc.NewRPCFunc(Validators, "height"),
 	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""),
 	"consensus_state":      rpc.NewRPCFunc(ConsensusState, ""),
-	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, ""),
+	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, "limit"),
 	"num_unconfirmed_txs":  rpc.NewRPCFunc(NumUnconfirmedTxs, ""),
 
 	// broadcast API

rpc/core/tx.go

@@ -6,6 +6,7 @@ import (
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	"github.com/tendermint/tendermint/state/txindex/null"
 	"github.com/tendermint/tendermint/types"
+	cmn "github.com/tendermint/tmlibs/common"
 	tmquery "github.com/tendermint/tmlibs/pubsub/query"
 )
 
@@ -104,7 +105,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 	}, nil
 }
 
-// TxSearch allows you to query for multiple transactions results.
+// TxSearch allows you to query for multiple transactions results. It returns a
+// list of transactions (maximum ?per_page entries) and the total count.
 //
 // ```shell
 // curl "localhost:46657/tx_search?query=\"account.owner='Ivan'\"&prove=true"
@@ -120,43 +122,46 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 //
 // ```json
 // {
-//   "result": [
-//     {
-//       "proof": {
-//         "Proof": {
-//           "aunts": [
-//             "J3LHbizt806uKnABNLwG4l7gXCA=",
-//             "iblMO/M1TnNtlAefJyNCeVhjAb0=",
-//             "iVk3ryurVaEEhdeS0ohAJZ3wtB8=",
-//             "5hqMkTeGqpct51ohX0lZLIdsn7Q=",
-//             "afhsNxFnLlZgFDoyPpdQSe0bR8g="
-//           ]
-//         },
-//         "Data": "mvZHHa7HhZ4aRT0xMDA=",
-//         "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776",
-//         "Total": 32,
-//         "Index": 31
-//       },
-//       "tx": "mvZHHa7HhZ4aRT0xMDA=",
-//       "tx_result": {},
-//       "index": 31,
-//       "height": 12,
-//       "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF"
-//     }
-//   ],
+//   "jsonrpc": "2.0",
 //   "id": "",
-//   "jsonrpc": "2.0"
+//   "result": {
+// 	   "txs": [
+//       {
+//         "proof": {
+//           "Proof": {
+//             "aunts": [
+//               "J3LHbizt806uKnABNLwG4l7gXCA=",
+//               "iblMO/M1TnNtlAefJyNCeVhjAb0=",
+//               "iVk3ryurVaEEhdeS0ohAJZ3wtB8=",
+//               "5hqMkTeGqpct51ohX0lZLIdsn7Q=",
+//               "afhsNxFnLlZgFDoyPpdQSe0bR8g="
+//             ]
+//           },
+//           "Data": "mvZHHa7HhZ4aRT0xMDA=",
+//           "RootHash": "F6541223AA46E428CB1070E9840D2C3DF3B6D776",
+//           "Total": 32,
+//           "Index": 31
+//         },
+//         "tx": "mvZHHa7HhZ4aRT0xMDA=",
+//         "tx_result": {},
+//         "index": 31,
+//         "height": 12,
+//         "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF"
+//       }
+//     ],
+//     "total_count": 1
+//   }
 // }
 // ```
 //
-// Returns transactions matching the given query.
-//
 // ### Query Parameters
 //
 // | Parameter | Type   | Default | Required | Description                                               |
 // |-----------+--------+---------+----------+-----------------------------------------------------------|
 // | query     | string | ""      | true     | Query                                                     |
 // | prove     | bool   | false   | false    | Include proofs of the transactions inclusion in the block |
+// | page      | int    | 1       | false    | Page number (1-based)                                     |
+// | per_page  | int    | 30      | false    | Number of entries per page (max: 100)                     |
 //
 // ### Returns
 //
@@ -166,7 +171,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) {
 // - `index`: `int` - index of the transaction
 // - `height`: `int` - height of the block where this transaction was in
 // - `hash`: `[]byte` - hash of the transaction
-func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) {
+func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSearch, error) {
 	// if index is disabled, return error
 	if _, ok := txIndexer.(*null.TxIndex); ok {
 		return nil, fmt.Errorf("Transaction indexing is disabled")
@@ -182,11 +187,15 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) {
 		return nil, err
 	}
 
-	// TODO: we may want to consider putting a maximum on this length and somehow
-	// informing the user that things were truncated.
-	apiResults := make([]*ctypes.ResultTx, len(results))
+	totalCount := len(results)
+	page = validatePage(page, perPage, totalCount)
+	perPage = validatePerPage(perPage)
+	skipCount := (page - 1) * perPage
+
+	apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount))
 	var proof types.TxProof
-	for i, r := range results {
+	for i := 0; i < len(apiResults); i++ {
+		r := results[skipCount+i]
 		height := r.Height
 		index := r.Index
 
@@ -205,5 +214,5 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) {
 		}
 	}
 
-	return apiResults, nil
+	return &ctypes.ResultTxSearch{Txs: apiResults, TotalCount: totalCount}, nil
 }

rpc/core/types/responses.go

@@ -172,6 +172,12 @@ type ResultTx struct {
 	Proof    types.TxProof          `json:"proof,omitempty"`
 }
 
+// Result of searching for txs
+type ResultTxSearch struct {
+	Txs        []*ResultTx `json:"txs"`
+	TotalCount int         `json:"total_count"`
+}
+
 // List of mempool txs
 type ResultUnconfirmedTxs struct {
 	N   int        `json:"n_txs"`