From da9eefd1116552a2b6f74a688b7a56ed1f6123b5 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 14 Jul 2021 09:22:53 +0000 Subject: [PATCH] rpc: add chunked rpc interface (backport #6445) (#6717) * rpc: add chunked rpc interface (#6445) (cherry picked from commit d9134063e7f556876bdbbcf56d822b3d7c5cee07) # Conflicts: # light/proxy/routes.go # node/node.go # rpc/core/net.go # rpc/core/routes.go * fix conflicts Co-authored-by: Sam Kleinman Co-authored-by: marbar3778 --- light/proxy/routes.go | 9 +++++++++ light/rpc/client.go | 4 ++++ node/node.go | 4 ++++ rpc/client/http/http.go | 9 +++++++++ rpc/client/interface.go | 1 + rpc/client/local/local.go | 4 ++++ rpc/client/mocks/client.go | 25 +++++++++++++++++++++++- rpc/client/rpc_test.go | 27 ++++++++++++++++++++++++++ rpc/core/env.go | 38 +++++++++++++++++++++++++++++++++++++ rpc/core/net.go | 26 +++++++++++++++++++++++++ rpc/core/routes.go | 1 + rpc/core/types/responses.go | 10 ++++++++++ 12 files changed, 157 insertions(+), 1 deletion(-) diff --git a/light/proxy/routes.go b/light/proxy/routes.go index e28b23e0c..2f53c8808 100644 --- a/light/proxy/routes.go +++ b/light/proxy/routes.go @@ -23,6 +23,7 @@ func RPCRoutes(c *lrpc.Client) map[string]*rpcserver.RPCFunc { "net_info": rpcserver.NewRPCFunc(makeNetInfoFunc(c), ""), "blockchain": rpcserver.NewRPCFunc(makeBlockchainInfoFunc(c), "minHeight,maxHeight"), "genesis": rpcserver.NewRPCFunc(makeGenesisFunc(c), ""), + "genesis_chunked": rpcserver.NewRPCFunc(makeGenesisChunkedFunc(c), ""), "block": rpcserver.NewRPCFunc(makeBlockFunc(c), "height"), "block_by_hash": rpcserver.NewRPCFunc(makeBlockByHashFunc(c), "hash"), "block_results": rpcserver.NewRPCFunc(makeBlockResultsFunc(c), "height"), @@ -92,6 +93,14 @@ func makeGenesisFunc(c *lrpc.Client) rpcGenesisFunc { } } +type rpcGenesisChunkedFunc func(ctx *rpctypes.Context, chunk uint) (*ctypes.ResultGenesisChunk, error) + +func makeGenesisChunkedFunc(c *lrpc.Client) rpcGenesisChunkedFunc { + return func(ctx *rpctypes.Context, chunk uint) (*ctypes.ResultGenesisChunk, error) { + return c.GenesisChunked(ctx.Context(), chunk) + } +} + type rpcBlockFunc func(ctx *rpctypes.Context, height *int64) (*ctypes.ResultBlock, error) func makeBlockFunc(c *lrpc.Client) rpcBlockFunc { diff --git a/light/rpc/client.go b/light/rpc/client.go index 079d5b6a7..6c2461b47 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -304,6 +304,10 @@ func (c *Client) Genesis(ctx context.Context) (*ctypes.ResultGenesis, error) { return c.next.Genesis(ctx) } +func (c *Client) GenesisChunked(ctx context.Context, id uint) (*ctypes.ResultGenesisChunk, error) { + return c.next.GenesisChunked(ctx, id) +} + // Block calls rpcclient#Block and then verifies the result. func (c *Client) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) { res, err := c.next.Block(ctx, height) diff --git a/node/node.go b/node/node.go index 205c1f747..2358fd35d 100644 --- a/node/node.go +++ b/node/node.go @@ -1028,6 +1028,10 @@ func (n *Node) ConfigureRPC() error { Config: *n.config.RPC, }) + if err := rpccore.InitGenesisChunks(); err != nil { + return err + } + return nil } diff --git a/rpc/client/http/http.go b/rpc/client/http/http.go index 10fada8a6..64c3cf727 100644 --- a/rpc/client/http/http.go +++ b/rpc/client/http/http.go @@ -394,6 +394,15 @@ func (c *baseRPCClient) Genesis(ctx context.Context) (*ctypes.ResultGenesis, err return result, nil } +func (c *baseRPCClient) GenesisChunked(ctx context.Context, id uint) (*ctypes.ResultGenesisChunk, error) { + result := new(ctypes.ResultGenesisChunk) + _, err := c.caller.Call(ctx, "genesis_chunked", map[string]interface{}{"chunk": id}, result) + if err != nil { + return nil, err + } + return result, nil +} + func (c *baseRPCClient) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) { result := new(ctypes.ResultBlock) params := make(map[string]interface{}) diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 9fa909272..cda5c541c 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -94,6 +94,7 @@ type SignClient interface { // HistoryClient provides access to data from genesis to now in large chunks. type HistoryClient interface { Genesis(context.Context) (*ctypes.ResultGenesis, error) + GenesisChunked(context.Context, uint) (*ctypes.ResultGenesisChunk, error) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error) } diff --git a/rpc/client/local/local.go b/rpc/client/local/local.go index 72f18e00f..62c8415a7 100644 --- a/rpc/client/local/local.go +++ b/rpc/client/local/local.go @@ -153,6 +153,10 @@ func (c *Local) Genesis(ctx context.Context) (*ctypes.ResultGenesis, error) { return core.Genesis(c.ctx) } +func (c *Local) GenesisChunked(ctx context.Context, id uint) (*ctypes.ResultGenesisChunk, error) { + return core.GenesisChunked(c.ctx, id) +} + func (c *Local) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) { return core.Block(c.ctx, height) } diff --git a/rpc/client/mocks/client.go b/rpc/client/mocks/client.go index 265ba796d..f8eb7a45c 100644 --- a/rpc/client/mocks/client.go +++ b/rpc/client/mocks/client.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.6.0. DO NOT EDIT. +// Code generated by mockery 2.7.4. DO NOT EDIT. package mocks @@ -436,6 +436,29 @@ func (_m *Client) Genesis(_a0 context.Context) (*coretypes.ResultGenesis, error) return r0, r1 } +// GenesisChunked provides a mock function with given fields: _a0, _a1 +func (_m *Client) GenesisChunked(_a0 context.Context, _a1 uint) (*coretypes.ResultGenesisChunk, error) { + ret := _m.Called(_a0, _a1) + + var r0 *coretypes.ResultGenesisChunk + if rf, ok := ret.Get(0).(func(context.Context, uint) *coretypes.ResultGenesisChunk); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*coretypes.ResultGenesisChunk) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, uint) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Health provides a mock function with given fields: _a0 func (_m *Client) Health(_a0 context.Context) (*coretypes.ResultHealth, error) { ret := _m.Called(_a0) diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 49f26b85f..fb2d441b3 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -2,6 +2,7 @@ package client_test import ( "context" + "encoding/base64" "fmt" "math" "net/http" @@ -14,6 +15,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" + tmjson "github.com/tendermint/tendermint/libs/json" "github.com/tendermint/tendermint/libs/log" tmmath "github.com/tendermint/tendermint/libs/math" mempl "github.com/tendermint/tendermint/mempool" @@ -186,6 +188,31 @@ func TestGenesisAndValidators(t *testing.T) { } } +func TestGenesisChunked(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, c := range GetClients() { + first, err := c.GenesisChunked(ctx, 0) + require.NoError(t, err) + + decoded := make([]string, 0, first.TotalChunks) + for i := 0; i < first.TotalChunks; i++ { + chunk, err := c.GenesisChunked(ctx, uint(i)) + require.NoError(t, err) + data, err := base64.StdEncoding.DecodeString(chunk.Data) + require.NoError(t, err) + decoded = append(decoded, string(data)) + + } + doc := []byte(strings.Join(decoded, "")) + + var out types.GenesisDoc + require.NoError(t, tmjson.Unmarshal(doc, &out), + "first: %+v, doc: %s", first, string(doc)) + } +} + func TestABCIQuery(t *testing.T) { for i, c := range GetClients() { // write something diff --git a/rpc/core/env.go b/rpc/core/env.go index c24c3d262..11a51bfe7 100644 --- a/rpc/core/env.go +++ b/rpc/core/env.go @@ -1,12 +1,14 @@ package core import ( + "encoding/base64" "fmt" "time" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/crypto" + tmjson "github.com/tendermint/tendermint/libs/json" "github.com/tendermint/tendermint/libs/log" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" @@ -25,6 +27,10 @@ const ( // SubscribeTimeout is the maximum time we wait to subscribe for an event. // must be less than the server's write timeout (see rpcserver.DefaultConfig) SubscribeTimeout = 5 * time.Second + + // genesisChunkSize is the maximum size, in bytes, of each + // chunk in the genesis structure for the chunked API + genesisChunkSize = 16 * 1024 * 1024 // 16 ) var ( @@ -91,6 +97,9 @@ type Environment struct { Logger log.Logger Config cfg.RPCConfig + + // cache of chunked genesis data. + genChunks []string } //---------------------------------------------- @@ -130,6 +139,35 @@ func validatePerPage(perPagePtr *int) int { return perPage } +// InitGenesisChunks configures the environment and should be called on service +// startup. +func InitGenesisChunks() error { + if env.genChunks != nil { + return nil + } + + if env.GenDoc == nil { + return nil + } + + data, err := tmjson.Marshal(env.GenDoc) + if err != nil { + return err + } + + for i := 0; i < len(data); i += genesisChunkSize { + end := i + genesisChunkSize + + if end > len(data) { + end = len(data) + } + + env.genChunks = append(env.genChunks, base64.StdEncoding.EncodeToString(data[i:end])) + } + + return nil +} + func validateSkipCount(page, perPage int) int { skipCount := (page - 1) * perPage if skipCount < 0 { diff --git a/rpc/core/net.go b/rpc/core/net.go index a8aedf9e0..2a0e2c92d 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -94,9 +94,35 @@ func UnsafeDialPeers(ctx *rpctypes.Context, peers []string, persistent, uncondit // Genesis returns genesis file. // More: https://docs.tendermint.com/master/rpc/#/Info/genesis func Genesis(ctx *rpctypes.Context) (*ctypes.ResultGenesis, error) { + if len(env.genChunks) > 1 { + return nil, errors.New("genesis response is large, please use the genesis_chunked API instead") + } + return &ctypes.ResultGenesis{Genesis: env.GenDoc}, nil } +func GenesisChunked(ctx *rpctypes.Context, chunk uint) (*ctypes.ResultGenesisChunk, error) { + if env.genChunks == nil { + return nil, fmt.Errorf("service configuration error, genesis chunks are not initialized") + } + + if len(env.genChunks) == 0 { + return nil, fmt.Errorf("service configuration error, there are no chunks") + } + + id := int(chunk) + + if id > len(env.genChunks)-1 { + return nil, fmt.Errorf("there are %d chunks, %d is invalid", len(env.genChunks)-1, id) + } + + return &ctypes.ResultGenesisChunk{ + TotalChunks: len(env.genChunks), + ChunkNumber: id, + Data: env.genChunks[id], + }, nil +} + func getIDs(peers []string) ([]string, error) { ids := make([]string, 0, len(peers)) diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 5583d6f29..195c6089a 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,6 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "net_info": rpc.NewRPCFunc(NetInfo, ""), "blockchain": rpc.NewRPCFunc(BlockchainInfo, "minHeight,maxHeight"), "genesis": rpc.NewRPCFunc(Genesis, ""), + "genesis_chunked": rpc.NewRPCFunc(GenesisChunked, "chunk"), "block": rpc.NewRPCFunc(Block, "height"), "block_by_hash": rpc.NewRPCFunc(BlockByHash, "hash"), "block_results": rpc.NewRPCFunc(BlockResults, "height"), diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 593e0f797..68f3e2b96 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -23,6 +23,16 @@ type ResultGenesis struct { Genesis *types.GenesisDoc `json:"genesis"` } +// ResultGenesisChunk is the output format for the chunked/paginated +// interface. These chunks are produced by converting the genesis +// document to JSON and then splitting the resulting payload into +// 16 megabyte blocks and then base64 encoding each block. +type ResultGenesisChunk struct { + ChunkNumber int `json:"chunk"` + TotalChunks int `json:"total"` + Data string `json:"data"` +} + // Single block (with meta) type ResultBlock struct { BlockID types.BlockID `json:"block_id"`