From 569fd474c25f245cd0a715c97308a157e4764b8b Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Thu, 23 Feb 2017 00:21:13 -0500 Subject: [PATCH 01/84] added use of Cobra CLI squash --- cmd/tendermint/commands/gen_validator.go | 27 +++++++ cmd/tendermint/{ => commands}/init.go | 16 +++- cmd/tendermint/{ => commands}/probe_upnp.go | 16 +++- cmd/tendermint/commands/replay.go | 40 ++++++++++ .../{ => commands}/reset_priv_validator.go | 27 ++++++- cmd/tendermint/commands/root.go | 76 +++++++++++++++++++ cmd/tendermint/{ => commands}/run_node.go | 18 ++++- .../{ => commands}/show_validator.go | 16 +++- cmd/tendermint/commands/version.go | 21 +++++ cmd/tendermint/flags.go | 66 ---------------- cmd/tendermint/gen_validator.go | 15 ---- cmd/tendermint/log.go | 7 -- cmd/tendermint/main.go | 70 +---------------- 13 files changed, 246 insertions(+), 169 deletions(-) create mode 100644 cmd/tendermint/commands/gen_validator.go rename cmd/tendermint/{ => commands}/init.go (78%) rename cmd/tendermint/{ => commands}/probe_upnp.go (55%) create mode 100644 cmd/tendermint/commands/replay.go rename cmd/tendermint/{ => commands}/reset_priv_validator.go (57%) create mode 100644 cmd/tendermint/commands/root.go rename cmd/tendermint/{ => commands}/run_node.go (86%) rename cmd/tendermint/{ => commands}/show_validator.go (50%) create mode 100644 cmd/tendermint/commands/version.go delete mode 100644 cmd/tendermint/flags.go delete mode 100644 cmd/tendermint/gen_validator.go delete mode 100644 cmd/tendermint/log.go diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go new file mode 100644 index 000000000..a1217e1f0 --- /dev/null +++ b/cmd/tendermint/commands/gen_validator.go @@ -0,0 +1,27 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/types" +) + +var genValidatorCmd = &cobra.Command{ + Use: "gen_validator", + Short: "Generate new validator keypair", + Run: genValidator, +} + +func init() { + RootCmd.AddCommand(genValidatorCmd) +} + +func genValidator(cmd *cobra.Command, args []string) { + privValidator := types.GenPrivValidator() + privValidatorJSONBytes := wire.JSONBytesPretty(privValidator) + fmt.Printf(`%v +`, string(privValidatorJSONBytes)) +} diff --git a/cmd/tendermint/init.go b/cmd/tendermint/commands/init.go similarity index 78% rename from cmd/tendermint/init.go rename to cmd/tendermint/commands/init.go index 5ec1efefb..366ca4e84 100644 --- a/cmd/tendermint/init.go +++ b/cmd/tendermint/commands/init.go @@ -1,13 +1,25 @@ -package main +package commands import ( "os" + "github.com/spf13/cobra" + cmn "github.com/tendermint/go-common" "github.com/tendermint/tendermint/types" ) -func init_files() { +var initFilesCmd = &cobra.Command{ + Use: "init", + Short: "Initialize Tendermint", + Run: initFiles, +} + +func init() { + RootCmd.AddCommand(initFilesCmd) +} + +func initFiles(cmd *cobra.Command, args []string) { privValFile := config.GetString("priv_validator_file") if _, err := os.Stat(privValFile); os.IsNotExist(err) { privValidator := types.GenPrivValidator() diff --git a/cmd/tendermint/probe_upnp.go b/cmd/tendermint/commands/probe_upnp.go similarity index 55% rename from cmd/tendermint/probe_upnp.go rename to cmd/tendermint/commands/probe_upnp.go index 5dcadddcd..bff433bc9 100644 --- a/cmd/tendermint/probe_upnp.go +++ b/cmd/tendermint/commands/probe_upnp.go @@ -1,13 +1,25 @@ -package main +package commands import ( "encoding/json" "fmt" + "github.com/spf13/cobra" + "github.com/tendermint/go-p2p/upnp" ) -func probe_upnp() { +var probeUpnpCmd = &cobra.Command{ + Use: "probe_upnp", + Short: "Test UPnP functionality", + Run: probeUpnp, +} + +func init() { + RootCmd.AddCommand(probeUpnpCmd) +} + +func probeUpnp(cmd *cobra.Command, args []string) { capabilities, err := upnp.Probe() if err != nil { diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go new file mode 100644 index 000000000..4579ea5ef --- /dev/null +++ b/cmd/tendermint/commands/replay.go @@ -0,0 +1,40 @@ +package commands + +import ( + "fmt" + + "github.com/tendermint/tendermint/consensus" + + "github.com/spf13/cobra" +) + +var replayCmd = &cobra.Command{ + Use: "replay", + Short: "Replay messages from WAL", + Run: func(cmd *cobra.Command, args []string) { + + if len(args) > 1 { + consensus.RunReplayFile(config, args[1], false) + } else { + fmt.Println("replay requires an argument (walfile)") + } + }, +} + +var replayConsoleCmd = &cobra.Command{ + Use: "replay_console", + Short: "Replay messages from WAL in a console", + Run: func(cmd *cobra.Command, args []string) { + + if len(args) > 1 { + consensus.RunReplayFile(config, args[1], true) + } else { + fmt.Println("replay_console requires an argument (walfile)") + } + }, +} + +func init() { + RootCmd.AddCommand(replayCmd) + RootCmd.AddCommand(replayConsoleCmd) +} diff --git a/cmd/tendermint/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go similarity index 57% rename from cmd/tendermint/reset_priv_validator.go rename to cmd/tendermint/commands/reset_priv_validator.go index 5bf3ba69b..217c387a5 100644 --- a/cmd/tendermint/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -1,22 +1,41 @@ -package main +package commands import ( "os" + "github.com/spf13/cobra" + "github.com/tendermint/tendermint/types" ) +var resetAllCmd = &cobra.Command{ + Use: "unsafe_reset_all", + Short: "(unsafe) Remove all the data and WAL, reset this node's validator", + Run: resetAll, +} + +var resetPrivValidatorCmd = &cobra.Command{ + Use: "unsafe_reset_priv_validator", + Short: "(unsafe) Reset this node's validator", + Run: resetPrivValidator, +} + +func init() { + RootCmd.AddCommand(resetAllCmd) + RootCmd.AddCommand(resetPrivValidatorCmd) +} + // XXX: this is totally unsafe. // it's only suitable for testnets. -func reset_all() { - reset_priv_validator() +func resetAll(cmd *cobra.Command, args []string) { + resetPrivValidator(cmd, args) os.RemoveAll(config.GetString("db_dir")) os.Remove(config.GetString("cs_wal_file")) } // XXX: this is totally unsafe. // it's only suitable for testnets. -func reset_priv_validator() { +func resetPrivValidator(cmd *cobra.Command, args []string) { // Get PrivValidator var privValidator *types.PrivValidator privValidatorFile := config.GetString("priv_validator_file") diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go new file mode 100644 index 000000000..8916ad8a1 --- /dev/null +++ b/cmd/tendermint/commands/root.go @@ -0,0 +1,76 @@ +package commands + +import ( + "github.com/spf13/cobra" + + cfg "github.com/tendermint/go-config" + "github.com/tendermint/go-logger" + tmcfg "github.com/tendermint/tendermint/config/tendermint" +) + +var config cfg.Config +var log = logger.New("module", "main") + +//global flags +var ( + printHelp bool + moniker string + nodeLaddr string + seeds string + fastSync bool + skipUPNP bool + rpcLaddr string + grpcLaddr string + logLevel string + proxyApp string + abciTransport string + + pex bool +) + +var RootCmd = &cobra.Command{ + Use: "tendermint", + Short: "Tendermint Core (BFT Consensus) in Go", +} + +func init() { + + // Get configuration + config = tmcfg.GetConfig("") + + ///////////////////// + // parse flags + + // configuration options + RootCmd.PersistentFlags().StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name") + RootCmd.PersistentFlags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") + RootCmd.PersistentFlags().StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes") + RootCmd.PersistentFlags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") + RootCmd.PersistentFlags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") + RootCmd.PersistentFlags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") + RootCmd.PersistentFlags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") + RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") + RootCmd.PersistentFlags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + RootCmd.PersistentFlags().StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") + + // feature flags + RootCmd.PersistentFlags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)") + + //------------------ + + // Merge parsed flag values onto config + config.Set("moniker", moniker) + config.Set("node_laddr", nodeLaddr) + config.Set("seeds", seeds) + config.Set("fast_sync", fastSync) + config.Set("skip_upnp", skipUPNP) + config.Set("rpc_laddr", rpcLaddr) + config.Set("grpc_laddr", grpcLaddr) + config.Set("log_level", logLevel) + config.Set("proxy_app", proxyApp) + config.Set("abci", abciTransport) + config.Set("pex_reactor", pex) + + // set the log level + logger.SetLogLevel(config.GetString("log_level")) +} diff --git a/cmd/tendermint/run_node.go b/cmd/tendermint/commands/run_node.go similarity index 86% rename from cmd/tendermint/run_node.go rename to cmd/tendermint/commands/run_node.go index 1940a6056..b16c1ab07 100644 --- a/cmd/tendermint/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -1,22 +1,33 @@ -package main +package commands import ( "io/ioutil" "time" + "github.com/spf13/cobra" + . "github.com/tendermint/go-common" - cfg "github.com/tendermint/go-config" "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/types" ) +var runNodeCmd = &cobra.Command{ + Use: "node", + Short: "Run the tendermint node", + Run: runNode, +} + +func init() { + RootCmd.AddCommand(runNodeCmd) +} + // Users wishing to: // * Use an external signer for their validators // * Supply an in-proc abci app // should import github.com/tendermint/tendermint/node and implement // their own run_node to call node.NewNode (instead of node.NewNodeDefault) // with their custom priv validator and/or custom proxy.ClientCreator -func run_node(config cfg.Config) { +func runNode(cmd *cobra.Command, args []string) { // Wait until the genesis doc becomes available // This is for Mintnet compatibility. @@ -55,5 +66,4 @@ func run_node(config cfg.Config) { // Trap signal, run forever. n.RunForever() - } diff --git a/cmd/tendermint/show_validator.go b/cmd/tendermint/commands/show_validator.go similarity index 50% rename from cmd/tendermint/show_validator.go rename to cmd/tendermint/commands/show_validator.go index e6c68e31f..4aa80ae14 100644 --- a/cmd/tendermint/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -1,13 +1,25 @@ -package main +package commands import ( "fmt" + "github.com/spf13/cobra" + "github.com/tendermint/go-wire" "github.com/tendermint/tendermint/types" ) -func show_validator() { +var showValidatorCmd = &cobra.Command{ + Use: "show_validator", + Short: "Show this node's validator info", + Run: showValidator, +} + +func init() { + RootCmd.AddCommand(showValidatorCmd) +} + +func showValidator(cmd *cobra.Command, args []string) { privValidatorFile := config.GetString("priv_validator_file") privValidator := types.LoadOrGenPrivValidator(privValidatorFile) fmt.Println(string(wire.JSONBytesPretty(privValidator.PubKey))) diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go new file mode 100644 index 000000000..5c92160ea --- /dev/null +++ b/cmd/tendermint/commands/version.go @@ -0,0 +1,21 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/tendermint/tendermint/version" +) + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Show version info", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(version.Version) + }, +} + +func init() { + RootCmd.AddCommand(versionCmd) +} diff --git a/cmd/tendermint/flags.go b/cmd/tendermint/flags.go deleted file mode 100644 index fdf43b657..000000000 --- a/cmd/tendermint/flags.go +++ /dev/null @@ -1,66 +0,0 @@ -package main - -import ( - flag "github.com/spf13/pflag" - "os" - - cfg "github.com/tendermint/go-config" -) - -func parseFlags(config cfg.Config, args []string) { - var ( - printHelp bool - moniker string - nodeLaddr string - seeds string - fastSync bool - skipUPNP bool - rpcLaddr string - grpcLaddr string - logLevel string - proxyApp string - abciTransport string - - pex bool - ) - - // Declare flags - var flags = flag.NewFlagSet("main", flag.ExitOnError) - flags.BoolVar(&printHelp, "help", false, "Print this help message.") - - // configuration options - flags.StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name") - flags.StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") - flags.StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes") - flags.BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") - flags.BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") - flags.StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") - flags.StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") - flags.StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") - flags.StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), - "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - flags.StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") - - // feature flags - flags.BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)") - - flags.Parse(args) - if printHelp { - flags.PrintDefaults() - os.Exit(0) - } - - // Merge parsed flag values onto app. - config.Set("moniker", moniker) - config.Set("node_laddr", nodeLaddr) - config.Set("seeds", seeds) - config.Set("fast_sync", fastSync) - config.Set("skip_upnp", skipUPNP) - config.Set("rpc_laddr", rpcLaddr) - config.Set("grpc_laddr", grpcLaddr) - config.Set("log_level", logLevel) - config.Set("proxy_app", proxyApp) - config.Set("abci", abciTransport) - - config.Set("pex_reactor", pex) -} diff --git a/cmd/tendermint/gen_validator.go b/cmd/tendermint/gen_validator.go deleted file mode 100644 index 0fc0e1661..000000000 --- a/cmd/tendermint/gen_validator.go +++ /dev/null @@ -1,15 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/types" -) - -func gen_validator() { - privValidator := types.GenPrivValidator() - privValidatorJSONBytes := wire.JSONBytesPretty(privValidator) - fmt.Printf(`%v -`, string(privValidatorJSONBytes)) -} diff --git a/cmd/tendermint/log.go b/cmd/tendermint/log.go deleted file mode 100644 index ce626d2e9..000000000 --- a/cmd/tendermint/log.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import ( - "github.com/tendermint/go-logger" -) - -var log = logger.New("module", "main") diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 640f2c275..9c2b9aa33 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -4,76 +4,12 @@ import ( "fmt" "os" - cfg "github.com/tendermint/go-config" - "github.com/tendermint/go-logger" - tmcfg "github.com/tendermint/tendermint/config/tendermint" - "github.com/tendermint/tendermint/consensus" - "github.com/tendermint/tendermint/version" + cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" ) -var config cfg.Config - func main() { - - args := os.Args[1:] - if len(args) == 0 { - fmt.Println(`Tendermint - -Commands: - init Initialize tendermint - node Run the tendermint node - show_validator Show this node's validator info - gen_validator Generate new validator keypair - probe_upnp Test UPnP functionality - replay Replay messages from WAL - replay_console Replay messages from WAL in a console - unsafe_reset_all (unsafe) Remove all the data and WAL, reset this node's validator - unsafe_reset_priv_validator (unsafe) Reset this node's validator - version Show version info -`) - return - } - - // Get configuration - config = tmcfg.GetConfig("") - parseFlags(config, args[1:]) // Command line overrides - - // set the log level - logger.SetLogLevel(config.GetString("log_level")) - - switch args[0] { - case "node": - run_node(config) - case "replay": - if len(args) > 1 { - consensus.RunReplayFile(config, args[1], false) - } else { - fmt.Println("replay requires an argument (walfile)") - os.Exit(1) - } - case "replay_console": - if len(args) > 1 { - consensus.RunReplayFile(config, args[1], true) - } else { - fmt.Println("replay_console requires an argument (walfile)") - os.Exit(1) - } - case "init": - init_files() - case "show_validator": - show_validator() - case "gen_validator": - gen_validator() - case "probe_upnp": - probe_upnp() - case "unsafe_reset_all": - reset_all() - case "unsafe_reset_priv_validator": - reset_priv_validator() - case "version": - fmt.Println(version.Version) - default: - fmt.Printf("Unknown command %v\n", args[0]) + if err := cmd.RootCmd.Execute(); err != nil { + fmt.Println(err) os.Exit(1) } } From fa609366d4cc1c1603cd8eac52076eff2d1bac95 Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Thu, 23 Feb 2017 13:21:48 -0500 Subject: [PATCH 02/84] melekes change request --- cmd/tendermint/commands/replay.go | 4 +-- cmd/tendermint/commands/root.go | 49 ++--------------------------- cmd/tendermint/commands/run_node.go | 42 +++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 48 deletions(-) diff --git a/cmd/tendermint/commands/replay.go b/cmd/tendermint/commands/replay.go index 4579ea5ef..2f0e3266e 100644 --- a/cmd/tendermint/commands/replay.go +++ b/cmd/tendermint/commands/replay.go @@ -9,7 +9,7 @@ import ( ) var replayCmd = &cobra.Command{ - Use: "replay", + Use: "replay [walfile]", Short: "Replay messages from WAL", Run: func(cmd *cobra.Command, args []string) { @@ -22,7 +22,7 @@ var replayCmd = &cobra.Command{ } var replayConsoleCmd = &cobra.Command{ - Use: "replay_console", + Use: "replay_console [walfile]", Short: "Replay messages from WAL in a console", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 8916ad8a1..4d5c33aa1 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -11,22 +11,8 @@ import ( var config cfg.Config var log = logger.New("module", "main") -//global flags -var ( - printHelp bool - moniker string - nodeLaddr string - seeds string - fastSync bool - skipUPNP bool - rpcLaddr string - grpcLaddr string - logLevel string - proxyApp string - abciTransport string - - pex bool -) +//global flag +var logLevel string var RootCmd = &cobra.Command{ Use: "tendermint", @@ -38,38 +24,9 @@ func init() { // Get configuration config = tmcfg.GetConfig("") - ///////////////////// - // parse flags - - // configuration options - RootCmd.PersistentFlags().StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name") - RootCmd.PersistentFlags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") - RootCmd.PersistentFlags().StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes") - RootCmd.PersistentFlags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") - RootCmd.PersistentFlags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") - RootCmd.PersistentFlags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") - RootCmd.PersistentFlags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") + //parse flag and set config RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") - RootCmd.PersistentFlags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - RootCmd.PersistentFlags().StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") - - // feature flags - RootCmd.PersistentFlags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)") - - //------------------ - - // Merge parsed flag values onto config - config.Set("moniker", moniker) - config.Set("node_laddr", nodeLaddr) - config.Set("seeds", seeds) - config.Set("fast_sync", fastSync) - config.Set("skip_upnp", skipUPNP) - config.Set("rpc_laddr", rpcLaddr) - config.Set("grpc_laddr", grpcLaddr) config.Set("log_level", logLevel) - config.Set("proxy_app", proxyApp) - config.Set("abci", abciTransport) - config.Set("pex_reactor", pex) // set the log level logger.SetLogLevel(config.GetString("log_level")) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index b16c1ab07..10f0a5ad9 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -17,7 +17,49 @@ var runNodeCmd = &cobra.Command{ Run: runNode, } +//flags +var ( + printHelp bool + moniker string + nodeLaddr string + seeds string + fastSync bool + skipUPNP bool + rpcLaddr string + grpcLaddr string + proxyApp string + abciTransport string + pex bool +) + func init() { + + // configuration options + RootCmd.Flags().StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name") + RootCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") + RootCmd.Flags().StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes") + RootCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") + RootCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") + RootCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") + RootCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") + RootCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + RootCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") + + // feature flags + RootCmd.Flags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)") + + // Merge parsed flag values onto config + config.Set("moniker", moniker) + config.Set("node_laddr", nodeLaddr) + config.Set("seeds", seeds) + config.Set("fast_sync", fastSync) + config.Set("skip_upnp", skipUPNP) + config.Set("rpc_laddr", rpcLaddr) + config.Set("grpc_laddr", grpcLaddr) + config.Set("proxy_app", proxyApp) + config.Set("abci", abciTransport) + config.Set("pex_reactor", pex) + RootCmd.AddCommand(runNodeCmd) } From fab518fc98601348a160a29a154b2fc765e39821 Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Thu, 23 Feb 2017 19:25:23 -0500 Subject: [PATCH 03/84] flag fix, glide update squash --- cmd/tendermint/commands/root.go | 4 ++- cmd/tendermint/commands/run_node.go | 45 +++++++++++++++++++---------- cmd/tendermint/main.go | 4 +-- glide.lock | 2 ++ glide.yaml | 3 +- 5 files changed, 38 insertions(+), 20 deletions(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 4d5c33aa1..cee5b6604 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -17,6 +17,9 @@ var logLevel string var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", + PersistentPreRun: func(cmd *cobra.Command, args []string) { + config.Set("log_level", logLevel) + }, } func init() { @@ -26,7 +29,6 @@ func init() { //parse flag and set config RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") - config.Set("log_level", logLevel) // set the log level logger.SetLogLevel(config.GetString("log_level")) diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 10f0a5ad9..a04b52d09 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -12,14 +12,14 @@ import ( ) var runNodeCmd = &cobra.Command{ - Use: "node", - Short: "Run the tendermint node", - Run: runNode, + Use: "node", + Short: "Run the tendermint node", + PreRun: setConfigFlags, + Run: runNode, } //flags var ( - printHelp bool moniker string nodeLaddr string seeds string @@ -35,18 +35,33 @@ var ( func init() { // configuration options - RootCmd.Flags().StringVar(&moniker, "moniker", config.GetString("moniker"), "Node Name") - RootCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), "Node listen address. (0.0.0.0:0 means any interface, any port)") - RootCmd.Flags().StringVar(&seeds, "seeds", config.GetString("seeds"), "Comma delimited host:port seed nodes") - RootCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), "Fast blockchain syncing") - RootCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), "Skip UPNP configuration") - RootCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), "RPC listen address. Port required") - RootCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), "GRPC listen address (BroadcastTx only). Port required") - RootCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), "Proxy app address, or 'nilapp' or 'dummy' for local testing.") - RootCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci"), "Specify abci transport (socket | grpc)") + runNodeCmd.Flags().StringVar(&moniker, "moniker", config.GetString("moniker"), + "Node Name") + runNodeCmd.Flags().StringVar(&nodeLaddr, "node_laddr", config.GetString("node_laddr"), + "Node listen address. (0.0.0.0:0 means any interface, any port)") + runNodeCmd.Flags().StringVar(&seeds, "seeds", config.GetString("seeds"), + "Comma delimited host:port seed nodes") + runNodeCmd.Flags().BoolVar(&fastSync, "fast_sync", config.GetBool("fast_sync"), + "Fast blockchain syncing") + runNodeCmd.Flags().BoolVar(&skipUPNP, "skip_upnp", config.GetBool("skip_upnp"), + "Skip UPNP configuration") + runNodeCmd.Flags().StringVar(&rpcLaddr, "rpc_laddr", config.GetString("rpc_laddr"), + "RPC listen address. Port required") + runNodeCmd.Flags().StringVar(&grpcLaddr, "grpc_laddr", config.GetString("grpc_laddr"), + "GRPC listen address (BroadcastTx only). Port required") + runNodeCmd.Flags().StringVar(&proxyApp, "proxy_app", config.GetString("proxy_app"), + "Proxy app address, or 'nilapp' or 'dummy' for local testing.") + runNodeCmd.Flags().StringVar(&abciTransport, "abci", config.GetString("abci"), + "Specify abci transport (socket | grpc)") // feature flags - RootCmd.Flags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), "Enable Peer-Exchange (dev feature)") + runNodeCmd.Flags().BoolVar(&pex, "pex", config.GetBool("pex_reactor"), + "Enable Peer-Exchange (dev feature)") + + RootCmd.AddCommand(runNodeCmd) +} + +func setConfigFlags(cmd *cobra.Command, args []string) { // Merge parsed flag values onto config config.Set("moniker", moniker) @@ -59,8 +74,6 @@ func init() { config.Set("proxy_app", proxyApp) config.Set("abci", abciTransport) config.Set("pex_reactor", pex) - - RootCmd.AddCommand(runNodeCmd) } // Users wishing to: diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index 9c2b9aa33..cddae985b 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -4,11 +4,11 @@ import ( "fmt" "os" - cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" + "github.com/tendermint/tendermint/cmd/tendermint/commands" ) func main() { - if err := cmd.RootCmd.Execute(); err != nil { + if err := commands.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } diff --git a/glide.lock b/glide.lock index c39121639..6cfc97036 100644 --- a/glide.lock +++ b/glide.lock @@ -39,6 +39,8 @@ imports: version: d8ed2627bdf02c080bf22230dbb337003b7aba2d subpackages: - difflib +- name: github.com/spf13/cobra + version: 92ea23a837e66f46ac9e7d04fa826602b7b0a42d - name: github.com/spf13/pflag version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/stretchr/testify diff --git a/glide.yaml b/glide.yaml index e40293802..35d81761c 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,4 +1,4 @@ -package: github.com/tendermint/tendermint +eackage: github.com/tendermint/tendermint import: - package: github.com/tendermint/go-autofile version: develop @@ -39,6 +39,7 @@ import: - proto - package: github.com/gorilla/websocket version: ^1.1.0 +- package: github.com/spf13/cobra - package: github.com/spf13/pflag - package: github.com/pkg/errors version: ^0.8.0 From 4e743649be97cb8d6bc49e16e82cf27d46e7809f Mon Sep 17 00:00:00 2001 From: rigelrozanski Date: Mon, 6 Mar 2017 17:36:53 -0500 Subject: [PATCH 04/84] glide update --- glide.lock | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/glide.lock b/glide.lock index 6cfc97036..e79888029 100644 --- a/glide.lock +++ b/glide.lock @@ -1,8 +1,8 @@ -hash: 41f8fec708e98b7f8c4804be46008493199fa45e89b2d5dc237fd65fe431c62f -updated: 2017-03-06T04:01:33.319604992-05:00 +hash: 81cd41d28f9a747a71e6a47e8bc7d855846df29f359ffdcc8d1645c451112b31 +updated: 2017-03-06T17:34:23.99160606-05:00 imports: - name: github.com/btcsuite/btcd - version: d06c0bb181529331be8f8d9350288c420d9e60e4 + version: 583684b21bfbde9b5fc4403916fd7c807feb0289 subpackages: - btcec - name: github.com/BurntSushi/toml @@ -12,7 +12,7 @@ imports: subpackages: - spew - name: github.com/ebuchman/fail-test - version: 13f91f14c826314205cdbed1ec8ac8bf08e03381 + version: 95f809107225be108efcf10a3509e4ea6ceef3c4 - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf @@ -20,19 +20,21 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: 8ee79997227bf9b34611aee7946ae64735e6fd93 + version: 69b215d01a5606c843240eab4937eab3acee6530 subpackages: - proto - name: github.com/golang/snappy - version: d9eb7a3d35ec988b8585d4a0068e462c27d28380 + version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13 +- name: github.com/inconshreveable/mousetrap + version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: d228849504861217f796da67fae4f6e347643f15 + version: acb9493f2794fd0f820de7a27a217dafbb1b65ea - name: github.com/mattn/go-isatty - version: 30a891c33c7cde7b02a981314b4228ec99380cca + version: 9622e0cc9d8f9be434ca605520ff9a16808fee47 - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -40,7 +42,7 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: 92ea23a837e66f46ac9e7d04fa826602b7b0a42d + version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c - name: github.com/spf13/pflag version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/stretchr/testify @@ -49,7 +51,7 @@ imports: - assert - require - name: github.com/syndtr/goleveldb - version: 23851d93a2292dcc56e71a18ec9e0624d84a0f65 + version: 3c5717caf1475fd25964109a0fc640bd150fce43 subpackages: - leveldb - leveldb/cache @@ -125,7 +127,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 7c6cc321c680f03b9ef0764448e780704f486b51 + version: 40541ccb1c6e64c947ed6f606b8a6cb4b67d7436 subpackages: - curve25519 - nacl/box @@ -136,7 +138,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: 61557ac0112b576429a0df080e1c2cef5dfbb642 + version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 subpackages: - context - http2 @@ -146,11 +148,11 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: d75a52659825e75fff6158388dddc6a5b04f9ba5 + version: e48874b42435b4347fc52bdee0424a52abc974d7 subpackages: - unix - name: google.golang.org/grpc - version: cbcceb2942a489498cf22b2f918536e819d33f0a + version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 subpackages: - codes - credentials From d474baeeea6c22b289e7402449572f7c89ee21da Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 8 Mar 2017 14:05:32 +0400 Subject: [PATCH 05/84] update Dockerfile for 0.9.0 release --- DOCKER/Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 947301e75..3b710596d 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,7 +1,8 @@ FROM alpine:3.5 # This is the release of tendermint to pull in. -ENV TM_VERSION 0.8.0 +ENV TM_VERSION 0.9.0 +ENV TM_SHA256SUM 697033ea0f34f8b34a8a2b74c4dd730b47dd4efcfce65e53e953bdae8eb14364 # Tendermint will be looking for genesis file in /tendermint (unless you change # `genesis_file` in config.toml). You can put your config.toml and private @@ -25,11 +26,11 @@ RUN mkdir -p $DATA_ROOT && \ RUN apk add --no-cache bash curl jq RUN apk add --no-cache openssl && \ - wget https://s3-us-west-2.amazonaws.com/tendermint/${TM_VERSION}/tendermint_linux_amd64.zip && \ - echo "83f6bd52055ebc93434a68263c6666a4de41e0e543d0b5a06ad461262c460f4c tendermint_linux_amd64.zip" | sha256sum -c && \ - unzip -d /bin tendermint_linux_amd64.zip && \ + wget https://s3-us-west-2.amazonaws.com/tendermint/${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ + echo "${TM_SHA256SUM} tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \ + unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \ apk del openssl && \ - rm -f tendermint_linux_amd64.zip + rm -f tendermint_${TM_VERSION}_linux_amd64.zip # Expose the data directory as a volume since there's mutable state in there VOLUME $DATA_ROOT From a4154e76c5de81f8514dc41fe96fda24d4578d11 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 8 Mar 2017 14:09:54 +0400 Subject: [PATCH 06/84] update docker readme [ci skip] [circleci skip] --- DOCKER/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DOCKER/README.md b/DOCKER/README.md index bbb662637..b7b9f3815 100644 --- a/DOCKER/README.md +++ b/DOCKER/README.md @@ -1,7 +1,8 @@ # Supported tags and respective `Dockerfile` links -- `0.8.0`, `0.8`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c/DOCKER/Dockerfile) -- `develop` [(Dockerfile)]() +- `0.9.0`, `0.9`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/d474baeeea6c22b289e7402449572f7c89ee21da/DOCKER/Dockerfile) +- `0.8.0`, `0.8` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/bf64dd21fdb193e54d8addaaaa2ecf7ac371de8c/DOCKER/Dockerfile) +- `develop` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/master/DOCKER/Dockerfile.develop) `develop` tag points to the [develop](https://github.com/tendermint/tendermint/tree/develop) branch. From 55b3c22d997e22875a31bda268bc0516ef60af5c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 7 Mar 2017 13:43:51 -0500 Subject: [PATCH 07/84] publish.sh to push build to s3 --- scripts/publish.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 scripts/publish.sh diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 000000000..e87fea0d3 --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eu + +VERSION=$1 +DIST_DIR=$2 # ./build/dist + +# Get the version from the environment, or try to figure it out. +if [ -z $VERSION ]; then + VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) +fi +if [ -z "$VERSION" ]; then + echo "Please specify a version." + exit 1 +fi + +# copy to s3 +aws s3 cp --recursive ${DIST_DIR} s3://tendermint/${VERSION} --acl public-read --exclude "*" --include "*.zip" +aws s3 cp ${DIST_DIR}/tendermint_${VERSION}_SHA256SUMS s3://tendermint/0.9.0 --acl public-read + +exit 0 From 5420254b36f2cb68a9993124395f6cd7e7da1a87 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 9 Mar 2017 01:33:31 -0500 Subject: [PATCH 08/84] changelog: add prehistory --- CHANGELOG.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c83b375..7ebcfd364 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,10 +35,9 @@ type BlockMeta struct { } ``` +- `ValidatorSet.Proposer` is exposed as a field and persisted with the `State`. Use `GetProposer()` to initialize or update after validator-set changes. + - `tendermint gen_validator` command output is now pure JSON -- `ValidatorSet` data type: - - expose a `Proposer` field. Note this means the `Proposer` is persisted with the `State`. - - change `.Proposer()` to `.GetProposer()` FEATURES: @@ -218,3 +217,19 @@ BUG FIXES: - Various fixes to WAL and replay logic - Various race conditions + +## PreHistory + +Strict versioning only began with the release of v0.7.0, in late summer 2016. +The project itself began in early summer 2014 and was workable decentralized cryptocurrency software by the end of that year. +Through the course of 2015, in collaboration with Eris Industries (now Monax Indsutries), +many additional features were integrated, including an implementation from scratch of the Ethereum Virtual Machine. +That implementation now forms the heart of [ErisDB](https://github.com/eris-ltd/eris-db). +In the later half of 2015, the consensus algorithm was upgraded with a more asynchronous design and a more deterministic and robust implementation. + +By late 2015, frustration with the difficulty of forking a large monolithic stack to create alternative cryptocurrency designs led to the +invention of the Application Blockchain Interface (ABCI), then called the Tendermint Socket Protocol (TMSP). +The Ethereum Virtual Machine and various other transaction features were removed, and Tendermint was whittled down to a core consensus engine +driving an application running in another process. +The ABCI interface and implementation were iterated on and improved over the course of 2016, +until versioned history kicked in with v0.7.0. From 5f6de800a0386b1080a489220a8bc7ab2ebc5037 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 27 Mar 2017 15:17:10 +0400 Subject: [PATCH 09/84] rename TMROOT to TMHOME (Refs #431) --- DOCKER/Dockerfile | 2 +- DOCKER/Dockerfile.develop | 2 +- Makefile | 6 +++--- config/tendermint/config.go | 2 +- test/app/test.sh | 12 ++++++------ test/p2p/README.md | 2 +- test/p2p/peer.sh | 5 +++-- test/persist/test_failure_indices.sh | 6 +++--- test/persist/test_simple.sh | 6 +++--- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 3b710596d..08a7b2674 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -10,7 +10,7 @@ ENV TM_SHA256SUM 697033ea0f34f8b34a8a2b74c4dd730b47dd4efcfce65e53e953bdae8eb1436 # # The /tendermint/data dir is used by tendermint to store state. ENV DATA_ROOT /tendermint -ENV TMROOT $DATA_ROOT +ENV TMHOME $DATA_ROOT # Set user right away for determinism RUN addgroup tmuser && \ diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop index 19e101adb..82cd884ae 100644 --- a/DOCKER/Dockerfile.develop +++ b/DOCKER/Dockerfile.develop @@ -1,7 +1,7 @@ FROM alpine:3.5 ENV DATA_ROOT /tendermint -ENV TMROOT $DATA_ROOT +ENV TMHOME $DATA_ROOT RUN addgroup tmuser && \ adduser -S -G tmuser tmuser diff --git a/Makefile b/Makefile index bfdaf0077..3cafad386 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GOTOOLS = \ github.com/Masterminds/glide PACKAGES=$(shell go list ./... | grep -v '/vendor/') BUILD_TAGS?=tendermint -TMROOT = $${TMROOT:-$$HOME/.tendermint} +TMHOME = $${TMHOME:-$$HOME/.tendermint} all: install test @@ -61,8 +61,8 @@ update_deps: tools @go get -d -u ./... revision: - -echo `git rev-parse --verify HEAD` > $(TMROOT)/revision - -echo `git rev-parse --verify HEAD` >> $(TMROOT)/revision_history + -echo `git rev-parse --verify HEAD` > $(TMHOME)/revision + -echo `git rev-parse --verify HEAD` >> $(TMHOME)/revision_history tools: go get -u -v $(GOTOOLS) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index c210d6e01..991304b5e 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -11,7 +11,7 @@ import ( func getTMRoot(rootDir string) string { if rootDir == "" { - rootDir = os.Getenv("TMROOT") + rootDir = os.Getenv("TMHOME") } if rootDir == "" { rootDir = os.Getenv("HOME") + "/.tendermint" diff --git a/test/app/test.sh b/test/app/test.sh index 8d589cd61..0b40c43e0 100644 --- a/test/app/test.sh +++ b/test/app/test.sh @@ -8,10 +8,10 @@ set -e # TODO: install everything -export TMROOT=$HOME/.tendermint_app +export TMHOME=$HOME/.tendermint_app function dummy_over_socket(){ - rm -rf $TMROOT + rm -rf $TMHOME tendermint init echo "Starting dummy_over_socket" dummy > /dev/null & @@ -28,7 +28,7 @@ function dummy_over_socket(){ # start tendermint first function dummy_over_socket_reorder(){ - rm -rf $TMROOT + rm -rf $TMHOME tendermint init echo "Starting dummy_over_socket_reorder (ie. start tendermint first)" tendermint node > tendermint.log & @@ -46,7 +46,7 @@ function dummy_over_socket_reorder(){ function counter_over_socket() { - rm -rf $TMROOT + rm -rf $TMHOME tendermint init echo "Starting counter_over_socket" counter --serial > /dev/null & @@ -62,7 +62,7 @@ function counter_over_socket() { } function counter_over_grpc() { - rm -rf $TMROOT + rm -rf $TMHOME tendermint init echo "Starting counter_over_grpc" counter --serial --abci grpc > /dev/null & @@ -78,7 +78,7 @@ function counter_over_grpc() { } function counter_over_grpc_grpc() { - rm -rf $TMROOT + rm -rf $TMHOME tendermint init echo "Starting counter_over_grpc_grpc (ie. with grpc broadcast_tx)" counter --serial --abci grpc > /dev/null & diff --git a/test/p2p/README.md b/test/p2p/README.md index 5cd314d3a..5836fc61e 100644 --- a/test/p2p/README.md +++ b/test/p2p/README.md @@ -37,7 +37,7 @@ for i in $(seq 1 4); do --ip="172.57.0.$((100 + $i))" \ --name local_testnet_$i \ --entrypoint tendermint \ - -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \ + -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \ tendermint_tester node --seeds 172.57.0.101:46656,172.57.0.102:46656,172.57.0.103:46656,172.57.0.104:46656 --proxy_app=dummy done ``` diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index 5299a3c60..283228f76 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -21,7 +21,7 @@ if [[ "$CIRCLECI" == true ]]; then --ip=$(test/p2p/ip.sh $ID) \ --name "local_testnet_$ID" \ --entrypoint tendermint \ - -e TMROOT="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ + -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ --log-driver=syslog \ --log-opt syslog-address=udp://127.0.0.1:5514 \ --log-opt syslog-facility=daemon \ @@ -34,6 +34,7 @@ else --ip=$(test/p2p/ip.sh $ID) \ --name "local_testnet_$ID" \ --entrypoint tendermint \ - -e TMROOT="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ + -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ "$DOCKER_IMAGE" node $NODE_FLAGS --log_level=info --proxy_app="$APP_PROXY" fi + diff --git a/test/persist/test_failure_indices.sh b/test/persist/test_failure_indices.sh index 8a708e508..bca8b8ae9 100644 --- a/test/persist/test_failure_indices.sh +++ b/test/persist/test_failure_indices.sh @@ -1,15 +1,15 @@ #! /bin/bash -export TMROOT=$HOME/.tendermint_persist +export TMHOME=$HOME/.tendermint_persist -rm -rf "$TMROOT" +rm -rf "$TMHOME" tendermint init # use a unix socket so we can remove it RPC_ADDR="$(pwd)/rpc.sock" TM_CMD="tendermint node --log_level=debug --rpc_laddr=unix://$RPC_ADDR" # &> tendermint_${name}.log" -DUMMY_CMD="dummy --persist $TMROOT/dummy" # &> dummy_${name}.log" +DUMMY_CMD="dummy --persist $TMHOME/dummy" # &> dummy_${name}.log" function start_procs(){ diff --git a/test/persist/test_simple.sh b/test/persist/test_simple.sh index a483145b7..59bc38458 100644 --- a/test/persist/test_simple.sh +++ b/test/persist/test_simple.sh @@ -1,15 +1,15 @@ #! /bin/bash -export TMROOT=$HOME/.tendermint_persist +export TMHOME=$HOME/.tendermint_persist -rm -rf $TMROOT +rm -rf $TMHOME tendermint init function start_procs(){ name=$1 echo "Starting persistent dummy and tendermint" - dummy --persist $TMROOT/dummy &> "dummy_${name}.log" & + dummy --persist $TMHOME/dummy &> "dummy_${name}.log" & PID_DUMMY=$! tendermint node &> tendermint_${name}.log & PID_TENDERMINT=$! From 12ead6cc7e045202c01f5b08851c21b6b01dd640 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 27 Mar 2017 20:41:00 +0400 Subject: [PATCH 10/84] make changes backwards compatible (Refs #431) --- config/tendermint/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 991304b5e..b28170ce7 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -13,6 +13,10 @@ func getTMRoot(rootDir string) string { if rootDir == "" { rootDir = os.Getenv("TMHOME") } + if rootDir == "" { + // deprecated, use TMHOME (TODO: remove in TM 0.11.0) + rootDir = os.Getenv("TMROOT") + } if rootDir == "" { rootDir = os.Getenv("HOME") + "/.tendermint" } From 077cf13a1f58727e99df0e3942aa717f5b817aa4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 27 Mar 2017 15:41:45 -0400 Subject: [PATCH 11/84] consensus: timeout on replayLastBlock --- config/tendermint/config.go | 1 + config/tendermint_test/config.go | 1 + consensus/replay.go | 25 +++++++++++++++++++++---- consensus/state.go | 14 +++++++------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index c210d6e01..ea2f1d43a 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -79,6 +79,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("block_size", 10000) // max number of txs mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) + mapConfig.SetDefault("timeout_handshake", 10000) mapConfig.SetDefault("timeout_propose", 3000) mapConfig.SetDefault("timeout_propose_delta", 500) mapConfig.SetDefault("timeout_prevote", 1000) diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 55e3adb4e..26a483358 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -93,6 +93,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("block_size", 10000) mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) + mapConfig.SetDefault("timeout_handshake", 10000) mapConfig.SetDefault("timeout_propose", 2000) mapConfig.SetDefault("timeout_propose_delta", 1) mapConfig.SetDefault("timeout_prevote", 10) diff --git a/consensus/replay.go b/consensus/replay.go index 6c4e65a03..4bdc2e870 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -190,6 +190,8 @@ func (h *Handshaker) NBlocks() int { return h.nBlocks } +var ErrReplayLastBlockTimeout = errors.New("Timed out waiting for last block to be replayed") + // TODO: retry the handshake/replay if it fails ? func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // handshake is done via info request on the query conn @@ -207,7 +209,11 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // replay blocks up to the latest in the blockstore _, err = h.ReplayBlocks(appHash, blockHeight, proxyApp) - if err != nil { + if err == ErrReplayLastBlockTimeout { + log.Warn("Failed to sync via handshake. Trying other means. If they fail, please increase the timeout_handshake parameter") + return nil + + } else if err != nil { return errors.New(Fmt("Error on replay: %v", err)) } @@ -320,6 +326,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) { mempool := types.MockMempool{} cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool) + defer cs.Stop() evsw := types.NewEventSwitch() evsw.Start() @@ -328,9 +335,19 @@ func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, e newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1) // run through the WAL, commit new block, stop - cs.Start() - <-newBlockCh // TODO: use a timeout and return err? - cs.Stop() + if _, err := cs.Start(); err != nil { + return nil, err + } + + timeout := h.config.GetInt("timeout_handshake") + timer := time.NewTimer(time.Duration(timeout) * time.Millisecond) + log.Notice("Attempting to replay last block", "height", h.store.Height(), "timeout", timeout) + + select { + case <-newBlockCh: + case <-timer.C: + return nil, ErrReplayLastBlockTimeout + } h.nBlocks += 1 diff --git a/consensus/state.go b/consensus/state.go index 23eaff748..cca9d2ed4 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -343,13 +343,7 @@ func (cs *ConsensusState) OnStart() error { cs.BaseService.OnStart() walFile := cs.config.GetString("cs_wal_file") - err := EnsureDir(path.Dir(walFile), 0700) - if err != nil { - log.Error("Error ensuring ConsensusState wal dir", "error", err.Error()) - return err - } - err = cs.OpenWAL(walFile) - if err != nil { + if err := cs.OpenWAL(walFile); err != nil { log.Error("Error loading ConsensusState wal", "error", err.Error()) return err } @@ -404,6 +398,12 @@ func (cs *ConsensusState) Wait() { // Open file to log all consensus messages and timeouts for deterministic accountability func (cs *ConsensusState) OpenWAL(walFile string) (err error) { + err = EnsureDir(path.Dir(walFile), 0700) + if err != nil { + log.Error("Error ensuring ConsensusState wal dir", "error", err.Error()) + return err + } + cs.mtx.Lock() defer cs.mtx.Unlock() wal, err := NewWAL(walFile, cs.config.GetBool("cs_wal_light")) From 0413a87eb4ae087ed34bb38cad695067b1793894 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 27 Mar 2017 16:59:54 -0400 Subject: [PATCH 12/84] fix typo --- glide.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.yaml b/glide.yaml index 35d81761c..53d308233 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,4 +1,4 @@ -eackage: github.com/tendermint/tendermint +package: github.com/tendermint/tendermint import: - package: github.com/tendermint/go-autofile version: develop From 85e83934a10d10f08dfd7bc7f0bd55573fe1c7bb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 28 Mar 2017 12:07:32 -0400 Subject: [PATCH 13/84] fixes from review --- config/tendermint/config.go | 3 +++ consensus/replay.go | 2 +- consensus/state.go | 5 +++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index ea2f1d43a..fd9075e3b 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -79,6 +79,8 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("block_size", 10000) // max number of txs mapConfig.SetDefault("block_part_size", 65536) // part size 64K mapConfig.SetDefault("disable_data_hash", false) + + // all timeouts are in ms mapConfig.SetDefault("timeout_handshake", 10000) mapConfig.SetDefault("timeout_propose", 3000) mapConfig.SetDefault("timeout_propose_delta", 500) @@ -87,6 +89,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("timeout_precommit", 1000) mapConfig.SetDefault("timeout_precommit_delta", 500) mapConfig.SetDefault("timeout_commit", 1000) + // make progress asap (no `timeout_commit`) on full precommit votes mapConfig.SetDefault("skip_timeout_commit", false) mapConfig.SetDefault("mempool_recheck", true) diff --git a/consensus/replay.go b/consensus/replay.go index 4bdc2e870..ff11a390b 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -326,7 +326,6 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) { mempool := types.MockMempool{} cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool) - defer cs.Stop() evsw := types.NewEventSwitch() evsw.Start() @@ -338,6 +337,7 @@ func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, e if _, err := cs.Start(); err != nil { return nil, err } + defer cs.Stop() timeout := h.config.GetInt("timeout_handshake") timer := time.NewTimer(time.Duration(timeout) * time.Millisecond) diff --git a/consensus/state.go b/consensus/state.go index cca9d2ed4..d4c63a681 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -358,8 +358,9 @@ func (cs *ConsensusState) OnStart() error { // we may have lost some votes if the process crashed // reload from consensus log to catchup if err := cs.catchupReplay(cs.Height); err != nil { - log.Error("Error on catchup replay", "error", err.Error()) - // let's go for it anyways, maybe we're fine + log.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "error", err.Error()) + // NOTE: if we ever do return an error here, + // make sure to stop the timeoutTicker } // now start the receiveRoutine From 4fd1471f11ba814b0edb3a6a4e8debc2510edbe2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 28 Mar 2017 12:09:11 -0400 Subject: [PATCH 14/84] remove BaseService.OnStart --- blockchain/pool.go | 2 -- consensus/replay_file.go | 2 -- consensus/state.go | 1 - consensus/ticker.go | 1 - consensus/wal.go | 1 - node/node.go | 1 - proxy/multi_app_conn.go | 1 - 7 files changed, 9 deletions(-) diff --git a/blockchain/pool.go b/blockchain/pool.go index ef673b342..32db956c1 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -63,7 +63,6 @@ func NewBlockPool(start int, requestsCh chan<- BlockRequest, timeoutsCh chan<- s } func (pool *BlockPool) OnStart() error { - pool.BaseService.OnStart() go pool.makeRequestersRoutine() pool.startTime = time.Now() return nil @@ -409,7 +408,6 @@ func newBPRequester(pool *BlockPool, height int) *bpRequester { } func (bpr *bpRequester) OnStart() error { - bpr.BaseService.OnStart() go bpr.requestRoutine() return nil } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 6ff380880..5ad1b9457 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -127,8 +127,6 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { } func (cs *ConsensusState) startForReplay() { - // don't want to start full cs - cs.BaseService.OnStart() log.Warn("Replay commands are disabled until someone updates them and writes tests") /* TODO:! diff --git a/consensus/state.go b/consensus/state.go index d4c63a681..63616cbd3 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -340,7 +340,6 @@ func (cs *ConsensusState) LoadCommit(height int) *types.Commit { } func (cs *ConsensusState) OnStart() error { - cs.BaseService.OnStart() walFile := cs.config.GetString("cs_wal_file") if err := cs.OpenWAL(walFile); err != nil { diff --git a/consensus/ticker.go b/consensus/ticker.go index 06a8f7d20..b318597d3 100644 --- a/consensus/ticker.go +++ b/consensus/ticker.go @@ -45,7 +45,6 @@ func NewTimeoutTicker() TimeoutTicker { } func (t *timeoutTicker) OnStart() error { - t.BaseService.OnStart() go t.timeoutRoutine() diff --git a/consensus/wal.go b/consensus/wal.go index 99035ee2e..6d8eb3819 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -55,7 +55,6 @@ func NewWAL(walFile string, light bool) (*WAL, error) { } func (wal *WAL) OnStart() error { - wal.BaseService.OnStart() size, err := wal.group.Head.Size() if err != nil { return err diff --git a/node/node.go b/node/node.go index f9c69c289..6815c779b 100644 --- a/node/node.go +++ b/node/node.go @@ -194,7 +194,6 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato } func (n *Node) OnStart() error { - n.BaseService.OnStart() // Create & add listener protocol, address := ProtocolAndAddress(n.config.GetString("node_laddr")) diff --git a/proxy/multi_app_conn.go b/proxy/multi_app_conn.go index 353ade35c..81e01aa29 100644 --- a/proxy/multi_app_conn.go +++ b/proxy/multi_app_conn.go @@ -72,7 +72,6 @@ func (app *multiAppConn) Query() AppConnQuery { } func (app *multiAppConn) OnStart() error { - app.BaseService.OnStart() // query connection querycli, err := app.clientCreator.NewABCIClient() From 09f7dabd5ef31bb16c3fa6b511a7d36ecae00229 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 28 Mar 2017 14:06:03 -0400 Subject: [PATCH 15/84] update comment --- consensus/replay.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index ff11a390b..731e7d216 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -292,10 +292,11 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) { // App is further behind than it should be, so we need to replay blocks. - // We replay all blocks from appBlockHeight+1 to storeBlockHeight-1, - // and let the final block be replayed through ReplayBlocks. + // We replay all blocks from appBlockHeight+1. + // If useReplayFunc == true, stop short of the last block + // so it can be replayed using the WAL in ReplayBlocks. // Note that we don't have an old version of the state, - // so we by-pass state validation using applyBlock here. + // so we by-pass state validation using sm.ApplyBlock. var appHash []byte var err error From c3f1b08b6ab956b923e3ed6c664241cf48ead736 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 19 Jan 2017 13:33:58 +0400 Subject: [PATCH 16/84] tx indexing (Refs #237) save transactions to blockstore move to a separate module benchmark KVIndexer batch write transactions Benchmarks: ``` BenchmarkKVIndexerIndex-2 100000 516300 ns/op PASS ok github.com/tendermint/tendermint/blockchain/tx 56.506s 5,16 s for 10000 transactions 1 s for 2000 transactions ``` ``` BenchmarkKVIndexerIndex-2 h 3000000 8622 ns/op PASS ok github.com/tendermint/tendermint/blockchain/tx 34.210s 86 ms for 10000 transactions 16 ms for 2000 transactions ``` ``` BenchmarkKVIndexerIndex1-2 5000000 7160 ns/op BenchmarkKVIndexerIndex500-2 20000 1750411 ns/op BenchmarkKVIndexerIndex1000-2 10000 3573973 ns/op BenchmarkKVIndexerIndex2000-2 5000 7836851 ns/op BenchmarkKVIndexerIndex10000-2 1000 33438980 ns/op PASS ok github.com/tendermint/tendermint/blockchain/tx 209.482s 7,8 ms for 2000 transactions ``` [state] write test for ApplyBlock review comments - move txindexer to state - fix type save Tx Index as well do not store tx itself in the result --- blockchain/pool_test.go | 6 +-- consensus/state.go | 2 +- glide.lock | 10 +++- node/node.go | 2 +- state/execution.go | 102 +++++++++++++++++++++--------------- state/state.go | 17 ++++++ state/tx/indexer.go | 21 ++++++++ state/tx/indexer/batch.go | 32 +++++++++++ state/tx/indexer/error.go | 6 +++ state/tx/indexer/kv.go | 55 +++++++++++++++++++ state/tx/indexer/kv_test.go | 61 +++++++++++++++++++++ state/tx/indexer/null.go | 16 ++++++ types/tx_result.go | 14 +++++ 13 files changed, 295 insertions(+), 49 deletions(-) create mode 100644 state/tx/indexer.go create mode 100644 state/tx/indexer/batch.go create mode 100644 state/tx/indexer/error.go create mode 100644 state/tx/indexer/kv.go create mode 100644 state/tx/indexer/kv_test.go create mode 100644 state/tx/indexer/null.go create mode 100644 types/tx_result.go diff --git a/blockchain/pool_test.go b/blockchain/pool_test.go index 8e6dbee94..220bc5ce5 100644 --- a/blockchain/pool_test.go +++ b/blockchain/pool_test.go @@ -35,6 +35,7 @@ func TestBasic(t *testing.T) { requestsCh := make(chan BlockRequest, 100) pool := NewBlockPool(start, requestsCh, timeoutsCh) pool.Start() + defer pool.Stop() // Introduce each peer. go func() { @@ -76,8 +77,6 @@ func TestBasic(t *testing.T) { }() } } - - pool.Stop() } func TestTimeout(t *testing.T) { @@ -87,6 +86,7 @@ func TestTimeout(t *testing.T) { requestsCh := make(chan BlockRequest, 100) pool := NewBlockPool(start, requestsCh, timeoutsCh) pool.Start() + defer pool.Stop() for _, peer := range peers { log.Info("Peer", "id", peer.id) @@ -131,6 +131,4 @@ func TestTimeout(t *testing.T) { log.Info("TEST: Pulled new BlockRequest", "request", request) } } - - pool.Stop() } diff --git a/consensus/state.go b/consensus/state.go index 63616cbd3..d69435814 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -283,7 +283,7 @@ func NewConsensusState(config cfg.Config, state *sm.State, proxyAppConn proxy.Ap //---------------------------------------- // Public interface -// implements events.Eventable +// SetEventSwitch implements events.Eventable func (cs *ConsensusState) SetEventSwitch(evsw types.EventSwitch) { cs.evsw = evsw } diff --git a/glide.lock b/glide.lock index e79888029..71b549dbe 100644 --- a/glide.lock +++ b/glide.lock @@ -164,4 +164,12 @@ imports: - stats - tap - transport -testImports: [] +testImports: +- name: github.com/davecgh/go-spew + version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 + subpackages: + - spew +- name: github.com/pmezard/go-difflib + version: d8ed2627bdf02c080bf22230dbb337003b7aba2d + subpackages: + - difflib diff --git a/node/node.go b/node/node.go index 6815c779b..7a88f2d09 100644 --- a/node/node.go +++ b/node/node.go @@ -82,7 +82,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato } // reload the state (it may have been updated by the handshake) - state = sm.LoadState(stateDB) + state = sm.GetState(config, stateDB) // Generate node PrivKey privKey := crypto.GenPrivKeyEd25519() diff --git a/state/execution.go b/state/execution.go index aa9113011..0a82c6da9 100644 --- a/state/execution.go +++ b/state/execution.go @@ -2,26 +2,26 @@ package state import ( "errors" + "fmt" - "github.com/ebuchman/fail-test" - + fail "github.com/ebuchman/fail-test" abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" - "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" + txindexer "github.com/tendermint/tendermint/state/tx/indexer" "github.com/tendermint/tendermint/types" ) -//-------------------------------------------------- -// Execute the block - -// Execute the block to mutate State. -// Validates block and then executes Data.Txs in the block. -func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) error { - +// ExecBlock executes the block to mutate State. +// + validates the block +// + executes block.Txs on the proxyAppConn +// + updates validator sets +// + returns block.Txs results +func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) ([]*types.TxResult, error) { // Validate the block. if err := s.validateBlock(block); err != nil { - return ErrInvalidBlock(err) + return nil, ErrInvalidBlock(err) } // compute bitarray of validators that signed @@ -33,11 +33,11 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC nextValSet := valSet.Copy() // Execute the block txs - changedValidators, err := execBlockOnProxyApp(eventCache, proxyAppConn, block) + txResults, changedValidators, err := execBlockOnProxyApp(eventCache, proxyAppConn, block) if err != nil { // There was some error in proxyApp // TODO Report error and wait for proxyApp to be available. - return ErrProxyAppConn(err) + return nil, ErrProxyAppConn(err) } // update the validator set @@ -54,16 +54,22 @@ func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnC fail.Fail() // XXX - return nil + return txResults, nil } // Executes block's transactions on proxyAppConn. -// Returns a list of updates to the validator set +// Returns a list of transaction results and updates to the validator set // TODO: Generate a bitmap or otherwise store tx validity in state. -func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*abci.Validator, error) { - +func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*types.TxResult, []*abci.Validator, error) { var validTxs, invalidTxs = 0, 0 + txResults := make([]*types.TxResult, len(block.Txs)) + + txHashToIndexMap := make(map[string]int) + for index, tx := range block.Txs { + txHashToIndexMap[string(tx.Hash())] = index + } + // Execute transactions and get hash proxyCb := func(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { @@ -73,21 +79,28 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo // Blocks may include invalid txs. // reqDeliverTx := req.(abci.RequestDeliverTx) txError := "" - apTx := r.DeliverTx - if apTx.Code == abci.CodeType_OK { - validTxs += 1 + txResult := r.DeliverTx + if txResult.Code == abci.CodeType_OK { + validTxs++ } else { - log.Debug("Invalid tx", "code", r.DeliverTx.Code, "log", r.DeliverTx.Log) - invalidTxs += 1 - txError = apTx.Code.String() + log.Debug("Invalid tx", "code", txResult.Code, "log", txResult.Log) + invalidTxs++ + txError = txResult.Code.String() + } + + tx := types.Tx(req.GetDeliverTx().Tx) + index, ok := txHashToIndexMap[string(tx.Hash())] + if ok { + txResults[index] = &types.TxResult{uint64(block.Height), uint32(index), *txResult} } + // NOTE: if we count we can access the tx from the block instead of // pulling it from the req event := types.EventDataTx{ - Tx: req.GetDeliverTx().Tx, - Data: apTx.Data, - Code: apTx.Code, - Log: apTx.Log, + Tx: tx, + Data: txResult.Data, + Code: txResult.Code, + Log: txResult.Log, Error: txError, } types.FireEventTx(eventCache, event) @@ -99,7 +112,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo err := proxyAppConn.BeginBlockSync(block.Hash(), types.TM2PB.Header(block.Header)) if err != nil { log.Warn("Error in proxyAppConn.BeginBlock", "error", err) - return nil, err + return nil, nil, err } fail.Fail() // XXX @@ -109,7 +122,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.FailRand(len(block.Txs)) // XXX proxyAppConn.DeliverTxAsync(tx) if err := proxyAppConn.Error(); err != nil { - return nil, err + return nil, nil, err } } @@ -119,7 +132,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo respEndBlock, err := proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) - return nil, err + return nil, nil, err } fail.Fail() // XXX @@ -128,7 +141,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo if len(respEndBlock.Diffs) > 0 { log.Info("Update to validator set", "updates", abci.ValidatorsString(respEndBlock.Diffs)) } - return respEndBlock.Diffs, nil + return txResults, respEndBlock.Diffs, nil } func updateValidators(validators *types.ValidatorSet, changedValidators []*abci.Validator) error { @@ -218,26 +231,31 @@ func (s *State) validateBlock(block *types.Block) error { return nil } -//----------------------------------------------------------------------------- -// ApplyBlock executes the block, then commits and updates the mempool atomically - -// Execute and commit block against app, save block and state +// ApplyBlock executes the block, then commits and updates the mempool +// atomically, optionally indexing transaction results. func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, partsHeader types.PartSetHeader, mempool types.Mempool) error { - // Run the block on the State: - // + update validator sets - // + run txs on the proxyAppConn - err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) + txResults, err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) if err != nil { - return errors.New(Fmt("Exec failed for application: %v", err)) + return fmt.Errorf("Exec failed for application: %v", err) } // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) if err != nil { - return errors.New(Fmt("Commit failed for application: %v", err)) + return fmt.Errorf("Commit failed for application: %v", err) } + + batch := txindexer.NewBatch() + for i, r := range txResults { + if r != nil { + tx := block.Txs[i] + batch.Index(string(tx.Hash()), *r) + } + } + s.TxIndexer.Batch(batch) + return nil } @@ -272,7 +290,7 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl // Returns the application root hash (result of abci.Commit) func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) { var eventCache types.Fireable // nil - _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) + _, _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) if err != nil { log.Warn("Error executing block on proxy app", "height", block.Height, "err", err) return nil, err diff --git a/state/state.go b/state/state.go index c4c6d7489..892c38746 100644 --- a/state/state.go +++ b/state/state.go @@ -10,6 +10,8 @@ import ( cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/state/tx" + txindexer "github.com/tendermint/tendermint/state/tx/indexer" "github.com/tendermint/tendermint/types" ) @@ -38,6 +40,8 @@ type State struct { // AppHash is updated after Commit AppHash []byte + + TxIndexer tx.Indexer `json:"-"` // Transaction indexer. } func LoadState(db dbm.DB) *State { @@ -72,6 +76,7 @@ func (s *State) Copy() *State { Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), AppHash: s.AppHash, + TxIndexer: s.TxIndexer, // pointer here, not value } } @@ -125,12 +130,20 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { state = MakeGenesisStateFromFile(stateDB, config.GetString("genesis_file")) state.Save() } + + // Transaction indexing + store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) + state.TxIndexer = txindexer.NewKV(store) + return state } //----------------------------------------------------------------------------- // Genesis +// MakeGenesisStateFromFile reads and unmarshals state from the given file. +// +// Used during replay and in tests. func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State { genDocJSON, err := ioutil.ReadFile(genDocFile) if err != nil { @@ -143,6 +156,9 @@ func MakeGenesisStateFromFile(db dbm.DB, genDocFile string) *State { return MakeGenesisState(db, genDoc) } +// MakeGenesisState creates state from types.GenesisDoc. +// +// Used in tests. func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { if len(genDoc.Validators) == 0 { Exit(Fmt("The genesis file has no validators")) @@ -176,5 +192,6 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { Validators: types.NewValidatorSet(validators), LastValidators: types.NewValidatorSet(nil), AppHash: genDoc.AppHash, + TxIndexer: &txindexer.Null{}, // we do not need indexer during replay and in tests } } diff --git a/state/tx/indexer.go b/state/tx/indexer.go new file mode 100644 index 000000000..759f141ae --- /dev/null +++ b/state/tx/indexer.go @@ -0,0 +1,21 @@ +package tx + +import ( + txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/types" +) + +// Indexer interface defines methods to index and search transactions. +type Indexer interface { + + // Batch analyzes, indexes or stores a batch of transactions. + // + // NOTE We do not specify Index method for analyzing a single transaction + // here because it bears heavy perfomance loses. Almost all advanced indexers + // support batching. + Batch(b *txindexer.Batch) error + + // Tx returns specified transaction or nil if the transaction is not indexed + // or stored. + Tx(hash string) (*types.TxResult, error) +} diff --git a/state/tx/indexer/batch.go b/state/tx/indexer/batch.go new file mode 100644 index 000000000..3c64c8869 --- /dev/null +++ b/state/tx/indexer/batch.go @@ -0,0 +1,32 @@ +package indexer + +import "github.com/tendermint/tendermint/types" + +// A Batch groups together multiple Index operations you would like performed +// at the same time. The Batch structure is NOT thread-safe. You should only +// perform operations on a batch from a single thread at a time. Once batch +// execution has started, you may not modify it. +type Batch struct { + Ops map[string]types.TxResult +} + +// NewBatch creates a new Batch. +func NewBatch() *Batch { + return &Batch{ + Ops: make(map[string]types.TxResult), + } +} + +// Index adds or updates entry for the given hash. +func (b *Batch) Index(hash string, result types.TxResult) error { + if hash == "" { + return ErrorEmptyHash + } + b.Ops[hash] = result + return nil +} + +// Size returns the total number of operations inside the batch. +func (b *Batch) Size() int { + return len(b.Ops) +} diff --git a/state/tx/indexer/error.go b/state/tx/indexer/error.go new file mode 100644 index 000000000..9d20593d1 --- /dev/null +++ b/state/tx/indexer/error.go @@ -0,0 +1,6 @@ +package indexer + +import "errors" + +// ErrorEmptyHash indicates empty hash +var ErrorEmptyHash = errors.New("Transaction hash cannot be empty") diff --git a/state/tx/indexer/kv.go b/state/tx/indexer/kv.go new file mode 100644 index 000000000..0c86d0a35 --- /dev/null +++ b/state/tx/indexer/kv.go @@ -0,0 +1,55 @@ +package indexer + +import ( + "bytes" + "fmt" + + db "github.com/tendermint/go-db" + "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/types" +) + +// KV is a simplest possible indexer, backed by Key-Value storage (levelDB). +// It could only index transaction by its identifier. +type KV struct { + store db.DB +} + +// NewKV returns new instance of KV indexer. +func NewKV(store db.DB) *KV { + return &KV{store: store} +} + +// Tx gets transaction from the KV storage and returns it or nil if the +// transaction is not found. +func (indexer *KV) Tx(hash string) (*types.TxResult, error) { + if hash == "" { + return nil, ErrorEmptyHash + } + + rawBytes := indexer.store.Get([]byte(hash)) + if rawBytes == nil { + return nil, nil + } + + r := bytes.NewReader(rawBytes) + var n int + var err error + txResult := wire.ReadBinary(&types.TxResult{}, r, 0, &n, &err).(*types.TxResult) + if err != nil { + return nil, fmt.Errorf("Error reading TxResult: %v", err) + } + + return txResult, nil +} + +// Batch writes a batch of transactions into the KV storage. +func (indexer *KV) Batch(b *Batch) error { + storeBatch := indexer.store.NewBatch() + for hash, result := range b.Ops { + rawBytes := wire.BinaryBytes(&result) + storeBatch.Set([]byte(hash), rawBytes) + } + storeBatch.Write() + return nil +} diff --git a/state/tx/indexer/kv_test.go b/state/tx/indexer/kv_test.go new file mode 100644 index 000000000..0e3b0a7d7 --- /dev/null +++ b/state/tx/indexer/kv_test.go @@ -0,0 +1,61 @@ +package indexer + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/abci/types" + db "github.com/tendermint/go-db" + "github.com/tendermint/tendermint/types" +) + +func TestKVIndex(t *testing.T) { + indexer := &KV{store: db.NewMemDB()} + + tx := types.Tx("HELLO WORLD") + txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + hash := string(tx.Hash()) + + batch := NewBatch() + batch.Index(hash, *txResult) + err := indexer.Batch(batch) + require.Nil(t, err) + + loadedTxResult, err := indexer.Tx(hash) + require.Nil(t, err) + assert.Equal(t, txResult, loadedTxResult) +} + +func benchmarkKVIndex(txsCount int, b *testing.B) { + txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + + dir, err := ioutil.TempDir("", "tx_indexer_db") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + + store := db.NewDB("tx_indexer", "leveldb", dir) + indexer := &KV{store: store} + + batch := NewBatch() + for i := 0; i < txsCount; i++ { + batch.Index(fmt.Sprintf("hash%v", i), *txResult) + } + + b.ResetTimer() + + for n := 0; n < b.N; n++ { + err = indexer.Batch(batch) + } +} + +func BenchmarkKVIndex1(b *testing.B) { benchmarkKVIndex(1, b) } +func BenchmarkKVIndex500(b *testing.B) { benchmarkKVIndex(500, b) } +func BenchmarkKVIndex1000(b *testing.B) { benchmarkKVIndex(1000, b) } +func BenchmarkKVIndex2000(b *testing.B) { benchmarkKVIndex(2000, b) } +func BenchmarkKVIndex10000(b *testing.B) { benchmarkKVIndex(10000, b) } diff --git a/state/tx/indexer/null.go b/state/tx/indexer/null.go new file mode 100644 index 000000000..b196a7056 --- /dev/null +++ b/state/tx/indexer/null.go @@ -0,0 +1,16 @@ +package indexer + +import "github.com/tendermint/tendermint/types" + +// Null acts as a /dev/null. +type Null struct{} + +// Tx panics. +func (indexer *Null) Tx(hash string) (*types.TxResult, error) { + panic("You are trying to get the transaction from a null indexer") +} + +// Batch returns nil. +func (indexer *Null) Batch(batch *Batch) error { + return nil +} diff --git a/types/tx_result.go b/types/tx_result.go new file mode 100644 index 000000000..c3ec00304 --- /dev/null +++ b/types/tx_result.go @@ -0,0 +1,14 @@ +package types + +import ( + abci "github.com/tendermint/abci/types" +) + +// TxResult contains results of executing the transaction. +// +// One usage is indexing transaction results. +type TxResult struct { + Height uint64 + Index uint32 + DeliverTxResponse abci.ResponseDeliverTx +} From d62e85757f500bab3163962e08d01aeaf47ed325 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 21 Feb 2017 23:40:26 +0400 Subject: [PATCH 17/84] execution test --- state/execution_test.go | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 state/execution_test.go diff --git a/state/execution_test.go b/state/execution_test.go new file mode 100644 index 000000000..7e0b7332b --- /dev/null +++ b/state/execution_test.go @@ -0,0 +1,90 @@ +package state + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/abci/example/dummy" + crypto "github.com/tendermint/go-crypto" + dbm "github.com/tendermint/go-db" + cfg "github.com/tendermint/tendermint/config/tendermint_test" + "github.com/tendermint/tendermint/mempool" + "github.com/tendermint/tendermint/proxy" + txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/types" +) + +var ( + privKey = crypto.GenPrivKeyEd25519FromSecret([]byte("execution_test")) + chainID = "execution_chain" + testPartSize = 65536 + nTxsPerBlock = 10 +) + +func TestApplyBlock(t *testing.T) { + cc := proxy.NewLocalClientCreator(dummy.NewDummyApplication()) + config := cfg.ResetConfig("execution_test_") + proxyApp := proxy.NewAppConns(config, cc, nil) + _, err := proxyApp.Start() + require.Nil(t, err) + defer proxyApp.Stop() + mempool := mempool.NewMempool(config, proxyApp.Mempool()) + + state := state() + indexer := &dummyIndexer{0} + state.TxIndexer = indexer + + // make block + block := makeBlock(1, state) + + err = state.ApplyBlock(nil, proxyApp.Consensus(), block, block.MakePartSet(testPartSize).Header(), mempool) + + require.Nil(t, err) + assert.Equal(t, nTxsPerBlock, indexer.Indexed) // test indexing works + + // TODO check state and mempool +} + +//---------------------------------------------------------------------------- + +// make some bogus txs +func txsFunc(blockNum int) (txs []types.Tx) { + for i := 0; i < nTxsPerBlock; i++ { + txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)})) + } + return txs +} + +func state() *State { + return MakeGenesisState(dbm.NewMemDB(), &types.GenesisDoc{ + ChainID: chainID, + Validators: []types.GenesisValidator{ + types.GenesisValidator{privKey.PubKey(), 10000, "test"}, + }, + AppHash: nil, + }) +} + +func makeBlock(num int, state *State) *types.Block { + prevHash := state.LastBlockID.Hash + prevParts := types.PartSetHeader{} + valHash := state.Validators.Hash() + prevBlockID := types.BlockID{prevHash, prevParts} + block, _ := types.MakeBlock(num, chainID, txsFunc(num), new(types.Commit), + prevBlockID, valHash, state.AppHash, testPartSize) + return block +} + +// dummyIndexer increments counter every time we index transaction. +type dummyIndexer struct { + Indexed int +} + +func (indexer *dummyIndexer) Tx(hash string) (*types.TxResult, error) { + return nil, nil +} +func (indexer *dummyIndexer) Batch(batch *txindexer.Batch) error { + indexer.Indexed += batch.Size() + return nil +} From b08f29cb7186970a22d6a87fe6fa9d696fb54120 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 22 Feb 2017 12:15:53 +0400 Subject: [PATCH 18/84] add config option for tx indexing and disable it by default --- config/tendermint/config.go | 2 ++ config/tendermint_test/config.go | 2 ++ state/state.go | 10 ++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 3edf2df18..828e67ef6 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -101,6 +101,8 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal") + mapConfig.SetDefault("tx_indexer", "none") + return mapConfig } diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 26a483358..b6462cab1 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -107,6 +107,8 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", "") + mapConfig.SetDefault("tx_indexer", "none") + logger.SetLogLevel(mapConfig.GetString("log_level")) return mapConfig diff --git a/state/state.go b/state/state.go index 892c38746..347718f1e 100644 --- a/state/state.go +++ b/state/state.go @@ -44,6 +44,7 @@ type State struct { TxIndexer tx.Indexer `json:"-"` // Transaction indexer. } +// Used in tests. func LoadState(db dbm.DB) *State { return loadState(db, stateKey) } @@ -132,8 +133,13 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { } // Transaction indexing - store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) - state.TxIndexer = txindexer.NewKV(store) + switch config.GetString("tx_indexer") { + case "kv": + store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) + state.TxIndexer = txindexer.NewKV(store) + default: + state.TxIndexer = &txindexer.Null{} + } return state } From 63704454a39005d54a8ca9e7d202dab2f159ae37 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 22 Feb 2017 14:53:59 +0400 Subject: [PATCH 19/84] expose `/tx?hash="XXXXXXXXXXXX"` RPC call --- node/node.go | 28 +++++-- rpc/core/mempool.go | 8 +- rpc/core/pipe.go | 16 ++-- rpc/core/routes.go | 150 +++++++++--------------------------- rpc/core/types/responses.go | 2 + state/execution.go | 4 +- state/state.go | 12 +-- state/tx/indexer/null.go | 7 +- types/tx_result.go | 6 +- 9 files changed, 88 insertions(+), 145 deletions(-) diff --git a/node/node.go b/node/node.go index 7a88f2d09..0eea6278f 100644 --- a/node/node.go +++ b/node/node.go @@ -10,12 +10,12 @@ import ( abci "github.com/tendermint/abci/types" cmn "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" - "github.com/tendermint/go-crypto" + crypto "github.com/tendermint/go-crypto" dbm "github.com/tendermint/go-db" - "github.com/tendermint/go-p2p" - "github.com/tendermint/go-rpc" - "github.com/tendermint/go-rpc/server" - "github.com/tendermint/go-wire" + p2p "github.com/tendermint/go-p2p" + rpc "github.com/tendermint/go-rpc" + rpcserver "github.com/tendermint/go-rpc/server" + wire "github.com/tendermint/go-wire" bc "github.com/tendermint/tendermint/blockchain" "github.com/tendermint/tendermint/consensus" mempl "github.com/tendermint/tendermint/mempool" @@ -23,6 +23,8 @@ import ( rpccore "github.com/tendermint/tendermint/rpc/core" grpccore "github.com/tendermint/tendermint/rpc/grpc" sm "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/state/tx" + txindexer "github.com/tendermint/tendermint/state/tx/indexer" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" @@ -51,6 +53,7 @@ type Node struct { consensusReactor *consensus.ConsensusReactor // for participating in the consensus proxyApp proxy.AppConns // connection to the application rpcListeners []net.Listener // rpc servers + txIndexer tx.Indexer } func NewNodeDefault(config cfg.Config) *Node { @@ -82,7 +85,18 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato } // reload the state (it may have been updated by the handshake) - state = sm.GetState(config, stateDB) + state = sm.LoadState(stateDB) + + // Transaction indexing + var txIndexer tx.Indexer + switch config.GetString("tx_indexer") { + case "kv": + store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) + txIndexer = txindexer.NewKV(store) + default: + txIndexer = &txindexer.Null{} + } + state.TxIndexer = txIndexer // Generate node PrivKey privKey := crypto.GenPrivKeyEd25519() @@ -188,6 +202,7 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato consensusState: consensusState, consensusReactor: consensusReactor, proxyApp: proxyApp, + txIndexer: txIndexer, } node.BaseService = *cmn.NewBaseService(log, "Node", node) return node @@ -278,6 +293,7 @@ func (n *Node) ConfigureRPC() { rpccore.SetGenesisDoc(n.genesisDoc) rpccore.SetAddrBook(n.addrBook) rpccore.SetProxyAppQuery(n.proxyApp.Query()) + rpccore.SetTxIndexer(n.txIndexer) } func (n *Node) startRPC() ([]net.Listener, error) { diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index eefc226ad..dfe841365 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -4,9 +4,9 @@ import ( "fmt" "time" + abci "github.com/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - abci "github.com/tendermint/abci/types" ) //----------------------------------------------------------------------------- @@ -65,7 +65,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { if checkTxR.Code != abci.CodeType_OK { // CheckTx failed! return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR, + CheckTx: checkTxR, DeliverTx: nil, }, nil } @@ -84,13 +84,13 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { } log.Notice("DeliverTx passed ", "tx", []byte(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR, + CheckTx: checkTxR, DeliverTx: deliverTxR, }, nil case <-timer.C: log.Error("failed to include tx") return &ctypes.ResultBroadcastTxCommit{ - CheckTx: checkTxR, + CheckTx: checkTxR, DeliverTx: nil, }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 123b13dd8..c92216428 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -2,11 +2,12 @@ package core import ( cfg "github.com/tendermint/go-config" - "github.com/tendermint/go-crypto" - "github.com/tendermint/go-p2p" + crypto "github.com/tendermint/go-crypto" + p2p "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/state/tx" "github.com/tendermint/tendermint/types" ) @@ -42,9 +43,10 @@ var ( p2pSwitch P2P // objects - pubKey crypto.PubKey - genDoc *types.GenesisDoc // cache the genesis structure - addrBook *p2p.AddrBook + pubKey crypto.PubKey + genDoc *types.GenesisDoc // cache the genesis structure + addrBook *p2p.AddrBook + txIndexer tx.Indexer ) func SetConfig(c cfg.Config) { @@ -86,3 +88,7 @@ func SetAddrBook(book *p2p.AddrBook) { func SetProxyAppQuery(appConn proxy.AppConnQuery) { proxyAppQuery = appConn } + +func SetTxIndexer(indexer tx.Indexer) { + txIndexer = indexer +} diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 643b2bf02..e79fb0299 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -1,9 +1,12 @@ package core import ( + "strings" + rpc "github.com/tendermint/go-rpc/server" "github.com/tendermint/go-rpc/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix @@ -23,6 +26,7 @@ var Routes = map[string]*rpc.RPCFunc{ "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxsResult, ""), + "tx": rpc.NewRPCFunc(Tx, "hash"), // broadcast API "broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommitResult, "tx"), @@ -45,185 +49,105 @@ var Routes = map[string]*rpc.RPCFunc{ } func SubscribeResult(wsCtx rpctypes.WSRPCContext, event string) (ctypes.TMResult, error) { - if r, err := Subscribe(wsCtx, event); err != nil { - return nil, err - } else { - return r, nil - } + return Subscribe(wsCtx, event) } func UnsubscribeResult(wsCtx rpctypes.WSRPCContext, event string) (ctypes.TMResult, error) { - if r, err := Unsubscribe(wsCtx, event); err != nil { - return nil, err - } else { - return r, nil - } + return Unsubscribe(wsCtx, event) } func StatusResult() (ctypes.TMResult, error) { - if r, err := Status(); err != nil { - return nil, err - } else { - return r, nil - } + return Status() } func NetInfoResult() (ctypes.TMResult, error) { - if r, err := NetInfo(); err != nil { - return nil, err - } else { - return r, nil - } + return NetInfo() } func UnsafeDialSeedsResult(seeds []string) (ctypes.TMResult, error) { - if r, err := UnsafeDialSeeds(seeds); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeDialSeeds(seeds) } func BlockchainInfoResult(min, max int) (ctypes.TMResult, error) { - if r, err := BlockchainInfo(min, max); err != nil { - return nil, err - } else { - return r, nil - } + return BlockchainInfo(min, max) } func GenesisResult() (ctypes.TMResult, error) { - if r, err := Genesis(); err != nil { - return nil, err - } else { - return r, nil - } + return Genesis() } func BlockResult(height int) (ctypes.TMResult, error) { - if r, err := Block(height); err != nil { - return nil, err - } else { - return r, nil - } + return Block(height) } func CommitResult(height int) (ctypes.TMResult, error) { - if r, err := Commit(height); err != nil { - return nil, err - } else { - return r, nil - } + return Commit(height) } func ValidatorsResult() (ctypes.TMResult, error) { - if r, err := Validators(); err != nil { - return nil, err - } else { - return r, nil - } + return Validators() } func DumpConsensusStateResult() (ctypes.TMResult, error) { - if r, err := DumpConsensusState(); err != nil { - return nil, err - } else { - return r, nil - } + return DumpConsensusState() } func UnconfirmedTxsResult() (ctypes.TMResult, error) { - if r, err := UnconfirmedTxs(); err != nil { - return nil, err - } else { - return r, nil - } + return UnconfirmedTxs() } func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { - if r, err := NumUnconfirmedTxs(); err != nil { + return NumUnconfirmedTxs() +} + +// Tx allow user to query the transaction results. `nil` could mean the +// transaction is in the mempool, invalidated, or was not send in the first +// place. +func Tx(hash string) (ctypes.TMResult, error) { + r, err := txIndexer.Tx(strings.ToLower(hash)) + // nil-pointer interface values are forbidden in go-rpc + if r == (*types.TxResult)(nil) { return nil, err - } else { - return r, nil } + return r, err } func BroadcastTxCommitResult(tx []byte) (ctypes.TMResult, error) { - if r, err := BroadcastTxCommit(tx); err != nil { - return nil, err - } else { - return r, nil - } + return BroadcastTxCommit(tx) } func BroadcastTxSyncResult(tx []byte) (ctypes.TMResult, error) { - if r, err := BroadcastTxSync(tx); err != nil { - return nil, err - } else { - return r, nil - } + return BroadcastTxSync(tx) } func BroadcastTxAsyncResult(tx []byte) (ctypes.TMResult, error) { - if r, err := BroadcastTxAsync(tx); err != nil { - return nil, err - } else { - return r, nil - } + return BroadcastTxAsync(tx) } func ABCIQueryResult(path string, data []byte, prove bool) (ctypes.TMResult, error) { - if r, err := ABCIQuery(path, data, prove); err != nil { - return nil, err - } else { - return r, nil - } + return ABCIQuery(path, data, prove) } func ABCIInfoResult() (ctypes.TMResult, error) { - if r, err := ABCIInfo(); err != nil { - return nil, err - } else { - return r, nil - } + return ABCIInfo() } func UnsafeFlushMempoolResult() (ctypes.TMResult, error) { - if r, err := UnsafeFlushMempool(); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeFlushMempool() } func UnsafeSetConfigResult(typ, key, value string) (ctypes.TMResult, error) { - if r, err := UnsafeSetConfig(typ, key, value); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeSetConfig(typ, key, value) } func UnsafeStartCPUProfilerResult(filename string) (ctypes.TMResult, error) { - if r, err := UnsafeStartCPUProfiler(filename); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeStartCPUProfiler(filename) } func UnsafeStopCPUProfilerResult() (ctypes.TMResult, error) { - if r, err := UnsafeStopCPUProfiler(); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeStopCPUProfiler() } func UnsafeWriteHeapProfileResult(filename string) (ctypes.TMResult, error) { - if r, err := UnsafeWriteHeapProfile(filename); err != nil { - return nil, err - } else { - return r, nil - } + return UnsafeWriteHeapProfile(filename) } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index bcab4f59c..908a33f19 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -127,6 +127,7 @@ const ( ResultTypeBroadcastTx = byte(0x60) ResultTypeUnconfirmedTxs = byte(0x61) ResultTypeBroadcastTxCommit = byte(0x62) + ResultTypeTx = byte(0x63) // 0x7 bytes are for querying the application ResultTypeABCIQuery = byte(0x70) @@ -163,6 +164,7 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultDumpConsensusState{}, ResultTypeDumpConsensusState}, wire.ConcreteType{&ResultBroadcastTx{}, ResultTypeBroadcastTx}, wire.ConcreteType{&ResultBroadcastTxCommit{}, ResultTypeBroadcastTxCommit}, + wire.ConcreteType{&types.TxResult{}, ResultTypeTx}, wire.ConcreteType{&ResultUnconfirmedTxs{}, ResultTypeUnconfirmedTxs}, wire.ConcreteType{&ResultSubscribe{}, ResultTypeSubscribe}, wire.ConcreteType{&ResultUnsubscribe{}, ResultTypeUnsubscribe}, diff --git a/state/execution.go b/state/execution.go index 0a82c6da9..bc125d4e3 100644 --- a/state/execution.go +++ b/state/execution.go @@ -1,6 +1,7 @@ package state import ( + "encoding/hex" "errors" "fmt" @@ -251,7 +252,8 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn for i, r := range txResults { if r != nil { tx := block.Txs[i] - batch.Index(string(tx.Hash()), *r) + // dd2e325f79f7e5f77788759d278c1d4b370c842e => {"height":2405, "index":0, ...} + batch.Index(hex.EncodeToString(tx.Hash()), *r) } } s.TxIndexer.Batch(batch) diff --git a/state/state.go b/state/state.go index 347718f1e..cecb38418 100644 --- a/state/state.go +++ b/state/state.go @@ -44,13 +44,12 @@ type State struct { TxIndexer tx.Indexer `json:"-"` // Transaction indexer. } -// Used in tests. func LoadState(db dbm.DB) *State { return loadState(db, stateKey) } func loadState(db dbm.DB, key []byte) *State { - s := &State{db: db} + s := &State{db: db, TxIndexer: &txindexer.Null{}} buf := db.Get(key) if len(buf) == 0 { return nil @@ -132,15 +131,6 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { state.Save() } - // Transaction indexing - switch config.GetString("tx_indexer") { - case "kv": - store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) - state.TxIndexer = txindexer.NewKV(store) - default: - state.TxIndexer = &txindexer.Null{} - } - return state } diff --git a/state/tx/indexer/null.go b/state/tx/indexer/null.go index b196a7056..c9df50edb 100644 --- a/state/tx/indexer/null.go +++ b/state/tx/indexer/null.go @@ -1,13 +1,16 @@ package indexer -import "github.com/tendermint/tendermint/types" +import ( + "errors" + "github.com/tendermint/tendermint/types" +) // Null acts as a /dev/null. type Null struct{} // Tx panics. func (indexer *Null) Tx(hash string) (*types.TxResult, error) { - panic("You are trying to get the transaction from a null indexer") + return nil, errors.New("Indexing is disabled (set `tx_indexer=kv` in config)") } // Batch returns nil. diff --git a/types/tx_result.go b/types/tx_result.go index c3ec00304..c1fc717d3 100644 --- a/types/tx_result.go +++ b/types/tx_result.go @@ -8,7 +8,7 @@ import ( // // One usage is indexing transaction results. type TxResult struct { - Height uint64 - Index uint32 - DeliverTxResponse abci.ResponseDeliverTx + Height uint64 `json:"height"` + Index uint32 `json:"index"` + DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` } From 3478de50a166e1411f0576e0ac0f4e2f4233632c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 10 Apr 2017 22:41:07 +0400 Subject: [PATCH 20/84] no need for map - tx responses should arrive in order (Refs #237) ``` me: so we are executing them in order and receiving them in order and there is no way we could receive them out of order (due to network or something else), correct? ebuchman: if we receive them out of order, ABCI is broken ebuchman: so it is possible, if the ABCI server we're talking to is not implementing the spec ebuchman: but that shouldn't justify us building a map ``` --- state/execution.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/state/execution.go b/state/execution.go index bc125d4e3..5f0df6133 100644 --- a/state/execution.go +++ b/state/execution.go @@ -65,11 +65,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo var validTxs, invalidTxs = 0, 0 txResults := make([]*types.TxResult, len(block.Txs)) - - txHashToIndexMap := make(map[string]int) - for index, tx := range block.Txs { - txHashToIndexMap[string(tx.Hash())] = index - } + txIndex := 0 // Execute transactions and get hash proxyCb := func(req *abci.Request, res *abci.Response) { @@ -89,16 +85,13 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo txError = txResult.Code.String() } - tx := types.Tx(req.GetDeliverTx().Tx) - index, ok := txHashToIndexMap[string(tx.Hash())] - if ok { - txResults[index] = &types.TxResult{uint64(block.Height), uint32(index), *txResult} - } + txResults[txIndex] = &types.TxResult{uint64(block.Height), uint32(txIndex), *txResult} + txIndex++ // NOTE: if we count we can access the tx from the block instead of // pulling it from the req event := types.EventDataTx{ - Tx: tx, + Tx: types.Tx(req.GetDeliverTx().Tx), Data: txResult.Data, Code: txResult.Code, Log: txResult.Log, From e4e17a2c956cfaebbe9e9d1565a94ff20a23003f Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 10 Apr 2017 21:16:41 +0200 Subject: [PATCH 21/84] Play well with go-{rpc,crypto,data}:develop --- benchmarks/simu/counter.go | 4 +- glide.lock | 16 +++---- glide.yaml | 2 + rpc/client/httpclient.go | 32 +++++++------ rpc/test/client_test.go | 95 ++++++++++++++++---------------------- rpc/test/helpers.go | 8 ++-- 6 files changed, 76 insertions(+), 81 deletions(-) diff --git a/benchmarks/simu/counter.go b/benchmarks/simu/counter.go index ca155bbd8..36d1e35df 100644 --- a/benchmarks/simu/counter.go +++ b/benchmarks/simu/counter.go @@ -37,7 +37,9 @@ func main() { for i := 0; ; i++ { binary.BigEndian.PutUint64(buf, uint64(i)) //txBytes := hex.EncodeToString(buf[:n]) - request := rpctypes.NewRPCRequest("fakeid", "broadcast_tx", Arr(buf[:8])) + request := rpctypes.NewRPCRequest("fakeid", + "broadcast_tx", + map[string]interface{}{"tx": buf[:8]}) reqBytes := wire.JSONBytes(request) //fmt.Println("!!", string(reqBytes)) fmt.Print(".") diff --git a/glide.lock b/glide.lock index e79888029..b018ab6e9 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 81cd41d28f9a747a71e6a47e8bc7d855846df29f359ffdcc8d1645c451112b31 -updated: 2017-03-06T17:34:23.99160606-05:00 +hash: 3fd28c04658701e65d8f27e245d15c61056199201fd5af6d18385491fc579a70 +updated: 2017-04-10T20:38:02.65047007+02:00 imports: - name: github.com/btcsuite/btcd version: 583684b21bfbde9b5fc4403916fd7c807feb0289 @@ -16,7 +16,7 @@ imports: - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: 909568be09de550ed094403c2bf8a261b5bb730a + version: 100ba4e885062801d56799d78530b73b178a78f3 subpackages: - proto - name: github.com/golang/protobuf @@ -66,7 +66,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 1236e8fb6eee3a63909f4014a8e84385ead7933d + version: af792eac777de757cd496349a5f6b5313738fcbc subpackages: - client - example/counter @@ -83,15 +83,15 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: dcb015dff6c7af21e65c8e2f3b450df19d38c777 + version: 6af2364fa91ef2f3afc8ba0db33b66d9d3ae006c subpackages: - test - name: github.com/tendermint/go-config version: 620dcbbd7d587cf3599dedbf329b64311b0c307a - name: github.com/tendermint/go-crypto - version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a + version: 750b25c47a5782f5f2b773ed9e706cb82b3ccef4 - name: github.com/tendermint/go-data - version: 32271140e8fd5abdbb22e268d7a02421fa382f0b + version: e7fcc6d081ec8518912fcdc103188275f83a3ee5 - name: github.com/tendermint/go-db version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72 - name: github.com/tendermint/go-events @@ -109,7 +109,7 @@ imports: subpackages: - upnp - name: github.com/tendermint/go-rpc - version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7 + version: a416c37ebd389dcc320d8f41bdcdc575bdc0a826 subpackages: - client - server diff --git a/glide.yaml b/glide.yaml index 53d308233..cdb083e69 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,6 +10,8 @@ import: version: develop - package: github.com/tendermint/go-crypto version: develop +- package: github.com/tendermint/go-data + version: develop - package: github.com/tendermint/go-db version: develop - package: github.com/tendermint/go-events diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index bb4e6d3a8..07059bca0 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -22,7 +22,7 @@ out the server for test code (mock). */ type HTTP struct { remote string - rpc *rpcclient.ClientJSONRPC + rpc *rpcclient.JSONRPCClient *WSEvents } @@ -30,7 +30,7 @@ type HTTP struct { // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { return &HTTP{ - rpc: rpcclient.NewClientJSONRPC(remote), + rpc: rpcclient.NewJSONRPCClient(remote), remote: remote, WSEvents: newWSEvents(remote, wsEndpoint), } @@ -50,7 +50,7 @@ func (c *HTTP) _assertIsEventSwitch() types.EventSwitch { func (c *HTTP) Status() (*ctypes.ResultStatus, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("status", []interface{}{}, tmResult) + _, err := c.rpc.Call("status", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Status") } @@ -60,7 +60,7 @@ func (c *HTTP) Status() (*ctypes.ResultStatus, error) { func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_info", []interface{}{}, tmResult) + _, err := c.rpc.Call("abci_info", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIInfo") } @@ -69,7 +69,9 @@ func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult) + _, err := c.rpc.Call("abci_query", + map[string]interface{}{"path": path, "data": data, "prove": prove}, + tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIQuery") } @@ -78,7 +80,7 @@ func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultAB func (c *HTTP) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("broadcast_tx_commit", []interface{}{tx}, tmResult) + _, err := c.rpc.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) if err != nil { return nil, errors.Wrap(err, "broadcast_tx_commit") } @@ -95,7 +97,7 @@ func (c *HTTP) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call(route, []interface{}{tx}, tmResult) + _, err := c.rpc.Call(route, map[string]interface{}{"tx": tx}, tmResult) if err != nil { return nil, errors.Wrap(err, route) } @@ -104,7 +106,7 @@ func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("net_info", nil, tmResult) + _, err := c.rpc.Call("net_info", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "NetInfo") } @@ -113,7 +115,7 @@ func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("dump_consensus_state", nil, tmResult) + _, err := c.rpc.Call("dump_consensus_state", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "DumpConsensusState") } @@ -122,7 +124,9 @@ func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult) + _, err := c.rpc.Call("blockchain", + map[string]interface{}{"minHeight": minHeight, "maxHeight": maxHeight}, + tmResult) if err != nil { return nil, errors.Wrap(err, "BlockchainInfo") } @@ -131,7 +135,7 @@ func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchai func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("genesis", nil, tmResult) + _, err := c.rpc.Call("genesis", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Genesis") } @@ -140,7 +144,7 @@ func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("block", []interface{}{height}, tmResult) + _, err := c.rpc.Call("block", map[string]interface{}{"height": height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Block") } @@ -149,7 +153,7 @@ func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("commit", []interface{}{height}, tmResult) + _, err := c.rpc.Call("commit", map[string]interface{}{"height": height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Commit") } @@ -158,7 +162,7 @@ func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { func (c *HTTP) Validators() (*ctypes.ResultValidators, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("validators", nil, tmResult) + _, err := c.rpc.Call("validators", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Validators") } diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 1141de4ee..43409c723 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" + rpc "github.com/tendermint/go-rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" ) @@ -25,24 +26,20 @@ import ( // status func TestURIStatus(t *testing.T) { - tmResult := new(ctypes.TMResult) - _, err := GetURIClient().Call("status", map[string]interface{}{}, tmResult) - require.Nil(t, err) - testStatus(t, tmResult) + testStatus(t, GetURIClient()) } func TestJSONStatus(t *testing.T) { - tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("status", []interface{}{}, tmResult) - require.Nil(t, err) - testStatus(t, tmResult) + testStatus(t, GetJSONClient()) } -func testStatus(t *testing.T, statusI interface{}) { +func testStatus(t *testing.T, client rpc.HTTPClient) { chainID := GetConfig().GetString("chain_id") + tmResult := new(ctypes.TMResult) + _, err := client.Call("status", map[string]interface{}{}, tmResult) + require.Nil(t, err) - tmRes := statusI.(*ctypes.TMResult) - status := (*tmRes).(*ctypes.ResultStatus) + status := (*tmResult).(*ctypes.ResultStatus) assert.Equal(t, chainID, status.NodeInfo.Network) } @@ -59,28 +56,22 @@ func randBytes(t *testing.T) []byte { } func TestURIBroadcastTxSync(t *testing.T) { - config.Set("block_size", 0) - defer config.Set("block_size", -1) - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetURIClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) - require.Nil(t, err) - testBroadcastTxSync(t, tmResult, tx) + testBroadcastTxSync(t, GetURIClient()) } func TestJSONBroadcastTxSync(t *testing.T) { + testBroadcastTxSync(t, GetJSONClient()) +} + +func testBroadcastTxSync(t *testing.T, client rpc.HTTPClient) { config.Set("block_size", 0) defer config.Set("block_size", -1) tmResult := new(ctypes.TMResult) tx := randBytes(t) - _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) + _, err := client.Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) require.Nil(t, err) - testBroadcastTxSync(t, tmResult, tx) -} -func testBroadcastTxSync(t *testing.T, resI interface{}, tx []byte) { - tmRes := resI.(*ctypes.TMResult) - res := (*tmRes).(*ctypes.ResultBroadcastTx) + res := (*tmResult).(*ctypes.ResultBroadcastTx) require.Equal(t, abci.CodeType_OK, res.Code) mem := node.MempoolReactor().Mempool require.Equal(t, 1, mem.Size()) @@ -98,34 +89,31 @@ func testTxKV(t *testing.T) ([]byte, []byte, []byte) { return k, v, []byte(Fmt("%s=%s", k, v)) } -func sendTx(t *testing.T) ([]byte, []byte) { +func sendTx(t *testing.T, client rpc.HTTPClient) ([]byte, []byte) { tmResult := new(ctypes.TMResult) k, v, tx := testTxKV(t) - _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) require.Nil(t, err) return k, v } func TestURIABCIQuery(t *testing.T) { - k, v := sendTx(t) - time.Sleep(time.Second) - tmResult := new(ctypes.TMResult) - _, err := GetURIClient().Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) - require.Nil(t, err) - testABCIQuery(t, tmResult, v) + testABCIQuery(t, GetURIClient()) } func TestJSONABCIQuery(t *testing.T) { - k, v := sendTx(t) + testABCIQuery(t, GetURIClient()) +} + +func testABCIQuery(t *testing.T, client rpc.HTTPClient) { + k, _ := sendTx(t, client) + time.Sleep(time.Millisecond * 100) tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("abci_query", []interface{}{"", k, false}, tmResult) + _, err := client.Call("abci_query", + map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) require.Nil(t, err) - testABCIQuery(t, tmResult, v) -} -func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { - tmRes := statusI.(*ctypes.TMResult) - resQuery := (*tmRes).(*ctypes.ResultABCIQuery) + resQuery := (*tmResult).(*ctypes.ResultABCIQuery) require.EqualValues(t, 0, resQuery.Response.Code) // XXX: specific to value returned by the dummy @@ -136,25 +124,22 @@ func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { // broadcast tx commit func TestURIBroadcastTxCommit(t *testing.T) { - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetURIClient().Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) - require.Nil(t, err) - testBroadcastTxCommit(t, tmResult, tx) + testBroadcastTxCommit(t, GetURIClient()) } func TestJSONBroadcastTxCommit(t *testing.T) { - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) - require.Nil(t, err) - testBroadcastTxCommit(t, tmResult, tx) + testBroadcastTxCommit(t, GetJSONClient()) } -func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { +func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { require := require.New(t) - tmRes := resI.(*ctypes.TMResult) - res := (*tmRes).(*ctypes.ResultBroadcastTxCommit) + + tmResult := new(ctypes.TMResult) + tx := randBytes(t) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + require.Nil(err) + + res := (*tmResult).(*ctypes.ResultBroadcastTxCommit) checkTx := res.CheckTx require.Equal(abci.CodeType_OK, checkTx.Code) deliverTx := res.DeliverTx @@ -240,7 +225,7 @@ func TestWSTxEvent(t *testing.T) { // send an tx tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) + _, err := GetJSONClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) require.Nil(err) waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error { @@ -310,7 +295,9 @@ func TestURIUnsafeSetConfig(t *testing.T) { func TestJSONUnsafeSetConfig(t *testing.T) { for _, testCase := range testCasesUnsafeSetConfig { tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult) + _, err := GetJSONClient().Call("unsafe_set_config", + map[string]interface{}{"type": testCase[0], "key": testCase[1], "value": testCase[2]}, + tmResult) require.Nil(t, err) } testUnsafeSetConfig(t) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index cf64ac155..349980e9c 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -72,15 +72,15 @@ func GetConfig() cfg.Config { } // GetURIClient gets a uri client pointing to the test tendermint rpc -func GetURIClient() *client.ClientURI { +func GetURIClient() *client.URIClient { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewClientURI(rpcAddr) + return client.NewURIClient(rpcAddr) } // GetJSONClient gets a http/json client pointing to the test tendermint rpc -func GetJSONClient() *client.ClientJSONRPC { +func GetJSONClient() *client.JSONRPCClient { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewClientJSONRPC(rpcAddr) + return client.NewJSONRPCClient(rpcAddr) } func GetGRPCClient() core_grpc.BroadcastAPIClient { From d3069b0f5b24e0a62d9ab7f1c88acdeb6b5ac94b Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 10 Apr 2017 22:46:03 +0200 Subject: [PATCH 22/84] Update abci develop --- glide.lock | 44 ++++++++++++++++++++++++------------ scripts/install_abci_apps.sh | 2 +- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/glide.lock b/glide.lock index b018ab6e9..f3065569d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ hash: 3fd28c04658701e65d8f27e245d15c61056199201fd5af6d18385491fc579a70 -updated: 2017-04-10T20:38:02.65047007+02:00 +updated: 2017-04-10T22:34:33.346948802+02:00 imports: - name: github.com/btcsuite/btcd - version: 583684b21bfbde9b5fc4403916fd7c807feb0289 + version: 4b348c1d33373d672edd83fc576892d0e46686d2 subpackages: - btcec - name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca + version: b26d9c308763d68093482582cea63d69be07a0f0 - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -20,9 +20,10 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: 69b215d01a5606c843240eab4937eab3acee6530 + version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef subpackages: - proto + - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket @@ -32,9 +33,9 @@ imports: - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: acb9493f2794fd0f820de7a27a217dafbb1b65ea + version: ded68f7a9561c023e790de24279db7ebf473ea80 - name: github.com/mattn/go-isatty - version: 9622e0cc9d8f9be434ca605520ff9a16808fee47 + version: fc9e8d8ef48496124e79ae0df75490096eccf6fe - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -42,16 +43,16 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c + version: 5deb57bbca49eb370538fc295ba4b2988f9f5e09 - name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 + version: 9a906f17374922ed0f74e1b2f593d3723f2ffb00 - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require - name: github.com/syndtr/goleveldb - version: 3c5717caf1475fd25964109a0fc640bd150fce43 + version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 subpackages: - leveldb - leveldb/cache @@ -66,7 +67,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: af792eac777de757cd496349a5f6b5313738fcbc + version: 31eafe8f8eba6b8817edd74df399f508540da528 subpackages: - client - example/counter @@ -105,7 +106,7 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: 97a5ed2d1a17eaee8717b8a32cfaf7a9a82a273d + version: ebd3929c0db9e42268cd677a7782451be7d23327 subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -127,7 +128,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 40541ccb1c6e64c947ed6f606b8a6cb4b67d7436 + version: 9ef620b9ca2f82b55030ffd4f41327fa9e77a92c subpackages: - curve25519 - nacl/box @@ -138,7 +139,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 + version: d1e1b351919c6738fdeb9893d5c998b161464f0c subpackages: - context - http2 @@ -148,20 +149,33 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: e48874b42435b4347fc52bdee0424a52abc974d7 + version: f3918c30c5c2cb527c0b071a27c35120a6c0719a subpackages: - unix +- name: golang.org/x/text + version: f4b4367115ec2de254587813edaa901bc1c723a8 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: google.golang.org/genproto + version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 + subpackages: + - googleapis/rpc/status - name: google.golang.org/grpc - version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 + version: b47cbd158b4721c318abbb389929ef005c322118 subpackages: - codes - credentials - grpclog - internal + - keepalive - metadata - naming - peer - stats + - status - tap - transport testImports: [] diff --git a/scripts/install_abci_apps.sh b/scripts/install_abci_apps.sh index 6da2e2ed2..b2c0b085b 100644 --- a/scripts/install_abci_apps.sh +++ b/scripts/install_abci_apps.sh @@ -1,6 +1,6 @@ #! /bin/bash -go get github.com/tendermint/abci/... +go get -d github.com/tendermint/abci # get the abci commit used by tendermint COMMIT=`bash scripts/glide/parse.sh abci` From d7c5690f177b2b31b99d6ff2ad1d484585894c86 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 10 Apr 2017 17:18:22 -0400 Subject: [PATCH 23/84] index by bytes. add TxID to broadcast_tx responses --- rpc/core/mempool.go | 6 +++++- rpc/core/routes.go | 14 +++----------- rpc/core/tx.go | 13 +++++++++++++ rpc/core/types/responses.go | 9 ++++++++- state/execution.go | 3 +-- state/tx/indexer.go | 2 +- state/tx/indexer/batch.go | 6 +++--- state/tx/indexer/kv.go | 6 +++--- state/tx/indexer/kv_test.go | 4 ++-- state/tx/indexer/null.go | 4 ++-- 10 files changed, 41 insertions(+), 26 deletions(-) create mode 100644 rpc/core/tx.go diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index dfe841365..f78f9d8b0 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -18,7 +18,7 @@ func BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { if err != nil { return nil, fmt.Errorf("Error broadcasting transaction: %v", err) } - return &ctypes.ResultBroadcastTx{}, nil + return &ctypes.ResultBroadcastTx{TxID: tx.Hash()}, nil } // Returns with the response from CheckTx @@ -36,6 +36,7 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { Code: r.Code, Data: r.Data, Log: r.Log, + TxID: tx.Hash(), }, nil } @@ -67,6 +68,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: nil, + TxID: tx.Hash(), }, nil } @@ -86,12 +88,14 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: deliverTxR, + TxID: tx.Hash(), }, nil case <-timer.C: log.Error("failed to include tx") return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: nil, + TxID: tx.Hash(), }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index e79fb0299..8de828826 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -1,12 +1,9 @@ package core import ( - "strings" - rpc "github.com/tendermint/go-rpc/server" "github.com/tendermint/go-rpc/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" - "github.com/tendermint/tendermint/types" ) // TODO: better system than "unsafe" prefix @@ -22,11 +19,11 @@ var Routes = map[string]*rpc.RPCFunc{ "genesis": rpc.NewRPCFunc(GenesisResult, ""), "block": rpc.NewRPCFunc(BlockResult, "height"), "commit": rpc.NewRPCFunc(CommitResult, "height"), + "tx": rpc.NewRPCFunc(TxResult, "hash"), "validators": rpc.NewRPCFunc(ValidatorsResult, ""), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), "num_unconfirmed_txs": rpc.NewRPCFunc(NumUnconfirmedTxsResult, ""), - "tx": rpc.NewRPCFunc(Tx, "hash"), // broadcast API "broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommitResult, "tx"), @@ -103,13 +100,8 @@ func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { // Tx allow user to query the transaction results. `nil` could mean the // transaction is in the mempool, invalidated, or was not send in the first // place. -func Tx(hash string) (ctypes.TMResult, error) { - r, err := txIndexer.Tx(strings.ToLower(hash)) - // nil-pointer interface values are forbidden in go-rpc - if r == (*types.TxResult)(nil) { - return nil, err - } - return r, err +func TxResult(hash []byte) (ctypes.TMResult, error) { + return Tx(hash) } func BroadcastTxCommitResult(tx []byte) (ctypes.TMResult, error) { diff --git a/rpc/core/tx.go b/rpc/core/tx.go new file mode 100644 index 000000000..bffeced7c --- /dev/null +++ b/rpc/core/tx.go @@ -0,0 +1,13 @@ +package core + +import ( + ctypes "github.com/tendermint/tendermint/rpc/core/types" +) + +func Tx(hash []byte) (*ctypes.ResultTx, error) { + r, err := txIndexer.Tx(hash) + if err != nil { + return nil, err + } + return &ctypes.ResultTx{*r}, nil +} diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 908a33f19..2ec762e36 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -67,11 +67,18 @@ type ResultBroadcastTx struct { Code abci.CodeType `json:"code"` Data []byte `json:"data"` Log string `json:"log"` + + TxID []byte `json:"tx_id"` } type ResultBroadcastTxCommit struct { CheckTx *abci.ResponseCheckTx `json:"check_tx"` DeliverTx *abci.ResponseDeliverTx `json:"deliver_tx"` + TxID []byte `json:"tx_id"` +} + +type ResultTx struct { + types.TxResult } type ResultUnconfirmedTxs struct { @@ -164,7 +171,7 @@ var _ = wire.RegisterInterface( wire.ConcreteType{&ResultDumpConsensusState{}, ResultTypeDumpConsensusState}, wire.ConcreteType{&ResultBroadcastTx{}, ResultTypeBroadcastTx}, wire.ConcreteType{&ResultBroadcastTxCommit{}, ResultTypeBroadcastTxCommit}, - wire.ConcreteType{&types.TxResult{}, ResultTypeTx}, + wire.ConcreteType{&ResultTx{}, ResultTypeTx}, wire.ConcreteType{&ResultUnconfirmedTxs{}, ResultTypeUnconfirmedTxs}, wire.ConcreteType{&ResultSubscribe{}, ResultTypeSubscribe}, wire.ConcreteType{&ResultUnsubscribe{}, ResultTypeUnsubscribe}, diff --git a/state/execution.go b/state/execution.go index 5f0df6133..545225be1 100644 --- a/state/execution.go +++ b/state/execution.go @@ -1,7 +1,6 @@ package state import ( - "encoding/hex" "errors" "fmt" @@ -246,7 +245,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn if r != nil { tx := block.Txs[i] // dd2e325f79f7e5f77788759d278c1d4b370c842e => {"height":2405, "index":0, ...} - batch.Index(hex.EncodeToString(tx.Hash()), *r) + batch.Index(tx.Hash(), *r) } } s.TxIndexer.Batch(batch) diff --git a/state/tx/indexer.go b/state/tx/indexer.go index 759f141ae..be7f304c4 100644 --- a/state/tx/indexer.go +++ b/state/tx/indexer.go @@ -17,5 +17,5 @@ type Indexer interface { // Tx returns specified transaction or nil if the transaction is not indexed // or stored. - Tx(hash string) (*types.TxResult, error) + Tx(hash []byte) (*types.TxResult, error) } diff --git a/state/tx/indexer/batch.go b/state/tx/indexer/batch.go index 3c64c8869..78137f598 100644 --- a/state/tx/indexer/batch.go +++ b/state/tx/indexer/batch.go @@ -18,11 +18,11 @@ func NewBatch() *Batch { } // Index adds or updates entry for the given hash. -func (b *Batch) Index(hash string, result types.TxResult) error { - if hash == "" { +func (b *Batch) Index(hash []byte, result types.TxResult) error { + if len(hash) == 0 { return ErrorEmptyHash } - b.Ops[hash] = result + b.Ops[string(hash)] = result return nil } diff --git a/state/tx/indexer/kv.go b/state/tx/indexer/kv.go index 0c86d0a35..4f6fb3581 100644 --- a/state/tx/indexer/kv.go +++ b/state/tx/indexer/kv.go @@ -22,12 +22,12 @@ func NewKV(store db.DB) *KV { // Tx gets transaction from the KV storage and returns it or nil if the // transaction is not found. -func (indexer *KV) Tx(hash string) (*types.TxResult, error) { - if hash == "" { +func (indexer *KV) Tx(hash []byte) (*types.TxResult, error) { + if len(hash) == 0 { return nil, ErrorEmptyHash } - rawBytes := indexer.store.Get([]byte(hash)) + rawBytes := indexer.store.Get(hash) if rawBytes == nil { return nil, nil } diff --git a/state/tx/indexer/kv_test.go b/state/tx/indexer/kv_test.go index 0e3b0a7d7..c797151af 100644 --- a/state/tx/indexer/kv_test.go +++ b/state/tx/indexer/kv_test.go @@ -18,7 +18,7 @@ func TestKVIndex(t *testing.T) { tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} - hash := string(tx.Hash()) + hash := tx.Hash() batch := NewBatch() batch.Index(hash, *txResult) @@ -44,7 +44,7 @@ func benchmarkKVIndex(txsCount int, b *testing.B) { batch := NewBatch() for i := 0; i < txsCount; i++ { - batch.Index(fmt.Sprintf("hash%v", i), *txResult) + batch.Index([]byte(fmt.Sprintf("hash%v", i)), *txResult) } b.ResetTimer() diff --git a/state/tx/indexer/null.go b/state/tx/indexer/null.go index c9df50edb..a322602d0 100644 --- a/state/tx/indexer/null.go +++ b/state/tx/indexer/null.go @@ -9,8 +9,8 @@ import ( type Null struct{} // Tx panics. -func (indexer *Null) Tx(hash string) (*types.TxResult, error) { - return nil, errors.New("Indexing is disabled (set `tx_indexer=kv` in config)") +func (indexer *Null) Tx(hash []byte) (*types.TxResult, error) { + return nil, errors.New(`Indexing is disabled (set 'tx_indexer = "kv"' in config)`) } // Batch returns nil. From 7fbe8e47d46c8da547f714624ca0f5d09aef3230 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 10 Apr 2017 17:32:48 -0400 Subject: [PATCH 24/84] fix tests --- rpc/client/mock/abci.go | 4 ++-- state/execution_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go index 13e187412..6f6fa1d47 100644 --- a/rpc/client/mock/abci.go +++ b/rpc/client/mock/abci.go @@ -45,7 +45,7 @@ func (a ABCIApp) BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error if c.IsOK() { go func() { a.App.DeliverTx(tx) }() } - return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log}, nil + return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log, tx.Hash()}, nil } func (a ABCIApp) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { @@ -54,7 +54,7 @@ func (a ABCIApp) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) if c.IsOK() { go func() { a.App.DeliverTx(tx) }() } - return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log}, nil + return &ctypes.ResultBroadcastTx{c.Code, c.Data, c.Log, tx.Hash()}, nil } // ABCIMock will send all abci related request to the named app, diff --git a/state/execution_test.go b/state/execution_test.go index 7e0b7332b..8b9cf4a73 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -81,7 +81,7 @@ type dummyIndexer struct { Indexed int } -func (indexer *dummyIndexer) Tx(hash string) (*types.TxResult, error) { +func (indexer *dummyIndexer) Tx(hash []byte) (*types.TxResult, error) { return nil, nil } func (indexer *dummyIndexer) Batch(batch *txindexer.Batch) error { From d1fc37ff9e8b8034aa3c029fc59f56768f373d73 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 11 Apr 2017 12:57:06 +0200 Subject: [PATCH 25/84] Undo last two commits --- benchmarks/simu/counter.go | 4 +- glide.lock | 56 ++++++++------------- glide.yaml | 2 - rpc/client/httpclient.go | 32 ++++++------ rpc/test/client_test.go | 95 ++++++++++++++++++++---------------- rpc/test/helpers.go | 8 +-- scripts/install_abci_apps.sh | 2 +- 7 files changed, 95 insertions(+), 104 deletions(-) diff --git a/benchmarks/simu/counter.go b/benchmarks/simu/counter.go index 36d1e35df..ca155bbd8 100644 --- a/benchmarks/simu/counter.go +++ b/benchmarks/simu/counter.go @@ -37,9 +37,7 @@ func main() { for i := 0; ; i++ { binary.BigEndian.PutUint64(buf, uint64(i)) //txBytes := hex.EncodeToString(buf[:n]) - request := rpctypes.NewRPCRequest("fakeid", - "broadcast_tx", - map[string]interface{}{"tx": buf[:8]}) + request := rpctypes.NewRPCRequest("fakeid", "broadcast_tx", Arr(buf[:8])) reqBytes := wire.JSONBytes(request) //fmt.Println("!!", string(reqBytes)) fmt.Print(".") diff --git a/glide.lock b/glide.lock index f3065569d..e79888029 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: 3fd28c04658701e65d8f27e245d15c61056199201fd5af6d18385491fc579a70 -updated: 2017-04-10T22:34:33.346948802+02:00 +hash: 81cd41d28f9a747a71e6a47e8bc7d855846df29f359ffdcc8d1645c451112b31 +updated: 2017-03-06T17:34:23.99160606-05:00 imports: - name: github.com/btcsuite/btcd - version: 4b348c1d33373d672edd83fc576892d0e46686d2 + version: 583684b21bfbde9b5fc4403916fd7c807feb0289 subpackages: - btcec - name: github.com/BurntSushi/toml - version: b26d9c308763d68093482582cea63d69be07a0f0 + version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -16,14 +16,13 @@ imports: - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: 100ba4e885062801d56799d78530b73b178a78f3 + version: 909568be09de550ed094403c2bf8a261b5bb730a subpackages: - proto - name: github.com/golang/protobuf - version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef + version: 69b215d01a5606c843240eab4937eab3acee6530 subpackages: - proto - - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket @@ -33,9 +32,9 @@ imports: - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: ded68f7a9561c023e790de24279db7ebf473ea80 + version: acb9493f2794fd0f820de7a27a217dafbb1b65ea - name: github.com/mattn/go-isatty - version: fc9e8d8ef48496124e79ae0df75490096eccf6fe + version: 9622e0cc9d8f9be434ca605520ff9a16808fee47 - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -43,16 +42,16 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: 5deb57bbca49eb370538fc295ba4b2988f9f5e09 + version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c - name: github.com/spf13/pflag - version: 9a906f17374922ed0f74e1b2f593d3723f2ffb00 + version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require - name: github.com/syndtr/goleveldb - version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 + version: 3c5717caf1475fd25964109a0fc640bd150fce43 subpackages: - leveldb - leveldb/cache @@ -67,7 +66,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 31eafe8f8eba6b8817edd74df399f508540da528 + version: 1236e8fb6eee3a63909f4014a8e84385ead7933d subpackages: - client - example/counter @@ -84,15 +83,15 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 6af2364fa91ef2f3afc8ba0db33b66d9d3ae006c + version: dcb015dff6c7af21e65c8e2f3b450df19d38c777 subpackages: - test - name: github.com/tendermint/go-config version: 620dcbbd7d587cf3599dedbf329b64311b0c307a - name: github.com/tendermint/go-crypto - version: 750b25c47a5782f5f2b773ed9e706cb82b3ccef4 + version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a - name: github.com/tendermint/go-data - version: e7fcc6d081ec8518912fcdc103188275f83a3ee5 + version: 32271140e8fd5abdbb22e268d7a02421fa382f0b - name: github.com/tendermint/go-db version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72 - name: github.com/tendermint/go-events @@ -106,11 +105,11 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: ebd3929c0db9e42268cd677a7782451be7d23327 + version: 97a5ed2d1a17eaee8717b8a32cfaf7a9a82a273d subpackages: - upnp - name: github.com/tendermint/go-rpc - version: a416c37ebd389dcc320d8f41bdcdc575bdc0a826 + version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7 subpackages: - client - server @@ -128,7 +127,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 9ef620b9ca2f82b55030ffd4f41327fa9e77a92c + version: 40541ccb1c6e64c947ed6f606b8a6cb4b67d7436 subpackages: - curve25519 - nacl/box @@ -139,7 +138,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: d1e1b351919c6738fdeb9893d5c998b161464f0c + version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 subpackages: - context - http2 @@ -149,33 +148,20 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: f3918c30c5c2cb527c0b071a27c35120a6c0719a + version: e48874b42435b4347fc52bdee0424a52abc974d7 subpackages: - unix -- name: golang.org/x/text - version: f4b4367115ec2de254587813edaa901bc1c723a8 - subpackages: - - secure/bidirule - - transform - - unicode/bidi - - unicode/norm -- name: google.golang.org/genproto - version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 - subpackages: - - googleapis/rpc/status - name: google.golang.org/grpc - version: b47cbd158b4721c318abbb389929ef005c322118 + version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 subpackages: - codes - credentials - grpclog - internal - - keepalive - metadata - naming - peer - stats - - status - tap - transport testImports: [] diff --git a/glide.yaml b/glide.yaml index cdb083e69..53d308233 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,8 +10,6 @@ import: version: develop - package: github.com/tendermint/go-crypto version: develop -- package: github.com/tendermint/go-data - version: develop - package: github.com/tendermint/go-db version: develop - package: github.com/tendermint/go-events diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 07059bca0..bb4e6d3a8 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -22,7 +22,7 @@ out the server for test code (mock). */ type HTTP struct { remote string - rpc *rpcclient.JSONRPCClient + rpc *rpcclient.ClientJSONRPC *WSEvents } @@ -30,7 +30,7 @@ type HTTP struct { // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { return &HTTP{ - rpc: rpcclient.NewJSONRPCClient(remote), + rpc: rpcclient.NewClientJSONRPC(remote), remote: remote, WSEvents: newWSEvents(remote, wsEndpoint), } @@ -50,7 +50,7 @@ func (c *HTTP) _assertIsEventSwitch() types.EventSwitch { func (c *HTTP) Status() (*ctypes.ResultStatus, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("status", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("status", []interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Status") } @@ -60,7 +60,7 @@ func (c *HTTP) Status() (*ctypes.ResultStatus, error) { func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_info", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("abci_info", []interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIInfo") } @@ -69,9 +69,7 @@ func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_query", - map[string]interface{}{"path": path, "data": data, "prove": prove}, - tmResult) + _, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIQuery") } @@ -80,7 +78,7 @@ func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultAB func (c *HTTP) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + _, err := c.rpc.Call("broadcast_tx_commit", []interface{}{tx}, tmResult) if err != nil { return nil, errors.Wrap(err, "broadcast_tx_commit") } @@ -97,7 +95,7 @@ func (c *HTTP) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call(route, map[string]interface{}{"tx": tx}, tmResult) + _, err := c.rpc.Call(route, []interface{}{tx}, tmResult) if err != nil { return nil, errors.Wrap(err, route) } @@ -106,7 +104,7 @@ func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("net_info", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("net_info", nil, tmResult) if err != nil { return nil, errors.Wrap(err, "NetInfo") } @@ -115,7 +113,7 @@ func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("dump_consensus_state", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("dump_consensus_state", nil, tmResult) if err != nil { return nil, errors.Wrap(err, "DumpConsensusState") } @@ -124,9 +122,7 @@ func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("blockchain", - map[string]interface{}{"minHeight": minHeight, "maxHeight": maxHeight}, - tmResult) + _, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult) if err != nil { return nil, errors.Wrap(err, "BlockchainInfo") } @@ -135,7 +131,7 @@ func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchai func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("genesis", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("genesis", nil, tmResult) if err != nil { return nil, errors.Wrap(err, "Genesis") } @@ -144,7 +140,7 @@ func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("block", map[string]interface{}{"height": height}, tmResult) + _, err := c.rpc.Call("block", []interface{}{height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Block") } @@ -153,7 +149,7 @@ func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("commit", map[string]interface{}{"height": height}, tmResult) + _, err := c.rpc.Call("commit", []interface{}{height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Commit") } @@ -162,7 +158,7 @@ func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { func (c *HTTP) Validators() (*ctypes.ResultValidators, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("validators", map[string]interface{}{}, tmResult) + _, err := c.rpc.Call("validators", nil, tmResult) if err != nil { return nil, errors.Wrap(err, "Validators") } diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 43409c723..1141de4ee 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" - rpc "github.com/tendermint/go-rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" ) @@ -26,20 +25,24 @@ import ( // status func TestURIStatus(t *testing.T) { - testStatus(t, GetURIClient()) + tmResult := new(ctypes.TMResult) + _, err := GetURIClient().Call("status", map[string]interface{}{}, tmResult) + require.Nil(t, err) + testStatus(t, tmResult) } func TestJSONStatus(t *testing.T) { - testStatus(t, GetJSONClient()) + tmResult := new(ctypes.TMResult) + _, err := GetJSONClient().Call("status", []interface{}{}, tmResult) + require.Nil(t, err) + testStatus(t, tmResult) } -func testStatus(t *testing.T, client rpc.HTTPClient) { +func testStatus(t *testing.T, statusI interface{}) { chainID := GetConfig().GetString("chain_id") - tmResult := new(ctypes.TMResult) - _, err := client.Call("status", map[string]interface{}{}, tmResult) - require.Nil(t, err) - status := (*tmResult).(*ctypes.ResultStatus) + tmRes := statusI.(*ctypes.TMResult) + status := (*tmRes).(*ctypes.ResultStatus) assert.Equal(t, chainID, status.NodeInfo.Network) } @@ -56,22 +59,28 @@ func randBytes(t *testing.T) []byte { } func TestURIBroadcastTxSync(t *testing.T) { - testBroadcastTxSync(t, GetURIClient()) + config.Set("block_size", 0) + defer config.Set("block_size", -1) + tmResult := new(ctypes.TMResult) + tx := randBytes(t) + _, err := GetURIClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) + require.Nil(t, err) + testBroadcastTxSync(t, tmResult, tx) } func TestJSONBroadcastTxSync(t *testing.T) { - testBroadcastTxSync(t, GetJSONClient()) -} - -func testBroadcastTxSync(t *testing.T, client rpc.HTTPClient) { config.Set("block_size", 0) defer config.Set("block_size", -1) tmResult := new(ctypes.TMResult) tx := randBytes(t) - _, err := client.Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) + _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) require.Nil(t, err) + testBroadcastTxSync(t, tmResult, tx) +} - res := (*tmResult).(*ctypes.ResultBroadcastTx) +func testBroadcastTxSync(t *testing.T, resI interface{}, tx []byte) { + tmRes := resI.(*ctypes.TMResult) + res := (*tmRes).(*ctypes.ResultBroadcastTx) require.Equal(t, abci.CodeType_OK, res.Code) mem := node.MempoolReactor().Mempool require.Equal(t, 1, mem.Size()) @@ -89,31 +98,34 @@ func testTxKV(t *testing.T) ([]byte, []byte, []byte) { return k, v, []byte(Fmt("%s=%s", k, v)) } -func sendTx(t *testing.T, client rpc.HTTPClient) ([]byte, []byte) { +func sendTx(t *testing.T) ([]byte, []byte) { tmResult := new(ctypes.TMResult) k, v, tx := testTxKV(t) - _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) require.Nil(t, err) return k, v } func TestURIABCIQuery(t *testing.T) { - testABCIQuery(t, GetURIClient()) + k, v := sendTx(t) + time.Sleep(time.Second) + tmResult := new(ctypes.TMResult) + _, err := GetURIClient().Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) + require.Nil(t, err) + testABCIQuery(t, tmResult, v) } func TestJSONABCIQuery(t *testing.T) { - testABCIQuery(t, GetURIClient()) -} - -func testABCIQuery(t *testing.T, client rpc.HTTPClient) { - k, _ := sendTx(t, client) - time.Sleep(time.Millisecond * 100) + k, v := sendTx(t) tmResult := new(ctypes.TMResult) - _, err := client.Call("abci_query", - map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) + _, err := GetJSONClient().Call("abci_query", []interface{}{"", k, false}, tmResult) require.Nil(t, err) + testABCIQuery(t, tmResult, v) +} - resQuery := (*tmResult).(*ctypes.ResultABCIQuery) +func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { + tmRes := statusI.(*ctypes.TMResult) + resQuery := (*tmRes).(*ctypes.ResultABCIQuery) require.EqualValues(t, 0, resQuery.Response.Code) // XXX: specific to value returned by the dummy @@ -124,22 +136,25 @@ func testABCIQuery(t *testing.T, client rpc.HTTPClient) { // broadcast tx commit func TestURIBroadcastTxCommit(t *testing.T) { - testBroadcastTxCommit(t, GetURIClient()) + tmResult := new(ctypes.TMResult) + tx := randBytes(t) + _, err := GetURIClient().Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + require.Nil(t, err) + testBroadcastTxCommit(t, tmResult, tx) } func TestJSONBroadcastTxCommit(t *testing.T) { - testBroadcastTxCommit(t, GetJSONClient()) -} - -func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { - require := require.New(t) - tmResult := new(ctypes.TMResult) tx := randBytes(t) - _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) - require.Nil(err) + _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) + require.Nil(t, err) + testBroadcastTxCommit(t, tmResult, tx) +} - res := (*tmResult).(*ctypes.ResultBroadcastTxCommit) +func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { + require := require.New(t) + tmRes := resI.(*ctypes.TMResult) + res := (*tmRes).(*ctypes.ResultBroadcastTxCommit) checkTx := res.CheckTx require.Equal(abci.CodeType_OK, checkTx.Code) deliverTx := res.DeliverTx @@ -225,7 +240,7 @@ func TestWSTxEvent(t *testing.T) { // send an tx tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) + _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) require.Nil(err) waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error { @@ -295,9 +310,7 @@ func TestURIUnsafeSetConfig(t *testing.T) { func TestJSONUnsafeSetConfig(t *testing.T) { for _, testCase := range testCasesUnsafeSetConfig { tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("unsafe_set_config", - map[string]interface{}{"type": testCase[0], "key": testCase[1], "value": testCase[2]}, - tmResult) + _, err := GetJSONClient().Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult) require.Nil(t, err) } testUnsafeSetConfig(t) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 349980e9c..cf64ac155 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -72,15 +72,15 @@ func GetConfig() cfg.Config { } // GetURIClient gets a uri client pointing to the test tendermint rpc -func GetURIClient() *client.URIClient { +func GetURIClient() *client.ClientURI { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewURIClient(rpcAddr) + return client.NewClientURI(rpcAddr) } // GetJSONClient gets a http/json client pointing to the test tendermint rpc -func GetJSONClient() *client.JSONRPCClient { +func GetJSONClient() *client.ClientJSONRPC { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewJSONRPCClient(rpcAddr) + return client.NewClientJSONRPC(rpcAddr) } func GetGRPCClient() core_grpc.BroadcastAPIClient { diff --git a/scripts/install_abci_apps.sh b/scripts/install_abci_apps.sh index b2c0b085b..6da2e2ed2 100644 --- a/scripts/install_abci_apps.sh +++ b/scripts/install_abci_apps.sh @@ -1,6 +1,6 @@ #! /bin/bash -go get -d github.com/tendermint/abci +go get github.com/tendermint/abci/... # get the abci commit used by tendermint COMMIT=`bash scripts/glide/parse.sh abci` From ac86e664c76fde358441ecb6c34b833e5607a5bb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 11 Apr 2017 15:44:36 -0400 Subject: [PATCH 26/84] Revert "Undo last two commits" This reverts commit d1fc37ff9e8b8034aa3c029fc59f56768f373d73. --- benchmarks/simu/counter.go | 4 +- glide.lock | 56 +++++++++++++-------- glide.yaml | 2 + rpc/client/httpclient.go | 32 ++++++------ rpc/test/client_test.go | 95 ++++++++++++++++-------------------- rpc/test/helpers.go | 8 +-- scripts/install_abci_apps.sh | 2 +- 7 files changed, 104 insertions(+), 95 deletions(-) diff --git a/benchmarks/simu/counter.go b/benchmarks/simu/counter.go index ca155bbd8..36d1e35df 100644 --- a/benchmarks/simu/counter.go +++ b/benchmarks/simu/counter.go @@ -37,7 +37,9 @@ func main() { for i := 0; ; i++ { binary.BigEndian.PutUint64(buf, uint64(i)) //txBytes := hex.EncodeToString(buf[:n]) - request := rpctypes.NewRPCRequest("fakeid", "broadcast_tx", Arr(buf[:8])) + request := rpctypes.NewRPCRequest("fakeid", + "broadcast_tx", + map[string]interface{}{"tx": buf[:8]}) reqBytes := wire.JSONBytes(request) //fmt.Println("!!", string(reqBytes)) fmt.Print(".") diff --git a/glide.lock b/glide.lock index e79888029..f3065569d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,12 @@ -hash: 81cd41d28f9a747a71e6a47e8bc7d855846df29f359ffdcc8d1645c451112b31 -updated: 2017-03-06T17:34:23.99160606-05:00 +hash: 3fd28c04658701e65d8f27e245d15c61056199201fd5af6d18385491fc579a70 +updated: 2017-04-10T22:34:33.346948802+02:00 imports: - name: github.com/btcsuite/btcd - version: 583684b21bfbde9b5fc4403916fd7c807feb0289 + version: 4b348c1d33373d672edd83fc576892d0e46686d2 subpackages: - btcec - name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca + version: b26d9c308763d68093482582cea63d69be07a0f0 - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -16,13 +16,14 @@ imports: - name: github.com/go-stack/stack version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: 909568be09de550ed094403c2bf8a261b5bb730a + version: 100ba4e885062801d56799d78530b73b178a78f3 subpackages: - proto - name: github.com/golang/protobuf - version: 69b215d01a5606c843240eab4937eab3acee6530 + version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef subpackages: - proto + - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket @@ -32,9 +33,9 @@ imports: - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: acb9493f2794fd0f820de7a27a217dafbb1b65ea + version: ded68f7a9561c023e790de24279db7ebf473ea80 - name: github.com/mattn/go-isatty - version: 9622e0cc9d8f9be434ca605520ff9a16808fee47 + version: fc9e8d8ef48496124e79ae0df75490096eccf6fe - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -42,16 +43,16 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c + version: 5deb57bbca49eb370538fc295ba4b2988f9f5e09 - name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 + version: 9a906f17374922ed0f74e1b2f593d3723f2ffb00 - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require - name: github.com/syndtr/goleveldb - version: 3c5717caf1475fd25964109a0fc640bd150fce43 + version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 subpackages: - leveldb - leveldb/cache @@ -66,7 +67,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 1236e8fb6eee3a63909f4014a8e84385ead7933d + version: 31eafe8f8eba6b8817edd74df399f508540da528 subpackages: - client - example/counter @@ -83,15 +84,15 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: dcb015dff6c7af21e65c8e2f3b450df19d38c777 + version: 6af2364fa91ef2f3afc8ba0db33b66d9d3ae006c subpackages: - test - name: github.com/tendermint/go-config version: 620dcbbd7d587cf3599dedbf329b64311b0c307a - name: github.com/tendermint/go-crypto - version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a + version: 750b25c47a5782f5f2b773ed9e706cb82b3ccef4 - name: github.com/tendermint/go-data - version: 32271140e8fd5abdbb22e268d7a02421fa382f0b + version: e7fcc6d081ec8518912fcdc103188275f83a3ee5 - name: github.com/tendermint/go-db version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72 - name: github.com/tendermint/go-events @@ -105,11 +106,11 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: 97a5ed2d1a17eaee8717b8a32cfaf7a9a82a273d + version: ebd3929c0db9e42268cd677a7782451be7d23327 subpackages: - upnp - name: github.com/tendermint/go-rpc - version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7 + version: a416c37ebd389dcc320d8f41bdcdc575bdc0a826 subpackages: - client - server @@ -127,7 +128,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 40541ccb1c6e64c947ed6f606b8a6cb4b67d7436 + version: 9ef620b9ca2f82b55030ffd4f41327fa9e77a92c subpackages: - curve25519 - nacl/box @@ -138,7 +139,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 + version: d1e1b351919c6738fdeb9893d5c998b161464f0c subpackages: - context - http2 @@ -148,20 +149,33 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: e48874b42435b4347fc52bdee0424a52abc974d7 + version: f3918c30c5c2cb527c0b071a27c35120a6c0719a subpackages: - unix +- name: golang.org/x/text + version: f4b4367115ec2de254587813edaa901bc1c723a8 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: google.golang.org/genproto + version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 + subpackages: + - googleapis/rpc/status - name: google.golang.org/grpc - version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 + version: b47cbd158b4721c318abbb389929ef005c322118 subpackages: - codes - credentials - grpclog - internal + - keepalive - metadata - naming - peer - stats + - status - tap - transport testImports: [] diff --git a/glide.yaml b/glide.yaml index 53d308233..cdb083e69 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,6 +10,8 @@ import: version: develop - package: github.com/tendermint/go-crypto version: develop +- package: github.com/tendermint/go-data + version: develop - package: github.com/tendermint/go-db version: develop - package: github.com/tendermint/go-events diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index bb4e6d3a8..07059bca0 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -22,7 +22,7 @@ out the server for test code (mock). */ type HTTP struct { remote string - rpc *rpcclient.ClientJSONRPC + rpc *rpcclient.JSONRPCClient *WSEvents } @@ -30,7 +30,7 @@ type HTTP struct { // and the websocket path (which always seems to be "/websocket") func NewHTTP(remote, wsEndpoint string) *HTTP { return &HTTP{ - rpc: rpcclient.NewClientJSONRPC(remote), + rpc: rpcclient.NewJSONRPCClient(remote), remote: remote, WSEvents: newWSEvents(remote, wsEndpoint), } @@ -50,7 +50,7 @@ func (c *HTTP) _assertIsEventSwitch() types.EventSwitch { func (c *HTTP) Status() (*ctypes.ResultStatus, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("status", []interface{}{}, tmResult) + _, err := c.rpc.Call("status", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Status") } @@ -60,7 +60,7 @@ func (c *HTTP) Status() (*ctypes.ResultStatus, error) { func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_info", []interface{}{}, tmResult) + _, err := c.rpc.Call("abci_info", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIInfo") } @@ -69,7 +69,9 @@ func (c *HTTP) ABCIInfo() (*ctypes.ResultABCIInfo, error) { func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultABCIQuery, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("abci_query", []interface{}{path, data, prove}, tmResult) + _, err := c.rpc.Call("abci_query", + map[string]interface{}{"path": path, "data": data, "prove": prove}, + tmResult) if err != nil { return nil, errors.Wrap(err, "ABCIQuery") } @@ -78,7 +80,7 @@ func (c *HTTP) ABCIQuery(path string, data []byte, prove bool) (*ctypes.ResultAB func (c *HTTP) BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("broadcast_tx_commit", []interface{}{tx}, tmResult) + _, err := c.rpc.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) if err != nil { return nil, errors.Wrap(err, "broadcast_tx_commit") } @@ -95,7 +97,7 @@ func (c *HTTP) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call(route, []interface{}{tx}, tmResult) + _, err := c.rpc.Call(route, map[string]interface{}{"tx": tx}, tmResult) if err != nil { return nil, errors.Wrap(err, route) } @@ -104,7 +106,7 @@ func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("net_info", nil, tmResult) + _, err := c.rpc.Call("net_info", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "NetInfo") } @@ -113,7 +115,7 @@ func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("dump_consensus_state", nil, tmResult) + _, err := c.rpc.Call("dump_consensus_state", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "DumpConsensusState") } @@ -122,7 +124,9 @@ func (c *HTTP) DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchainInfo, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("blockchain", []interface{}{minHeight, maxHeight}, tmResult) + _, err := c.rpc.Call("blockchain", + map[string]interface{}{"minHeight": minHeight, "maxHeight": maxHeight}, + tmResult) if err != nil { return nil, errors.Wrap(err, "BlockchainInfo") } @@ -131,7 +135,7 @@ func (c *HTTP) BlockchainInfo(minHeight, maxHeight int) (*ctypes.ResultBlockchai func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("genesis", nil, tmResult) + _, err := c.rpc.Call("genesis", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Genesis") } @@ -140,7 +144,7 @@ func (c *HTTP) Genesis() (*ctypes.ResultGenesis, error) { func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("block", []interface{}{height}, tmResult) + _, err := c.rpc.Call("block", map[string]interface{}{"height": height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Block") } @@ -149,7 +153,7 @@ func (c *HTTP) Block(height int) (*ctypes.ResultBlock, error) { func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("commit", []interface{}{height}, tmResult) + _, err := c.rpc.Call("commit", map[string]interface{}{"height": height}, tmResult) if err != nil { return nil, errors.Wrap(err, "Commit") } @@ -158,7 +162,7 @@ func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { func (c *HTTP) Validators() (*ctypes.ResultValidators, error) { tmResult := new(ctypes.TMResult) - _, err := c.rpc.Call("validators", nil, tmResult) + _, err := c.rpc.Call("validators", map[string]interface{}{}, tmResult) if err != nil { return nil, errors.Wrap(err, "Validators") } diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 1141de4ee..43409c723 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" + rpc "github.com/tendermint/go-rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" ) @@ -25,24 +26,20 @@ import ( // status func TestURIStatus(t *testing.T) { - tmResult := new(ctypes.TMResult) - _, err := GetURIClient().Call("status", map[string]interface{}{}, tmResult) - require.Nil(t, err) - testStatus(t, tmResult) + testStatus(t, GetURIClient()) } func TestJSONStatus(t *testing.T) { - tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("status", []interface{}{}, tmResult) - require.Nil(t, err) - testStatus(t, tmResult) + testStatus(t, GetJSONClient()) } -func testStatus(t *testing.T, statusI interface{}) { +func testStatus(t *testing.T, client rpc.HTTPClient) { chainID := GetConfig().GetString("chain_id") + tmResult := new(ctypes.TMResult) + _, err := client.Call("status", map[string]interface{}{}, tmResult) + require.Nil(t, err) - tmRes := statusI.(*ctypes.TMResult) - status := (*tmRes).(*ctypes.ResultStatus) + status := (*tmResult).(*ctypes.ResultStatus) assert.Equal(t, chainID, status.NodeInfo.Network) } @@ -59,28 +56,22 @@ func randBytes(t *testing.T) []byte { } func TestURIBroadcastTxSync(t *testing.T) { - config.Set("block_size", 0) - defer config.Set("block_size", -1) - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetURIClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) - require.Nil(t, err) - testBroadcastTxSync(t, tmResult, tx) + testBroadcastTxSync(t, GetURIClient()) } func TestJSONBroadcastTxSync(t *testing.T) { + testBroadcastTxSync(t, GetJSONClient()) +} + +func testBroadcastTxSync(t *testing.T, client rpc.HTTPClient) { config.Set("block_size", 0) defer config.Set("block_size", -1) tmResult := new(ctypes.TMResult) tx := randBytes(t) - _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) + _, err := client.Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) require.Nil(t, err) - testBroadcastTxSync(t, tmResult, tx) -} -func testBroadcastTxSync(t *testing.T, resI interface{}, tx []byte) { - tmRes := resI.(*ctypes.TMResult) - res := (*tmRes).(*ctypes.ResultBroadcastTx) + res := (*tmResult).(*ctypes.ResultBroadcastTx) require.Equal(t, abci.CodeType_OK, res.Code) mem := node.MempoolReactor().Mempool require.Equal(t, 1, mem.Size()) @@ -98,34 +89,31 @@ func testTxKV(t *testing.T) ([]byte, []byte, []byte) { return k, v, []byte(Fmt("%s=%s", k, v)) } -func sendTx(t *testing.T) ([]byte, []byte) { +func sendTx(t *testing.T, client rpc.HTTPClient) ([]byte, []byte) { tmResult := new(ctypes.TMResult) k, v, tx := testTxKV(t) - _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) require.Nil(t, err) return k, v } func TestURIABCIQuery(t *testing.T) { - k, v := sendTx(t) - time.Sleep(time.Second) - tmResult := new(ctypes.TMResult) - _, err := GetURIClient().Call("abci_query", map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) - require.Nil(t, err) - testABCIQuery(t, tmResult, v) + testABCIQuery(t, GetURIClient()) } func TestJSONABCIQuery(t *testing.T) { - k, v := sendTx(t) + testABCIQuery(t, GetURIClient()) +} + +func testABCIQuery(t *testing.T, client rpc.HTTPClient) { + k, _ := sendTx(t, client) + time.Sleep(time.Millisecond * 100) tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("abci_query", []interface{}{"", k, false}, tmResult) + _, err := client.Call("abci_query", + map[string]interface{}{"path": "", "data": k, "prove": false}, tmResult) require.Nil(t, err) - testABCIQuery(t, tmResult, v) -} -func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { - tmRes := statusI.(*ctypes.TMResult) - resQuery := (*tmRes).(*ctypes.ResultABCIQuery) + resQuery := (*tmResult).(*ctypes.ResultABCIQuery) require.EqualValues(t, 0, resQuery.Response.Code) // XXX: specific to value returned by the dummy @@ -136,25 +124,22 @@ func testABCIQuery(t *testing.T, statusI interface{}, value []byte) { // broadcast tx commit func TestURIBroadcastTxCommit(t *testing.T) { - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetURIClient().Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) - require.Nil(t, err) - testBroadcastTxCommit(t, tmResult, tx) + testBroadcastTxCommit(t, GetURIClient()) } func TestJSONBroadcastTxCommit(t *testing.T) { - tmResult := new(ctypes.TMResult) - tx := randBytes(t) - _, err := GetJSONClient().Call("broadcast_tx_commit", []interface{}{tx}, tmResult) - require.Nil(t, err) - testBroadcastTxCommit(t, tmResult, tx) + testBroadcastTxCommit(t, GetJSONClient()) } -func testBroadcastTxCommit(t *testing.T, resI interface{}, tx []byte) { +func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { require := require.New(t) - tmRes := resI.(*ctypes.TMResult) - res := (*tmRes).(*ctypes.ResultBroadcastTxCommit) + + tmResult := new(ctypes.TMResult) + tx := randBytes(t) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + require.Nil(err) + + res := (*tmResult).(*ctypes.ResultBroadcastTxCommit) checkTx := res.CheckTx require.Equal(abci.CodeType_OK, checkTx.Code) deliverTx := res.DeliverTx @@ -240,7 +225,7 @@ func TestWSTxEvent(t *testing.T) { // send an tx tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("broadcast_tx_sync", []interface{}{tx}, tmResult) + _, err := GetJSONClient().Call("broadcast_tx_sync", map[string]interface{}{"tx": tx}, tmResult) require.Nil(err) waitForEvent(t, wsc, eid, true, func() {}, func(eid string, b interface{}) error { @@ -310,7 +295,9 @@ func TestURIUnsafeSetConfig(t *testing.T) { func TestJSONUnsafeSetConfig(t *testing.T) { for _, testCase := range testCasesUnsafeSetConfig { tmResult := new(ctypes.TMResult) - _, err := GetJSONClient().Call("unsafe_set_config", []interface{}{testCase[0], testCase[1], testCase[2]}, tmResult) + _, err := GetJSONClient().Call("unsafe_set_config", + map[string]interface{}{"type": testCase[0], "key": testCase[1], "value": testCase[2]}, + tmResult) require.Nil(t, err) } testUnsafeSetConfig(t) diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index cf64ac155..349980e9c 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -72,15 +72,15 @@ func GetConfig() cfg.Config { } // GetURIClient gets a uri client pointing to the test tendermint rpc -func GetURIClient() *client.ClientURI { +func GetURIClient() *client.URIClient { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewClientURI(rpcAddr) + return client.NewURIClient(rpcAddr) } // GetJSONClient gets a http/json client pointing to the test tendermint rpc -func GetJSONClient() *client.ClientJSONRPC { +func GetJSONClient() *client.JSONRPCClient { rpcAddr := GetConfig().GetString("rpc_laddr") - return client.NewClientJSONRPC(rpcAddr) + return client.NewJSONRPCClient(rpcAddr) } func GetGRPCClient() core_grpc.BroadcastAPIClient { diff --git a/scripts/install_abci_apps.sh b/scripts/install_abci_apps.sh index 6da2e2ed2..b2c0b085b 100644 --- a/scripts/install_abci_apps.sh +++ b/scripts/install_abci_apps.sh @@ -1,6 +1,6 @@ #! /bin/bash -go get github.com/tendermint/abci/... +go get -d github.com/tendermint/abci # get the abci commit used by tendermint COMMIT=`bash scripts/glide/parse.sh abci` From c0f026a9b3b9932f0a0e7133f17a7f1ef24baf50 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Tue, 11 Apr 2017 13:42:45 +0200 Subject: [PATCH 27/84] update go-rpc to fix race condition --- glide.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glide.lock b/glide.lock index f3065569d..321122e6f 100644 --- a/glide.lock +++ b/glide.lock @@ -110,7 +110,7 @@ imports: subpackages: - upnp - name: github.com/tendermint/go-rpc - version: a416c37ebd389dcc320d8f41bdcdc575bdc0a826 + version: 9d18cbe74e66f875afa36d2fa3be280e4a2dc9e6 subpackages: - client - server From 9775ecde996c0e7468b485d895a4f22ac373cfa1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 11 Apr 2017 15:25:28 -0400 Subject: [PATCH 28/84] update glide --- glide.lock | 48 ++++++++++++++++++------------------------------ 1 file changed, 18 insertions(+), 30 deletions(-) diff --git a/glide.lock b/glide.lock index 321122e6f..9b11967a2 100644 --- a/glide.lock +++ b/glide.lock @@ -1,12 +1,14 @@ -hash: 3fd28c04658701e65d8f27e245d15c61056199201fd5af6d18385491fc579a70 -updated: 2017-04-10T22:34:33.346948802+02:00 +hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 +updated: 2017-04-11T15:24:16.619608243-04:00 imports: - name: github.com/btcsuite/btcd - version: 4b348c1d33373d672edd83fc576892d0e46686d2 + version: b8df516b4b267acf2de46be593a9d948d1d2c420 subpackages: - btcec +- name: github.com/btcsuite/fastsha256 + version: 637e656429416087660c84436a2a035d69d54e2e - name: github.com/BurntSushi/toml - version: b26d9c308763d68093482582cea63d69be07a0f0 + version: 99064174e013895bbd9b025c31100bd1d9b590ca - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -20,10 +22,9 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef + version: 69b215d01a5606c843240eab4937eab3acee6530 subpackages: - proto - - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket @@ -33,9 +34,9 @@ imports: - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: ded68f7a9561c023e790de24279db7ebf473ea80 + version: 9fdad7c47650b7d2e1da50644c1f4ba7f172f252 - name: github.com/mattn/go-isatty - version: fc9e8d8ef48496124e79ae0df75490096eccf6fe + version: 56b76bdf51f7708750eac80fa38b952bb9f32639 - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -43,16 +44,16 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: 5deb57bbca49eb370538fc295ba4b2988f9f5e09 + version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c - name: github.com/spf13/pflag - version: 9a906f17374922ed0f74e1b2f593d3723f2ffb00 + version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require - name: github.com/syndtr/goleveldb - version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 + version: 3c5717caf1475fd25964109a0fc640bd150fce43 subpackages: - leveldb - leveldb/cache @@ -106,7 +107,7 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: ebd3929c0db9e42268cd677a7782451be7d23327 + version: c39e001a957caf768f06c85c840debb8282c3aaa subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -116,7 +117,7 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: f530b7af7a8b06e612c2063bff6ace49060a085e + version: ad797c70affa2c81fccc5edaed63ac25144397c6 - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: @@ -128,7 +129,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 9ef620b9ca2f82b55030ffd4f41327fa9e77a92c + version: 1f22c0103821b9390939b6776727195525381532 subpackages: - curve25519 - nacl/box @@ -139,7 +140,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: d1e1b351919c6738fdeb9893d5c998b161464f0c + version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 subpackages: - context - http2 @@ -149,33 +150,20 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: f3918c30c5c2cb527c0b071a27c35120a6c0719a + version: 50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e subpackages: - unix -- name: golang.org/x/text - version: f4b4367115ec2de254587813edaa901bc1c723a8 - subpackages: - - secure/bidirule - - transform - - unicode/bidi - - unicode/norm -- name: google.golang.org/genproto - version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 - subpackages: - - googleapis/rpc/status - name: google.golang.org/grpc - version: b47cbd158b4721c318abbb389929ef005c322118 + version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 subpackages: - codes - credentials - grpclog - internal - - keepalive - metadata - naming - peer - stats - - status - tap - transport testImports: [] From 28307fd4c96b50519059c27b48fcf264ebbd9b15 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 3 Apr 2017 15:45:09 +0200 Subject: [PATCH 29/84] Add proof generation for one tx --- types/tx.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/types/tx.go b/types/tx.go index a67206d4e..e7c454f09 100644 --- a/types/tx.go +++ b/types/tx.go @@ -1,6 +1,9 @@ package types import ( + "bytes" + "errors" + "github.com/tendermint/go-merkle" ) @@ -30,3 +33,55 @@ func (txs Txs) Hash() []byte { return merkle.SimpleHashFromTwoHashes(left, right) } } + +// Index returns the index of this transaction in the list, or -1 if not found +func (txs Txs) Index(tx Tx) int { + return -1 +} + +// Proof returns a simple merkle proof for this node. +// +// Panics if i < 0 or i >= len(txs) +// +// TODO: optimize this! +func (txs Txs) Proof(i int) TxProof { + l := len(txs) + hashables := make([]merkle.Hashable, l) + for i := 0; i < l; i++ { + hashables[i] = txs[i] + } + root, proofs := merkle.SimpleProofsFromHashables(hashables) + + return TxProof{ + Index: i, + Total: l, + RootHash: root, + Data: txs[i], + Proof: *proofs[i], + } +} + +type TxProof struct { + Index, Total int + RootHash []byte + Data Tx + Proof merkle.SimpleProof +} + +func (tp TxProof) LeafHash() []byte { + return tp.Data.Hash() +} + +// Validate returns nil if it matches the dataHash, and is internally consistent +// otherwise, returns a sensible error +func (tp TxProof) Validate(dataHash []byte) error { + if !bytes.Equal(dataHash, tp.RootHash) { + return errors.New("Proof matches different data hash") + } + + valid := tp.Proof.Verify(tp.Index, tp.Total, tp.LeafHash(), tp.RootHash) + if !valid { + return errors.New("Proof is not internally consistent") + } + return nil +} From fd68bc7cfda1592114f574e2bd2101ae62acd255 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 3 Apr 2017 16:18:03 +0200 Subject: [PATCH 30/84] Test Tx proofs secure --- types/tx_test.go | 105 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 types/tx_test.go diff --git a/types/tx_test.go b/types/tx_test.go new file mode 100644 index 000000000..2d4abc5b0 --- /dev/null +++ b/types/tx_test.go @@ -0,0 +1,105 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + cmn "github.com/tendermint/go-common" + ctest "github.com/tendermint/go-common/test" + wire "github.com/tendermint/go-wire" +) + +func makeTxs(cnt, size int) Txs { + txs := make(Txs, cnt) + for i := 0; i < cnt; i++ { + txs[i] = cmn.RandBytes(size) + } + return txs +} + +func randInt(low, high int) int { + off := cmn.RandInt() % (high - low) + return low + off +} + +func TestValidTxProof(t *testing.T) { + assert := assert.New(t) + cases := []struct { + txs Txs + }{ + {Txs{{1, 4, 34, 87, 163, 1}}}, + {Txs{{5, 56, 165, 2}, {4, 77}}}, + {Txs{Tx("foo"), Tx("bar"), Tx("baz")}}, + {makeTxs(20, 5)}, + {makeTxs(7, 81)}, + {makeTxs(61, 15)}, + } + + for h, tc := range cases { + txs := tc.txs + root := txs.Hash() + // make sure valid proof for every tx + for i := range txs { + leaf := txs[i] + leafHash := leaf.Hash() + proof := txs.Proof(i) + assert.Equal(i, proof.Index, "%d: %d", h, i) + assert.Equal(len(txs), proof.Total, "%d: %d", h, i) + assert.Equal(root, proof.RootHash, "%d: %d", h, i) + assert.Equal(leaf, proof.Data, "%d: %d", h, i) + assert.Equal(leafHash, proof.LeafHash(), "%d: %d", h, i) + assert.Nil(proof.Validate(root), "%d: %d", h, i) + assert.NotNil(proof.Validate([]byte("foobar")), "%d: %d", h, i) + + // read-write must also work + var p2 TxProof + bin := wire.BinaryBytes(proof) + err := wire.ReadBinaryBytes(bin, &p2) + if assert.Nil(err, "%d: %d: %+v", h, i, err) { + assert.Nil(p2.Validate(root), "%d: %d", h, i) + } + } + } +} + +func TestTxProofUnchangable(t *testing.T) { + // run the other test a bunch... + for i := 0; i < 4; i++ { + testTxProofUnchangable(t) + } +} + +func testTxProofUnchangable(t *testing.T) { + assert := assert.New(t) + + // make some proof + txs := makeTxs(randInt(2, 100), randInt(16, 128)) + root := txs.Hash() + i := randInt(0, len(txs)-1) + proof := txs.Proof(i) + + // make sure it is valid to start with + assert.Nil(proof.Validate(root)) + bin := wire.BinaryBytes(proof) + + // try mutating the data and make sure nothing breaks + for j := 0; j < 50; j++ { + bad := ctest.MutateByteSlice(bin) + assertBadProof(t, root, bad) + } +} + +// this make sure the proof doesn't deserialize into something valid +func assertBadProof(t *testing.T, root []byte, bad []byte) { + // we kind of expect this to panic sometimes... (bad, go-wire, bad) + defer func() { + recover() + }() + + var proof TxProof + err := wire.ReadBinaryBytes(bad, &proof) + if err == nil { + err = proof.Validate(root) + assert.NotNil(t, err, "%+v", err) + } +} From 285a2a70618d62e1dfed62f458a46723140c3440 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 3 Apr 2017 16:57:34 +0200 Subject: [PATCH 31/84] More thorough testing of mutated bytes, use fixed go-wire --- glide.lock | 2 +- types/tx_test.go | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index 9b11967a2..674b8b86d 100644 --- a/glide.lock +++ b/glide.lock @@ -117,7 +117,7 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: ad797c70affa2c81fccc5edaed63ac25144397c6 + version: 09dae074245a8042aa689d084af774e6ad6a90bb - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: diff --git a/types/tx_test.go b/types/tx_test.go index 2d4abc5b0..3be669638 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -1,6 +1,7 @@ package types import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -64,7 +65,7 @@ func TestValidTxProof(t *testing.T) { func TestTxProofUnchangable(t *testing.T) { // run the other test a bunch... - for i := 0; i < 4; i++ { + for i := 0; i < 40; i++ { testTxProofUnchangable(t) } } @@ -83,14 +84,16 @@ func testTxProofUnchangable(t *testing.T) { bin := wire.BinaryBytes(proof) // try mutating the data and make sure nothing breaks - for j := 0; j < 50; j++ { + for j := 0; j < 500; j++ { bad := ctest.MutateByteSlice(bin) - assertBadProof(t, root, bad) + if !bytes.Equal(bad, bin) { + assertBadProof(t, root, bad, proof) + } } } // this make sure the proof doesn't deserialize into something valid -func assertBadProof(t *testing.T, root []byte, bad []byte) { +func assertBadProof(t *testing.T, root []byte, bad []byte, good TxProof) { // we kind of expect this to panic sometimes... (bad, go-wire, bad) defer func() { recover() @@ -100,6 +103,11 @@ func assertBadProof(t *testing.T, root []byte, bad []byte) { err := wire.ReadBinaryBytes(bad, &proof) if err == nil { err = proof.Validate(root) - assert.NotNil(t, err, "%+v", err) + if err == nil { + // okay, this can happen if we have a slightly different total + // (where the path ends up the same), if it is something else, we have + // a real problem + assert.NotEqual(t, proof.Total, good.Total, "bad: %#v\ngood: %#v", proof, good) + } } } From 705e7bd577f18b89d846595f26c681956f31fd42 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Mon, 3 Apr 2017 17:21:30 +0200 Subject: [PATCH 32/84] Implemented and tested Txs.Index, hopefully better coverage --- types/tx.go | 5 +++++ types/tx_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/types/tx.go b/types/tx.go index e7c454f09..bcff2633f 100644 --- a/types/tx.go +++ b/types/tx.go @@ -36,6 +36,11 @@ func (txs Txs) Hash() []byte { // Index returns the index of this transaction in the list, or -1 if not found func (txs Txs) Index(tx Tx) int { + for i := range txs { + if bytes.Equal(txs[i], tx) { + return i + } + } return -1 } diff --git a/types/tx_test.go b/types/tx_test.go index 3be669638..aa20f7443 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -23,6 +23,20 @@ func randInt(low, high int) int { return low + off } +func TestTxIndex(t *testing.T) { + assert := assert.New(t) + for i := 0; i < 20; i++ { + txs := makeTxs(15, 60) + for j := 0; j < len(txs); j++ { + tx := txs[j] + idx := txs.Index(tx) + assert.Equal(j, idx) + } + assert.Equal(-1, txs.Index(nil)) + assert.Equal(-1, txs.Index(Tx("foodnwkf"))) + } +} + func TestValidTxProof(t *testing.T) { assert := assert.New(t) cases := []struct { From c34e136be9590d1e1331ce00456f600afd51b8cf Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 13:44:10 -0400 Subject: [PATCH 33/84] update go-rpc to support map and array params --- glide.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 9b11967a2..9c2a25a31 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 -updated: 2017-04-11T15:24:16.619608243-04:00 +updated: 2017-04-12T13:43:29.528413882-04:00 imports: - name: github.com/btcsuite/btcd version: b8df516b4b267acf2de46be593a9d948d1d2c420 @@ -111,7 +111,7 @@ imports: subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 9d18cbe74e66f875afa36d2fa3be280e4a2dc9e6 + version: c3295f4878019ff3fdfcac37a4c0e4bcf4bb02a7 subpackages: - client - server From cf875a51fde67db42deed3e5d573aebf98a0af30 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Wed, 22 Mar 2017 23:15:03 +0100 Subject: [PATCH 34/84] Fix draw_deps in Makefile and add resulting dependency graph --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 3cafad386..e11152fce 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ test100: @for i in {1..100}; do make test; done draw_deps: - # requires brew install graphviz - go get github.com/hirokidaichi/goviz - goviz -i ./cmd/tendermint | dot -Tpng -o huge.png + # requires brew install graphviz or apt-get install graphviz + go get github.com/RobotsAndPencils/goviz + @goviz -i github.com/tendermint/tendermint/cmd/tendermint -d 3 | dot -Tpng -o dependency-graph.png list_deps: @go list -f '{{join .Deps "\n"}}' ./... | \ From 6bcbd14d5a532078dae4c312e6ff50dd5f4f60e0 Mon Sep 17 00:00:00 2001 From: Adrian Brink Date: Thu, 23 Mar 2017 12:52:04 +0100 Subject: [PATCH 35/84] Add line counting badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index a1ba8fb34..3fb5d8645 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 )](https://godoc.org/github.com/tendermint/tendermint) [![chat](https://img.shields.io/badge/slack-join%20chat-pink.svg)](http://forum.tendermint.com:3000/) [![license](https://img.shields.io/github/license/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/blob/master/LICENSE) +[![](https://tokei.rs/b1/github/tendermint/tendermint?category=lines)](https://github.com/tendermint/tendermint) + Branch | Tests | Coverage | Report Card ----------|-------|----------|------------- From 2a59cda77e05094f815d63fa34045282b472c704 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 18:18:17 -0400 Subject: [PATCH 36/84] /tx returns tx bytes --- rpc/core/tx.go | 17 ++++++++++++++++- rpc/core/types/responses.go | 5 ++++- state/execution.go | 7 ++----- types/tx.go | 10 ++++++++++ types/tx_result.go | 14 -------------- 5 files changed, 32 insertions(+), 21 deletions(-) delete mode 100644 types/tx_result.go diff --git a/rpc/core/tx.go b/rpc/core/tx.go index bffeced7c..5e066d0d8 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -1,6 +1,8 @@ package core import ( + "fmt" + ctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -9,5 +11,18 @@ func Tx(hash []byte) (*ctypes.ResultTx, error) { if err != nil { return nil, err } - return &ctypes.ResultTx{*r}, nil + + if r == nil { + return &ctypes.ResultTx{}, fmt.Errorf("Tx (%X) not found", hash) + } + + block := blockStore.LoadBlock(int(r.Height)) + tx := block.Data.Txs[int(r.Index)] + + return &ctypes.ResultTx{ + Height: r.Height, + Index: r.Index, + DeliverTx: r.DeliverTx, + Tx: tx, + }, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 2ec762e36..8f6fb0ab7 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -78,7 +78,10 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { - types.TxResult + Height uint64 `json:"height"` + Index uint32 `json:"index"` + DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` + Tx types.Tx `json:"tx"` } type ResultUnconfirmedTxs struct { diff --git a/state/execution.go b/state/execution.go index 545225be1..75903d402 100644 --- a/state/execution.go +++ b/state/execution.go @@ -242,11 +242,8 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn batch := txindexer.NewBatch() for i, r := range txResults { - if r != nil { - tx := block.Txs[i] - // dd2e325f79f7e5f77788759d278c1d4b370c842e => {"height":2405, "index":0, ...} - batch.Index(tx.Hash(), *r) - } + tx := block.Txs[i] + batch.Index(tx.Hash(), *r) } s.TxIndexer.Batch(batch) diff --git a/types/tx.go b/types/tx.go index a67206d4e..d2b4ee8df 100644 --- a/types/tx.go +++ b/types/tx.go @@ -1,6 +1,7 @@ package types import ( + abci "github.com/tendermint/abci/types" "github.com/tendermint/go-merkle" ) @@ -30,3 +31,12 @@ func (txs Txs) Hash() []byte { return merkle.SimpleHashFromTwoHashes(left, right) } } + +// TxResult contains results of executing the transaction. +// +// One usage is indexing transaction results. +type TxResult struct { + Height uint64 `json:"height"` + Index uint32 `json:"index"` + DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` +} diff --git a/types/tx_result.go b/types/tx_result.go deleted file mode 100644 index c1fc717d3..000000000 --- a/types/tx_result.go +++ /dev/null @@ -1,14 +0,0 @@ -package types - -import ( - abci "github.com/tendermint/abci/types" -) - -// TxResult contains results of executing the transaction. -// -// One usage is indexing transaction results. -type TxResult struct { - Height uint64 `json:"height"` - Index uint32 `json:"index"` - DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` -} From ffe6d58a58b4dfcf3fbf8656f597d2259808b3ca Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 18:33:48 -0400 Subject: [PATCH 37/84] add Height to ResultBroadcastTxCommit and EventDataTx --- rpc/core/mempool.go | 1 + rpc/core/types/responses.go | 1 + state/execution.go | 11 ++++++----- types/events.go | 13 +++++++------ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index f78f9d8b0..c4fc3b1a1 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -89,6 +89,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { CheckTx: checkTxR, DeliverTx: deliverTxR, TxID: tx.Hash(), + Height: deliverTxRes.Height, }, nil case <-timer.C: log.Error("failed to include tx") diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 8f6fb0ab7..f5dc5f1d7 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -75,6 +75,7 @@ type ResultBroadcastTxCommit struct { CheckTx *abci.ResponseCheckTx `json:"check_tx"` DeliverTx *abci.ResponseDeliverTx `json:"deliver_tx"` TxID []byte `json:"tx_id"` + Height int `json:"height"` } type ResultTx struct { diff --git a/state/execution.go b/state/execution.go index 75903d402..802d12c49 100644 --- a/state/execution.go +++ b/state/execution.go @@ -90,11 +90,12 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo // NOTE: if we count we can access the tx from the block instead of // pulling it from the req event := types.EventDataTx{ - Tx: types.Tx(req.GetDeliverTx().Tx), - Data: txResult.Data, - Code: txResult.Code, - Log: txResult.Log, - Error: txError, + Height: block.Height, + Tx: types.Tx(req.GetDeliverTx().Tx), + Data: txResult.Data, + Code: txResult.Code, + Log: txResult.Log, + Error: txError, } types.FireEventTx(eventCache, event) } diff --git a/types/events.go b/types/events.go index aa5896e9b..114979047 100644 --- a/types/events.go +++ b/types/events.go @@ -2,10 +2,10 @@ package types import ( // for registering TMEventData as events.EventData + abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" "github.com/tendermint/go-events" "github.com/tendermint/go-wire" - abci "github.com/tendermint/abci/types" ) // Functions to generate eventId strings @@ -73,11 +73,12 @@ type EventDataNewBlockHeader struct { // All txs fire EventDataTx type EventDataTx struct { - Tx Tx `json:"tx"` - Data []byte `json:"data"` - Log string `json:"log"` - Code abci.CodeType `json:"code"` - Error string `json:"error"` // this is redundant information for now + Height int `json:"height"` + Tx Tx `json:"tx"` + Data []byte `json:"data"` + Log string `json:"log"` + Code abci.CodeType `json:"code"` + Error string `json:"error"` // this is redundant information for now } // NOTE: This goes into the replay WAL From a1387c7c179c15944030323b1e80112c9a46f034 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 18:45:37 -0400 Subject: [PATCH 38/84] remove expected panic in test --- types/tx_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/types/tx_test.go b/types/tx_test.go index aa20f7443..7688a9bf1 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -108,11 +108,6 @@ func testTxProofUnchangable(t *testing.T) { // this make sure the proof doesn't deserialize into something valid func assertBadProof(t *testing.T, root []byte, bad []byte, good TxProof) { - // we kind of expect this to panic sometimes... (bad, go-wire, bad) - defer func() { - recover() - }() - var proof TxProof err := wire.ReadBinaryBytes(bad, &proof) if err == nil { From 6899c91ebed16b3cee91af6b5c1e73b5caa24cc1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 18:55:00 -0400 Subject: [PATCH 39/84] add optional 'prove' flag to /tx --- rpc/core/routes.go | 6 +++--- rpc/core/tx.go | 9 ++++++++- rpc/core/types/responses.go | 1 + 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 8de828826..38e609601 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,7 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "genesis": rpc.NewRPCFunc(GenesisResult, ""), "block": rpc.NewRPCFunc(BlockResult, "height"), "commit": rpc.NewRPCFunc(CommitResult, "height"), - "tx": rpc.NewRPCFunc(TxResult, "hash"), + "tx": rpc.NewRPCFunc(TxResult, "hash,prove"), "validators": rpc.NewRPCFunc(ValidatorsResult, ""), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), @@ -100,8 +100,8 @@ func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { // Tx allow user to query the transaction results. `nil` could mean the // transaction is in the mempool, invalidated, or was not send in the first // place. -func TxResult(hash []byte) (ctypes.TMResult, error) { - return Tx(hash) +func TxResult(hash []byte, prove bool) (ctypes.TMResult, error) { + return Tx(hash, prove) } func BroadcastTxCommitResult(tx []byte) (ctypes.TMResult, error) { diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 5e066d0d8..14f87ece2 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -4,9 +4,10 @@ import ( "fmt" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/types" ) -func Tx(hash []byte) (*ctypes.ResultTx, error) { +func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { r, err := txIndexer.Tx(hash) if err != nil { return nil, err @@ -19,10 +20,16 @@ func Tx(hash []byte) (*ctypes.ResultTx, error) { block := blockStore.LoadBlock(int(r.Height)) tx := block.Data.Txs[int(r.Index)] + var proof types.TxProof + if prove { + proof = block.Data.Txs.Proof(int(r.Index)) + } + return &ctypes.ResultTx{ Height: r.Height, Index: r.Index, DeliverTx: r.DeliverTx, Tx: tx, + Proof: proof, }, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index f5dc5f1d7..9ed2383b2 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -83,6 +83,7 @@ type ResultTx struct { Index uint32 `json:"index"` DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` Tx types.Tx `json:"tx"` + Proof types.TxProof `json:"proof,omitempty"` } type ResultUnconfirmedTxs struct { From 585ce45a5e253743da7b72d9a2385c01059fc6d5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 12 Apr 2017 19:12:22 -0400 Subject: [PATCH 40/84] rpc: dial_seeds msg. addresses #403 --- rpc/core/net.go | 11 ++++++++++- rpc/core/types/responses.go | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/rpc/core/net.go b/rpc/core/net.go index df491ba80..31d9c34ea 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -1,6 +1,8 @@ package core import ( + "fmt" + ctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -31,10 +33,17 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { // Dial given list of seeds func UnsafeDialSeeds(seeds []string) (*ctypes.ResultDialSeeds, error) { + + if len(seeds) == 0 { + return &ctypes.ResultDialSeeds{}, fmt.Errorf("No seeds provided") + } // starts go routines to dial each seed after random delays log.Info("DialSeeds", "addrBook", addrBook, "seeds", seeds) err := p2pSwitch.DialSeeds(addrBook, seeds) - return &ctypes.ResultDialSeeds{}, err + if err != nil { + return &ctypes.ResultDialSeeds{}, err + } + return &ctypes.ResultDialSeeds{"Dialing seeds in progress. See /net_info for details"}, nil } //----------------------------------------------------------------------------- diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index bcab4f59c..82087e252 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -45,6 +45,7 @@ type ResultNetInfo struct { } type ResultDialSeeds struct { + Log string `json:"log"` } type Peer struct { From 257d81ddd16574aaeb225b070d86c255769569e1 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 13:35:16 -0400 Subject: [PATCH 41/84] rpc/core/types: uintX -> int --- rpc/core/types/responses.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 9ed2383b2..50b36099d 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -79,8 +79,8 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { - Height uint64 `json:"height"` - Index uint32 `json:"index"` + Height int `json:"height"` + Index int `json:"index"` DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` Tx types.Tx `json:"tx"` Proof types.TxProof `json:"proof,omitempty"` From 23dbd0c8c3dedcaadeb218184e6f57dad2f5cb11 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 13:36:12 -0400 Subject: [PATCH 42/84] enable tx_indexer by default --- config/tendermint/config.go | 2 +- config/tendermint_test/config.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 828e67ef6..19f7386b7 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -101,7 +101,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal") - mapConfig.SetDefault("tx_indexer", "none") + mapConfig.SetDefault("tx_indexer", "kv") return mapConfig } diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index b6462cab1..9cb8ee327 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -107,7 +107,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", "") - mapConfig.SetDefault("tx_indexer", "none") + mapConfig.SetDefault("tx_indexer", "kv") logger.SetLogLevel(mapConfig.GetString("log_level")) From df35989742225af5977d4bbd35d2aadfdce35bb4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 13:47:48 -0400 Subject: [PATCH 43/84] /tx can take height+index or hash --- rpc/core/routes.go | 6 +++--- rpc/core/tx.go | 42 +++++++++++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 38e609601..4dc5aef8a 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,7 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "genesis": rpc.NewRPCFunc(GenesisResult, ""), "block": rpc.NewRPCFunc(BlockResult, "height"), "commit": rpc.NewRPCFunc(CommitResult, "height"), - "tx": rpc.NewRPCFunc(TxResult, "hash,prove"), + "tx": rpc.NewRPCFunc(TxResult, "hash,height,index,prove"), "validators": rpc.NewRPCFunc(ValidatorsResult, ""), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), @@ -100,8 +100,8 @@ func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { // Tx allow user to query the transaction results. `nil` could mean the // transaction is in the mempool, invalidated, or was not send in the first // place. -func TxResult(hash []byte, prove bool) (ctypes.TMResult, error) { - return Tx(hash, prove) +func TxResult(hash []byte, height, index int, prove bool) (ctypes.TMResult, error) { + return Tx(hash, height, index, prove) } func BroadcastTxCommitResult(tx []byte) (ctypes.TMResult, error) { diff --git a/rpc/core/tx.go b/rpc/core/tx.go index 14f87ece2..e6e4e3dfe 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -3,32 +3,48 @@ package core import ( "fmt" + abci "github.com/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" ) -func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { - r, err := txIndexer.Tx(hash) - if err != nil { - return nil, err - } +func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { + var deliverTx abci.ResponseDeliverTx + if len(hash) > 0 { + if height != 0 || index != 0 { + return nil, fmt.Errorf("Invalid args. If hash is provided, height and index should not be") + } + + r, err := txIndexer.Tx(hash) + if err != nil { + return nil, err + } + + if r == nil { + return &ctypes.ResultTx{}, fmt.Errorf("Tx (%X) not found", hash) + } - if r == nil { - return &ctypes.ResultTx{}, fmt.Errorf("Tx (%X) not found", hash) + height = int(r.Height) // XXX + index = int(r.Index) + deliverTx = r.DeliverTx } - block := blockStore.LoadBlock(int(r.Height)) - tx := block.Data.Txs[int(r.Index)] + block := blockStore.LoadBlock(height) + + if index >= len(block.Data.Txs) { + return nil, fmt.Errorf("Index (%d) is out of range for block (%d) with %d txs", index, height, len(block.Data.Txs)) + } + tx := block.Data.Txs[index] var proof types.TxProof if prove { - proof = block.Data.Txs.Proof(int(r.Index)) + proof = block.Data.Txs.Proof(index) } return &ctypes.ResultTx{ - Height: r.Height, - Index: r.Index, - DeliverTx: r.DeliverTx, + Height: height, + Index: index, + DeliverTx: deliverTx, Tx: tx, Proof: proof, }, nil From 90d1ed87fd2a752c9d88632b51b919255bbc3c91 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 14:18:20 -0400 Subject: [PATCH 44/84] add tx_indexer to NodeInfo --- node/node.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/node.go b/node/node.go index 0eea6278f..7fe874c01 100644 --- a/node/node.go +++ b/node/node.go @@ -376,6 +376,7 @@ func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd255 cmn.Fmt("p2p_version=%v", p2p.Version), cmn.Fmt("consensus_version=%v", consensus.Version), cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version), + cmn.Fmt("tx_indexer=%v", config.GetString("tx_indexer")), }, } From 58c860ba11c8e14f99a2bf8059fecf579e499d23 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 14:18:35 -0400 Subject: [PATCH 45/84] rpc/test: /tx --- rpc/test/client_test.go | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 43409c723..63a0dbfbf 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -149,6 +149,51 @@ func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { // TODO: find tx in block } +//-------------------------------------------------------------------------------- +// query tx + +func TestURITx(t *testing.T) { + testTx(t, GetURIClient()) +} + +func TestJSONTx(t *testing.T) { + testTx(t, GetJSONClient()) +} + +func testTx(t *testing.T, client rpc.HTTPClient) { + require := require.New(t) + + // first we broadcast a tx + tmResult := new(ctypes.TMResult) + tx := randBytes(t) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + require.Nil(err) + + res := (*tmResult).(*ctypes.ResultBroadcastTxCommit) + checkTx := res.CheckTx + require.Equal(abci.CodeType_OK, checkTx.Code) + deliverTx := res.DeliverTx + require.Equal(abci.CodeType_OK, deliverTx.Code) + mem := node.MempoolReactor().Mempool + require.Equal(0, mem.Size()) + + // now we query for the tx. + // since there's only one tx, we know index=0. + tmResult = new(ctypes.TMResult) + _, err = client.Call("tx", map[string]interface{}{"height": res.Height}, tmResult) + require.Nil(err) + + res2 := (*tmResult).(*ctypes.ResultTx) + require.Equal(res2.Tx, types.Tx(tx), "tx is not correct") + + // TODO: a query with height and hash should fail + + // TODO: a query with just hash should work same way + + // TODO: verify proof + +} + //-------------------------------------------------------------------------------- // Test the websocket service From a4ee7d25d10e4cf876cbd367aaaaa10c98545a15 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 13 Apr 2017 20:20:41 +0200 Subject: [PATCH 46/84] Add TxIndexEnabled method to ResultStatus --- rpc/core/types/responses.go | 15 +++++++++++++ rpc/core/types/responses_test.go | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 rpc/core/types/responses_test.go diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 50b36099d..1449164fd 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -1,6 +1,8 @@ package core_types import ( + "strings" + abci "github.com/tendermint/abci/types" "github.com/tendermint/go-crypto" "github.com/tendermint/go-p2p" @@ -38,6 +40,19 @@ type ResultStatus struct { LatestBlockTime int64 `json:"latest_block_time"` // nano } +func (s *ResultStatus) TxIndexEnabled() bool { + if s == nil || s.NodeInfo == nil { + return false + } + for _, s := range s.NodeInfo.Other { + info := strings.Split(s, "=") + if len(info) == 2 && info[0] == "tx_indexer" { + return info[1] == "kv" + } + } + return false +} + type ResultNetInfo struct { Listening bool `json:"listening"` Listeners []string `json:"listeners"` diff --git a/rpc/core/types/responses_test.go b/rpc/core/types/responses_test.go new file mode 100644 index 000000000..13728e42b --- /dev/null +++ b/rpc/core/types/responses_test.go @@ -0,0 +1,38 @@ +package core_types + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/go-p2p" +) + +func TestStatusIndexer(t *testing.T) { + assert := assert.New(t) + + var status *ResultStatus + assert.False(status.TxIndexEnabled()) + + status = &ResultStatus{} + assert.False(status.TxIndexEnabled()) + + status.NodeInfo = &p2p.NodeInfo{} + assert.False(status.TxIndexEnabled()) + + cases := []struct { + expected bool + other []string + }{ + {false, nil}, + {false, []string{}}, + {false, []string{"a=b"}}, + {false, []string{"tx_indexeriskv", "some=dood"}}, + {true, []string{"tx_indexer=kv", "tx_indexer=other"}}, + {true, []string{"^(*^(", "tx_indexer=kv", "a=n=b=d="}}, + } + + for _, tc := range cases { + status.NodeInfo.Other = tc.other + assert.Equal(tc.expected, status.TxIndexEnabled()) + } +} From b33a7c46ceaa3920701ed8832721c9838a04c2fb Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 13 Apr 2017 20:43:16 +0200 Subject: [PATCH 47/84] Break new Tx rpc endpoint --- rpc/test/client_test.go | 69 ++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 63a0dbfbf..034e2e58a 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -161,11 +161,11 @@ func TestJSONTx(t *testing.T) { } func testTx(t *testing.T, client rpc.HTTPClient) { - require := require.New(t) + assert, require := assert.New(t), require.New(t) // first we broadcast a tx tmResult := new(ctypes.TMResult) - tx := randBytes(t) + tx := types.Tx(randBytes(t)) _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) require.Nil(err) @@ -177,20 +177,59 @@ func testTx(t *testing.T, client rpc.HTTPClient) { mem := node.MempoolReactor().Mempool require.Equal(0, mem.Size()) - // now we query for the tx. - // since there's only one tx, we know index=0. - tmResult = new(ctypes.TMResult) - _, err = client.Call("tx", map[string]interface{}{"height": res.Height}, tmResult) - require.Nil(err) - - res2 := (*tmResult).(*ctypes.ResultTx) - require.Equal(res2.Tx, types.Tx(tx), "tx is not correct") - - // TODO: a query with height and hash should fail - - // TODO: a query with just hash should work same way + cases := []struct { + valid bool + height int + index int + hash []byte + }{ + // only on proper height, index match + {true, res.Height, 0, nil}, + {false, res.Height, 1, nil}, + {false, res.Height, -7, nil}, + {false, -10, -100, nil}, + {false, res.Height + 1, 0, nil}, + + // on proper hash match + {true, 0, 0, tx.Hash()}, + {false, res.Height, 0, tx.Hash()}, // TODO: or shall we allow this???? + // with extra data is an error + {false, 10, 0, tx.Hash()}, + {false, 0, 2, tx.Hash()}, + {false, 0, 0, []byte("jkh8y0fw")}, + {false, 0, 0, nil}, + + // missing height and hash fails + {false, 0, 0, nil}, + {false, 0, 1, nil}, + } - // TODO: verify proof + for _, tc := range cases { + // now we query for the tx. + // since there's only one tx, we know index=0. + tmResult = new(ctypes.TMResult) + query := map[string]interface{}{ + "height": tc.height, + "index": tc.index, + "hash": tc.hash, + } + _, err = client.Call("tx", query, tmResult) + if !tc.valid { + require.NotNil(err) + } else { + require.Nil(err) + res2 := (*tmResult).(*ctypes.ResultTx) + assert.Equal(tx, res2.Tx, "tx is not correct") + assert.Equal(res.Height, res2.Height) + assert.Equal(0, res2.Index) + assert.Equal(abci.CodeType_OK, res2.DeliverTx.Code) + // time to verify the proof + proof := res2.Proof + if assert.Equal(tx, proof.Data) { + assert.True(proof.Proof.Verify(proof.Index, proof.Total, tx.Hash(), proof.RootHash)) + } + } + } } From c848056438869ef7f52d70cb811ca9a3aa91f4a8 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 15:18:58 -0400 Subject: [PATCH 48/84] rpc: better arg validation for /tx --- rpc/core/tx.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rpc/core/tx.go b/rpc/core/tx.go index e6e4e3dfe..e578a25bf 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -29,9 +29,13 @@ func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { deliverTx = r.DeliverTx } + if height <= 0 || height > blockStore.Height() { + return nil, fmt.Errorf("Invalid height (%d) for blockStore at height %d", height, blockStore.Height()) + } + block := blockStore.LoadBlock(height) - if index >= len(block.Data.Txs) { + if index < 0 || index >= len(block.Data.Txs) { return nil, fmt.Errorf("Index (%d) is out of range for block (%d) with %d txs", index, height, len(block.Data.Txs)) } tx := block.Data.Txs[index] From 20458564b2d2b29af6d351146f71a0382ac8d0e4 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 13 Apr 2017 21:49:21 +0200 Subject: [PATCH 49/84] Expose Tx method in the clients --- rpc/client/httpclient.go | 15 +++++++++++++++ rpc/client/interface.go | 1 + rpc/client/localclient.go | 4 ++++ rpc/client/rpc_test.go | 35 ++++++++++++++++++++++------------- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 07059bca0..cc46cf69c 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -160,6 +160,21 @@ func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { return (*tmResult).(*ctypes.ResultCommit), nil } +func (c *HTTP) Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { + tmResult := new(ctypes.TMResult) + query := map[string]interface{}{ + "height": height, + "index": index, + "hash": hash, + "prove": prove, + } + _, err := c.rpc.Call("tx", query, tmResult) + if err != nil { + return nil, errors.Wrap(err, "Tx") + } + return (*tmResult).(*ctypes.ResultTx), nil +} + func (c *HTTP) Validators() (*ctypes.ResultValidators, error) { tmResult := new(ctypes.TMResult) _, err := c.rpc.Call("validators", map[string]interface{}{}, tmResult) diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 9a5ba668b..26ce5242d 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -44,6 +44,7 @@ type SignClient interface { Block(height int) (*ctypes.ResultBlock, error) Commit(height int) (*ctypes.ResultCommit, error) Validators() (*ctypes.ResultValidators, error) + Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) } // HistoryClient shows us data from genesis to now in large chunks. diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 0a83db05d..41bb0c508 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -103,3 +103,7 @@ func (c Local) Commit(height int) (*ctypes.ResultCommit, error) { func (c Local) Validators() (*ctypes.ResultValidators, error) { return core.Validators() } + +func (c Local) Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { + return core.Tx(hash, height, index, prove) +} diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index c5f32d97f..9e2543764 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -3,7 +3,6 @@ package client_test import ( "strings" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -11,6 +10,7 @@ import ( merktest "github.com/tendermint/merkleeyes/testutil" "github.com/tendermint/tendermint/rpc/client" rpctest "github.com/tendermint/tendermint/rpc/test" + "github.com/tendermint/tendermint/types" ) func getHTTPClient() *client.HTTP { @@ -117,49 +117,58 @@ func TestAppCalls(t *testing.T) { // write something k, v, tx := merktest.MakeTxKV() - _, err = c.BroadcastTxCommit(tx) + bres, err := c.BroadcastTxCommit(tx) require.Nil(err, "%d: %+v", i, err) + require.True(bres.DeliverTx.GetCode().IsOK()) + txh := bres.Height + apph := txh + 1 // this is where the tx will be applied to the state + // wait before querying - time.Sleep(time.Second * 1) + client.WaitForHeight(c, apph, nil) qres, err := c.ABCIQuery("/key", k, false) if assert.Nil(err) && assert.True(qres.Response.Code.IsOK()) { data := qres.Response // assert.Equal(k, data.GetKey()) // only returned for proofs assert.Equal(v, data.GetValue()) } - // +/- 1 making my head hurt - h := int(qres.Response.Height) - 1 + + // make sure we can lookup the tx with proof + // ptx, err := c.Tx(bres.TxID, txh, 0, true) + ptx, err := c.Tx(bres.TxID, 0, 0, true) + require.Nil(err, "%d: %+v", i, err) + assert.Equal(txh, ptx.Height) + assert.Equal(types.Tx(tx), ptx.Tx) // and we can even check the block is added - block, err := c.Block(h) + block, err := c.Block(apph) require.Nil(err, "%d: %+v", i, err) appHash := block.BlockMeta.Header.AppHash assert.True(len(appHash) > 0) - assert.EqualValues(h, block.BlockMeta.Header.Height) + assert.EqualValues(apph, block.BlockMeta.Header.Height) // check blockchain info, now that we know there is info // TODO: is this commented somewhere that they are returned // in order of descending height??? - info, err := c.BlockchainInfo(h-2, h) + info, err := c.BlockchainInfo(apph, apph) require.Nil(err, "%d: %+v", i, err) - assert.True(info.LastHeight > 2) - if assert.Equal(3, len(info.BlockMetas)) { + assert.True(info.LastHeight >= apph) + if assert.Equal(1, len(info.BlockMetas)) { lastMeta := info.BlockMetas[0] - assert.EqualValues(h, lastMeta.Header.Height) + assert.EqualValues(apph, lastMeta.Header.Height) bMeta := block.BlockMeta assert.Equal(bMeta.Header.AppHash, lastMeta.Header.AppHash) assert.Equal(bMeta.BlockID, lastMeta.BlockID) } // and get the corresponding commit with the same apphash - commit, err := c.Commit(h) + commit, err := c.Commit(apph) require.Nil(err, "%d: %+v", i, err) cappHash := commit.Header.AppHash assert.Equal(appHash, cappHash) assert.NotNil(commit.Commit) // compare the commits (note Commit(2) has commit from Block(3)) - commit2, err := c.Commit(h - 1) + commit2, err := c.Commit(apph - 1) require.Nil(err, "%d: %+v", i, err) assert.Equal(block.Block.LastCommit, commit2.Commit) From 4ee9acb8a776a1b004f173f3cb89656fc2423b50 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 13 Apr 2017 21:20:21 +0200 Subject: [PATCH 50/84] Improve tx tests for both prove true/false --- rpc/test/client_test.go | 51 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 034e2e58a..3a7ea796b 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -182,29 +182,35 @@ func testTx(t *testing.T, client rpc.HTTPClient) { height int index int hash []byte + prove bool }{ // only on proper height, index match - {true, res.Height, 0, nil}, - {false, res.Height, 1, nil}, - {false, res.Height, -7, nil}, - {false, -10, -100, nil}, - {false, res.Height + 1, 0, nil}, + {true, res.Height, 0, nil, false}, + {true, res.Height, 0, nil, true}, + {false, res.Height, 1, nil, false}, + {false, res.Height, -7, nil, true}, + {false, -10, -100, nil, false}, + {false, res.Height + 1, 0, nil, true}, // on proper hash match - {true, 0, 0, tx.Hash()}, - {false, res.Height, 0, tx.Hash()}, // TODO: or shall we allow this???? + {true, 0, 0, tx.Hash(), false}, + {true, 0, 0, tx.Hash(), true}, + {false, res.Height, 0, tx.Hash(), false}, // TODO: or shall we allow this???? + {false, res.Height, 0, tx.Hash(), true}, // TODO: or shall we allow this???? // with extra data is an error - {false, 10, 0, tx.Hash()}, - {false, 0, 2, tx.Hash()}, - {false, 0, 0, []byte("jkh8y0fw")}, - {false, 0, 0, nil}, + {false, 10, 0, tx.Hash(), false}, + {false, 0, 2, tx.Hash(), true}, + {false, 0, 0, []byte("jkh8y0fw"), false}, + {false, 0, 0, nil, true}, // missing height and hash fails - {false, 0, 0, nil}, - {false, 0, 1, nil}, + {false, 0, 0, nil, false}, + {false, 0, 1, nil, true}, } - for _, tc := range cases { + for i, tc := range cases { + idx := fmt.Sprintf("%d", i) + // now we query for the tx. // since there's only one tx, we know index=0. tmResult = new(ctypes.TMResult) @@ -212,21 +218,22 @@ func testTx(t *testing.T, client rpc.HTTPClient) { "height": tc.height, "index": tc.index, "hash": tc.hash, + "prove": tc.prove, } _, err = client.Call("tx", query, tmResult) if !tc.valid { - require.NotNil(err) + require.NotNil(err, idx) } else { - require.Nil(err) + require.Nil(err, idx) res2 := (*tmResult).(*ctypes.ResultTx) - assert.Equal(tx, res2.Tx, "tx is not correct") - assert.Equal(res.Height, res2.Height) - assert.Equal(0, res2.Index) - assert.Equal(abci.CodeType_OK, res2.DeliverTx.Code) + assert.Equal(tx, res2.Tx, idx) + assert.Equal(res.Height, res2.Height, idx) + assert.Equal(0, res2.Index, idx) + assert.Equal(abci.CodeType_OK, res2.DeliverTx.Code, idx) // time to verify the proof proof := res2.Proof - if assert.Equal(tx, proof.Data) { - assert.True(proof.Proof.Verify(proof.Index, proof.Total, tx.Hash(), proof.RootHash)) + if tc.prove && assert.Equal(tx, proof.Data, idx) { + assert.True(proof.Proof.Verify(proof.Index, proof.Total, tx.Hash(), proof.RootHash), idx) } } } From 6e065affe582998d826e86b33549f40a0c2dd8ec Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 16:04:20 -0400 Subject: [PATCH 51/84] rpc: /tx allows height+hash --- rpc/core/tx.go | 50 ++++++++++++++++++++++++++++--------- rpc/core/types/responses.go | 10 ++++---- rpc/test/client_test.go | 13 +++++----- types/tx.go | 10 ++++++++ 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/rpc/core/tx.go b/rpc/core/tx.go index e578a25bf..e7c416da5 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -5,16 +5,30 @@ import ( abci "github.com/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/state/tx/indexer" "github.com/tendermint/tendermint/types" ) func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { - var deliverTx abci.ResponseDeliverTx - if len(hash) > 0 { - if height != 0 || index != 0 { - return nil, fmt.Errorf("Invalid args. If hash is provided, height and index should not be") - } + // if index is disabled, we need a height + _, indexerDisabled := txIndexer.(*indexer.Null) + if indexerDisabled && height == 0 { + return nil, fmt.Errorf("TxIndexer is disabled. Please specify a height to search for the tx by hash or index") + } + + // hash and index must not be passed together + if len(hash) > 0 && index != 0 { + return nil, fmt.Errorf("Invalid args. Only one of hash and index may be provided") + } + + // results + var txResult abci.ResponseDeliverTx + var tx types.Tx + + // if indexer is enabled and we have a hash, + // fetch the tx result and set the height and index + if !indexerDisabled && len(hash) > 0 { r, err := txIndexer.Tx(hash) if err != nil { return nil, err @@ -26,19 +40,31 @@ func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { height = int(r.Height) // XXX index = int(r.Index) - deliverTx = r.DeliverTx + txResult = r.DeliverTx } + // height must be valid if height <= 0 || height > blockStore.Height() { return nil, fmt.Errorf("Invalid height (%d) for blockStore at height %d", height, blockStore.Height()) } block := blockStore.LoadBlock(height) + // index must be valid if index < 0 || index >= len(block.Data.Txs) { return nil, fmt.Errorf("Index (%d) is out of range for block (%d) with %d txs", index, height, len(block.Data.Txs)) } - tx := block.Data.Txs[index] + + // if indexer is disabled and we have a hash, + // search for it in the list of txs + if indexerDisabled && len(hash) > 0 { + index = block.Data.Txs.IndexByHash(hash) + if index < 0 { + return nil, fmt.Errorf("Tx hash %X not found in block %d", hash, height) + } + + } + tx = block.Data.Txs[index] var proof types.TxProof if prove { @@ -46,10 +72,10 @@ func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { } return &ctypes.ResultTx{ - Height: height, - Index: index, - DeliverTx: deliverTx, - Tx: tx, - Proof: proof, + Height: height, + Index: index, + TxResult: txResult, + Tx: tx, + Proof: proof, }, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 1449164fd..6979e0e6c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -94,11 +94,11 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { - Height int `json:"height"` - Index int `json:"index"` - DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` - Tx types.Tx `json:"tx"` - Proof types.TxProof `json:"proof,omitempty"` + Height int `json:"height"` + Index int `json:"index"` + TxResult abci.ResponseDeliverTx `json:"tx_result"` + Tx types.Tx `json:"tx"` + Proof types.TxProof `json:"proof,omitempty"` } type ResultUnconfirmedTxs struct { diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 3a7ea796b..2b06f77fe 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -165,8 +165,9 @@ func testTx(t *testing.T, client rpc.HTTPClient) { // first we broadcast a tx tmResult := new(ctypes.TMResult) - tx := types.Tx(randBytes(t)) - _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": tx}, tmResult) + txBytes := randBytes(t) + tx := types.Tx(txBytes) + _, err := client.Call("broadcast_tx_commit", map[string]interface{}{"tx": txBytes}, tmResult) require.Nil(err) res := (*tmResult).(*ctypes.ResultBroadcastTxCommit) @@ -195,10 +196,10 @@ func testTx(t *testing.T, client rpc.HTTPClient) { // on proper hash match {true, 0, 0, tx.Hash(), false}, {true, 0, 0, tx.Hash(), true}, - {false, res.Height, 0, tx.Hash(), false}, // TODO: or shall we allow this???? - {false, res.Height, 0, tx.Hash(), true}, // TODO: or shall we allow this???? + {true, res.Height, 0, tx.Hash(), false}, + {true, res.Height, 0, tx.Hash(), true}, + {true, 100, 0, tx.Hash(), false}, // with indexer disabled, height is overwritten // with extra data is an error - {false, 10, 0, tx.Hash(), false}, {false, 0, 2, tx.Hash(), true}, {false, 0, 0, []byte("jkh8y0fw"), false}, {false, 0, 0, nil, true}, @@ -229,7 +230,7 @@ func testTx(t *testing.T, client rpc.HTTPClient) { assert.Equal(tx, res2.Tx, idx) assert.Equal(res.Height, res2.Height, idx) assert.Equal(0, res2.Index, idx) - assert.Equal(abci.CodeType_OK, res2.DeliverTx.Code, idx) + assert.Equal(abci.CodeType_OK, res2.TxResult.Code, idx) // time to verify the proof proof := res2.Proof if tc.prove && assert.Equal(tx, proof.Data, idx) { diff --git a/types/tx.go b/types/tx.go index c47774b8f..91f6fbc01 100644 --- a/types/tx.go +++ b/types/tx.go @@ -45,6 +45,16 @@ func (txs Txs) Index(tx Tx) int { return -1 } +// Index returns the index of this transaction hash in the list, or -1 if not found +func (txs Txs) IndexByHash(hash []byte) int { + for i := range txs { + if bytes.Equal(txs[i].Hash(), hash) { + return i + } + } + return -1 +} + // Proof returns a simple merkle proof for this node. // // Panics if i < 0 or i >= len(txs) From c648bd5b3119dc0ca24ef975493678e9df6774d9 Mon Sep 17 00:00:00 2001 From: Ethan Frey Date: Thu, 13 Apr 2017 22:26:07 +0200 Subject: [PATCH 52/84] Test /tx with indexer disabled --- rpc/test/client_test.go | 59 ++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 2b06f77fe..46fbf6993 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -13,7 +13,9 @@ import ( abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" rpc "github.com/tendermint/go-rpc/client" + "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" + "github.com/tendermint/tendermint/state/tx/indexer" "github.com/tendermint/tendermint/types" ) @@ -153,14 +155,19 @@ func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { // query tx func TestURITx(t *testing.T) { - testTx(t, GetURIClient()) + testTx(t, GetURIClient(), true) } func TestJSONTx(t *testing.T) { - testTx(t, GetJSONClient()) + testTx(t, GetJSONClient(), true) } -func testTx(t *testing.T, client rpc.HTTPClient) { +func TestZZZZTxNoIndexer(t *testing.T) { + core.SetTxIndexer(&indexer.Null{}) + testTx(t, GetJSONClient(), false) +} + +func testTx(t *testing.T, client rpc.HTTPClient, withIndexer bool) { assert, require := assert.New(t), require.New(t) // first we broadcast a tx @@ -179,34 +186,35 @@ func testTx(t *testing.T, client rpc.HTTPClient) { require.Equal(0, mem.Size()) cases := []struct { - valid bool - height int - index int - hash []byte - prove bool + validWithIndexer bool + validNoIndexer bool + height int + index int + hash []byte + prove bool }{ // only on proper height, index match - {true, res.Height, 0, nil, false}, - {true, res.Height, 0, nil, true}, - {false, res.Height, 1, nil, false}, - {false, res.Height, -7, nil, true}, - {false, -10, -100, nil, false}, - {false, res.Height + 1, 0, nil, true}, + {true, true, res.Height, 0, nil, false}, + {true, true, res.Height, 0, nil, true}, + {false, false, res.Height, 1, nil, false}, + {false, false, res.Height, -7, nil, true}, + {false, false, -10, -100, nil, false}, + {false, false, res.Height + 1, 0, nil, true}, // on proper hash match - {true, 0, 0, tx.Hash(), false}, - {true, 0, 0, tx.Hash(), true}, - {true, res.Height, 0, tx.Hash(), false}, - {true, res.Height, 0, tx.Hash(), true}, - {true, 100, 0, tx.Hash(), false}, // with indexer disabled, height is overwritten + {true, false, 0, 0, tx.Hash(), false}, + {true, false, 0, 0, tx.Hash(), true}, + {true, true, res.Height, 0, tx.Hash(), false}, + {true, true, res.Height, 0, tx.Hash(), true}, + {true, false, 100, 0, tx.Hash(), false}, // with indexer enabled, height is overwritten // with extra data is an error - {false, 0, 2, tx.Hash(), true}, - {false, 0, 0, []byte("jkh8y0fw"), false}, - {false, 0, 0, nil, true}, + {false, false, 0, 2, tx.Hash(), true}, + {false, false, 0, 0, []byte("jkh8y0fw"), false}, + {false, false, 0, 0, nil, true}, // missing height and hash fails - {false, 0, 0, nil, false}, - {false, 0, 1, nil, true}, + {false, false, 0, 0, nil, false}, + {false, false, 0, 1, nil, true}, } for i, tc := range cases { @@ -222,7 +230,8 @@ func testTx(t *testing.T, client rpc.HTTPClient) { "prove": tc.prove, } _, err = client.Call("tx", query, tmResult) - if !tc.valid { + valid := (withIndexer && tc.validWithIndexer) || (!withIndexer && tc.validNoIndexer) + if !valid { require.NotNil(err, idx) } else { require.Nil(err, idx) From 16bffdf7ab4b6f0699966bf0f24bacb7a5d32d60 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 13 Apr 2017 16:38:44 -0400 Subject: [PATCH 53/84] rpc/test: restore txindexer after setting null --- rpc/test/client_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 46fbf6993..8a41d397b 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -156,15 +156,18 @@ func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { func TestURITx(t *testing.T) { testTx(t, GetURIClient(), true) + + core.SetTxIndexer(&indexer.Null{}) + testTx(t, GetJSONClient(), false) + core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) } func TestJSONTx(t *testing.T) { testTx(t, GetJSONClient(), true) -} -func TestZZZZTxNoIndexer(t *testing.T) { core.SetTxIndexer(&indexer.Null{}) testTx(t, GetJSONClient(), false) + core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) } func testTx(t *testing.T, client rpc.HTTPClient, withIndexer bool) { From 00847cdc6bc0681281315c85b310507722048feb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 15 Apr 2017 02:22:03 -0400 Subject: [PATCH 54/84] blockpool: fix removePeer bug --- blockchain/pool.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/blockchain/pool.go b/blockchain/pool.go index 32db956c1..0deacd263 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -240,7 +240,9 @@ func (pool *BlockPool) RemovePeer(peerID string) { func (pool *BlockPool) removePeer(peerID string) { for _, requester := range pool.requesters { if requester.getPeerID() == peerID { - pool.numPending++ + if requester.getBlock() != nil { + pool.numPending++ + } go requester.redo() // pick another peer and ... } } From d572bb0c5db1b98e2443f78086120c797d999208 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 19:29:02 -0400 Subject: [PATCH 55/84] state/txindex and pkg per indexer impl --- state/execution.go | 6 +-- state/execution_test.go | 6 +-- state/state.go | 10 ++-- state/tx/indexer.go | 21 -------- state/tx/indexer/batch.go | 32 ----------- state/tx/indexer/error.go | 6 --- state/tx/indexer/null.go | 19 ------- state/txindex/indexer.go | 60 +++++++++++++++++++++ state/{tx/indexer => txindex/kv}/kv.go | 27 +++++----- state/{tx/indexer => txindex/kv}/kv_test.go | 31 +++++------ state/txindex/null/null.go | 21 ++++++++ 11 files changed, 122 insertions(+), 117 deletions(-) delete mode 100644 state/tx/indexer.go delete mode 100644 state/tx/indexer/batch.go delete mode 100644 state/tx/indexer/error.go delete mode 100644 state/tx/indexer/null.go create mode 100644 state/txindex/indexer.go rename state/{tx/indexer => txindex/kv}/kv.go (52%) rename state/{tx/indexer => txindex/kv}/kv_test.go (57%) create mode 100644 state/txindex/null/null.go diff --git a/state/execution.go b/state/execution.go index 802d12c49..f6898cb4d 100644 --- a/state/execution.go +++ b/state/execution.go @@ -9,7 +9,7 @@ import ( . "github.com/tendermint/go-common" crypto "github.com/tendermint/go-crypto" "github.com/tendermint/tendermint/proxy" - txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" ) @@ -241,12 +241,12 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn return fmt.Errorf("Commit failed for application: %v", err) } - batch := txindexer.NewBatch() + batch := txindex.NewBatch() for i, r := range txResults { tx := block.Txs[i] batch.Index(tx.Hash(), *r) } - s.TxIndexer.Batch(batch) + s.TxIndexer.AddBatch(batch) return nil } diff --git a/state/execution_test.go b/state/execution_test.go index 8b9cf4a73..2da801964 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -11,7 +11,7 @@ import ( cfg "github.com/tendermint/tendermint/config/tendermint_test" "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" - txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" ) @@ -81,10 +81,10 @@ type dummyIndexer struct { Indexed int } -func (indexer *dummyIndexer) Tx(hash []byte) (*types.TxResult, error) { +func (indexer *dummyIndexer) Get(hash []byte) (*types.TxResult, error) { return nil, nil } -func (indexer *dummyIndexer) Batch(batch *txindexer.Batch) error { +func (indexer *dummyIndexer) AddBatch(batch *txindex.Batch) error { indexer.Indexed += batch.Size() return nil } diff --git a/state/state.go b/state/state.go index cecb38418..14b324d5f 100644 --- a/state/state.go +++ b/state/state.go @@ -10,8 +10,8 @@ import ( cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" "github.com/tendermint/go-wire" - "github.com/tendermint/tendermint/state/tx" - txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex" + "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" ) @@ -41,7 +41,7 @@ type State struct { // AppHash is updated after Commit AppHash []byte - TxIndexer tx.Indexer `json:"-"` // Transaction indexer. + TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. } func LoadState(db dbm.DB) *State { @@ -49,7 +49,7 @@ func LoadState(db dbm.DB) *State { } func loadState(db dbm.DB, key []byte) *State { - s := &State{db: db, TxIndexer: &txindexer.Null{}} + s := &State{db: db, TxIndexer: &null.TxIndex{}} buf := db.Get(key) if len(buf) == 0 { return nil @@ -188,6 +188,6 @@ func MakeGenesisState(db dbm.DB, genDoc *types.GenesisDoc) *State { Validators: types.NewValidatorSet(validators), LastValidators: types.NewValidatorSet(nil), AppHash: genDoc.AppHash, - TxIndexer: &txindexer.Null{}, // we do not need indexer during replay and in tests + TxIndexer: &null.TxIndex{}, // we do not need indexer during replay and in tests } } diff --git a/state/tx/indexer.go b/state/tx/indexer.go deleted file mode 100644 index be7f304c4..000000000 --- a/state/tx/indexer.go +++ /dev/null @@ -1,21 +0,0 @@ -package tx - -import ( - txindexer "github.com/tendermint/tendermint/state/tx/indexer" - "github.com/tendermint/tendermint/types" -) - -// Indexer interface defines methods to index and search transactions. -type Indexer interface { - - // Batch analyzes, indexes or stores a batch of transactions. - // - // NOTE We do not specify Index method for analyzing a single transaction - // here because it bears heavy perfomance loses. Almost all advanced indexers - // support batching. - Batch(b *txindexer.Batch) error - - // Tx returns specified transaction or nil if the transaction is not indexed - // or stored. - Tx(hash []byte) (*types.TxResult, error) -} diff --git a/state/tx/indexer/batch.go b/state/tx/indexer/batch.go deleted file mode 100644 index 78137f598..000000000 --- a/state/tx/indexer/batch.go +++ /dev/null @@ -1,32 +0,0 @@ -package indexer - -import "github.com/tendermint/tendermint/types" - -// A Batch groups together multiple Index operations you would like performed -// at the same time. The Batch structure is NOT thread-safe. You should only -// perform operations on a batch from a single thread at a time. Once batch -// execution has started, you may not modify it. -type Batch struct { - Ops map[string]types.TxResult -} - -// NewBatch creates a new Batch. -func NewBatch() *Batch { - return &Batch{ - Ops: make(map[string]types.TxResult), - } -} - -// Index adds or updates entry for the given hash. -func (b *Batch) Index(hash []byte, result types.TxResult) error { - if len(hash) == 0 { - return ErrorEmptyHash - } - b.Ops[string(hash)] = result - return nil -} - -// Size returns the total number of operations inside the batch. -func (b *Batch) Size() int { - return len(b.Ops) -} diff --git a/state/tx/indexer/error.go b/state/tx/indexer/error.go deleted file mode 100644 index 9d20593d1..000000000 --- a/state/tx/indexer/error.go +++ /dev/null @@ -1,6 +0,0 @@ -package indexer - -import "errors" - -// ErrorEmptyHash indicates empty hash -var ErrorEmptyHash = errors.New("Transaction hash cannot be empty") diff --git a/state/tx/indexer/null.go b/state/tx/indexer/null.go deleted file mode 100644 index a322602d0..000000000 --- a/state/tx/indexer/null.go +++ /dev/null @@ -1,19 +0,0 @@ -package indexer - -import ( - "errors" - "github.com/tendermint/tendermint/types" -) - -// Null acts as a /dev/null. -type Null struct{} - -// Tx panics. -func (indexer *Null) Tx(hash []byte) (*types.TxResult, error) { - return nil, errors.New(`Indexing is disabled (set 'tx_indexer = "kv"' in config)`) -} - -// Batch returns nil. -func (indexer *Null) Batch(batch *Batch) error { - return nil -} diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go new file mode 100644 index 000000000..a962eba7d --- /dev/null +++ b/state/txindex/indexer.go @@ -0,0 +1,60 @@ +package txindex + +import ( + "errors" + + "github.com/tendermint/tendermint/types" +) + +// Indexer interface defines methods to index and search transactions. +type TxIndexer interface { + + // Batch analyzes, indexes or stores a batch of transactions. + // + // NOTE We do not specify Index method for analyzing a single transaction + // here because it bears heavy perfomance loses. Almost all advanced indexers + // support batching. + AddBatch(b *Batch) error + + // Tx returns specified transaction or nil if the transaction is not indexed + // or stored. + Get(hash []byte) (*types.TxResult, error) +} + +//---------------------------------------------------- +// Txs are written as a batch + +// A Batch groups together multiple Index operations you would like performed +// at the same time. The Batch structure is NOT thread-safe. You should only +// perform operations on a batch from a single thread at a time. Once batch +// execution has started, you may not modify it. +type Batch struct { + Ops map[string]types.TxResult +} + +// NewBatch creates a new Batch. +func NewBatch() *Batch { + return &Batch{ + Ops: make(map[string]types.TxResult), + } +} + +// Index adds or updates entry for the given hash. +func (b *Batch) Index(hash []byte, result types.TxResult) error { + if len(hash) == 0 { + return ErrorEmptyHash + } + b.Ops[string(hash)] = result + return nil +} + +// Size returns the total number of operations inside the batch. +func (b *Batch) Size() int { + return len(b.Ops) +} + +//---------------------------------------------------- +// Errors + +// ErrorEmptyHash indicates empty hash +var ErrorEmptyHash = errors.New("Transaction hash cannot be empty") diff --git a/state/tx/indexer/kv.go b/state/txindex/kv/kv.go similarity index 52% rename from state/tx/indexer/kv.go rename to state/txindex/kv/kv.go index 4f6fb3581..b76e6a3b0 100644 --- a/state/tx/indexer/kv.go +++ b/state/txindex/kv/kv.go @@ -1,4 +1,4 @@ -package indexer +package kv import ( "bytes" @@ -6,28 +6,29 @@ import ( db "github.com/tendermint/go-db" "github.com/tendermint/go-wire" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" ) -// KV is a simplest possible indexer, backed by Key-Value storage (levelDB). +// TxIndex is the simplest possible indexer, backed by Key-Value storage (levelDB). // It could only index transaction by its identifier. -type KV struct { +type TxIndex struct { store db.DB } -// NewKV returns new instance of KV indexer. -func NewKV(store db.DB) *KV { - return &KV{store: store} +// NewTxIndex returns new instance of TxIndex. +func NewTxIndex(store db.DB) *TxIndex { + return &TxIndex{store: store} } -// Tx gets transaction from the KV storage and returns it or nil if the +// Get gets transaction from the TxIndex storage and returns it or nil if the // transaction is not found. -func (indexer *KV) Tx(hash []byte) (*types.TxResult, error) { +func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { if len(hash) == 0 { - return nil, ErrorEmptyHash + return nil, txindex.ErrorEmptyHash } - rawBytes := indexer.store.Get(hash) + rawBytes := txi.store.Get(hash) if rawBytes == nil { return nil, nil } @@ -43,9 +44,9 @@ func (indexer *KV) Tx(hash []byte) (*types.TxResult, error) { return txResult, nil } -// Batch writes a batch of transactions into the KV storage. -func (indexer *KV) Batch(b *Batch) error { - storeBatch := indexer.store.NewBatch() +// Batch writes a batch of transactions into the TxIndex storage. +func (txi *TxIndex) AddBatch(b *txindex.Batch) error { + storeBatch := txi.store.NewBatch() for hash, result := range b.Ops { rawBytes := wire.BinaryBytes(&result) storeBatch.Set([]byte(hash), rawBytes) diff --git a/state/tx/indexer/kv_test.go b/state/txindex/kv/kv_test.go similarity index 57% rename from state/tx/indexer/kv_test.go rename to state/txindex/kv/kv_test.go index c797151af..022e227e3 100644 --- a/state/tx/indexer/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -1,4 +1,4 @@ -package indexer +package kv import ( "fmt" @@ -10,27 +10,28 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/abci/types" db "github.com/tendermint/go-db" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" ) -func TestKVIndex(t *testing.T) { - indexer := &KV{store: db.NewMemDB()} +func TestTxIndex(t *testing.T) { + indexer := &TxIndex{store: db.NewMemDB()} tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} hash := tx.Hash() - batch := NewBatch() + batch := txindex.NewBatch() batch.Index(hash, *txResult) - err := indexer.Batch(batch) + err := indexer.AddBatch(batch) require.Nil(t, err) - loadedTxResult, err := indexer.Tx(hash) + loadedTxResult, err := indexer.Get(hash) require.Nil(t, err) assert.Equal(t, txResult, loadedTxResult) } -func benchmarkKVIndex(txsCount int, b *testing.B) { +func benchmarkTxIndex(txsCount int, b *testing.B) { txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} dir, err := ioutil.TempDir("", "tx_indexer_db") @@ -40,9 +41,9 @@ func benchmarkKVIndex(txsCount int, b *testing.B) { defer os.RemoveAll(dir) store := db.NewDB("tx_indexer", "leveldb", dir) - indexer := &KV{store: store} + indexer := &TxIndex{store: store} - batch := NewBatch() + batch := txindex.NewBatch() for i := 0; i < txsCount; i++ { batch.Index([]byte(fmt.Sprintf("hash%v", i)), *txResult) } @@ -50,12 +51,12 @@ func benchmarkKVIndex(txsCount int, b *testing.B) { b.ResetTimer() for n := 0; n < b.N; n++ { - err = indexer.Batch(batch) + err = indexer.AddBatch(batch) } } -func BenchmarkKVIndex1(b *testing.B) { benchmarkKVIndex(1, b) } -func BenchmarkKVIndex500(b *testing.B) { benchmarkKVIndex(500, b) } -func BenchmarkKVIndex1000(b *testing.B) { benchmarkKVIndex(1000, b) } -func BenchmarkKVIndex2000(b *testing.B) { benchmarkKVIndex(2000, b) } -func BenchmarkKVIndex10000(b *testing.B) { benchmarkKVIndex(10000, b) } +func BenchmarkTxIndex1(b *testing.B) { benchmarkTxIndex(1, b) } +func BenchmarkTxIndex500(b *testing.B) { benchmarkTxIndex(500, b) } +func BenchmarkTxIndex1000(b *testing.B) { benchmarkTxIndex(1000, b) } +func BenchmarkTxIndex2000(b *testing.B) { benchmarkTxIndex(2000, b) } +func BenchmarkTxIndex10000(b *testing.B) { benchmarkTxIndex(10000, b) } diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go new file mode 100644 index 000000000..e48e34cf5 --- /dev/null +++ b/state/txindex/null/null.go @@ -0,0 +1,21 @@ +package null + +import ( + "errors" + + "github.com/tendermint/tendermint/state/txindex" + "github.com/tendermint/tendermint/types" +) + +// TxIndex acts as a /dev/null. +type TxIndex struct{} + +// Tx panics. +func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { + return nil, errors.New(`Indexing is disabled (set 'tx_indexer = "kv"' in config)`) +} + +// Batch returns nil. +func (txi *TxIndex) AddBatch(batch *txindex.Batch) error { + return nil +} From f4d0076344e8cfdc7e405b2f2872d0e2ddbd3800 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 19:56:41 -0400 Subject: [PATCH 56/84] TxResult includes Tx. /tx only works if indexer active --- node/node.go | 13 +++---- rpc/client/httpclient.go | 8 ++--- rpc/client/interface.go | 2 +- rpc/client/localclient.go | 4 +-- rpc/client/rpc_test.go | 4 +-- rpc/core/pipe.go | 6 ++-- rpc/core/routes.go | 6 ++-- rpc/core/tx.go | 68 ++++++++----------------------------- rpc/test/client_test.go | 55 +++++++++++------------------- state/execution.go | 7 +++- state/txindex/kv/kv_test.go | 5 +-- types/tx.go | 7 ++-- 12 files changed, 68 insertions(+), 117 deletions(-) diff --git a/node/node.go b/node/node.go index 7fe874c01..7b39189fe 100644 --- a/node/node.go +++ b/node/node.go @@ -23,8 +23,9 @@ import ( rpccore "github.com/tendermint/tendermint/rpc/core" grpccore "github.com/tendermint/tendermint/rpc/grpc" sm "github.com/tendermint/tendermint/state" - "github.com/tendermint/tendermint/state/tx" - txindexer "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex" + "github.com/tendermint/tendermint/state/txindex/kv" + "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" @@ -53,7 +54,7 @@ type Node struct { consensusReactor *consensus.ConsensusReactor // for participating in the consensus proxyApp proxy.AppConns // connection to the application rpcListeners []net.Listener // rpc servers - txIndexer tx.Indexer + txIndexer txindex.TxIndexer } func NewNodeDefault(config cfg.Config) *Node { @@ -88,13 +89,13 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato state = sm.LoadState(stateDB) // Transaction indexing - var txIndexer tx.Indexer + var txIndexer txindex.TxIndexer switch config.GetString("tx_indexer") { case "kv": store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) - txIndexer = txindexer.NewKV(store) + txIndexer = kv.NewTxIndex(store) default: - txIndexer = &txindexer.Null{} + txIndexer = &null.TxIndex{} } state.TxIndexer = txIndexer diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index cc46cf69c..04595e766 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -160,13 +160,11 @@ func (c *HTTP) Commit(height int) (*ctypes.ResultCommit, error) { return (*tmResult).(*ctypes.ResultCommit), nil } -func (c *HTTP) Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { +func (c *HTTP) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { tmResult := new(ctypes.TMResult) query := map[string]interface{}{ - "height": height, - "index": index, - "hash": hash, - "prove": prove, + "hash": hash, + "prove": prove, } _, err := c.rpc.Call("tx", query, tmResult) if err != nil { diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 26ce5242d..2ba890798 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -44,7 +44,7 @@ type SignClient interface { Block(height int) (*ctypes.ResultBlock, error) Commit(height int) (*ctypes.ResultCommit, error) Validators() (*ctypes.ResultValidators, error) - Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) + Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) } // HistoryClient shows us data from genesis to now in large chunks. diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 41bb0c508..d0f0d11b1 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -104,6 +104,6 @@ func (c Local) Validators() (*ctypes.ResultValidators, error) { return core.Validators() } -func (c Local) Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { - return core.Tx(hash, height, index, prove) +func (c Local) Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { + return core.Tx(hash, prove) } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 9e2543764..18f0f1aab 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -133,8 +133,8 @@ func TestAppCalls(t *testing.T) { } // make sure we can lookup the tx with proof - // ptx, err := c.Tx(bres.TxID, txh, 0, true) - ptx, err := c.Tx(bres.TxID, 0, 0, true) + // ptx, err := c.Tx(bres.TxID, true) + ptx, err := c.Tx(bres.TxID, true) require.Nil(err, "%d: %+v", i, err) assert.Equal(txh, ptx.Height) assert.Equal(types.Tx(tx), ptx.Tx) diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index c92216428..4993ed992 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -7,7 +7,7 @@ import ( p2p "github.com/tendermint/go-p2p" "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/proxy" - "github.com/tendermint/tendermint/state/tx" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" ) @@ -46,7 +46,7 @@ var ( pubKey crypto.PubKey genDoc *types.GenesisDoc // cache the genesis structure addrBook *p2p.AddrBook - txIndexer tx.Indexer + txIndexer txindex.TxIndexer ) func SetConfig(c cfg.Config) { @@ -89,6 +89,6 @@ func SetProxyAppQuery(appConn proxy.AppConnQuery) { proxyAppQuery = appConn } -func SetTxIndexer(indexer tx.Indexer) { +func SetTxIndexer(indexer txindex.TxIndexer) { txIndexer = indexer } diff --git a/rpc/core/routes.go b/rpc/core/routes.go index 4dc5aef8a..38e609601 100644 --- a/rpc/core/routes.go +++ b/rpc/core/routes.go @@ -19,7 +19,7 @@ var Routes = map[string]*rpc.RPCFunc{ "genesis": rpc.NewRPCFunc(GenesisResult, ""), "block": rpc.NewRPCFunc(BlockResult, "height"), "commit": rpc.NewRPCFunc(CommitResult, "height"), - "tx": rpc.NewRPCFunc(TxResult, "hash,height,index,prove"), + "tx": rpc.NewRPCFunc(TxResult, "hash,prove"), "validators": rpc.NewRPCFunc(ValidatorsResult, ""), "dump_consensus_state": rpc.NewRPCFunc(DumpConsensusStateResult, ""), "unconfirmed_txs": rpc.NewRPCFunc(UnconfirmedTxsResult, ""), @@ -100,8 +100,8 @@ func NumUnconfirmedTxsResult() (ctypes.TMResult, error) { // Tx allow user to query the transaction results. `nil` could mean the // transaction is in the mempool, invalidated, or was not send in the first // place. -func TxResult(hash []byte, height, index int, prove bool) (ctypes.TMResult, error) { - return Tx(hash, height, index, prove) +func TxResult(hash []byte, prove bool) (ctypes.TMResult, error) { + return Tx(hash, prove) } func BroadcastTxCommitResult(tx []byte) (ctypes.TMResult, error) { diff --git a/rpc/core/tx.go b/rpc/core/tx.go index e7c416da5..7f3cdd037 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -3,79 +3,41 @@ package core import ( "fmt" - abci "github.com/tendermint/abci/types" ctypes "github.com/tendermint/tendermint/rpc/core/types" - "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" ) -func Tx(hash []byte, height, index int, prove bool) (*ctypes.ResultTx, error) { +func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { - // if index is disabled, we need a height - _, indexerDisabled := txIndexer.(*indexer.Null) - if indexerDisabled && height == 0 { - return nil, fmt.Errorf("TxIndexer is disabled. Please specify a height to search for the tx by hash or index") + // if index is disabled, return error + if _, ok := txIndexer.(*null.TxIndex); ok { + return nil, fmt.Errorf("Transaction indexing is disabled.") } - // hash and index must not be passed together - if len(hash) > 0 && index != 0 { - return nil, fmt.Errorf("Invalid args. Only one of hash and index may be provided") + r, err := txIndexer.Get(hash) + if err != nil { + return nil, err } - // results - var txResult abci.ResponseDeliverTx - var tx types.Tx - - // if indexer is enabled and we have a hash, - // fetch the tx result and set the height and index - if !indexerDisabled && len(hash) > 0 { - r, err := txIndexer.Tx(hash) - if err != nil { - return nil, err - } - - if r == nil { - return &ctypes.ResultTx{}, fmt.Errorf("Tx (%X) not found", hash) - } - - height = int(r.Height) // XXX - index = int(r.Index) - txResult = r.DeliverTx - } - - // height must be valid - if height <= 0 || height > blockStore.Height() { - return nil, fmt.Errorf("Invalid height (%d) for blockStore at height %d", height, blockStore.Height()) + if r == nil { + return nil, fmt.Errorf("Tx (%X) not found", hash) } - block := blockStore.LoadBlock(height) - - // index must be valid - if index < 0 || index >= len(block.Data.Txs) { - return nil, fmt.Errorf("Index (%d) is out of range for block (%d) with %d txs", index, height, len(block.Data.Txs)) - } - - // if indexer is disabled and we have a hash, - // search for it in the list of txs - if indexerDisabled && len(hash) > 0 { - index = block.Data.Txs.IndexByHash(hash) - if index < 0 { - return nil, fmt.Errorf("Tx hash %X not found in block %d", hash, height) - } - - } - tx = block.Data.Txs[index] + height := int(r.Height) // XXX + index := int(r.Index) var proof types.TxProof if prove { + block := blockStore.LoadBlock(height) proof = block.Data.Txs.Proof(index) } return &ctypes.ResultTx{ Height: height, Index: index, - TxResult: txResult, - Tx: tx, + TxResult: r.Result, + Tx: r.Tx, Proof: proof, }, nil } diff --git a/rpc/test/client_test.go b/rpc/test/client_test.go index 8a41d397b..50e326050 100644 --- a/rpc/test/client_test.go +++ b/rpc/test/client_test.go @@ -15,7 +15,7 @@ import ( rpc "github.com/tendermint/go-rpc/client" "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" - "github.com/tendermint/tendermint/state/tx/indexer" + "github.com/tendermint/tendermint/state/txindex/null" "github.com/tendermint/tendermint/types" ) @@ -157,7 +157,7 @@ func testBroadcastTxCommit(t *testing.T, client rpc.HTTPClient) { func TestURITx(t *testing.T) { testTx(t, GetURIClient(), true) - core.SetTxIndexer(&indexer.Null{}) + core.SetTxIndexer(&null.TxIndex{}) testTx(t, GetJSONClient(), false) core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) } @@ -165,7 +165,7 @@ func TestURITx(t *testing.T) { func TestJSONTx(t *testing.T) { testTx(t, GetJSONClient(), true) - core.SetTxIndexer(&indexer.Null{}) + core.SetTxIndexer(&null.TxIndex{}) testTx(t, GetJSONClient(), false) core.SetTxIndexer(node.ConsensusState().GetState().TxIndexer) } @@ -188,36 +188,21 @@ func testTx(t *testing.T, client rpc.HTTPClient, withIndexer bool) { mem := node.MempoolReactor().Mempool require.Equal(0, mem.Size()) + txHash := tx.Hash() + txHash2 := types.Tx("a different tx").Hash() + cases := []struct { - validWithIndexer bool - validNoIndexer bool - height int - index int - hash []byte - prove bool + valid bool + hash []byte + prove bool }{ - // only on proper height, index match - {true, true, res.Height, 0, nil, false}, - {true, true, res.Height, 0, nil, true}, - {false, false, res.Height, 1, nil, false}, - {false, false, res.Height, -7, nil, true}, - {false, false, -10, -100, nil, false}, - {false, false, res.Height + 1, 0, nil, true}, - - // on proper hash match - {true, false, 0, 0, tx.Hash(), false}, - {true, false, 0, 0, tx.Hash(), true}, - {true, true, res.Height, 0, tx.Hash(), false}, - {true, true, res.Height, 0, tx.Hash(), true}, - {true, false, 100, 0, tx.Hash(), false}, // with indexer enabled, height is overwritten - // with extra data is an error - {false, false, 0, 2, tx.Hash(), true}, - {false, false, 0, 0, []byte("jkh8y0fw"), false}, - {false, false, 0, 0, nil, true}, - - // missing height and hash fails - {false, false, 0, 0, nil, false}, - {false, false, 0, 1, nil, true}, + // only valid if correct hash provided + {true, txHash, false}, + {true, txHash, true}, + {false, txHash2, false}, + {false, txHash2, true}, + {false, nil, false}, + {false, nil, true}, } for i, tc := range cases { @@ -227,13 +212,11 @@ func testTx(t *testing.T, client rpc.HTTPClient, withIndexer bool) { // since there's only one tx, we know index=0. tmResult = new(ctypes.TMResult) query := map[string]interface{}{ - "height": tc.height, - "index": tc.index, - "hash": tc.hash, - "prove": tc.prove, + "hash": tc.hash, + "prove": tc.prove, } _, err = client.Call("tx", query, tmResult) - valid := (withIndexer && tc.validWithIndexer) || (!withIndexer && tc.validNoIndexer) + valid := (withIndexer && tc.valid) if !valid { require.NotNil(err, idx) } else { diff --git a/state/execution.go b/state/execution.go index f6898cb4d..05496aad5 100644 --- a/state/execution.go +++ b/state/execution.go @@ -84,7 +84,12 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo txError = txResult.Code.String() } - txResults[txIndex] = &types.TxResult{uint64(block.Height), uint32(txIndex), *txResult} + txResults[txIndex] = &types.TxResult{ + Height: uint64(block.Height), + Index: uint32(txIndex), + Tx: req.GetDeliverTx().Tx, + Result: *txResult, + } txIndex++ // NOTE: if we count we can access the tx from the block instead of diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 022e227e3..7d4a84dd1 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -18,7 +18,7 @@ func TestTxIndex(t *testing.T) { indexer := &TxIndex{store: db.NewMemDB()} tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + txResult := &types.TxResult{1, 1, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} hash := tx.Hash() batch := txindex.NewBatch() @@ -32,7 +32,8 @@ func TestTxIndex(t *testing.T) { } func benchmarkTxIndex(txsCount int, b *testing.B) { - txResult := &types.TxResult{1, 1, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + tx := types.Tx("HELLO WORLD") + txResult := &types.TxResult{1, 1, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} dir, err := ioutil.TempDir("", "tx_indexer_db") if err != nil { diff --git a/types/tx.go b/types/tx.go index 91f6fbc01..df7f0e71a 100644 --- a/types/tx.go +++ b/types/tx.go @@ -106,7 +106,8 @@ func (tp TxProof) Validate(dataHash []byte) error { // // One usage is indexing transaction results. type TxResult struct { - Height uint64 `json:"height"` - Index uint32 `json:"index"` - DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"` + Height uint64 `json:"height"` + Index uint32 `json:"index"` + Tx Tx `json:"tx"` + Result abci.ResponseDeliverTx `json:"result"` } From b6a04a3456792826fff39095414000b1a6b59cc5 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 20:10:02 -0400 Subject: [PATCH 57/84] more fixes from review --- node/node.go | 24 ++++++++++++++---------- rpc/core/types/responses.go | 2 +- rpc/core/types/responses_test.go | 4 ++-- state/execution.go | 5 ++--- state/execution_test.go | 4 ++-- state/txindex/indexer.go | 13 ++++--------- state/txindex/kv/kv.go | 4 ++-- state/txindex/kv/kv_test.go | 5 ++--- 8 files changed, 29 insertions(+), 32 deletions(-) diff --git a/node/node.go b/node/node.go index 7b39189fe..b75898eb3 100644 --- a/node/node.go +++ b/node/node.go @@ -217,7 +217,7 @@ func (n *Node) OnStart() error { n.sw.AddListener(l) // Start the switch - n.sw.SetNodeInfo(makeNodeInfo(n.config, n.sw, n.privKey)) + n.sw.SetNodeInfo(n.makeNodeInfo()) n.sw.SetNodePrivKey(n.privKey) _, err := n.sw.Start() if err != nil { @@ -365,35 +365,39 @@ func (n *Node) ProxyApp() proxy.AppConns { return n.proxyApp } -func makeNodeInfo(config cfg.Config, sw *p2p.Switch, privKey crypto.PrivKeyEd25519) *p2p.NodeInfo { +func (n *Node) makeNodeInfo() *p2p.NodeInfo { + txIndexerStatus := "on" + if _, ok := n.txIndexer.(*null.TxIndex); ok { + txIndexerStatus = "off" + } nodeInfo := &p2p.NodeInfo{ - PubKey: privKey.PubKey().(crypto.PubKeyEd25519), - Moniker: config.GetString("moniker"), - Network: config.GetString("chain_id"), + PubKey: n.privKey.PubKey().(crypto.PubKeyEd25519), + Moniker: n.config.GetString("moniker"), + Network: n.config.GetString("chain_id"), Version: version.Version, Other: []string{ cmn.Fmt("wire_version=%v", wire.Version), cmn.Fmt("p2p_version=%v", p2p.Version), cmn.Fmt("consensus_version=%v", consensus.Version), cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version), - cmn.Fmt("tx_indexer=%v", config.GetString("tx_indexer")), + cmn.Fmt("tx_indexer=%v", txIndexerStatus), }, } // include git hash in the nodeInfo if available - if rev, err := cmn.ReadFile(config.GetString("revision_file")); err == nil { + if rev, err := cmn.ReadFile(n.config.GetString("revision_file")); err == nil { nodeInfo.Other = append(nodeInfo.Other, cmn.Fmt("revision=%v", string(rev))) } - if !sw.IsListening() { + if !n.sw.IsListening() { return nodeInfo } - p2pListener := sw.Listeners()[0] + p2pListener := n.sw.Listeners()[0] p2pHost := p2pListener.ExternalAddress().IP.String() p2pPort := p2pListener.ExternalAddress().Port - rpcListenAddr := config.GetString("rpc_laddr") + rpcListenAddr := n.config.GetString("rpc_laddr") // We assume that the rpcListener has the same ExternalAddress. // This is probably true because both P2P and RPC listeners use UPnP, diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 6979e0e6c..f73d0e010 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -47,7 +47,7 @@ func (s *ResultStatus) TxIndexEnabled() bool { for _, s := range s.NodeInfo.Other { info := strings.Split(s, "=") if len(info) == 2 && info[0] == "tx_indexer" { - return info[1] == "kv" + return info[1] == "on" } } return false diff --git a/rpc/core/types/responses_test.go b/rpc/core/types/responses_test.go index 13728e42b..d1e8aa1b7 100644 --- a/rpc/core/types/responses_test.go +++ b/rpc/core/types/responses_test.go @@ -27,8 +27,8 @@ func TestStatusIndexer(t *testing.T) { {false, []string{}}, {false, []string{"a=b"}}, {false, []string{"tx_indexeriskv", "some=dood"}}, - {true, []string{"tx_indexer=kv", "tx_indexer=other"}}, - {true, []string{"^(*^(", "tx_indexer=kv", "a=n=b=d="}}, + {true, []string{"tx_indexer=on", "tx_indexer=other"}}, + {true, []string{"^(*^(", "tx_indexer=on", "a=n=b=d="}}, } for _, tc := range cases { diff --git a/state/execution.go b/state/execution.go index 05496aad5..2e980faa5 100644 --- a/state/execution.go +++ b/state/execution.go @@ -247,9 +247,8 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn } batch := txindex.NewBatch() - for i, r := range txResults { - tx := block.Txs[i] - batch.Index(tx.Hash(), *r) + for _, r := range txResults { + batch.Add(*r) } s.TxIndexer.AddBatch(batch) diff --git a/state/execution_test.go b/state/execution_test.go index 2da801964..299c6baa2 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -49,7 +49,7 @@ func TestApplyBlock(t *testing.T) { //---------------------------------------------------------------------------- // make some bogus txs -func txsFunc(blockNum int) (txs []types.Tx) { +func makeTxs(blockNum int) (txs []types.Tx) { for i := 0; i < nTxsPerBlock; i++ { txs = append(txs, types.Tx([]byte{byte(blockNum), byte(i)})) } @@ -71,7 +71,7 @@ func makeBlock(num int, state *State) *types.Block { prevParts := types.PartSetHeader{} valHash := state.Validators.Hash() prevBlockID := types.BlockID{prevHash, prevParts} - block, _ := types.MakeBlock(num, chainID, txsFunc(num), new(types.Commit), + block, _ := types.MakeBlock(num, chainID, makeTxs(num), new(types.Commit), prevBlockID, valHash, state.AppHash, testPartSize) return block } diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index a962eba7d..f492dac2c 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -29,22 +29,17 @@ type TxIndexer interface { // perform operations on a batch from a single thread at a time. Once batch // execution has started, you may not modify it. type Batch struct { - Ops map[string]types.TxResult + Ops []types.TxResult } // NewBatch creates a new Batch. func NewBatch() *Batch { - return &Batch{ - Ops: make(map[string]types.TxResult), - } + return &Batch{} } // Index adds or updates entry for the given hash. -func (b *Batch) Index(hash []byte, result types.TxResult) error { - if len(hash) == 0 { - return ErrorEmptyHash - } - b.Ops[string(hash)] = result +func (b *Batch) Add(result types.TxResult) error { + b.Ops = append(b.Ops, result) return nil } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index b76e6a3b0..03acc8dae 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -47,9 +47,9 @@ func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { // Batch writes a batch of transactions into the TxIndex storage. func (txi *TxIndex) AddBatch(b *txindex.Batch) error { storeBatch := txi.store.NewBatch() - for hash, result := range b.Ops { + for _, result := range b.Ops { rawBytes := wire.BinaryBytes(&result) - storeBatch.Set([]byte(hash), rawBytes) + storeBatch.Set(result.Tx.Hash(), rawBytes) } storeBatch.Write() return nil diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 7d4a84dd1..0bd45a824 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -1,7 +1,6 @@ package kv import ( - "fmt" "io/ioutil" "os" "testing" @@ -22,7 +21,7 @@ func TestTxIndex(t *testing.T) { hash := tx.Hash() batch := txindex.NewBatch() - batch.Index(hash, *txResult) + batch.Add(*txResult) err := indexer.AddBatch(batch) require.Nil(t, err) @@ -46,7 +45,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { batch := txindex.NewBatch() for i := 0; i < txsCount; i++ { - batch.Index([]byte(fmt.Sprintf("hash%v", i)), *txResult) + batch.Add(*txResult) } b.ResetTimer() From 45876d082878f1bf40981c236a941b311c125d10 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 20:23:58 -0400 Subject: [PATCH 58/84] NewBatch takes size, batch.Add doesn't use append --- state/execution.go | 2 +- state/txindex/indexer.go | 10 ++++++---- state/txindex/kv/kv_test.go | 9 +++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/state/execution.go b/state/execution.go index 2e980faa5..25d5dcd3a 100644 --- a/state/execution.go +++ b/state/execution.go @@ -246,7 +246,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn return fmt.Errorf("Commit failed for application: %v", err) } - batch := txindex.NewBatch() + batch := txindex.NewBatch(block.NumTxs) for _, r := range txResults { batch.Add(*r) } diff --git a/state/txindex/indexer.go b/state/txindex/indexer.go index f492dac2c..1c311830a 100644 --- a/state/txindex/indexer.go +++ b/state/txindex/indexer.go @@ -33,13 +33,15 @@ type Batch struct { } // NewBatch creates a new Batch. -func NewBatch() *Batch { - return &Batch{} +func NewBatch(n int) *Batch { + return &Batch{ + Ops: make([]types.TxResult, n), + } } -// Index adds or updates entry for the given hash. +// Index adds or updates entry for the given result.Index. func (b *Batch) Add(result types.TxResult) error { - b.Ops = append(b.Ops, result) + b.Ops[result.Index] = result return nil } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 0bd45a824..cd881dfc1 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -17,10 +17,10 @@ func TestTxIndex(t *testing.T) { indexer := &TxIndex{store: db.NewMemDB()} tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 1, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} hash := tx.Hash() - batch := txindex.NewBatch() + batch := txindex.NewBatch(1) batch.Add(*txResult) err := indexer.AddBatch(batch) require.Nil(t, err) @@ -32,7 +32,7 @@ func TestTxIndex(t *testing.T) { func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 1, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} + txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} dir, err := ioutil.TempDir("", "tx_indexer_db") if err != nil { @@ -43,8 +43,9 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { store := db.NewDB("tx_indexer", "leveldb", dir) indexer := &TxIndex{store: store} - batch := txindex.NewBatch() + batch := txindex.NewBatch(txsCount) for i := 0; i < txsCount; i++ { + txResult.Index += 1 batch.Add(*txResult) } From 9d2de2b7566c70e741f584cda5c6f66853639f13 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 20:55:40 -0400 Subject: [PATCH 59/84] tx_indexer -> tx_index --- config/tendermint/config.go | 2 +- config/tendermint_test/config.go | 2 +- node/node.go | 6 +++--- rpc/core/types/responses.go | 2 +- rpc/core/types/responses_test.go | 6 +++--- state/txindex/kv/kv_test.go | 4 ++-- state/txindex/null/null.go | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/config/tendermint/config.go b/config/tendermint/config.go index 19f7386b7..5ddde460d 100644 --- a/config/tendermint/config.go +++ b/config/tendermint/config.go @@ -101,7 +101,7 @@ func GetConfig(rootDir string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", rootDir+"/data/mempool.wal") - mapConfig.SetDefault("tx_indexer", "kv") + mapConfig.SetDefault("tx_index", "kv") return mapConfig } diff --git a/config/tendermint_test/config.go b/config/tendermint_test/config.go index 9cb8ee327..9d405dc95 100644 --- a/config/tendermint_test/config.go +++ b/config/tendermint_test/config.go @@ -107,7 +107,7 @@ func ResetConfig(localPath string) cfg.Config { mapConfig.SetDefault("mempool_broadcast", true) mapConfig.SetDefault("mempool_wal_dir", "") - mapConfig.SetDefault("tx_indexer", "kv") + mapConfig.SetDefault("tx_index", "kv") logger.SetLogLevel(mapConfig.GetString("log_level")) diff --git a/node/node.go b/node/node.go index b75898eb3..990779486 100644 --- a/node/node.go +++ b/node/node.go @@ -90,9 +90,9 @@ func NewNode(config cfg.Config, privValidator *types.PrivValidator, clientCreato // Transaction indexing var txIndexer txindex.TxIndexer - switch config.GetString("tx_indexer") { + switch config.GetString("tx_index") { case "kv": - store := dbm.NewDB("tx_indexer", config.GetString("db_backend"), config.GetString("db_dir")) + store := dbm.NewDB("tx_index", config.GetString("db_backend"), config.GetString("db_dir")) txIndexer = kv.NewTxIndex(store) default: txIndexer = &null.TxIndex{} @@ -381,7 +381,7 @@ func (n *Node) makeNodeInfo() *p2p.NodeInfo { cmn.Fmt("p2p_version=%v", p2p.Version), cmn.Fmt("consensus_version=%v", consensus.Version), cmn.Fmt("rpc_version=%v/%v", rpc.Version, rpccore.Version), - cmn.Fmt("tx_indexer=%v", txIndexerStatus), + cmn.Fmt("tx_index=%v", txIndexerStatus), }, } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index f73d0e010..a95f259bf 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -46,7 +46,7 @@ func (s *ResultStatus) TxIndexEnabled() bool { } for _, s := range s.NodeInfo.Other { info := strings.Split(s, "=") - if len(info) == 2 && info[0] == "tx_indexer" { + if len(info) == 2 && info[0] == "tx_index" { return info[1] == "on" } } diff --git a/rpc/core/types/responses_test.go b/rpc/core/types/responses_test.go index d1e8aa1b7..69ee4faec 100644 --- a/rpc/core/types/responses_test.go +++ b/rpc/core/types/responses_test.go @@ -26,9 +26,9 @@ func TestStatusIndexer(t *testing.T) { {false, nil}, {false, []string{}}, {false, []string{"a=b"}}, - {false, []string{"tx_indexeriskv", "some=dood"}}, - {true, []string{"tx_indexer=on", "tx_indexer=other"}}, - {true, []string{"^(*^(", "tx_indexer=on", "a=n=b=d="}}, + {false, []string{"tx_indexiskv", "some=dood"}}, + {true, []string{"tx_index=on", "tx_index=other"}}, + {true, []string{"^(*^(", "tx_index=on", "a=n=b=d="}}, } for _, tc := range cases { diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index cd881dfc1..9a1898d7e 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -34,13 +34,13 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeType_OK, Log: ""}} - dir, err := ioutil.TempDir("", "tx_indexer_db") + dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { b.Fatal(err) } defer os.RemoveAll(dir) - store := db.NewDB("tx_indexer", "leveldb", dir) + store := db.NewDB("tx_index", "leveldb", dir) indexer := &TxIndex{store: store} batch := txindex.NewBatch(txsCount) diff --git a/state/txindex/null/null.go b/state/txindex/null/null.go index e48e34cf5..4999bbdeb 100644 --- a/state/txindex/null/null.go +++ b/state/txindex/null/null.go @@ -12,7 +12,7 @@ type TxIndex struct{} // Tx panics. func (txi *TxIndex) Get(hash []byte) (*types.TxResult, error) { - return nil, errors.New(`Indexing is disabled (set 'tx_indexer = "kv"' in config)`) + return nil, errors.New(`Indexing is disabled (set 'tx_index = "kv"' in config)`) } // Batch returns nil. From cb279bf662cc64af57b4758404ae0373e4fb18ae Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 15:33:19 -0400 Subject: [PATCH 60/84] state: ABCIResponses, s.Save() in ApplyBlock --- blockchain/reactor.go | 1 - consensus/state.go | 6 +-- state/execution.go | 85 ++++++++++++++++++++--------------------- state/state.go | 89 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 54 deletions(-) diff --git a/blockchain/reactor.go b/blockchain/reactor.go index 90258825e..f88bccc3d 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -251,7 +251,6 @@ FOR_LOOP: // TODO This is bad, are we zombie? cmn.PanicQ(cmn.Fmt("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) } - bcR.state.Save() } } continue FOR_LOOP diff --git a/consensus/state.go b/consensus/state.go index d69435814..fdbf43099 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1221,7 +1221,7 @@ func (cs *ConsensusState) finalizeCommit(height int) { stateCopy := cs.state.Copy() eventCache := types.NewEventCache(cs.evsw) - // Execute and commit the block, and update the mempool. + // Execute and commit the block, update and save the state, and update the mempool. // All calls to the proxyAppConn should come here. // NOTE: the block.AppHash wont reflect these txs until the next block err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) @@ -1233,14 +1233,10 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX // Fire off event for new block. - // TODO: Handle app failure. See #177 types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header}) eventCache.Flush() - // Save the state. - stateCopy.Save() - fail.Fail() // XXX // NewHeightStep! diff --git a/state/execution.go b/state/execution.go index 25d5dcd3a..943a64f7f 100644 --- a/state/execution.go +++ b/state/execution.go @@ -13,59 +13,39 @@ import ( "github.com/tendermint/tendermint/types" ) -// ExecBlock executes the block to mutate State. +//-------------------------------------------------- +// Execute the block + +// ExecBlock executes the block, but does NOT mutate State. // + validates the block // + executes block.Txs on the proxyAppConn -// + updates validator sets -// + returns block.Txs results -func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, blockPartsHeader types.PartSetHeader) ([]*types.TxResult, error) { +func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) (*ABCIResponses, error) { // Validate the block. if err := s.validateBlock(block); err != nil { return nil, ErrInvalidBlock(err) } - // compute bitarray of validators that signed - signed := commitBitArrayFromBlock(block) - _ = signed // TODO send on begin block - - // copy the valset - valSet := s.Validators.Copy() - nextValSet := valSet.Copy() - // Execute the block txs - txResults, changedValidators, err := execBlockOnProxyApp(eventCache, proxyAppConn, block) + abciResponses, err := execBlockOnProxyApp(eventCache, proxyAppConn, block) if err != nil { // There was some error in proxyApp // TODO Report error and wait for proxyApp to be available. return nil, ErrProxyAppConn(err) } - // update the validator set - err = updateValidators(nextValSet, changedValidators) - if err != nil { - log.Warn("Error changing validator set", "error", err) - // TODO: err or carry on? - } - - // All good! - // Update validator accums and set state variables - nextValSet.IncrementAccum(1) - s.SetBlockAndValidators(block.Header, blockPartsHeader, valSet, nextValSet) - - fail.Fail() // XXX - - return txResults, nil + return abciResponses, nil } // Executes block's transactions on proxyAppConn. // Returns a list of transaction results and updates to the validator set // TODO: Generate a bitmap or otherwise store tx validity in state. -func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) ([]*types.TxResult, []*abci.Validator, error) { +func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) (*ABCIResponses, error) { var validTxs, invalidTxs = 0, 0 - txResults := make([]*types.TxResult, len(block.Txs)) txIndex := 0 + abciResponses := NewABCIResponses(block) + // Execute transactions and get hash proxyCb := func(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { @@ -84,12 +64,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo txError = txResult.Code.String() } - txResults[txIndex] = &types.TxResult{ - Height: uint64(block.Height), - Index: uint32(txIndex), - Tx: req.GetDeliverTx().Tx, - Result: *txResult, - } + abciResponses.TxResults[txIndex] = &types.TxResult{uint64(block.Height), uint32(txIndex), *txResult} txIndex++ // NOTE: if we count we can access the tx from the block instead of @@ -111,7 +86,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo err := proxyAppConn.BeginBlockSync(block.Hash(), types.TM2PB.Header(block.Header)) if err != nil { log.Warn("Error in proxyAppConn.BeginBlock", "error", err) - return nil, nil, err + return nil, err } fail.Fail() // XXX @@ -121,7 +96,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.FailRand(len(block.Txs)) // XXX proxyAppConn.DeliverTxAsync(tx) if err := proxyAppConn.Error(); err != nil { - return nil, nil, err + return nil, err } } @@ -131,7 +106,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo respEndBlock, err := proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) - return nil, nil, err + return nil, err } fail.Fail() // XXX @@ -140,7 +115,9 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo if len(respEndBlock.Diffs) > 0 { log.Info("Update to validator set", "updates", abci.ValidatorsString(respEndBlock.Diffs)) } - return txResults, respEndBlock.Diffs, nil + abciResponses.Validators = respEndBlock.Diffs + + return abciResponses, nil } func updateValidators(validators *types.ValidatorSet, changedValidators []*abci.Validator) error { @@ -230,16 +207,30 @@ func (s *State) validateBlock(block *types.Block) error { return nil } -// ApplyBlock executes the block, then commits and updates the mempool -// atomically, optionally indexing transaction results. +//----------------------------------------------------------------------------- +// ApplyBlock executes the block, updates state w/ ABCI responses, +// then commits and updates the mempool atomically, then saves state. +// Transaction results are optionally indexed. + +// Execute and commit block against app, save block and state func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, partsHeader types.PartSetHeader, mempool types.Mempool) error { - txResults, err := s.ExecBlock(eventCache, proxyAppConn, block, partsHeader) + abciResponses, err := s.ExecBlock(eventCache, proxyAppConn, block) if err != nil { return fmt.Errorf("Exec failed for application: %v", err) } + fail.Fail() // XXX + + // save the results before we commit + s.SaveABCIResponses(abciResponses) + + fail.Fail() // XXX + + // now update the block and validators + s.SetBlockAndValidators(block.Header, partsHeader) + // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) if err != nil { @@ -252,6 +243,11 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn } s.TxIndexer.AddBatch(batch) + fail.Fail() // XXX + + // save the state + s.Save() + return nil } @@ -284,9 +280,10 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl // Apply and commit a block, but without all the state validation. // Returns the application root hash (result of abci.Commit) +// TODO handle abciResponses func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) { var eventCache types.Fireable // nil - _, _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) + _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) if err != nil { log.Warn("Error executing block on proxy app", "height", block.Height, "err", err) return nil, err diff --git a/state/state.go b/state/state.go index 14b324d5f..1b90d3406 100644 --- a/state/state.go +++ b/state/state.go @@ -6,6 +6,7 @@ import ( "sync" "time" + abci "github.com/tendermint/abci/types" . "github.com/tendermint/go-common" cfg "github.com/tendermint/go-config" dbm "github.com/tendermint/go-db" @@ -16,7 +17,8 @@ import ( ) var ( - stateKey = []byte("stateKey") + stateKey = []byte("stateKey") + abciResponsesKey = []byte("abciResponsesKey") ) //----------------------------------------------------------------------------- @@ -31,7 +33,7 @@ type State struct { GenesisDoc *types.GenesisDoc ChainID string - // updated at end of ExecBlock + // updated at end of SetBlockAndValidators LastBlockHeight int // Genesis state has this set to 0. So, Block(H=0) does not exist. LastBlockID types.BlockID LastBlockTime time.Time @@ -42,6 +44,10 @@ type State struct { AppHash []byte TxIndexer txindex.TxIndexer `json:"-"` // Transaction indexer. + + // Intermediate results from processing + // Persisted separately from the state + abciResponses *ABCIResponses } func LoadState(db dbm.DB) *State { @@ -62,6 +68,8 @@ func loadState(db dbm.DB, key []byte) *State { } // TODO: ensure that buf is completely read. } + + s.LoadABCIResponses() return s } @@ -76,7 +84,8 @@ func (s *State) Copy() *State { Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), AppHash: s.AppHash, - TxIndexer: s.TxIndexer, // pointer here, not value + abciResponses: s.abciResponses, // pointer here, not value + TxIndexer: s.TxIndexer, // pointer here, not value } } @@ -86,6 +95,37 @@ func (s *State) Save() { s.db.SetSync(stateKey, s.Bytes()) } +// Sets the ABCIResponses in the state and writes them to disk +func (s *State) SaveABCIResponses(abciResponses *ABCIResponses) { + s.abciResponses = abciResponses + + // save the validators to the db + s.db.SetSync(abciResponsesKey, s.abciResponses.Bytes()) + + // save the tx results using the TxIndexer + batch := txindexer.NewBatch() + for i, r := range s.abciResponses.TxResults { + tx := s.abciResponses.Txs[i] + batch.Index(tx.Hash(), *r) + } + s.TxIndexer.Batch(batch) +} + +func (s *State) LoadABCIResponses() { + s.abciResponses = new(ABCIResponses) + + buf := s.db.Get(abciResponsesKey) + if len(buf) != 0 { + r, n, err := bytes.NewReader(buf), new(int), new(error) + wire.ReadBinaryPtr(&s.abciResponses.Validators, r, 0, n, err) + if *err != nil { + // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED + Exit(Fmt("Data has been corrupted or its spec has changed: %v\n", *err)) + } + // TODO: ensure that buf is completely read. + } +} + func (s *State) Equals(s2 *State) bool { return bytes.Equal(s.Bytes(), s2.Bytes()) } @@ -101,7 +141,21 @@ func (s *State) Bytes() []byte { // Mutate state variables to match block and validators // after running EndBlock -func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, prevValSet, nextValSet *types.ValidatorSet) { +func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader) { + + // copy the valset + prevValSet := s.Validators.Copy() + nextValSet := prevValSet.Copy() + + // update the validator set + err := updateValidators(nextValSet, s.abciResponses.Validators) + if err != nil { + log.Warn("Error changing validator set", "error", err) + // TODO: err or carry on? + } + // Update validator accums and set state variables + nextValSet.IncrementAccum(1) + s.setBlockAndValidators(header.Height, types.BlockID{header.Hash(), blockPartsHeader}, header.Time, prevValSet, nextValSet) @@ -134,6 +188,33 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { return state } +//-------------------------------------------------- +// ABCIResponses holds intermediate state during block processing + +type ABCIResponses struct { + Validators []*abci.Validator // changes to the validator set + + Txs types.Txs // for reference later + TxResults []*types.TxResult // results of the txs, populated in the proxyCb +} + +func NewABCIResponses(block *types.Block) *ABCIResponses { + return &ABCIResponses{ + Txs: block.Data.Txs, + TxResults: make([]*types.TxResult, block.NumTxs), + } +} + +// Serialize the list of validators +func (a *ABCIResponses) Bytes() []byte { + buf, n, err := new(bytes.Buffer), new(int), new(error) + wire.WriteBinary(a.Validators, buf, n, err) + if *err != nil { + PanicCrisis(*err) + } + return buf.Bytes() +} + //----------------------------------------------------------------------------- // Genesis From 54b26869d51c6cce9df44aa34ff46e8cf5965459 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 16:27:22 -0400 Subject: [PATCH 61/84] consensus/wal: #HEIGHT -> #ENDHEIGHT --- consensus/replay.go | 14 +++++++------- consensus/replay_test.go | 2 +- consensus/state.go | 6 ++++++ consensus/test_data/build.sh | 8 ++++---- consensus/test_data/empty_block.cswal | 2 +- consensus/test_data/many_blocks.cswal | 12 ++++++------ consensus/test_data/small_block1.cswal | 2 +- consensus/test_data/small_block2.cswal | 2 +- consensus/wal.go | 12 +++--------- 9 files changed, 30 insertions(+), 30 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 731e7d216..d532828f8 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -100,19 +100,19 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { cs.replayMode = true defer func() { cs.replayMode = false }() - // Ensure that height+1 doesn't exist - gr, found, err := cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight+1)) + // Ensure that ENDHEIGHT for this height doesn't exist + gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight)) if found { - return errors.New(Fmt("WAL should not contain height %d.", csHeight+1)) + return errors.New(Fmt("WAL should not contain height %d.", csHeight)) } if gr != nil { gr.Close() } - // Search for height marker - gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) + // Search for last height marker + gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1)) if err == io.EOF { - log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight) + log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight-1) return nil } else if err != nil { return err @@ -147,7 +147,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { //-------------------------------------------------------------------------------- // Parses marker lines of the form: -// #HEIGHT: 12345 +// #ENDHEIGHT: 12345 func makeHeightSearchFunc(height int) auto.SearchFunc { return func(line string) (int, error) { line = strings.TrimRight(line, "\n") diff --git a/consensus/replay_test.go b/consensus/replay_test.go index c70b60fa0..43204ab72 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -443,7 +443,7 @@ func buildTMStateFromChain(config cfg.Config, state *sm.State, chain []*types.Bl func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) { // Search for height marker - gr, found, err := wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(1)) + gr, found, err := wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(0)) if err != nil { return nil, nil, err } diff --git a/consensus/state.go b/consensus/state.go index fdbf43099..9c652c95a 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1202,6 +1202,12 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX + if cs.wal != nil { + cs.wal.writeEndHeight(height) + } + + fail.Fail() // XXX + // Save to blockStore. if cs.blockStore.Height() < block.Height { // NOTE: the seenCommit is local justification to commit this block, diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh index ea0c9604a..2759c0e38 100644 --- a/consensus/test_data/build.sh +++ b/consensus/test_data/build.sh @@ -27,7 +27,7 @@ killall tendermint # /q would print up to and including the match, then quit. # /Q doesn't include the match. # http://unix.stackexchange.com/questions/11305/grep-show-all-the-file-up-to-the-match -sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal +sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/empty_block.cswal reset } @@ -41,7 +41,7 @@ sleep 7 killall tendermint kill -9 $PID -sed '/HEIGHT: 7/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/many_blocks.cswal +sed '/ENDHEIGHT: 6/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/many_blocks.cswal reset } @@ -56,7 +56,7 @@ sleep 10 killall tendermint kill -9 $PID -sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block1.cswal +sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block1.cswal reset } @@ -73,7 +73,7 @@ sleep 5 killall tendermint kill -9 $PID -sed '/HEIGHT: 2/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block2.cswal +sed '/ENDHEIGHT: 1/Q' ~/.tendermint/data/cs.wal/wal > consensus/test_data/small_block2.cswal reset } diff --git a/consensus/test_data/empty_block.cswal b/consensus/test_data/empty_block.cswal index a3a3585ce..aa5b232c9 100644 --- a/consensus/test_data/empty_block.cswal +++ b/consensus/test_data/empty_block.cswal @@ -1,4 +1,4 @@ -#HEIGHT: 1 +#ENDHEIGHT: 0 {"time":"2016-12-18T05:05:33.502Z","msg":[3,{"duration":974084551,"height":1,"round":0,"step":1}]} {"time":"2016-12-18T05:05:33.505Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-12-18T05:05:33.505Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"71D2DA2336A9F84C22A28FF6C67F35F3478FC0AF"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"62C0F2BCCB491399EEDAF8E85837ADDD4E25BAB7A84BFC4F0E88594531FBC6D4755DEC7E6427F04AD7EB8BB89502762AB4380C7BBA93A4C297E6180EC78E3504"]}}],"peer_key":""}]} diff --git a/consensus/test_data/many_blocks.cswal b/consensus/test_data/many_blocks.cswal index 9ef06c32c..fd103cb1e 100644 --- a/consensus/test_data/many_blocks.cswal +++ b/consensus/test_data/many_blocks.cswal @@ -1,4 +1,4 @@ -#HEIGHT: 1 +#ENDHEIGHT: 0 {"time":"2017-02-17T23:54:19.013Z","msg":[3,{"duration":969121813,"height":1,"round":0,"step":1}]} {"time":"2017-02-17T23:54:19.014Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2017-02-17T23:54:19.014Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"2E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"105A5A834E9AE2FA2191CAB5CB20D63594BA7859BD3EB92F055C8A35476D71F0D89F9FD5B0FF030D021533C71A81BF6E8F026BF4A37FC637CF38CA35291A9D00"]}}],"peer_key":""}]} @@ -8,7 +8,7 @@ {"time":"2017-02-17T23:54:19.016Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPrecommit"}]} {"time":"2017-02-17T23:54:19.016Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":1,"round":0,"type":2,"block_id":{"hash":"3F32EE37F9EA674A2173CAD651836A8EE628B5C7","parts":{"total":1,"hash":"2E32C8D500E936D27A47FCE3FF4BE7C1AFB3FAE1"}},"signature":[1,"2B1070A5AB9305612A3AE74A8036D82B5E49E0DBBFBC7D723DB985CC8A8E72A52FF8E34D85273FEB8B901945CA541FA5142C3C4D43A04E9205ACECF53FD19B01"]}}],"peer_key":""}]} {"time":"2017-02-17T23:54:19.017Z","msg":[1,{"height":1,"round":0,"step":"RoundStepCommit"}]} -#HEIGHT: 2 +#ENDHEIGHT: 1 {"time":"2017-02-17T23:54:19.019Z","msg":[1,{"height":2,"round":0,"step":"RoundStepNewHeight"}]} {"time":"2017-02-17T23:54:20.017Z","msg":[3,{"duration":998073370,"height":2,"round":0,"step":1}]} {"time":"2017-02-17T23:54:20.018Z","msg":[1,{"height":2,"round":0,"step":"RoundStepPropose"}]} @@ -19,7 +19,7 @@ {"time":"2017-02-17T23:54:20.021Z","msg":[1,{"height":2,"round":0,"step":"RoundStepPrecommit"}]} {"time":"2017-02-17T23:54:20.021Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":2,"round":0,"type":2,"block_id":{"hash":"32310D174A99844713693C9815D2CA660364E028","parts":{"total":1,"hash":"D008E9014CDDEA8EC95E1E99E21333241BD52DFC"}},"signature":[1,"AA9F03D0707752301D7CBFCF4F0BCDBD666A46C1CAED3910BD64A3C5C2874AAF328172646C951C5E2FD962359C382A3CBBA2C73EC9B533668C6386995B83EC08"]}}],"peer_key":""}]} {"time":"2017-02-17T23:54:20.022Z","msg":[1,{"height":2,"round":0,"step":"RoundStepCommit"}]} -#HEIGHT: 3 +#ENDHEIGHT: 2 {"time":"2017-02-17T23:54:20.025Z","msg":[1,{"height":3,"round":0,"step":"RoundStepNewHeight"}]} {"time":"2017-02-17T23:54:21.022Z","msg":[3,{"duration":997103974,"height":3,"round":0,"step":1}]} {"time":"2017-02-17T23:54:21.024Z","msg":[1,{"height":3,"round":0,"step":"RoundStepPropose"}]} @@ -30,7 +30,7 @@ {"time":"2017-02-17T23:54:21.028Z","msg":[1,{"height":3,"round":0,"step":"RoundStepPrecommit"}]} {"time":"2017-02-17T23:54:21.028Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":3,"round":0,"type":2,"block_id":{"hash":"37AF6866DA8C3167CFC280FAE47B6ED441B00D5B","parts":{"total":1,"hash":"2E5DE5777A5AD899CD2531304F42A470509DE989"}},"signature":[1,"C900519E305EC03392E7D197D5FAB535DB240C9C0BA5375A1679C75BAAA07C7410C0EF43CF97D98F2C08A1D739667D5ACFF6233A1FAE75D3DA275AEA422EFD0F"]}}],"peer_key":""}]} {"time":"2017-02-17T23:54:21.028Z","msg":[1,{"height":3,"round":0,"step":"RoundStepCommit"}]} -#HEIGHT: 4 +#ENDHEIGHT: 3 {"time":"2017-02-17T23:54:21.032Z","msg":[1,{"height":4,"round":0,"step":"RoundStepNewHeight"}]} {"time":"2017-02-17T23:54:22.028Z","msg":[3,{"duration":996302067,"height":4,"round":0,"step":1}]} {"time":"2017-02-17T23:54:22.030Z","msg":[1,{"height":4,"round":0,"step":"RoundStepPropose"}]} @@ -41,7 +41,7 @@ {"time":"2017-02-17T23:54:22.033Z","msg":[1,{"height":4,"round":0,"step":"RoundStepPrecommit"}]} {"time":"2017-02-17T23:54:22.033Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":4,"round":0,"type":2,"block_id":{"hash":"04715E223BF4327FFA9B0D5AD849B74A099D5DEC","parts":{"total":1,"hash":"24CEBCBEB833F56D47AD14354071B3B7A243068A"}},"signature":[1,"F544743F17479A61F94B0F68C63D254BD60493D78E818D48A5859133619AEE5E92C47CAD89C654DF64E0911C3152091E047555D5F14655D95B9681AE9B336505"]}}],"peer_key":""}]} {"time":"2017-02-17T23:54:22.034Z","msg":[1,{"height":4,"round":0,"step":"RoundStepCommit"}]} -#HEIGHT: 5 +#ENDHEIGHT: 4 {"time":"2017-02-17T23:54:22.036Z","msg":[1,{"height":5,"round":0,"step":"RoundStepNewHeight"}]} {"time":"2017-02-17T23:54:23.034Z","msg":[3,{"duration":997096276,"height":5,"round":0,"step":1}]} {"time":"2017-02-17T23:54:23.035Z","msg":[1,{"height":5,"round":0,"step":"RoundStepPropose"}]} @@ -52,7 +52,7 @@ {"time":"2017-02-17T23:54:23.038Z","msg":[1,{"height":5,"round":0,"step":"RoundStepPrecommit"}]} {"time":"2017-02-17T23:54:23.038Z","msg":[2,{"msg":[20,{"Vote":{"validator_address":"D028C9981F7A87F3093672BF0D5B0E2A1B3ED456","validator_index":0,"height":5,"round":0,"type":2,"block_id":{"hash":"FDC6D837995BEBBBFCBF3E7D7CF44F8FDA448543","parts":{"total":1,"hash":"A52BAA9C2E52E633A1605F4B930205613E3E7A2F"}},"signature":[1,"DF51D23D5D2C57598F67791D953A6C2D9FC5865A3048ADA4469B37500D2996B95732E0DC6F99EAEAEA12B4818CE355C7B701D16857D2AC767D740C2E30E9260C"]}}],"peer_key":""}]} {"time":"2017-02-17T23:54:23.038Z","msg":[1,{"height":5,"round":0,"step":"RoundStepCommit"}]} -#HEIGHT: 6 +#ENDHEIGHT: 5 {"time":"2017-02-17T23:54:23.041Z","msg":[1,{"height":6,"round":0,"step":"RoundStepNewHeight"}]} {"time":"2017-02-17T23:54:24.038Z","msg":[3,{"duration":997341910,"height":6,"round":0,"step":1}]} {"time":"2017-02-17T23:54:24.040Z","msg":[1,{"height":6,"round":0,"step":"RoundStepPropose"}]} diff --git a/consensus/test_data/small_block1.cswal b/consensus/test_data/small_block1.cswal index 90103dff3..d4eff73f1 100644 --- a/consensus/test_data/small_block1.cswal +++ b/consensus/test_data/small_block1.cswal @@ -1,4 +1,4 @@ -#HEIGHT: 1 +#ENDHEIGHT: 0 {"time":"2016-12-18T05:05:38.593Z","msg":[3,{"duration":970717663,"height":1,"round":0,"step":1}]} {"time":"2016-12-18T05:05:38.595Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-12-18T05:05:38.595Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":1,"hash":"A434EC796DF1CECC01296E953839C4675863A4E5"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"39563C3C7EDD9855B2971457A5DABF05CFDAF52805658847EB1F05115B8341344A77761CC85E670AF1B679DA9FC0905231957174699FE8326DBE7706209BDD0B"]}}],"peer_key":""}]} diff --git a/consensus/test_data/small_block2.cswal b/consensus/test_data/small_block2.cswal index 1be6c4c93..b5d1d282b 100644 --- a/consensus/test_data/small_block2.cswal +++ b/consensus/test_data/small_block2.cswal @@ -1,4 +1,4 @@ -#HEIGHT: 1 +#ENDHEIGHT: 0 {"time":"2016-12-18T05:05:43.641Z","msg":[3,{"duration":969409681,"height":1,"round":0,"step":1}]} {"time":"2016-12-18T05:05:43.643Z","msg":[1,{"height":1,"round":0,"step":"RoundStepPropose"}]} {"time":"2016-12-18T05:05:43.643Z","msg":[2,{"msg":[17,{"Proposal":{"height":1,"round":0,"block_parts_header":{"total":5,"hash":"C916905C3C444501DDDAA1BF52E959B7531E762E"},"pol_round":-1,"pol_block_id":{"hash":"","parts":{"total":0,"hash":""}},"signature":[1,"F1A8E9928889C68FD393F3983B5362AECA4A95AA13FE3C78569B2515EC046893CB718071CAF54F3F1507DCD851B37CD5557EA17BB5471D2DC6FB5AC5FBB72E02"]}}],"peer_key":""}]} diff --git a/consensus/wal.go b/consensus/wal.go index 6d8eb3819..a89eff5e4 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -59,7 +59,7 @@ func (wal *WAL) OnStart() error { if err != nil { return err } else if size == 0 { - wal.writeHeight(1) + wal.writeEndHeight(0) } _, err = wal.group.Start() return err @@ -83,12 +83,6 @@ func (wal *WAL) Save(wmsg WALMessage) { } } } - // Write #HEIGHT: XYZ if new height - if edrs, ok := wmsg.(types.EventDataRoundState); ok { - if edrs.Step == RoundStepNewHeight.String() { - wal.writeHeight(edrs.Height) - } - } // Write the wal message var wmsgBytes = wire.JSONBytes(TimedWALMessage{time.Now(), wmsg}) err := wal.group.WriteLine(string(wmsgBytes)) @@ -101,8 +95,8 @@ func (wal *WAL) Save(wmsg WALMessage) { } } -func (wal *WAL) writeHeight(height int) { - wal.group.WriteLine(Fmt("#HEIGHT: %v", height)) +func (wal *WAL) writeEndHeight(height int) { + wal.group.WriteLine(Fmt("#ENDHEIGHT: %v", height)) // TODO: only flush when necessary if err := wal.group.Flush(); err != nil { From ed03cb5c17c2cc58068c7ca7fb6e3e2a21381c42 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 16:40:54 -0400 Subject: [PATCH 62/84] consensus/replay: remove timeout --- consensus/replay.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index d532828f8..4ed5573ee 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -332,23 +332,13 @@ func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, e evsw.Start() defer evsw.Stop() cs.SetEventSwitch(evsw) - newBlockCh := subscribeToEvent(evsw, "consensus-replay", types.EventStringNewBlock(), 1) + log.Notice("Attempting to replay last block", "height", h.store.Height()) // run through the WAL, commit new block, stop if _, err := cs.Start(); err != nil { return nil, err } - defer cs.Stop() - - timeout := h.config.GetInt("timeout_handshake") - timer := time.NewTimer(time.Duration(timeout) * time.Millisecond) - log.Notice("Attempting to replay last block", "height", h.store.Height(), "timeout", timeout) - - select { - case <-newBlockCh: - case <-timer.C: - return nil, ErrReplayLastBlockTimeout - } + cs.Stop() h.nBlocks += 1 From 3a973b80ac0ca266c56f4eb2bb2d11185cf1a1db Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 17:20:03 -0400 Subject: [PATCH 63/84] update glide --- glide.lock | 18 +++++------------- state/state.go | 2 +- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/glide.lock b/glide.lock index 63b3875d0..f0ede402e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 -updated: 2017-04-11T15:24:16.619608243-04:00 +updated: 2017-04-14T17:17:31.933202871-04:00 imports: - name: github.com/btcsuite/btcd version: b8df516b4b267acf2de46be593a9d948d1d2c420 @@ -85,7 +85,7 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 6af2364fa91ef2f3afc8ba0db33b66d9d3ae006c + version: 714fdaee3bb3f8670e721a75c5ddda8787b256dd subpackages: - test - name: github.com/tendermint/go-config @@ -111,13 +111,13 @@ imports: subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 9d18cbe74e66f875afa36d2fa3be280e4a2dc9e6 + version: 4671c44b2d124f7f6f6243dbfbf4ae2bf42ee809 subpackages: - client - server - types - name: github.com/tendermint/go-wire - version: 09dae074245a8042aa689d084af774e6ad6a90bb + version: 50889e2b4a9ba65b67be86a486f25853d514b937 - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: @@ -166,12 +166,4 @@ imports: - stats - tap - transport -testImports: -- name: github.com/davecgh/go-spew - version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 - subpackages: - - spew -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib +testImports: [] diff --git a/state/state.go b/state/state.go index 1b90d3406..8a27104cd 100644 --- a/state/state.go +++ b/state/state.go @@ -147,7 +147,7 @@ func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader typ prevValSet := s.Validators.Copy() nextValSet := prevValSet.Copy() - // update the validator set + // update the validator set with the latest abciResponses err := updateValidators(nextValSet, s.abciResponses.Validators) if err != nil { log.Warn("Error changing validator set", "error", err) From 1684ec163fad41a9d97512a6e08082e966cd6567 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 20:21:50 -0400 Subject: [PATCH 64/84] ABCIResponses not needed as field in state --- state/execution.go | 14 +++++++------- state/state.go | 41 +++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/state/execution.go b/state/execution.go index 943a64f7f..fb0eac02e 100644 --- a/state/execution.go +++ b/state/execution.go @@ -43,7 +43,6 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo var validTxs, invalidTxs = 0, 0 txIndex := 0 - abciResponses := NewABCIResponses(block) // Execute transactions and get hash @@ -64,7 +63,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo txError = txResult.Code.String() } - abciResponses.TxResults[txIndex] = &types.TxResult{uint64(block.Height), uint32(txIndex), *txResult} + abciResponses.DeliverTx[txIndex] = txResult txIndex++ // NOTE: if we count we can access the tx from the block instead of @@ -103,7 +102,7 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.Fail() // XXX // End block - respEndBlock, err := proxyAppConn.EndBlockSync(uint64(block.Height)) + abciResponses.EndBlock, err = proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { log.Warn("Error in proxyAppConn.EndBlock", "error", err) return nil, err @@ -111,11 +110,12 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo fail.Fail() // XXX + valDiff := abciResponses.EndBlock.Diffs + log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) - if len(respEndBlock.Diffs) > 0 { - log.Info("Update to validator set", "updates", abci.ValidatorsString(respEndBlock.Diffs)) + if len(valDiff) > 0 { + log.Info("Update to validator set", "updates", abci.ValidatorsString(valDiff)) } - abciResponses.Validators = respEndBlock.Diffs return abciResponses, nil } @@ -229,7 +229,7 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn fail.Fail() // XXX // now update the block and validators - s.SetBlockAndValidators(block.Header, partsHeader) + s.SetBlockAndValidators(block.Header, partsHeader, abciResponses) // lock mempool, commit state, update mempoool err = s.CommitStateUpdateMempool(proxyAppConn, block, mempool) diff --git a/state/state.go b/state/state.go index 8a27104cd..604f0892a 100644 --- a/state/state.go +++ b/state/state.go @@ -68,8 +68,6 @@ func loadState(db dbm.DB, key []byte) *State { } // TODO: ensure that buf is completely read. } - - s.LoadABCIResponses() return s } @@ -84,8 +82,7 @@ func (s *State) Copy() *State { Validators: s.Validators.Copy(), LastValidators: s.LastValidators.Copy(), AppHash: s.AppHash, - abciResponses: s.abciResponses, // pointer here, not value - TxIndexer: s.TxIndexer, // pointer here, not value + TxIndexer: s.TxIndexer, // pointer here, not value } } @@ -96,34 +93,36 @@ func (s *State) Save() { } // Sets the ABCIResponses in the state and writes them to disk +// in case we crash after app.Commit and before s.Save() func (s *State) SaveABCIResponses(abciResponses *ABCIResponses) { - s.abciResponses = abciResponses // save the validators to the db - s.db.SetSync(abciResponsesKey, s.abciResponses.Bytes()) + s.db.SetSync(abciResponsesKey, abciResponses.Bytes()) // save the tx results using the TxIndexer + // NOTE: these may be overwriting, but the values should be the same. batch := txindexer.NewBatch() - for i, r := range s.abciResponses.TxResults { - tx := s.abciResponses.Txs[i] - batch.Index(tx.Hash(), *r) + for i, d := range abciResponses.DeliverTx { + tx := abciResponses.txs[i] + batch.Index(tx.Hash(), types.TxResult{uint64(abciResponses.height), uint32(i), *d}) } s.TxIndexer.Batch(batch) } -func (s *State) LoadABCIResponses() { - s.abciResponses = new(ABCIResponses) +func (s *State) LoadABCIResponses() *ABCIResponses { + abciResponses := new(ABCIResponses) buf := s.db.Get(abciResponsesKey) if len(buf) != 0 { r, n, err := bytes.NewReader(buf), new(int), new(error) - wire.ReadBinaryPtr(&s.abciResponses.Validators, r, 0, n, err) + wire.ReadBinaryPtr(&abciResponses.EndBlock.Diffs, r, 0, n, err) if *err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED Exit(Fmt("Data has been corrupted or its spec has changed: %v\n", *err)) } // TODO: ensure that buf is completely read. } + return abciResponses } func (s *State) Equals(s2 *State) bool { @@ -141,14 +140,14 @@ func (s *State) Bytes() []byte { // Mutate state variables to match block and validators // after running EndBlock -func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader) { +func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, abciResponses *ABCIResponses) { // copy the valset prevValSet := s.Validators.Copy() nextValSet := prevValSet.Copy() // update the validator set with the latest abciResponses - err := updateValidators(nextValSet, s.abciResponses.Validators) + err := updateValidators(nextValSet, abciResponses.EndBlock.Diffs) if err != nil { log.Warn("Error changing validator set", "error", err) // TODO: err or carry on? @@ -192,23 +191,25 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { // ABCIResponses holds intermediate state during block processing type ABCIResponses struct { - Validators []*abci.Validator // changes to the validator set + height int + txs types.Txs // for reference later - Txs types.Txs // for reference later - TxResults []*types.TxResult // results of the txs, populated in the proxyCb + DeliverTx []*abci.ResponseDeliverTx // results of the txs, populated in the proxyCb + EndBlock abci.ResponseEndBlock // changes to the validator set } func NewABCIResponses(block *types.Block) *ABCIResponses { return &ABCIResponses{ - Txs: block.Data.Txs, - TxResults: make([]*types.TxResult, block.NumTxs), + height: block.Height, + txs: block.Data.Txs, + DeliverTx: make([]*abci.ResponseDeliverTx, block.NumTxs), } } // Serialize the list of validators func (a *ABCIResponses) Bytes() []byte { buf, n, err := new(bytes.Buffer), new(int), new(error) - wire.WriteBinary(a.Validators, buf, n, err) + wire.WriteBinary(a.EndBlock, buf, n, err) if *err != nil { PanicCrisis(*err) } From 5109746b1cd597ed7f2b2d9a1c6818bd45989bab Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 14 Apr 2017 20:30:15 -0400 Subject: [PATCH 65/84] Handshake uses ApplyBlock, no ConsensuState --- consensus/replay.go | 64 ++++++++++++++++++++++++++++++--------------- consensus/state.go | 24 +++++++++++------ state/execution.go | 14 ++++++++++ state/state.go | 33 +++++++++-------------- state/state_test.go | 31 ++++++++++++++++++++++ 5 files changed, 116 insertions(+), 50 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 4ed5573ee..53f876084 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -101,6 +101,8 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { defer func() { cs.replayMode = false }() // Ensure that ENDHEIGHT for this height doesn't exist + // NOTE: This is just a sanity check. As far as we know things work fine without it, + // and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT). gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight)) if found { return errors.New(Fmt("WAL should not contain height %d.", csHeight)) @@ -273,15 +275,18 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p } else if appBlockHeight == stateBlockHeight { // We haven't run Commit (both the state and app are one block behind), - // so run through consensus with the real app + // so ApplyBlock with the real app. + // NOTE: We could instead use the cs.WAL on cs.Start, + // but we'd have to allow the WAL to replay a block that wrote it's ENDHEIGHT log.Info("Replay last block using real app") - return h.replayLastBlock(proxyApp.Consensus()) + return h.replayLastBlock(storeBlockHeight, proxyApp.Consensus()) } else if appBlockHeight == storeBlockHeight { - // We ran Commit, but didn't save the state, so run through consensus with mock app - mockApp := newMockProxyApp(appHash) + // We ran Commit, but didn't save the state, so ApplyBlock with mock app + abciResponses := h.state.LoadABCIResponses() + mockApp := newMockProxyApp(appHash, abciResponses) log.Info("Replay last block using mock app") - return h.replayLastBlock(mockApp) + return h.replayLastBlock(storeBlockHeight, mockApp) } } @@ -323,26 +328,21 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store return appHash, h.checkAppHash(appHash) } -// Replay the last block through the consensus and return the AppHash from after Commit. -func (h *Handshaker) replayLastBlock(proxyApp proxy.AppConnConsensus) ([]byte, error) { +// ApplyBlock on the proxyApp with the last block. +func (h *Handshaker) replayLastBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) { mempool := types.MockMempool{} - cs := NewConsensusState(h.config, h.state, proxyApp, h.store, mempool) - evsw := types.NewEventSwitch() - evsw.Start() - defer evsw.Stop() - cs.SetEventSwitch(evsw) + var eventCache types.Fireable // nil + block := h.store.LoadBlock(height) + meta := h.store.LoadBlockMeta(height) - log.Notice("Attempting to replay last block", "height", h.store.Height()) - // run through the WAL, commit new block, stop - if _, err := cs.Start(); err != nil { + if err := h.state.ApplyBlock(eventCache, proxyApp, block, meta.BlockID.PartsHeader, mempool); err != nil { return nil, err } - cs.Stop() h.nBlocks += 1 - return cs.state.AppHash, nil + return h.state.AppHash, nil } func (h *Handshaker) checkAppHash(appHash []byte) error { @@ -354,9 +354,14 @@ func (h *Handshaker) checkAppHash(appHash []byte) error { } //-------------------------------------------------------------------------------- - -func newMockProxyApp(appHash []byte) proxy.AppConnConsensus { - clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{appHash: appHash}) +// mockProxyApp uses ABCIResponses to give the right results +// Useful because we don't want to call Commit() twice for the same block on the real app. + +func newMockProxyApp(appHash []byte, abciResponses *sm.ABCIResponses) proxy.AppConnConsensus { + clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{ + appHash: appHash, + abciResponses: abciResponses, + }) cli, _ := clientCreator.NewABCIClient() return proxy.NewAppConnConsensus(cli) } @@ -364,7 +369,24 @@ func newMockProxyApp(appHash []byte) proxy.AppConnConsensus { type mockProxyApp struct { abci.BaseApplication - appHash []byte + appHash []byte + txCount int + abciResponses *sm.ABCIResponses +} + +func (mock *mockProxyApp) DeliverTx(tx []byte) abci.Result { + r := mock.abciResponses.DeliverTx[mock.txCount] + mock.txCount += 1 + return abci.Result{ + r.Code, + r.Data, + r.Log, + } +} + +func (mock *mockProxyApp) EndBlock(height uint64) abci.ResponseEndBlock { + mock.txCount = 0 + return mock.abciResponses.EndBlock } func (mock *mockProxyApp) Commit() abci.Result { diff --git a/consensus/state.go b/consensus/state.go index 9c652c95a..f1423e2f2 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1202,12 +1202,6 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX - if cs.wal != nil { - cs.wal.writeEndHeight(height) - } - - fail.Fail() // XXX - // Save to blockStore. if cs.blockStore.Height() < block.Height { // NOTE: the seenCommit is local justification to commit this block, @@ -1222,13 +1216,22 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX + // Finish writing to the WAL for this height. + // NOTE: ConsensusState should not be started again + // until we successfully call ApplyBlock (eg. in Handshake after restart) + if cs.wal != nil { + cs.wal.writeEndHeight(height) + } + + fail.Fail() // XXX + // Create a copy of the state for staging // and an event cache for txs stateCopy := cs.state.Copy() eventCache := types.NewEventCache(cs.evsw) // Execute and commit the block, update and save the state, and update the mempool. - // All calls to the proxyAppConn should come here. + // All calls to the proxyAppConn come here. // NOTE: the block.AppHash wont reflect these txs until the next block err := stateCopy.ApplyBlock(eventCache, cs.proxyAppConn, block, blockParts.Header(), cs.mempool) if err != nil { @@ -1238,7 +1241,12 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX - // Fire off event for new block. + // Fire event for new block. + // NOTE: If we fail before firing, these events will never fire + // + // Some options (for which they may fire more than once. I guess that's fine): + // * Fire before persisting state, in ApplyBlock + // * Fire on start up if we haven't written any new WAL msgs types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header}) eventCache.Flush() diff --git a/state/execution.go b/state/execution.go index fb0eac02e..d978c1956 100644 --- a/state/execution.go +++ b/state/execution.go @@ -223,6 +223,9 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn fail.Fail() // XXX + // index txs. This could run in the background + s.indexTxs(abciResponses) + // save the results before we commit s.SaveABCIResponses(abciResponses) @@ -278,6 +281,17 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl return nil } +func (s *State) indexTxs(abciResponses *ABCIResponses) { + // save the tx results using the TxIndexer + // NOTE: these may be overwriting, but the values should be the same. + batch := txindexer.NewBatch() + for i, d := range abciResponses.DeliverTx { + tx := abciResponses.txs[i] + batch.Index(tx.Hash(), types.TxResult{uint64(abciResponses.Height), uint32(i), *d}) + } + s.TxIndexer.Batch(batch) +} + // Apply and commit a block, but without all the state validation. // Returns the application root hash (result of abci.Commit) // TODO handle abciResponses diff --git a/state/state.go b/state/state.go index 604f0892a..eb7159e24 100644 --- a/state/state.go +++ b/state/state.go @@ -64,7 +64,7 @@ func loadState(db dbm.DB, key []byte) *State { wire.ReadBinaryPtr(&s, r, 0, n, err) if *err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED - Exit(Fmt("Data has been corrupted or its spec has changed: %v\n", *err)) + Exit(Fmt("LoadState: Data has been corrupted or its spec has changed: %v\n", *err)) } // TODO: ensure that buf is completely read. } @@ -95,18 +95,8 @@ func (s *State) Save() { // Sets the ABCIResponses in the state and writes them to disk // in case we crash after app.Commit and before s.Save() func (s *State) SaveABCIResponses(abciResponses *ABCIResponses) { - // save the validators to the db s.db.SetSync(abciResponsesKey, abciResponses.Bytes()) - - // save the tx results using the TxIndexer - // NOTE: these may be overwriting, but the values should be the same. - batch := txindexer.NewBatch() - for i, d := range abciResponses.DeliverTx { - tx := abciResponses.txs[i] - batch.Index(tx.Hash(), types.TxResult{uint64(abciResponses.height), uint32(i), *d}) - } - s.TxIndexer.Batch(batch) } func (s *State) LoadABCIResponses() *ABCIResponses { @@ -115,10 +105,10 @@ func (s *State) LoadABCIResponses() *ABCIResponses { buf := s.db.Get(abciResponsesKey) if len(buf) != 0 { r, n, err := bytes.NewReader(buf), new(int), new(error) - wire.ReadBinaryPtr(&abciResponses.EndBlock.Diffs, r, 0, n, err) + wire.ReadBinaryPtr(abciResponses, r, 0, n, err) if *err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED - Exit(Fmt("Data has been corrupted or its spec has changed: %v\n", *err)) + Exit(Fmt("LoadABCIResponses: Data has been corrupted or its spec has changed: %v\n", *err)) } // TODO: ensure that buf is completely read. } @@ -191,25 +181,26 @@ func GetState(config cfg.Config, stateDB dbm.DB) *State { // ABCIResponses holds intermediate state during block processing type ABCIResponses struct { - height int - txs types.Txs // for reference later + Height int - DeliverTx []*abci.ResponseDeliverTx // results of the txs, populated in the proxyCb - EndBlock abci.ResponseEndBlock // changes to the validator set + DeliverTx []*abci.ResponseDeliverTx + EndBlock abci.ResponseEndBlock + + txs types.Txs // for reference later } func NewABCIResponses(block *types.Block) *ABCIResponses { return &ABCIResponses{ - height: block.Height, - txs: block.Data.Txs, + Height: block.Height, DeliverTx: make([]*abci.ResponseDeliverTx, block.NumTxs), + txs: block.Data.Txs, } } -// Serialize the list of validators +// Serialize the ABCIResponse func (a *ABCIResponses) Bytes() []byte { buf, n, err := new(bytes.Buffer), new(int), new(error) - wire.WriteBinary(a.EndBlock, buf, n, err) + wire.WriteBinary(*a, buf, n, err) if *err != nil { PanicCrisis(*err) } diff --git a/state/state_test.go b/state/state_test.go index a534cb695..dca83e801 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -1,8 +1,12 @@ package state import ( + "fmt" "testing" + "github.com/stretchr/testify/assert" + abci "github.com/tendermint/abci/types" + "github.com/tendermint/go-crypto" dbm "github.com/tendermint/go-db" "github.com/tendermint/tendermint/config/tendermint_test" ) @@ -40,3 +44,30 @@ func TestStateSaveLoad(t *testing.T) { t.Fatal("expected state and its copy to be identical. got %v\n expected %v\n", loadedState, state) } } + +func TestABCIResponsesSaveLoad(t *testing.T) { + assert := assert.New(t) + + config := tendermint_test.ResetConfig("state_") + stateDB := dbm.NewDB("state", config.GetString("db_backend"), config.GetString("db_dir")) + state := GetState(config, stateDB) + + state.LastBlockHeight += 1 + + // build mock responses + block := makeBlock(2, state) + abciResponses := NewABCIResponses(block) + abciResponses.DeliverTx[0] = &abci.ResponseDeliverTx{Data: []byte("foo")} + abciResponses.DeliverTx[1] = &abci.ResponseDeliverTx{Data: []byte("bar"), Log: "ok"} + abciResponses.EndBlock = abci.ResponseEndBlock{Diffs: []*abci.Validator{ + { + PubKey: crypto.GenPrivKeyEd25519().PubKey().Bytes(), + Power: 10, + }, + }} + abciResponses.txs = nil + + state.SaveABCIResponses(abciResponses) + abciResponses2 := state.LoadABCIResponses() + assert.Equal(abciResponses, abciResponses2, fmt.Sprintf("ABCIResponses don't match: Got %v, Expected %v", abciResponses2, abciResponses)) +} From 935f70a346b2c76db8d4806d99be6660bea544a2 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 15 Apr 2017 01:33:30 -0400 Subject: [PATCH 66/84] comments and cleanup --- consensus/replay.go | 28 ++++++++++++++-------------- consensus/state.go | 13 ++++++++++--- state/execution.go | 10 +--------- state/state.go | 2 +- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 53f876084..0adb333c7 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -105,7 +105,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { // and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT). gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight)) if found { - return errors.New(Fmt("WAL should not contain height %d.", csHeight)) + return errors.New(Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight)) } if gr != nil { gr.Close() @@ -114,13 +114,13 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { // Search for last height marker gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1)) if err == io.EOF { - log.Warn("Replay: wal.group.Search returned EOF", "height", csHeight-1) + log.Warn("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1) return nil } else if err != nil { return err } if !found { - return errors.New(Fmt("WAL does not contain height %d.", csHeight)) + return errors.New(Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) } defer gr.Close() @@ -275,18 +275,18 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p } else if appBlockHeight == stateBlockHeight { // We haven't run Commit (both the state and app are one block behind), - // so ApplyBlock with the real app. + // so replayBlock with the real app. // NOTE: We could instead use the cs.WAL on cs.Start, // but we'd have to allow the WAL to replay a block that wrote it's ENDHEIGHT log.Info("Replay last block using real app") - return h.replayLastBlock(storeBlockHeight, proxyApp.Consensus()) + return h.replayBlock(storeBlockHeight, proxyApp.Consensus()) } else if appBlockHeight == storeBlockHeight { - // We ran Commit, but didn't save the state, so ApplyBlock with mock app + // We ran Commit, but didn't save the state, so replayBlock with mock app abciResponses := h.state.LoadABCIResponses() mockApp := newMockProxyApp(appHash, abciResponses) log.Info("Replay last block using mock app") - return h.replayLastBlock(storeBlockHeight, mockApp) + return h.replayBlock(storeBlockHeight, mockApp) } } @@ -295,18 +295,18 @@ func (h *Handshaker) ReplayBlocks(appHash []byte, appBlockHeight int, proxyApp p return nil, nil } -func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, useReplayFunc bool) ([]byte, error) { +func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, storeBlockHeight int, mutateState bool) ([]byte, error) { // App is further behind than it should be, so we need to replay blocks. // We replay all blocks from appBlockHeight+1. - // If useReplayFunc == true, stop short of the last block - // so it can be replayed using the WAL in ReplayBlocks. // Note that we don't have an old version of the state, - // so we by-pass state validation using sm.ApplyBlock. + // so we by-pass state validation/mutation using sm.ApplyBlock. + // If mutateState == true, stop short of the last block + // so it can be replayed with a real state.ApplyBlock var appHash []byte var err error finalBlock := storeBlockHeight - if useReplayFunc { + if mutateState { finalBlock -= 1 } for i := appBlockHeight + 1; i <= finalBlock; i++ { @@ -320,7 +320,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store h.nBlocks += 1 } - if useReplayFunc { + if mutateState { // sync the final block return h.ReplayBlocks(appHash, finalBlock, proxyApp) } @@ -329,7 +329,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store } // ApplyBlock on the proxyApp with the last block. -func (h *Handshaker) replayLastBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) { +func (h *Handshaker) replayBlock(height int, proxyApp proxy.AppConnConsensus) ([]byte, error) { mempool := types.MockMempool{} var eventCache types.Fireable // nil diff --git a/consensus/state.go b/consensus/state.go index f1423e2f2..6ff97ddc8 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1217,8 +1217,12 @@ func (cs *ConsensusState) finalizeCommit(height int) { fail.Fail() // XXX // Finish writing to the WAL for this height. - // NOTE: ConsensusState should not be started again - // until we successfully call ApplyBlock (eg. in Handshake after restart) + // NOTE: If we fail before writing this, we'll never write it, + // and just recover by running ApplyBlock in the Handshake. + // If we moved it before persisting the block, we'd have to allow + // WAL replay for blocks with an #ENDHEIGHT + // As is, ConsensusState should not be started again + // until we successfully call ApplyBlock (ie. here or in Handshake after restart) if cs.wal != nil { cs.wal.writeEndHeight(height) } @@ -1244,9 +1248,10 @@ func (cs *ConsensusState) finalizeCommit(height int) { // Fire event for new block. // NOTE: If we fail before firing, these events will never fire // - // Some options (for which they may fire more than once. I guess that's fine): + // TODO: Either // * Fire before persisting state, in ApplyBlock // * Fire on start up if we haven't written any new WAL msgs + // Both options mean we may fire more than once. Is that fine ? types.FireEventNewBlock(cs.evsw, types.EventDataNewBlock{block}) types.FireEventNewBlockHeader(cs.evsw, types.EventDataNewBlockHeader{block.Header}) eventCache.Flush() @@ -1256,6 +1261,8 @@ func (cs *ConsensusState) finalizeCommit(height int) { // NewHeightStep! cs.updateToState(stateCopy) + fail.Fail() // XXX + // cs.StartTime is already set. // Schedule Round0 to start soon. cs.scheduleRound0(&cs.RoundState) diff --git a/state/execution.go b/state/execution.go index d978c1956..959231eed 100644 --- a/state/execution.go +++ b/state/execution.go @@ -88,19 +88,14 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo return nil, err } - fail.Fail() // XXX - // Run txs of block for _, tx := range block.Txs { - fail.FailRand(len(block.Txs)) // XXX proxyAppConn.DeliverTxAsync(tx) if err := proxyAppConn.Error(); err != nil { return nil, err } } - fail.Fail() // XXX - // End block abciResponses.EndBlock, err = proxyAppConn.EndBlockSync(uint64(block.Height)) if err != nil { @@ -108,8 +103,6 @@ func execBlockOnProxyApp(eventCache types.Fireable, proxyAppConn proxy.AppConnCo return nil, err } - fail.Fail() // XXX - valDiff := abciResponses.EndBlock.Diffs log.Info("Executed block", "height", block.Height, "valid txs", validTxs, "invalid txs", invalidTxs) @@ -292,9 +285,8 @@ func (s *State) indexTxs(abciResponses *ABCIResponses) { s.TxIndexer.Batch(batch) } -// Apply and commit a block, but without all the state validation. +// Apply and commit a block on the proxyApp without validating or mutating the state // Returns the application root hash (result of abci.Commit) -// TODO handle abciResponses func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) { var eventCache types.Fireable // nil _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) diff --git a/state/state.go b/state/state.go index eb7159e24..7d0f24558 100644 --- a/state/state.go +++ b/state/state.go @@ -186,7 +186,7 @@ type ABCIResponses struct { DeliverTx []*abci.ResponseDeliverTx EndBlock abci.ResponseEndBlock - txs types.Txs // for reference later + txs types.Txs // reference for indexing results by hash } func NewABCIResponses(block *types.Block) *ABCIResponses { From f9d00967444e4ca4c716dec171e71ae8c892f935 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sat, 15 Apr 2017 02:02:33 -0400 Subject: [PATCH 67/84] support #HEIGHT based WAL --- consensus/replay.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 0adb333c7..f43c5ecad 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -115,12 +115,31 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1)) if err == io.EOF { log.Warn("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", csHeight-1) - return nil + // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead + // TODO (0.10.0): remove this + gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) + if err == io.EOF { + log.Warn("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight) + return nil + } else if err != nil { + return err + } } else if err != nil { return err } if !found { - return errors.New(Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) + // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead + // TODO (0.10.0): remove this + gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) + if err == io.EOF { + log.Warn("Replay: wal.group.Search returned EOF", "#HEIGHT", csHeight) + return nil + } else if err != nil { + return err + } + + // TODO (0.10.0): uncomment + // return errors.New(Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) } defer gr.Close() From cd9e9e9f455328db4399ae191e147bdb5f79f885 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 17 Apr 2017 15:24:44 -0700 Subject: [PATCH 68/84] s/ExecBlock/ValExecBlock/g; s/sm.ApplyBlock/sm.ExecCommitBlock/g --- consensus/replay.go | 4 ++-- state/execution.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index f43c5ecad..441b27018 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -318,7 +318,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store // App is further behind than it should be, so we need to replay blocks. // We replay all blocks from appBlockHeight+1. // Note that we don't have an old version of the state, - // so we by-pass state validation/mutation using sm.ApplyBlock. + // so we by-pass state validation/mutation using sm.ExecCommitBlock. // If mutateState == true, stop short of the last block // so it can be replayed with a real state.ApplyBlock @@ -331,7 +331,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store for i := appBlockHeight + 1; i <= finalBlock; i++ { log.Info("Applying block", "height", i) block := h.store.LoadBlock(i) - appHash, err = sm.ApplyBlock(proxyApp.Consensus(), block) + appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block) if err != nil { return nil, err } diff --git a/state/execution.go b/state/execution.go index 959231eed..4874fdc70 100644 --- a/state/execution.go +++ b/state/execution.go @@ -16,10 +16,10 @@ import ( //-------------------------------------------------- // Execute the block -// ExecBlock executes the block, but does NOT mutate State. +// ValExecBlock executes the block, but does NOT mutate State. // + validates the block // + executes block.Txs on the proxyAppConn -func (s *State) ExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) (*ABCIResponses, error) { +func (s *State) ValExecBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block) (*ABCIResponses, error) { // Validate the block. if err := s.validateBlock(block); err != nil { return nil, ErrInvalidBlock(err) @@ -201,15 +201,15 @@ func (s *State) validateBlock(block *types.Block) error { } //----------------------------------------------------------------------------- -// ApplyBlock executes the block, updates state w/ ABCI responses, +// ApplyBlock validates & executes the block, updates state w/ ABCI responses, // then commits and updates the mempool atomically, then saves state. // Transaction results are optionally indexed. -// Execute and commit block against app, save block and state +// Validate, execute, and commit block against app, save block and state func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConnConsensus, block *types.Block, partsHeader types.PartSetHeader, mempool types.Mempool) error { - abciResponses, err := s.ExecBlock(eventCache, proxyAppConn, block) + abciResponses, err := s.ValExecBlock(eventCache, proxyAppConn, block) if err != nil { return fmt.Errorf("Exec failed for application: %v", err) } @@ -285,9 +285,9 @@ func (s *State) indexTxs(abciResponses *ABCIResponses) { s.TxIndexer.Batch(batch) } -// Apply and commit a block on the proxyApp without validating or mutating the state +// Exec and commit a block on the proxyApp without validating or mutating the state // Returns the application root hash (result of abci.Commit) -func ApplyBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) { +func ExecCommitBlock(appConnConsensus proxy.AppConnConsensus, block *types.Block) ([]byte, error) { var eventCache types.Fireable // nil _, err := execBlockOnProxyApp(eventCache, appConnConsensus, block) if err != nil { From 2ba3656ffd50213894eb189557069fd477f85652 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 17 Apr 2017 21:10:38 -0400 Subject: [PATCH 69/84] wal: gr.Close() --- consensus/replay.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 441b27018..0ebbb735f 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -104,12 +104,12 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { // NOTE: This is just a sanity check. As far as we know things work fine without it, // and Handshake could reuse ConsensusState if it weren't for this check (since we can crash after writing ENDHEIGHT). gr, found, err := cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight)) - if found { - return errors.New(Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight)) - } if gr != nil { gr.Close() } + if found { + return errors.New(Fmt("WAL should not contain #ENDHEIGHT %d.", csHeight)) + } // Search for last height marker gr, found, err = cs.wal.group.Search("#ENDHEIGHT: ", makeHeightSearchFunc(csHeight-1)) @@ -128,6 +128,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { return err } if !found { + gr.Close() // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead // TODO (0.10.0): remove this gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) From cb2ed5bb7c293c5dfba9b88325caf1f38dc0c46d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 17 Apr 2017 21:14:35 -0400 Subject: [PATCH 70/84] fixes from review --- consensus/replay.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index 0ebbb735f..ab13d86f8 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -320,8 +320,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store // We replay all blocks from appBlockHeight+1. // Note that we don't have an old version of the state, // so we by-pass state validation/mutation using sm.ExecCommitBlock. - // If mutateState == true, stop short of the last block - // so it can be replayed with a real state.ApplyBlock + // If mutateState == true, the final block is replayed with h.replayBlock() var appHash []byte var err error @@ -342,7 +341,7 @@ func (h *Handshaker) replayBlocks(proxyApp proxy.AppConns, appBlockHeight, store if mutateState { // sync the final block - return h.ReplayBlocks(appHash, finalBlock, proxyApp) + return h.replayBlock(storeBlockHeight, proxyApp.Consensus()) } return appHash, h.checkAppHash(appHash) From cf4074cc807d77f4042dff940d6710c68e427247 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 17 Apr 2017 19:40:40 -0700 Subject: [PATCH 71/84] defer gr.Close() fixes --- consensus/replay.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/consensus/replay.go b/consensus/replay.go index ab13d86f8..bd0975f4d 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -126,9 +126,10 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { } } else if err != nil { return err + } else { + defer gr.Close() } if !found { - gr.Close() // if we upgraded from 0.9 to 0.9.1, we may have #HEIGHT instead // TODO (0.10.0): remove this gr, found, err = cs.wal.group.Search("#HEIGHT: ", makeHeightSearchFunc(csHeight)) @@ -137,12 +138,13 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { return nil } else if err != nil { return err + } else { + defer gr.Close() } // TODO (0.10.0): uncomment // return errors.New(Fmt("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1)) } - defer gr.Close() log.Notice("Catchup by replaying consensus messages", "height", csHeight) From 29a893b193f6e94a0b10c169d110d0ae48ddcd27 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 10:44:13 -0400 Subject: [PATCH 72/84] update comment --- state/state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/state/state.go b/state/state.go index 7d0f24558..086b0e710 100644 --- a/state/state.go +++ b/state/state.go @@ -132,7 +132,8 @@ func (s *State) Bytes() []byte { // after running EndBlock func (s *State) SetBlockAndValidators(header *types.Header, blockPartsHeader types.PartSetHeader, abciResponses *ABCIResponses) { - // copy the valset + // copy the valset so we can apply changes from EndBlock + // and update s.LastValidators and s.Validators prevValSet := s.Validators.Copy() nextValSet := prevValSet.Copy() From 52d03d0071c737e1048ea4bd8ef12f4cd490211c Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 21:35:00 -0400 Subject: [PATCH 73/84] post rebase fixes --- state/execution.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/state/execution.go b/state/execution.go index 4874fdc70..0b1aff699 100644 --- a/state/execution.go +++ b/state/execution.go @@ -233,12 +233,6 @@ func (s *State) ApplyBlock(eventCache types.Fireable, proxyAppConn proxy.AppConn return fmt.Errorf("Commit failed for application: %v", err) } - batch := txindex.NewBatch(block.NumTxs) - for _, r := range txResults { - batch.Add(*r) - } - s.TxIndexer.AddBatch(batch) - fail.Fail() // XXX // save the state @@ -277,12 +271,17 @@ func (s *State) CommitStateUpdateMempool(proxyAppConn proxy.AppConnConsensus, bl func (s *State) indexTxs(abciResponses *ABCIResponses) { // save the tx results using the TxIndexer // NOTE: these may be overwriting, but the values should be the same. - batch := txindexer.NewBatch() + batch := txindex.NewBatch(len(abciResponses.DeliverTx)) for i, d := range abciResponses.DeliverTx { tx := abciResponses.txs[i] - batch.Index(tx.Hash(), types.TxResult{uint64(abciResponses.Height), uint32(i), *d}) + batch.Add(types.TxResult{ + Height: uint64(abciResponses.Height), + Index: uint32(i), + Tx: tx, + Result: *d, + }) } - s.TxIndexer.Batch(batch) + s.TxIndexer.AddBatch(batch) } // Exec and commit a block on the proxyApp without validating or mutating the state From aecf860b26d014820c8557ed05089a42b012eb68 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 22:27:24 -0400 Subject: [PATCH 74/84] update glide --- glide.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index f0ede402e..97ee77f47 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 -updated: 2017-04-14T17:17:31.933202871-04:00 +updated: 2017-04-18T22:25:07.879254903-04:00 imports: - name: github.com/btcsuite/btcd version: b8df516b4b267acf2de46be593a9d948d1d2c420 @@ -68,7 +68,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: 31eafe8f8eba6b8817edd74df399f508540da528 + version: f2851145792559491cf5a3f8fc412a91594deaf2 subpackages: - client - example/counter @@ -85,7 +85,7 @@ imports: - name: github.com/tendermint/go-clist version: 3baa390bbaf7634251c42ad69a8682e7e3990552 - name: github.com/tendermint/go-common - version: 714fdaee3bb3f8670e721a75c5ddda8787b256dd + version: f9e3db037330c8a8d61d3966de8473eaf01154fa subpackages: - test - name: github.com/tendermint/go-config @@ -95,7 +95,7 @@ imports: - name: github.com/tendermint/go-data version: e7fcc6d081ec8518912fcdc103188275f83a3ee5 - name: github.com/tendermint/go-db - version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72 + version: 9643f60bc2578693844aacf380a7c32e4c029fee - name: github.com/tendermint/go-events version: f8ffbfb2be3483e9e7927495590a727f51c0c11f - name: github.com/tendermint/go-flowrate @@ -107,7 +107,7 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: c39e001a957caf768f06c85c840debb8282c3aaa + version: e6b7e66bbed094f9ff44f2f7ca31a10078faee75 subpackages: - upnp - name: github.com/tendermint/go-rpc @@ -117,7 +117,7 @@ imports: - server - types - name: github.com/tendermint/go-wire - version: 50889e2b4a9ba65b67be86a486f25853d514b937 + version: c1c9a57ab8038448ddea1714c0698f8051e5748c - name: github.com/tendermint/log15 version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6 subpackages: From 480b24d9f04b9f0bf547b1800e8f3a244c930498 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 23:05:46 -0400 Subject: [PATCH 75/84] CHANGELOG [ci skip] --- CHANGELOG.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ebcfd364..a722d4548 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.9.1 (April 18, 2017) + +FEATURES: + +- Transaction indexing - txs are indexed by their hash using a simple key-value store; easily extended to more advanced indexers +- New `/tx?hash=X` endpoint to query for transactions and their DeliverTx result by hash. Optionally returns a proof of the tx's inclusion in the block + +IMPROVEMENTS: + +- CLI now uses Cobra framework +- TMROOT is now TMHOME (TMROOT will stop working in 0.10.0) +- `/broadcast_tx_XXX` also returns the TxID (can be used to query for the tx) +- `/broadcast_tx_commit` also returns the height the block was committed in +- ABCIResponses struct persisted to disk before calling Commit; makes handshake replay much cleaner +- WAL uses #ENDHEIGHT instead of #HEIGHT (#HEIGHT will stop working in 0.10.0) +- Peers included via `--seeds`, under `seeds` in the config, or in `/dial_seeds` are now persistent, and will be reconnected to if the connection breaks + +BUG FIXES: + +- Fix bug in fast-sync where we stop syncing after a peer is removed, even if they're re-added later +- Fix handshake replay to handle validator set changes and results of DeliverTx when we crash after app.Commit but before state.Save() + ## 0.9.0 (March 6, 2017) BREAKING CHANGES: From 083fe959e25421fca3d41298d9111167a3b47122 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 23:14:10 -0400 Subject: [PATCH 76/84] Update README [ci skip] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3fb5d8645..24b3aef2e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Branch | Tests | Coverage | Report Card develop | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/develop.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/develop) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/develop/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/develop)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/develop) master | [![CircleCI](https://circleci.com/gh/tendermint/tendermint/tree/master.svg?style=shield)](https://circleci.com/gh/tendermint/tendermint/tree/master) | [![codecov](https://codecov.io/gh/tendermint/tendermint/branch/master/graph/badge.svg)](https://codecov.io/gh/tendermint/tendermint) | [![Go Report Card](https://goreportcard.com/badge/github.com/tendermint/tendermint/tree/master)](https://goreportcard.com/report/github.com/tendermint/tendermint/tree/master) -_NOTE: This is yet pre-alpha non-production-quality software._ +_NOTE: This is alpha software. Please contact us if you intend to run it in production._ Tendermint Core is Byzantine Fault Tolerant (BFT) middleware that takes a state transition machine, written in any programming language, and securely replicates it on many machines. From a75353da6dcd8d01e24ebff026cbbd0cbe35acc4 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Tue, 18 Apr 2017 23:21:57 -0400 Subject: [PATCH 77/84] version bump --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 074876b9a..cd1726092 100644 --- a/version/version.go +++ b/version/version.go @@ -2,6 +2,6 @@ package version const Maj = "0" const Min = "9" -const Fix = "0" +const Fix = "1" -const Version = "0.9.0" +const Version = "0.9.1" From 757a548edf77a9d3a54f6e7a2e07e696b6235977 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 20 Apr 2017 17:56:35 -0400 Subject: [PATCH 78/84] update glide --- glide.lock | 50 +++++++++++++++++++++++++++++------------------ test/test_libs.sh | 2 ++ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/glide.lock b/glide.lock index 97ee77f47..25530268e 100644 --- a/glide.lock +++ b/glide.lock @@ -1,14 +1,12 @@ hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 -updated: 2017-04-18T22:25:07.879254903-04:00 +updated: 2017-04-20T17:44:26.582287887-04:00 imports: - name: github.com/btcsuite/btcd - version: b8df516b4b267acf2de46be593a9d948d1d2c420 + version: 4b348c1d33373d672edd83fc576892d0e46686d2 subpackages: - btcec -- name: github.com/btcsuite/fastsha256 - version: 637e656429416087660c84436a2a035d69d54e2e - name: github.com/BurntSushi/toml - version: 99064174e013895bbd9b025c31100bd1d9b590ca + version: b26d9c308763d68093482582cea63d69be07a0f0 - name: github.com/davecgh/go-spew version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: @@ -22,9 +20,10 @@ imports: subpackages: - proto - name: github.com/golang/protobuf - version: 69b215d01a5606c843240eab4937eab3acee6530 + version: 2bba0603135d7d7f5cb73b2125beeda19c09f4ef subpackages: - proto + - ptypes/any - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket @@ -34,9 +33,9 @@ imports: - name: github.com/jmhodges/levigo version: c42d9e0ca023e2198120196f842701bb4c55d7b9 - name: github.com/mattn/go-colorable - version: 9fdad7c47650b7d2e1da50644c1f4ba7f172f252 + version: ded68f7a9561c023e790de24279db7ebf473ea80 - name: github.com/mattn/go-isatty - version: 56b76bdf51f7708750eac80fa38b952bb9f32639 + version: fc9e8d8ef48496124e79ae0df75490096eccf6fe - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/pmezard/go-difflib @@ -44,16 +43,16 @@ imports: subpackages: - difflib - name: github.com/spf13/cobra - version: fcd0c5a1df88f5d6784cb4feead962c3f3d0b66c + version: 10f6b9d7e1631a54ad07c5c0fb71c28a1abfd3c2 - name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 + version: 2300d0f8576fe575f71aaa5b9bbe4e1b0dc2eb51 - name: github.com/stretchr/testify version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require - name: github.com/syndtr/goleveldb - version: 3c5717caf1475fd25964109a0fc640bd150fce43 + version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 subpackages: - leveldb - leveldb/cache @@ -68,7 +67,7 @@ imports: - leveldb/table - leveldb/util - name: github.com/tendermint/abci - version: f2851145792559491cf5a3f8fc412a91594deaf2 + version: 56e13d87f4e3ec1ea756957d6b23caa6ebcf0998 subpackages: - client - example/counter @@ -91,7 +90,7 @@ imports: - name: github.com/tendermint/go-config version: 620dcbbd7d587cf3599dedbf329b64311b0c307a - name: github.com/tendermint/go-crypto - version: 750b25c47a5782f5f2b773ed9e706cb82b3ccef4 + version: 0ca2c6fdb0706001ca4c4b9b80c9f428e8cf39da - name: github.com/tendermint/go-data version: e7fcc6d081ec8518912fcdc103188275f83a3ee5 - name: github.com/tendermint/go-db @@ -107,11 +106,11 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: e6b7e66bbed094f9ff44f2f7ca31a10078faee75 + version: 17124989a93774833df33107fbf17157a7f8ef31 subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 4671c44b2d124f7f6f6243dbfbf4ae2bf42ee809 + version: 1a42f946dc6bcd88f9f58c7f2fb86f785584d793 subpackages: - client - server @@ -129,7 +128,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 1f22c0103821b9390939b6776727195525381532 + version: 0242f07995e684be54f2a2776327141acf1cef91 subpackages: - curve25519 - nacl/box @@ -140,7 +139,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: d379faa25cbdc04d653984913a2ceb43b0bc46d7 + version: 5602c733f70afc6dcec6766be0d5034d4c4f14de subpackages: - context - http2 @@ -150,20 +149,33 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: 50c6bc5e4292a1d4e65c6e9be5f53be28bcbe28e + version: f3918c30c5c2cb527c0b071a27c35120a6c0719a subpackages: - unix +- name: golang.org/x/text + version: 19e3104b43db45fca0303f489a9536087b184802 + subpackages: + - secure/bidirule + - transform + - unicode/bidi + - unicode/norm +- name: google.golang.org/genproto + version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 + subpackages: + - googleapis/rpc/status - name: google.golang.org/grpc - version: 7b399ed358736bc5522021cdc7d79a8ee9ac6f98 + version: 6914ab1e338c92da4218a23d27fcd03d0ad78d46 subpackages: - codes - credentials - grpclog - internal + - keepalive - metadata - naming - peer - stats + - status - tap - transport testImports: [] diff --git a/test/test_libs.sh b/test/test_libs.sh index 4a531bf35..d08a4659c 100644 --- a/test/test_libs.sh +++ b/test/test_libs.sh @@ -12,6 +12,8 @@ fi # libs we depend on #################### +# some libs are tested with go, others with make +# TODO: should be all make (post repo merge) LIBS_GO_TEST=(go-clist go-common go-config go-crypto go-db go-events go-merkle go-p2p) LIBS_MAKE_TEST=(go-rpc go-wire abci) From f5b77d50b5caaa5c0f9acf758fde234e2c3d06ad Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 20 Apr 2017 17:56:49 -0400 Subject: [PATCH 79/84] fix setting log level --- cmd/tendermint/commands/root.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index cee5b6604..0cbaa289f 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -3,13 +3,14 @@ package commands import ( "github.com/spf13/cobra" - cfg "github.com/tendermint/go-config" "github.com/tendermint/go-logger" tmcfg "github.com/tendermint/tendermint/config/tendermint" ) -var config cfg.Config -var log = logger.New("module", "main") +var ( + config = tmcfg.GetConfig("") + log = logger.New("module", "main") +) //global flag var logLevel string @@ -18,18 +19,13 @@ var RootCmd = &cobra.Command{ Use: "tendermint", Short: "Tendermint Core (BFT Consensus) in Go", PersistentPreRun: func(cmd *cobra.Command, args []string) { + // set the log level in the config and logger config.Set("log_level", logLevel) + logger.SetLogLevel(logLevel) }, } func init() { - - // Get configuration - config = tmcfg.GetConfig("") - //parse flag and set config RootCmd.PersistentFlags().StringVar(&logLevel, "log_level", config.GetString("log_level"), "Log level") - - // set the log level - logger.SetLogLevel(config.GetString("log_level")) } From 7cf773e2d37b2b5a08bc94fb125cfd346b834824 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Thu, 20 Apr 2017 18:22:42 -0400 Subject: [PATCH 80/84] cli: testnet cmd inits files for testnet --- CHANGELOG.md | 1 + cmd/tendermint/commands/testnet.go | 93 ++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 cmd/tendermint/commands/testnet.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a722d4548..7be3fbc6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ FEATURES: - Transaction indexing - txs are indexed by their hash using a simple key-value store; easily extended to more advanced indexers - New `/tx?hash=X` endpoint to query for transactions and their DeliverTx result by hash. Optionally returns a proof of the tx's inclusion in the block +- `tendermint testnet` command initializes files for a testnet IMPROVEMENTS: diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go new file mode 100644 index 000000000..0a2e00ad0 --- /dev/null +++ b/cmd/tendermint/commands/testnet.go @@ -0,0 +1,93 @@ +package commands + +import ( + "fmt" + "path" + "time" + + "github.com/spf13/cobra" + + cmn "github.com/tendermint/go-common" + "github.com/tendermint/tendermint/types" +) + +var testnetFilesCmd = &cobra.Command{ + Use: "testnet", + Short: "Initialize files for a Tendermint testnet", + Run: testnetFiles, +} + +//flags +var ( + nValidators int + dataDir string +) + +func init() { + testnetFilesCmd.Flags().IntVar(&nValidators, "n", 4, + "Number of validators to initialize the testnet with") + testnetFilesCmd.Flags().StringVar(&dataDir, "dir", "mytestnet", + "Directory to store initialization data for the testnet") + + RootCmd.AddCommand(testnetFilesCmd) +} + +func testnetFiles(cmd *cobra.Command, args []string) { + + genVals := make([]types.GenesisValidator, nValidators) + + // Initialize core dir and priv_validator.json's + for i := 0; i < nValidators; i++ { + mach := cmn.Fmt("mach%d", i) + err := initMachCoreDirectory(dataDir, mach) + if err != nil { + cmn.Exit(err.Error()) + } + // Read priv_validator.json to populate vals + privValFile := path.Join(dataDir, mach, "priv_validator.json") + privVal := types.LoadPrivValidator(privValFile) + genVals[i] = types.GenesisValidator{ + PubKey: privVal.PubKey, + Amount: 1, + Name: mach, + } + } + + // Generate genesis doc from generated validators + genDoc := &types.GenesisDoc{ + GenesisTime: time.Now(), + ChainID: "chain-" + cmn.RandStr(6), + Validators: genVals, + } + + // Write genesis file. + for i := 0; i < nValidators; i++ { + mach := cmn.Fmt("mach%d", i) + genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json")) + } + + fmt.Println(cmn.Fmt("Successfully initialized %v node directories", nValidators)) +} + +// Initialize per-machine core directory +func initMachCoreDirectory(base, mach string) error { + dir := path.Join(base, mach) + err := cmn.EnsureDir(dir, 0777) + if err != nil { + return err + } + + // Create priv_validator.json file if not present + ensurePrivValidator(path.Join(dir, "priv_validator.json")) + return nil + +} + +func ensurePrivValidator(file string) { + if cmn.FileExists(file) { + return + } + privValidator := types.GenPrivValidator() + privValidator.SetFile(file) + privValidator.Save() +} From 5e5fb37774664036cb51079af5291a4aec48ace4 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 21 Apr 2017 18:39:02 +0300 Subject: [PATCH 81/84] rename TxID to Hash --- CHANGELOG.md | 2 +- rpc/client/rpc_test.go | 4 ++-- rpc/core/mempool.go | 10 +++++----- rpc/core/types/responses.go | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a722d4548..7642668e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ IMPROVEMENTS: - CLI now uses Cobra framework - TMROOT is now TMHOME (TMROOT will stop working in 0.10.0) -- `/broadcast_tx_XXX` also returns the TxID (can be used to query for the tx) +- `/broadcast_tx_XXX` also returns the Hash (can be used to query for the tx) - `/broadcast_tx_commit` also returns the height the block was committed in - ABCIResponses struct persisted to disk before calling Commit; makes handshake replay much cleaner - WAL uses #ENDHEIGHT instead of #HEIGHT (#HEIGHT will stop working in 0.10.0) diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index 18f0f1aab..d9f5d3796 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -133,8 +133,8 @@ func TestAppCalls(t *testing.T) { } // make sure we can lookup the tx with proof - // ptx, err := c.Tx(bres.TxID, true) - ptx, err := c.Tx(bres.TxID, true) + // ptx, err := c.Tx(bres.Hash, true) + ptx, err := c.Tx(bres.Hash, true) require.Nil(err, "%d: %+v", i, err) assert.Equal(txh, ptx.Height) assert.Equal(types.Tx(tx), ptx.Tx) diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index c4fc3b1a1..4da83a1f6 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -18,7 +18,7 @@ func BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { if err != nil { return nil, fmt.Errorf("Error broadcasting transaction: %v", err) } - return &ctypes.ResultBroadcastTx{TxID: tx.Hash()}, nil + return &ctypes.ResultBroadcastTx{Hash: tx.Hash()}, nil } // Returns with the response from CheckTx @@ -36,7 +36,7 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { Code: r.Code, Data: r.Data, Log: r.Log, - TxID: tx.Hash(), + Hash: tx.Hash(), }, nil } @@ -68,7 +68,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: nil, - TxID: tx.Hash(), + Hash: tx.Hash(), }, nil } @@ -88,7 +88,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: deliverTxR, - TxID: tx.Hash(), + Hash: tx.Hash(), Height: deliverTxRes.Height, }, nil case <-timer.C: @@ -96,7 +96,7 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { return &ctypes.ResultBroadcastTxCommit{ CheckTx: checkTxR, DeliverTx: nil, - TxID: tx.Hash(), + Hash: tx.Hash(), }, fmt.Errorf("Timed out waiting for transaction to be included in a block") } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 940aa433f..7cab8535d 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -84,13 +84,13 @@ type ResultBroadcastTx struct { Data []byte `json:"data"` Log string `json:"log"` - TxID []byte `json:"tx_id"` + Hash []byte `json:"hash"` } type ResultBroadcastTxCommit struct { CheckTx *abci.ResponseCheckTx `json:"check_tx"` DeliverTx *abci.ResponseDeliverTx `json:"deliver_tx"` - TxID []byte `json:"tx_id"` + Hash []byte `json:"hash"` Height int `json:"height"` } From 53e2b9693a179d0add569170f4ba0e7696d22b06 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 21 Apr 2017 12:46:12 -0400 Subject: [PATCH 82/84] export ResetAll cmd --- cmd/tendermint/commands/reset_priv_validator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 217c387a5..5e3172a52 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -11,7 +11,7 @@ import ( var resetAllCmd = &cobra.Command{ Use: "unsafe_reset_all", Short: "(unsafe) Remove all the data and WAL, reset this node's validator", - Run: resetAll, + Run: ResetAll, } var resetPrivValidatorCmd = &cobra.Command{ @@ -27,7 +27,7 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. -func resetAll(cmd *cobra.Command, args []string) { +func ResetAll(cmd *cobra.Command, args []string) { resetPrivValidator(cmd, args) os.RemoveAll(config.GetString("db_dir")) os.Remove(config.GetString("cs_wal_file")) From e8cad948e366cd1d0a9ebef642073f4ade9899e9 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 21 Apr 2017 12:54:53 -0400 Subject: [PATCH 83/84] cli: ResetAll doesnt depend on cobra --- .../commands/reset_priv_validator.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 5e3172a52..b1fb334bc 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -5,13 +5,15 @@ import ( "github.com/spf13/cobra" + cfg "github.com/tendermint/go-config" + "github.com/tendermint/log15" "github.com/tendermint/tendermint/types" ) var resetAllCmd = &cobra.Command{ Use: "unsafe_reset_all", Short: "(unsafe) Remove all the data and WAL, reset this node's validator", - Run: ResetAll, + Run: resetAll, } var resetPrivValidatorCmd = &cobra.Command{ @@ -27,15 +29,23 @@ func init() { // XXX: this is totally unsafe. // it's only suitable for testnets. -func ResetAll(cmd *cobra.Command, args []string) { - resetPrivValidator(cmd, args) - os.RemoveAll(config.GetString("db_dir")) - os.Remove(config.GetString("cs_wal_file")) +func resetAll(cmd *cobra.Command, args []string) { + ResetAll(config, log) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { + ResetPrivValidator(config, log) +} + +// Exported so other CLI tools can use it +func ResetAll(c cfg.Config, l log15.Logger) { + ResetPrivValidator(c, l) + os.RemoveAll(c.GetString("db_dir")) +} + +func ResetPrivValidator(c cfg.Config, l log15.Logger) { // Get PrivValidator var privValidator *types.PrivValidator privValidatorFile := config.GetString("priv_validator_file") From 7f20eca8927b798acd00cc291246a130f3b4ad2d Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 21 Apr 2017 13:09:55 -0400 Subject: [PATCH 84/84] update glide --- glide.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index 25530268e..42712c232 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: d9724aa287c40d1b3856b6565f09235d809c8b2f7c6537c04f597137c0d6cd26 -updated: 2017-04-20T17:44:26.582287887-04:00 +updated: 2017-04-21T13:09:25.708801802-04:00 imports: - name: github.com/btcsuite/btcd version: 4b348c1d33373d672edd83fc576892d0e46686d2 @@ -106,11 +106,11 @@ imports: - name: github.com/tendermint/go-merkle version: 714d4d04557fd068a7c2a1748241ce8428015a96 - name: github.com/tendermint/go-p2p - version: 17124989a93774833df33107fbf17157a7f8ef31 + version: e8f33a47846708269d373f9c8080613d6c4f66b2 subpackages: - upnp - name: github.com/tendermint/go-rpc - version: 1a42f946dc6bcd88f9f58c7f2fb86f785584d793 + version: 2c8df0ee6b60d8ac33662df13a4e358c679e02bf subpackages: - client - server @@ -128,7 +128,7 @@ imports: - client - testutil - name: golang.org/x/crypto - version: 0242f07995e684be54f2a2776327141acf1cef91 + version: 96846453c37f0876340a66a47f3f75b1f3a6cd2d subpackages: - curve25519 - nacl/box @@ -139,7 +139,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: 5602c733f70afc6dcec6766be0d5034d4c4f14de + version: c8c74377599bd978aee1cf3b9b63a8634051cec2 subpackages: - context - http2 @@ -149,7 +149,7 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: f3918c30c5c2cb527c0b071a27c35120a6c0719a + version: ea9bcade75cb975a0b9738936568ab388b845617 subpackages: - unix - name: golang.org/x/text