diff --git a/.circleci/config.yml b/.circleci/config.yml index cb2577908..8dac95a8e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,19 +130,6 @@ jobs: paths: - "profiles/*" - test_libs: - <<: *defaults - steps: - - attach_workspace: - at: /tmp/workspace - - restore_cache: - key: v1-pkg-cache - - restore_cache: - key: v1-tree-{{ .Environment.CIRCLE_SHA1 }} - - run: - name: Run tests - command: bash test/test_libs.sh - test_persistence: <<: *defaults steps: @@ -205,14 +192,6 @@ workflows: - test_cover: requires: - setup_dependencies - - test_libs: - filters: - branches: - only: - - develop - - master - requires: - - setup_dependencies - test_persistence: requires: - setup_abci diff --git a/.gitignore b/.gitignore index 7b7b746fb..e76fb1fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ test/logs coverage.txt docs/_build docs/tools +docs/abci-spec.rst *.log scripts/wal2json/wal2json diff --git a/CHANGELOG.md b/CHANGELOG.md index aea9b5450..c5dc06753 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ BREAKING CHANGES: - Upgrade consensus for more real-time use of evidence FEATURES: -- Peer reputation management - Use the chain as its own CA for nodes and validators - Tooling to run multiple blockchains/apps, possibly in a single process - State syncing (without transaction replay) @@ -25,10 +24,36 @@ BUG FIXES: - Graceful handling/recovery for apps that have non-determinism or fail to halt - Graceful handling/recovery for violations of safety, or liveness -## 0.17.1 (March 27th, 2018) +## 0.18.0 (April 6th, 2018) + +BREAKING: + +- [types] Merkle tree uses different encoding for varints (see tmlibs v0.8.0) +- [types] ValidtorSet.GetByAddress returns -1 if no validator found +- [p2p] require all addresses come with an ID no matter what +- [rpc] Listening address must contain tcp:// or unix:// prefix + +FEATURES: + +- [rpc] StartHTTPAndTLSServer (not used yet) +- [rpc] Include validator's voting power in `/status` +- [rpc] `/tx` and `/tx_search` responses now include the transaction hash +- [rpc] Include peer NodeIDs in `/net_info` + +IMPROVEMENTS: +- [config] trim whitespace from elements of lists (like `persistent_peers`) +- [rpc] `/tx_search` results are sorted by height +- [p2p] do not try to connect to ourselves (ok, maybe only once) +- [p2p] seeds respond with a bias towards good peers BUG FIXES: +- [rpc] fix subscribing using an abci.ResponseDeliverTx tag +- [rpc] fix tx_indexers matchRange +- [rpc] fix unsubscribing (see tmlibs v0.8.0) +## 0.17.1 (March 27th, 2018) + +BUG FIXES: - [types] Actually support `app_state` in genesis as `AppStateJSON` ## 0.17.0 (March 27th, 2018) diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index c00318fba..cf45ac2f1 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,8 +1,8 @@ -FROM alpine:3.6 +FROM alpine:3.7 # This is the release of tendermint to pull in. -ENV TM_VERSION 0.15.0 -ENV TM_SHA256SUM 71cc271c67eca506ca492c8b90b090132f104bf5dbfe0af2702a50886e88de17 +ENV TM_VERSION 0.17.1 +ENV TM_SHA256SUM d57008c63d2d9176861137e38ed203da486febf20ae7d388fb810a75afff8f24 # Tendermint will be looking for genesis file in /tendermint (unless you change # `genesis_file` in config.toml). You can put your config.toml and private @@ -26,7 +26,7 @@ RUN mkdir -p $DATA_ROOT && \ RUN apk add --no-cache bash curl jq RUN apk add --no-cache openssl && \ - wget https://s3-us-west-2.amazonaws.com/tendermint/binaries/tendermint/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ + wget https://github.com/tendermint/tendermint/releases/download/v${TM_VERSION}/tendermint_${TM_VERSION}_linux_amd64.zip && \ echo "${TM_SHA256SUM} tendermint_${TM_VERSION}_linux_amd64.zip" | sha256sum -c && \ unzip -d /bin tendermint_${TM_VERSION}_linux_amd64.zip && \ apk del openssl && \ diff --git a/DOCKER/Dockerfile.develop b/DOCKER/Dockerfile.develop index e6ee6607e..4f8ae0b2e 100644 --- a/DOCKER/Dockerfile.develop +++ b/DOCKER/Dockerfile.develop @@ -1,4 +1,4 @@ -FROM alpine:3.6 +FROM alpine:3.7 ENV DATA_ROOT /tendermint ENV TMHOME $DATA_ROOT diff --git a/DOCKER/README.md b/DOCKER/README.md index 24cd38a51..2fe3db866 100644 --- a/DOCKER/README.md +++ b/DOCKER/README.md @@ -1,6 +1,7 @@ # Supported tags and respective `Dockerfile` links -- `0.15.0`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) +- `0.17.1`, `latest` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/208ac32fa266657bd6c304e84ec828aa252bb0b8/DOCKER/Dockerfile) +- `0.15.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/170777300ea92dc21a8aec1abc16cb51812513a4/DOCKER/Dockerfile) - `0.13.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/a28b3fff49dce2fb31f90abb2fc693834e0029c2/DOCKER/Dockerfile) - `0.12.1` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/457c688346b565e90735431619ca3ca597ef9007/DOCKER/Dockerfile) - `0.12.0` [(Dockerfile)](https://github.com/tendermint/tendermint/blob/70d8afa6e952e24c573ece345560a5971bf2cc0e/DOCKER/Dockerfile) diff --git a/Gopkg.lock b/Gopkg.lock index 894d98ead..9461b319b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -105,7 +105,7 @@ "json/scanner", "json/token" ] - revision = "f40e974e75af4e271d97ce0fc917af5898ae7bda" + revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -167,8 +167,8 @@ ".", "mem" ] - revision = "bb8f1927f2a9d3ab41c9340aa034f6b803f4359c" - version = "v1.0.2" + revision = "63644898a8da0bc22138abf860edaf5277b6102e" + version = "v1.1.0" [[projects]] name = "github.com/spf13/cast" @@ -179,8 +179,8 @@ [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" - version = "v0.0.1" + revision = "a1f051bc3eba734da4772d60e2d677f47cf93ef4" + version = "v0.0.2" [[projects]] branch = "master" @@ -226,7 +226,7 @@ "leveldb/table", "leveldb/util" ] - revision = "169b1b37be738edb2813dab48c97a549bcf99bb5" + revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad" [[projects]] name = "github.com/tendermint/abci" @@ -283,8 +283,8 @@ "pubsub/query", "test" ] - revision = "24da7009c3d8c019b40ba4287495749e3160caca" - version = "v0.7.1" + revision = "2e24b64fc121dcdf1cabceab8dc2f7257675483c" + version = "0.8.1" [[projects]] branch = "master" @@ -299,7 +299,7 @@ "ripemd160", "salsa20/salsa" ] - revision = "88942b9c40a4c9d203b82b3731787b672d6e809b" + revision = "b2aa35443fbc700ab74c586ae79b81c171851023" [[projects]] branch = "master" @@ -313,13 +313,13 @@ "lex/httplex", "trace" ] - revision = "6078986fec03a1dcc236c34816c71b0e05018fda" + revision = "b3c676e531a6dc479fa1b35ac961c13f5e2b4d2e" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "91ee8cde435411ca3f1cd365e8f20131aed4d0a1" + revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e" [[projects]] name = "golang.org/x/text" @@ -346,7 +346,7 @@ branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "f8c8703595236ae70fdf8789ecb656ea0bcdcf46" + revision = "35de2414665fc36f56b72d982c5af480d86de5ab" [[projects]] name = "google.golang.org/grpc" @@ -375,12 +375,12 @@ [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5" - version = "v2.1.1" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "4dca5dbd2d280d093d7c8fc423606ab86d6ad1b241b076a7716c2093b5a09231" + inputs-digest = "b7c02a311569ec5fe2197614444fb231ea60f3e65a11a20e318421f1752054d7" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 098d5ea9b..f6846793d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -82,9 +82,11 @@ source = "github.com/tendermint/go-amino" version = "~0.7.3" -[[constraint]] +[[override]] +# [[constraint]] name = "github.com/tendermint/tmlibs" - version = "~0.7.1" + version = "~0.8.1" + # branch = "develop" [[constraint]] name = "google.golang.org/grpc" diff --git a/Makefile b/Makefile index 851151eeb..3e51c834e 100644 --- a/Makefile +++ b/Makefile @@ -14,13 +14,13 @@ check: check_tools ensure_deps ### Build build: - go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ + CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint/ build_race: - go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint + CGO_ENABLED=0 go build -race $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' -o build/tendermint ./cmd/tendermint install: - go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint + CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags '$(BUILD_TAGS)' ./cmd/tendermint ######################################## ### Distribution @@ -119,11 +119,6 @@ test_integrations: make test_persistence make test_p2p -test_libs: - # checkout every github.com/tendermint dir and run its tests - # NOTE: on release-* or master branches only (set by Jenkins) - docker run --name run_libs -t tester bash test/test_libs.sh - test_release: @go test -tags release $(PACKAGES) @@ -186,4 +181,4 @@ metalinter_all: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_libs test_integrations test_release test100 vagrant_test fmt +.PHONY: check build build_race dist install check_tools get_tools update_tools get_vendor_deps draw_deps test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index 5ea20a785..6866e4d6c 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -34,14 +34,14 @@ var ( ) func init() { - LiteCmd.Flags().StringVar(&listenAddr, "laddr", ":8888", "Serve the proxy on the given port") + LiteCmd.Flags().StringVar(&listenAddr, "laddr", "tcp://localhost:8888", "Serve the proxy on the given address") LiteCmd.Flags().StringVar(&nodeAddr, "node", "tcp://localhost:46657", "Connect to a Tendermint node at this address") LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID") LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory") } func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) { - u, err := url.Parse(nodeAddr) + u, err := url.Parse(addr) if err != nil { return "", err } diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6ad92441b..0d50f9e4b 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -57,9 +57,8 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command { if err := n.Start(); err != nil { return fmt.Errorf("Failed to start node: %v", err) - } else { - logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) } + logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo()) // Trap signal, run forever. n.RunForever() diff --git a/codecov.yml b/codecov.yml index 6db0f15d5..b190853de 100644 --- a/codecov.yml +++ b/codecov.yml @@ -16,3 +16,8 @@ comment: require_changes: no require_base: no require_head: yes + +ignore: + - "docs" + - "DOCKER" + - "scripts" diff --git a/config/config.go b/config/config.go index 6b3f87b57..c87d56b3d 100644 --- a/config/config.go +++ b/config/config.go @@ -137,10 +137,6 @@ type BaseConfig struct { DBPath string `mapstructure:"db_dir"` } -func (c BaseConfig) ChainID() string { - return c.chainID -} - // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ @@ -161,32 +157,36 @@ func DefaultBaseConfig() BaseConfig { // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { - conf := DefaultBaseConfig() - conf.chainID = "tendermint_test" - conf.ProxyApp = "kvstore" - conf.FastSync = false - conf.DBBackend = "memdb" - return conf + cfg := DefaultBaseConfig() + cfg.chainID = "tendermint_test" + cfg.ProxyApp = "kvstore" + cfg.FastSync = false + cfg.DBBackend = "memdb" + return cfg +} + +func (cfg BaseConfig) ChainID() string { + return cfg.chainID } // GenesisFile returns the full path to the genesis.json file -func (b BaseConfig) GenesisFile() string { - return rootify(b.Genesis, b.RootDir) +func (cfg BaseConfig) GenesisFile() string { + return rootify(cfg.Genesis, cfg.RootDir) } // PrivValidatorFile returns the full path to the priv_validator.json file -func (b BaseConfig) PrivValidatorFile() string { - return rootify(b.PrivValidator, b.RootDir) +func (cfg BaseConfig) PrivValidatorFile() string { + return rootify(cfg.PrivValidator, cfg.RootDir) } // NodeKeyFile returns the full path to the node_key.json file -func (b BaseConfig) NodeKeyFile() string { - return rootify(b.NodeKey, b.RootDir) +func (cfg BaseConfig) NodeKeyFile() string { + return rootify(cfg.NodeKey, cfg.RootDir) } // DBDir returns the full path to the database directory -func (b BaseConfig) DBDir() string { - return rootify(b.DBPath, b.RootDir) +func (cfg BaseConfig) DBDir() string { + return rootify(cfg.DBPath, cfg.RootDir) } // DefaultLogLevel returns a default log level of "error" @@ -229,11 +229,11 @@ func DefaultRPCConfig() *RPCConfig { // TestRPCConfig returns a configuration for testing the RPC server func TestRPCConfig() *RPCConfig { - conf := DefaultRPCConfig() - conf.ListenAddress = "tcp://0.0.0.0:36657" - conf.GRPCListenAddress = "tcp://0.0.0.0:36658" - conf.Unsafe = true - return conf + cfg := DefaultRPCConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36657" + cfg.GRPCListenAddress = "tcp://0.0.0.0:36658" + cfg.Unsafe = true + return cfg } //----------------------------------------------------------------------------- @@ -313,16 +313,16 @@ func DefaultP2PConfig() *P2PConfig { // TestP2PConfig returns a configuration for testing the peer-to-peer layer func TestP2PConfig() *P2PConfig { - conf := DefaultP2PConfig() - conf.ListenAddress = "tcp://0.0.0.0:36656" - conf.SkipUPNP = true - conf.FlushThrottleTimeout = 10 - return conf + cfg := DefaultP2PConfig() + cfg.ListenAddress = "tcp://0.0.0.0:36656" + cfg.SkipUPNP = true + cfg.FlushThrottleTimeout = 10 + return cfg } // AddrBookFile returns the full path to the address book -func (p *P2PConfig) AddrBookFile() string { - return rootify(p.AddrBook, p.RootDir) +func (cfg *P2PConfig) AddrBookFile() string { + return rootify(cfg.AddrBook, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -351,14 +351,14 @@ func DefaultMempoolConfig() *MempoolConfig { // TestMempoolConfig returns a configuration for testing the Tendermint mempool func TestMempoolConfig() *MempoolConfig { - config := DefaultMempoolConfig() - config.CacheSize = 1000 - return config + cfg := DefaultMempoolConfig() + cfg.CacheSize = 1000 + return cfg } // WalDir returns the full path to the mempool's write-ahead log -func (m *MempoolConfig) WalDir() string { - return rootify(m.WalPath, m.RootDir) +func (cfg *MempoolConfig) WalDir() string { + return rootify(cfg.WalPath, cfg.RootDir) } //----------------------------------------------------------------------------- @@ -397,6 +397,44 @@ type ConsensusConfig struct { PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"` } +// DefaultConsensusConfig returns a default configuration for the consensus service +func DefaultConsensusConfig() *ConsensusConfig { + return &ConsensusConfig{ + WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), + WalLight: false, + TimeoutPropose: 3000, + TimeoutProposeDelta: 500, + TimeoutPrevote: 1000, + TimeoutPrevoteDelta: 500, + TimeoutPrecommit: 1000, + TimeoutPrecommitDelta: 500, + TimeoutCommit: 1000, + SkipTimeoutCommit: false, + MaxBlockSizeTxs: 10000, + MaxBlockSizeBytes: 1, // TODO + CreateEmptyBlocks: true, + CreateEmptyBlocksInterval: 0, + PeerGossipSleepDuration: 100, + PeerQueryMaj23SleepDuration: 2000, + } +} + +// TestConsensusConfig returns a configuration for testing the consensus service +func TestConsensusConfig() *ConsensusConfig { + cfg := DefaultConsensusConfig() + cfg.TimeoutPropose = 100 + cfg.TimeoutProposeDelta = 1 + cfg.TimeoutPrevote = 10 + cfg.TimeoutPrevoteDelta = 1 + cfg.TimeoutPrecommit = 10 + cfg.TimeoutPrecommitDelta = 1 + cfg.TimeoutCommit = 10 + cfg.SkipTimeoutCommit = true + cfg.PeerGossipSleepDuration = 5 + cfg.PeerQueryMaj23SleepDuration = 250 + return cfg +} + // WaitForTxs returns true if the consensus should wait for transactions before entering the propose step func (cfg *ConsensusConfig) WaitForTxs() bool { return !cfg.CreateEmptyBlocks || cfg.CreateEmptyBlocksInterval > 0 @@ -437,55 +475,17 @@ func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { return time.Duration(cfg.PeerQueryMaj23SleepDuration) * time.Millisecond } -// DefaultConsensusConfig returns a default configuration for the consensus service -func DefaultConsensusConfig() *ConsensusConfig { - return &ConsensusConfig{ - WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), - WalLight: false, - TimeoutPropose: 3000, - TimeoutProposeDelta: 500, - TimeoutPrevote: 1000, - TimeoutPrevoteDelta: 500, - TimeoutPrecommit: 1000, - TimeoutPrecommitDelta: 500, - TimeoutCommit: 1000, - SkipTimeoutCommit: false, - MaxBlockSizeTxs: 10000, - MaxBlockSizeBytes: 1, // TODO - CreateEmptyBlocks: true, - CreateEmptyBlocksInterval: 0, - PeerGossipSleepDuration: 100, - PeerQueryMaj23SleepDuration: 2000, - } -} - -// TestConsensusConfig returns a configuration for testing the consensus service -func TestConsensusConfig() *ConsensusConfig { - config := DefaultConsensusConfig() - config.TimeoutPropose = 100 - config.TimeoutProposeDelta = 1 - config.TimeoutPrevote = 10 - config.TimeoutPrevoteDelta = 1 - config.TimeoutPrecommit = 10 - config.TimeoutPrecommitDelta = 1 - config.TimeoutCommit = 10 - config.SkipTimeoutCommit = true - config.PeerGossipSleepDuration = 5 - config.PeerQueryMaj23SleepDuration = 250 - return config -} - // WalFile returns the full path to the write-ahead log file -func (c *ConsensusConfig) WalFile() string { - if c.walFile != "" { - return c.walFile +func (cfg *ConsensusConfig) WalFile() string { + if cfg.walFile != "" { + return cfg.walFile } - return rootify(c.WalPath, c.RootDir) + return rootify(cfg.WalPath, cfg.RootDir) } // SetWalFile sets the path to the write-ahead log file -func (c *ConsensusConfig) SetWalFile(walFile string) { - c.walFile = walFile +func (cfg *ConsensusConfig) SetWalFile(walFile string) { + cfg.walFile = walFile } //----------------------------------------------------------------------------- diff --git a/consensus/common_test.go b/consensus/common_test.go index 6d16958ca..287852e04 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -101,13 +101,13 @@ func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*v func incrementHeight(vss ...*validatorStub) { for _, vs := range vss { - vs.Height += 1 + vs.Height++ } } func incrementRound(vss ...*validatorStub) { for _, vs := range vss { - vs.Round += 1 + vs.Round++ } } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index d283ff4ba..31e4e1e62 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -200,7 +200,7 @@ func (app *CounterApplication) DeliverTx(tx []byte) abci.ResponseDeliverTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)} } - app.txCount += 1 + app.txCount++ return abci.ResponseDeliverTx{Code: code.CodeTypeOK} } @@ -211,7 +211,7 @@ func (app *CounterApplication) CheckTx(tx []byte) abci.ResponseCheckTx { Code: code.CodeTypeBadNonce, Log: fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)} } - app.mempoolTxCount += 1 + app.mempoolTxCount++ return abci.ResponseCheckTx{Code: code.CodeTypeOK} } @@ -225,9 +225,8 @@ func (app *CounterApplication) Commit() abci.ResponseCommit { app.mempoolTxCount = app.txCount if app.txCount == 0 { return abci.ResponseCommit{} - } else { - hash := make([]byte, 8) - binary.BigEndian.PutUint64(hash, uint64(app.txCount)) - return abci.ResponseCommit{Data: hash} } + hash := make([]byte, 8) + binary.BigEndian.PutUint64(hash, uint64(app.txCount)) + return abci.ResponseCommit{Data: hash} } diff --git a/consensus/reactor.go b/consensus/reactor.go index a265f76c0..70a79d86a 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -371,19 +371,21 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { } go func() { + var data interface{} + var ok bool for { select { - case data, ok := <-stepsCh: + case data, ok = <-stepsCh: if ok { // a receive from a closed channel returns the zero value immediately edrs := data.(types.TMEventData).Unwrap().(types.EventDataRoundState) conR.broadcastNewRoundStep(edrs.RoundState.(*cstypes.RoundState)) } - case data, ok := <-votesCh: + case data, ok = <-votesCh: if ok { edv := data.(types.TMEventData).Unwrap().(types.EventDataVote) conR.broadcastHasVoteMessage(edv.Vote) } - case data, ok := <-heartbeatsCh: + case data, ok = <-heartbeatsCh: if ok { edph := data.(types.TMEventData).Unwrap().(types.EventDataProposalHeartbeat) conR.broadcastProposalHeartbeatMessage(edph) @@ -392,6 +394,10 @@ func (conR *ConsensusReactor) startBroadcastRoutine() error { conR.eventBus.UnsubscribeAll(ctx, subscriber) return } + if !ok { + conR.eventBus.UnsubscribeAll(ctx, subscriber) + return + } } }() @@ -602,11 +608,9 @@ func (conR *ConsensusReactor) gossipDataForCatchup(logger log.Logger, rs *cstype logger.Debug("Sending block part for catchup failed") } return - } else { - //logger.Info("No parts to send in catch-up, sleeping") - time.Sleep(conR.conS.config.PeerGossipSleep()) - return } + //logger.Info("No parts to send in catch-up, sleeping") + time.Sleep(conR.conS.config.PeerGossipSleep()) } func (conR *ConsensusReactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { @@ -1083,36 +1087,49 @@ func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { // It returns the total number of votes (1 per block). This essentially means // the number of blocks for which peer has been sending us votes. func (ps *PeerState) RecordVote(vote *types.Vote) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastVoteHeight >= vote.Height { return ps.stats.votes } ps.stats.lastVoteHeight = vote.Height - ps.stats.votes += 1 + ps.stats.votes++ return ps.stats.votes } // VotesSent returns the number of blocks for which peer has been sending us // votes. func (ps *PeerState) VotesSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.votes } -// RecordVote updates internal statistics for this peer by recording the block part. -// It returns the total number of block parts (1 per block). This essentially means -// the number of blocks for which peer has been sending us block parts. +// RecordBlockPart updates internal statistics for this peer by recording the +// block part. It returns the total number of block parts (1 per block). This +// essentially means the number of blocks for which peer has been sending us +// block parts. func (ps *PeerState) RecordBlockPart(bp *BlockPartMessage) int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + if ps.stats.lastBlockPartHeight >= bp.Height { return ps.stats.blockParts } ps.stats.lastBlockPartHeight = bp.Height - ps.stats.blockParts += 1 + ps.stats.blockParts++ return ps.stats.blockParts } // BlockPartsSent returns the number of blocks for which peer has been sending // us block parts. func (ps *PeerState) BlockPartsSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.stats.blockParts } diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 26fc7e171..8448caf63 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -441,7 +441,7 @@ func waitForAndValidateBlockWithTx(t *testing.T, n int, activeVals map[string]st // but they should be in order. for _, tx := range newBlock.Data.Txs { assert.EqualValues(t, txs[ntxs], tx) - ntxs += 1 + ntxs++ } if ntxs == len(txs) { diff --git a/consensus/replay.go b/consensus/replay.go index 39dd592a4..5b5a48425 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -112,7 +112,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { } } if found { - return fmt.Errorf("WAL should not contain #ENDHEIGHT %d.", csHeight) + return fmt.Errorf("WAL should not contain #ENDHEIGHT %d", csHeight) } // Search for last height marker @@ -125,7 +125,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { return err } if !found { - return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d.", csHeight, csHeight-1) + return fmt.Errorf("Cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, csHeight-1) } defer gr.Close() // nolint: errcheck @@ -352,7 +352,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl var err error finalBlock := storeBlockHeight if mutateState { - finalBlock -= 1 + finalBlock-- } for i := appBlockHeight + 1; i <= finalBlock; i++ { h.logger.Info("Applying block", "height", i) @@ -362,7 +362,7 @@ func (h *Handshaker) replayBlocks(state sm.State, proxyApp proxy.AppConns, appBl return nil, err } - h.nBlocks += 1 + h.nBlocks++ } if mutateState { @@ -390,7 +390,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap return sm.State{}, err } - h.nBlocks += 1 + h.nBlocks++ return state, nil } @@ -429,7 +429,7 @@ type mockProxyApp struct { func (mock *mockProxyApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { r := mock.abciResponses.DeliverTx[mock.txCount] - mock.txCount += 1 + mock.txCount++ return *r } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 2fcd9b472..1fd4f415d 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -87,9 +87,9 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error { } if nextN > 0 { - nextN -= 1 + nextN-- } - pb.count += 1 + pb.count++ } return nil } @@ -153,7 +153,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil { return err } - pb.count += 1 + pb.count++ } return nil } @@ -197,13 +197,12 @@ func (pb *playback) replayConsoleLoop() int { if len(tokens) == 1 { return 0 + } + i, err := strconv.Atoi(tokens[1]) + if err != nil { + fmt.Println("next takes an integer argument") } else { - i, err := strconv.Atoi(tokens[1]) - if err != nil { - fmt.Println("next takes an integer argument") - } else { - return i - } + return i } case "back": diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 9d7bbc8e4..606df8260 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -382,9 +382,9 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { expectedBlocksToSync := NUM_BLOCKS - nBlocks if nBlocks == NUM_BLOCKS && mode > 0 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } else if nBlocks > 0 && mode == 1 { - expectedBlocksToSync += 1 + expectedBlocksToSync++ } if handshaker.NBlocks() != expectedBlocksToSync { @@ -533,7 +533,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { } blocks = append(blocks, block) commits = append(commits, thisBlockCommit) - height += 1 + height++ } case *types.PartSetHeader: thisBlockParts = types.NewPartSetFromHeader(*p) diff --git a/consensus/state.go b/consensus/state.go index b59baf1ee..57c7b32f0 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -494,7 +494,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { func (cs *ConsensusState) newStep() { rs := cs.RoundStateEvent() cs.wal.Save(rs) - cs.nSteps += 1 + cs.nSteps++ // newStep is called by updateToStep in NewConsensusState before the eventBus is set! if cs.eventBus != nil { cs.eventBus.PublishEventNewRoundStep(rs) @@ -720,11 +720,7 @@ func (cs *ConsensusState) needProofBlock(height int64) bool { func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { counter := 0 addr := cs.privValidator.GetAddress() - valIndex, v := cs.Validators.GetByAddress(addr) - if v == nil { - // not a validator - valIndex = -1 - } + valIndex, _ := cs.Validators.GetByAddress(addr) chainID := cs.state.ChainID for { rs := cs.GetRoundState() @@ -741,7 +737,7 @@ func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { } cs.privValidator.SignHeartbeat(chainID, heartbeat) cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat}) - counter += 1 + counter++ time.Sleep(proposalHeartbeatIntervalSeconds * time.Second) } } @@ -852,10 +848,10 @@ func (cs *ConsensusState) isProposalComplete() bool { // make sure we have the prevotes from it too if cs.Proposal.POLRound < 0 { return true - } else { - // if this is false the proposer is lying or we haven't received the POL yet - return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() } + // if this is false the proposer is lying or we haven't received the POL yet + return cs.Votes.Prevotes(cs.Proposal.POLRound).HasTwoThirdsMajority() + } // Create the next block to propose and return it. @@ -1498,12 +1494,11 @@ func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.Part cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) return vote - } else { - //if !cs.replayMode { - cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) - //} - return nil } + //if !cs.replayMode { + cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) + //} + return nil } //--------------------------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 6122e90a3..9d4f6b017 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -196,9 +196,8 @@ urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/statefuls urllib.urlretrieve(tools_repo+tools_branch+'/mintnet-kubernetes/assets/t_plus_k.png', filename=assets_dir+'/t_plus_k.png') urllib.urlretrieve(tools_repo+tools_branch+'/terraform-digitalocean/README.rst', filename=tools_dir+'/terraform-digitalocean.rst') -urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking-and-monitoring.rst') -# the readme for below is included in tm-bench -# urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/tm-monitor.rst') +urllib.urlretrieve(tools_repo+tools_branch+'/tm-bench/README.rst', filename=tools_dir+'/benchmarking.rst') +urllib.urlretrieve('https://raw.githubusercontent.com/tendermint/tools/master/tm-monitor/README.rst', filename='tools/monitoring.rst') #### abci spec ################################# diff --git a/docs/index.rst b/docs/index.rst index f2f70d194..a89adb296 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,7 +44,8 @@ Tendermint Tools tools/docker.rst tools/mintnet-kubernetes.rst tools/terraform-digitalocean.rst - tools/benchmarking-and-monitoring.rst + tools/benchmarking.rst + tools/monitoring.rst Tendermint 102 -------------- diff --git a/docs/specification/new-spec/light-client.md b/docs/specification/new-spec/light-client.md new file mode 100644 index 000000000..0ed9d36d4 --- /dev/null +++ b/docs/specification/new-spec/light-client.md @@ -0,0 +1,114 @@ +# Light client + +A light client is a process that connects to the Tendermint Full Node(s) and then tries to verify the Merkle proofs +about the blockchain application. In this document we describe mechanisms that ensures that the Tendermint light client +has the same level of security as Full Node processes (without being itself a Full Node). + +To be able to validate a Merkle proof, a light client needs to validate the blockchain header that contains the root app hash. +Validating a blockchain header in Tendermint consists in verifying that the header is committed (signed) by >2/3 of the +voting power of the corresponding validator set. As the validator set is a dynamic set (it is changing), one of the +core functionality of the light client is updating the current validator set, that is then used to verify the +blockchain header, and further the corresponding Merkle proofs. + +For the purpose of this light client specification, we assume that the Tendermint Full Node exposes the following functions over +Tendermint RPC: + +```golang +Header(height int64) (SignedHeader, error) // returns signed header for the given height +Validators(height int64) (ResultValidators, error) // returns validator set for the given height +LastHeader(valSetNumber int64) (SignedHeader, error) // returns last header signed by the validator set with the given validator set number + +type SignedHeader struct { + Header Header + Commit Commit + ValSetNumber int64 +} + +type ResultValidators struct { + BlockHeight int64 + Validators []Validator + // time the current validator set is initialised, i.e, time of the last validator change before header BlockHeight + ValSetTime int64 +} +``` + +We assume that Tendermint keeps track of the validator set changes and that each time a validator set is changed it is +being assigned the next sequence number. We can call this number the validator set sequence number. Tendermint also remembers +the Time from the header when the next validator set is initialised (starts to be in power), and we refer to this time +as validator set init time. +Furthermore, we assume that each validator set change is signed (committed) by the current validator set. More precisely, +given a block `H` that contains transactions that are modifying the current validator set, the Merkle root hash of the next +validator set (modified based on transactions from block H) will be in block `H+1` (and signed by the current validator +set), and then starting from the block `H+2`, it will be signed by the next validator set. + +Note that the real Tendermint RPC API is slightly different (for example, response messages contain more data and function +names are slightly different); we shortened (and modified) it for the purpose of this document to make the spec more +clear and simple. Furthermore, note that in case of the third function, the returned header has `ValSetNumber` equals to +`valSetNumber+1`. + + +Locally, light client manages the following state: + +```golang +valSet []Validator // current validator set (last known and verified validator set) +valSetNumber int64 // sequence number of the current validator set +valSetHash []byte // hash of the current validator set +valSetTime int64 // time when the current validator set is initialised +``` + +The light client is initialised with the trusted validator set, for example based on the known validator set hash, +validator set sequence number and the validator set init time. +The core of the light client logic is captured by the VerifyAndUpdate function that is used to 1) verify if the given header is valid, +and 2) update the validator set (when the given header is valid and it is more recent than the seen headers). + +```golang +VerifyAndUpdate(signedHeader SignedHeader): + assertThat signedHeader.valSetNumber >= valSetNumber + if isValid(signedHeader) and signedHeader.Header.Time <= valSetTime + UNBONDING_PERIOD then + setValidatorSet(signedHeader) + return true + else + updateValidatorSet(signedHeader.ValSetNumber) + return VerifyAndUpdate(signedHeader) + +isValid(signedHeader SignedHeader): + valSetOfTheHeader = Validators(signedHeader.Header.Height) + assertThat Hash(valSetOfTheHeader) == signedHeader.Header.ValSetHash + assertThat signedHeader is passing basic validation + if votingPower(signedHeader.Commit) > 2/3 * votingPower(valSetOfTheHeader) then return true + else + return false + +setValidatorSet(signedHeader SignedHeader): + nextValSet = Validators(signedHeader.Header.Height) + assertThat Hash(nextValSet) == signedHeader.Header.ValidatorsHash + valSet = nextValSet.Validators + valSetHash = signedHeader.Header.ValidatorsHash + valSetNumber = signedHeader.ValSetNumber + valSetTime = nextValSet.ValSetTime + +votingPower(commit Commit): + votingPower = 0 + for each precommit in commit.Precommits do: + if precommit.ValidatorAddress is in valSet and signature of the precommit verifies then + votingPower += valSet[precommit.ValidatorAddress].VotingPower + return votingPower + +votingPower(validatorSet []Validator): + for each validator in validatorSet do: + votingPower += validator.VotingPower + return votingPower + +updateValidatorSet(valSetNumberOfTheHeader): + while valSetNumber != valSetNumberOfTheHeader do + signedHeader = LastHeader(valSetNumber) + if isValid(signedHeader) then + setValidatorSet(signedHeader) + else return error + return +``` + +Note that in the logic above we assume that the light client will always go upward with respect to header verifications, +i.e., that it will always be used to verify more recent headers. In case a light client needs to be used to verify older +headers (go backward) the same mechanisms and similar logic can be used. In case a call to the FullNode or subsequent +checks fail, a light client need to implement some recovery strategy, for example connecting to other FullNode. diff --git a/lite/proxy/block.go b/lite/proxy/block.go index 60cd00f0d..4cff9ee68 100644 --- a/lite/proxy/block.go +++ b/lite/proxy/block.go @@ -11,11 +11,17 @@ import ( ) func ValidateBlockMeta(meta *types.BlockMeta, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil BlockMeta") + } // TODO: check the BlockID?? return ValidateHeader(meta.Header, check) } func ValidateBlock(meta *types.Block, check lite.Commit) error { + if meta == nil { + return errors.New("expecting a non-nil Block") + } err := ValidateHeader(meta.Header, check) if err != nil { return err @@ -27,6 +33,9 @@ func ValidateBlock(meta *types.Block, check lite.Commit) error { } func ValidateHeader(head *types.Header, check lite.Commit) error { + if head == nil { + return errors.New("expecting a non-nil Header") + } // make sure they are for the same height (obvious fail) if head.Height != check.Height() { return certerr.ErrHeightMismatch(head.Height, check.Height()) diff --git a/lite/proxy/validate_test.go b/lite/proxy/validate_test.go new file mode 100644 index 000000000..782a6aabb --- /dev/null +++ b/lite/proxy/validate_test.go @@ -0,0 +1,218 @@ +package proxy_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/tendermint/tendermint/lite" + "github.com/tendermint/tendermint/lite/proxy" + "github.com/tendermint/tendermint/types" +) + +var ( + deadBeefTxs = types.Txs{[]byte("DE"), []byte("AD"), []byte("BE"), []byte("EF")} + deadBeefHash = deadBeefTxs.Hash() + testTime1 = time.Date(2018, 1, 1, 1, 1, 1, 1, time.UTC) + testTime2 = time.Date(2017, 1, 2, 1, 1, 1, 1, time.UTC) +) + +var hdrHeight11 = &types.Header{ + Height: 11, + Time: testTime1, + ValidatorsHash: []byte("Tendermint"), +} + +func TestValidateBlock(t *testing.T) { + tests := []struct { + block *types.Block + commit lite.Commit + wantErr string + }{ + { + block: nil, wantErr: "non-nil Block", + }, + { + block: &types.Block{}, wantErr: "nil Header", + }, + { + block: &types.Block{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + block: &types.Block{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + block: &types.Block{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Header.Hash mismatch test + { + block: &types.Block{Header: hdrHeight11}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + block: &types.Block{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, + }, + // End Header.Hash mismatch test + + // Start Header.Data hash mismatch test + { + block: &types.Block{ + Header: &types.Header{Height: 11}, + Data: &types.Data{Txs: []types.Tx{[]byte("0xDE"), []byte("AD")}}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("0xDEADBEEF")}}, + }, + wantErr: "Data hash doesn't match header", + }, + { + block: &types.Block{ + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, + Data: &types.Data{Txs: deadBeefTxs}, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11}, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + }, + // End Header.Data hash mismatch test + } + + for i, tt := range tests { + err := proxy.ValidateBlock(tt.block, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf(t, "Unexpectedly passed", "#%d", i) + } else { + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(t, err, "#%d: expecting a nil error", i) + } +} + +func TestValidateBlockMeta(t *testing.T) { + tests := []struct { + meta *types.BlockMeta + commit lite.Commit + wantErr string + }{ + { + meta: nil, wantErr: "non-nil BlockMeta", + }, + { + meta: &types.BlockMeta{}, wantErr: "non-nil Header", + }, + { + meta: &types.BlockMeta{Header: new(types.Header)}, + }, + + // Start Header.Height mismatch test + { + meta: &types.BlockMeta{Header: &types.Header{Height: 10}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "don't match - 10 vs 11", + }, + + { + meta: &types.BlockMeta{Header: &types.Header{Height: 11}}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + }, + // End Header.Height mismatch test + + // Start Headers don't match test + { + meta: &types.BlockMeta{Header: hdrHeight11}, + commit: lite.Commit{Header: &types.Header{Height: 11}}, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{Header: hdrHeight11}, + commit: lite.Commit{Header: hdrHeight11}, + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, + ValidatorsHash: []byte("lite-test"), + // TODO: should be able to use empty time after Amino upgrade + Time: testTime1, + }, + }, + commit: lite.Commit{ + Header: &types.Header{Height: 11, DataHash: deadBeefHash}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefHash, + ValidatorsHash: []byte("Tendermint"), + Time: testTime1, + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefHash, + ValidatorsHash: []byte("Tendermint"), + Time: testTime2, + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + + { + meta: &types.BlockMeta{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefHash, + ValidatorsHash: []byte("Tendermint"), + Time: testTime2, + }, + }, + commit: lite.Commit{ + Header: &types.Header{ + Height: 11, DataHash: deadBeefHash, + ValidatorsHash: []byte("Tendermint-x"), + Time: testTime2, + }, + Commit: &types.Commit{BlockID: types.BlockID{Hash: []byte("DEADBEEF")}}, + }, + wantErr: "Headers don't match", + }, + // End Headers don't match test + } + + for i, tt := range tests { + err := proxy.ValidateBlockMeta(tt.meta, tt.commit) + if tt.wantErr != "" { + if err == nil { + assert.FailNowf(t, "Unexpectedly passed", "#%d: wanted error %q", i, tt.wantErr) + } else { + assert.Contains(t, err.Error(), tt.wantErr, "#%d should contain the substring\n\n", i) + } + continue + } + + assert.Nil(t, err, "#%d: expecting a nil error", i) + } +} diff --git a/node/node.go b/node/node.go index 1f745bf00..acde2e482 100644 --- a/node/node.go +++ b/node/node.go @@ -7,7 +7,6 @@ import ( "fmt" "net" "net/http" - "strings" abci "github.com/tendermint/abci/types" crypto "github.com/tendermint/go-crypto" @@ -277,19 +276,11 @@ func NewNode(config *cfg.Config, trustMetricStore = trust.NewTrustMetricStore(trustHistoryDB, trust.DefaultConfig()) trustMetricStore.SetLogger(p2pLogger) - var seeds []string - if config.P2P.Seeds != "" { - seeds = strings.Split(config.P2P.Seeds, ",") - } - var privatePeerIDs []string - if config.P2P.PrivatePeerIDs != "" { - privatePeerIDs = strings.Split(config.P2P.PrivatePeerIDs, ",") - } pexReactor := pex.NewPEXReactor(addrBook, &pex.PEXReactorConfig{ - Seeds: seeds, + Seeds: cmn.SplitAndTrim(config.P2P.Seeds, ",", " "), SeedMode: config.P2P.SeedMode, - PrivatePeerIDs: privatePeerIDs}) + PrivatePeerIDs: cmn.SplitAndTrim(config.P2P.PrivatePeerIDs, ",", " ")}) pexReactor.SetLogger(p2pLogger) sw.AddReactor("PEX", pexReactor) } @@ -339,7 +330,7 @@ func NewNode(config *cfg.Config, return nil, err } if config.TxIndex.IndexTags != "" { - txIndexer = kv.NewTxIndex(store, kv.IndexTags(strings.Split(config.TxIndex.IndexTags, ","))) + txIndexer = kv.NewTxIndex(store, kv.IndexTags(cmn.SplitAndTrim(config.TxIndex.IndexTags, ",", " "))) } else if config.TxIndex.IndexAllTags { txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) } else { @@ -414,9 +405,14 @@ func (n *Node) OnStart() error { } n.Logger.Info("P2P Node ID", "ID", nodeKey.ID(), "file", n.config.NodeKeyFile()) - // Start the switch - n.sw.SetNodeInfo(n.makeNodeInfo(nodeKey.PubKey())) + nodeInfo := n.makeNodeInfo(nodeKey.PubKey()) + n.sw.SetNodeInfo(nodeInfo) n.sw.SetNodeKey(nodeKey) + + // Add ourselves to addrbook to prevent dialing ourselves + n.addrBook.AddOurAddress(nodeInfo.NetAddress()) + + // Start the switch err = n.sw.Start() if err != nil { return err @@ -424,7 +420,7 @@ func (n *Node) OnStart() error { // Always connect to persistent peers if n.config.P2P.PersistentPeers != "" { - err = n.sw.DialPeersAsync(n.addrBook, strings.Split(n.config.P2P.PersistentPeers, ","), true) + err = n.sw.DialPeersAsync(n.addrBook, cmn.SplitAndTrim(n.config.P2P.PersistentPeers, ",", " "), true) if err != nil { return err } @@ -495,7 +491,7 @@ func (n *Node) ConfigureRPC() { func (n *Node) startRPC() ([]net.Listener, error) { n.ConfigureRPC() - listenAddrs := strings.Split(n.config.RPC.ListenAddress, ",") + listenAddrs := cmn.SplitAndTrim(n.config.RPC.ListenAddress, ",", " ") if n.config.RPC.Unsafe { rpccore.AddUnsafeRoutes() @@ -643,14 +639,13 @@ func loadGenesisDoc(db dbm.DB) (*types.GenesisDoc, error) { bytes := db.Get(genesisDocKey) if len(bytes) == 0 { return nil, errors.New("Genesis doc not found") - } else { - var genDoc *types.GenesisDoc - err := json.Unmarshal(bytes, &genDoc) - if err != nil { - cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) - } - return genDoc, nil } + var genDoc *types.GenesisDoc + err := json.Unmarshal(bytes, &genDoc) + if err != nil { + cmn.PanicCrisis(fmt.Sprintf("Failed to load genesis doc due to unmarshaling error: %v (bytes: %X)", err, bytes)) + } + return genDoc, nil } // panics if failed to marshal the given genesis document diff --git a/p2p/base_reactor.go b/p2p/base_reactor.go index f0dd14147..83c8efa4b 100644 --- a/p2p/base_reactor.go +++ b/p2p/base_reactor.go @@ -47,7 +47,7 @@ func NewBaseReactor(name string, impl Reactor) *BaseReactor { func (br *BaseReactor) SetSwitch(sw *Switch) { br.Switch = sw } -func (_ *BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } -func (_ *BaseReactor) AddPeer(peer Peer) {} -func (_ *BaseReactor) RemovePeer(peer Peer, reason interface{}) {} -func (_ *BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} +func (*BaseReactor) GetChannels() []*conn.ChannelDescriptor { return nil } +func (*BaseReactor) AddPeer(peer Peer) {} +func (*BaseReactor) RemovePeer(peer Peer, reason interface{}) {} +func (*BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {} diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 9acaf6175..9a3360f2d 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -7,7 +7,6 @@ import ( "io" "math" "net" - "runtime/debug" "sync/atomic" "time" @@ -230,8 +229,7 @@ func (c *MConnection) flush() { // Catch panics, usually caused by remote disconnects. func (c *MConnection) _recover() { if r := recover(); r != nil { - stack := debug.Stack() - err := cmn.StackError{r, stack} + err := cmn.ErrorWrap(r, "recovered from panic") c.stopForError(err) } } @@ -424,9 +422,8 @@ func (c *MConnection) sendMsgPacket() bool { // Nothing to send? if leastChannel == nil { return true - } else { - // c.Logger.Info("Found a msgPacket to send") } + // c.Logger.Info("Found a msgPacket to send") // Make & send a msgPacket from this channel n, err := leastChannel.writeMsgPacketTo(c.bufWriter) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index aa6db05bb..bc67abf3a 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -20,8 +20,8 @@ import ( "golang.org/x/crypto/nacl/secretbox" "golang.org/x/crypto/ripemd160" - "github.com/tendermint/go-crypto" - "github.com/tendermint/go-wire" + crypto "github.com/tendermint/go-crypto" + wire "github.com/tendermint/go-wire" cmn "github.com/tendermint/tmlibs/common" ) @@ -113,7 +113,7 @@ func (sc *SecretConnection) RemotePubKey() crypto.PubKey { // CONTRACT: data smaller than dataMaxSize is read atomically. func (sc *SecretConnection) Write(data []byte) (n int, err error) { for 0 < len(data) { - var frame []byte = make([]byte, totalFrameSize) + var frame = make([]byte, totalFrameSize) var chunk []byte if dataMaxSize < len(data) { chunk = data[:dataMaxSize] @@ -136,9 +136,8 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { _, err := sc.conn.Write(sealedFrame) if err != nil { return n, err - } else { - n += len(chunk) } + n += len(chunk) } return } @@ -200,26 +199,36 @@ func genEphKeys() (ephPub, ephPriv *[32]byte) { } func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[32]byte, err error) { - var err1, err2 error - - cmn.Parallel( - func() { - _, err1 = conn.Write(locEphPub[:]) + // Send our pubkey and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + var _, err1 = conn.Write(locEphPub[:]) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { - remEphPub = new([32]byte) - _, err2 = io.ReadFull(conn, remEphPub[:]) + func(_ int) (val interface{}, err error, abort bool) { + var _remEphPub [32]byte + var _, err2 = io.ReadFull(conn, _remEphPub[:]) + if err2 != nil { + return nil, err2, true // abort + } else { + return _remEphPub, nil, false + } }, ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return remEphPub, nil + // Otherwise: + var _remEphPub = trs.FirstValue().([32]byte) + return &_remEphPub, nil } func computeSharedSecret(remPubKey, locPrivKey *[32]byte) (shrSecret *[32]byte) { @@ -268,33 +277,42 @@ type authSigMessage struct { Sig crypto.Signature } -func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (*authSigMessage, error) { - var recvMsg authSigMessage - var err1, err2 error - - cmn.Parallel( - func() { +func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature crypto.Signature) (recvMsg *authSigMessage, err error) { + // Send our info and receive theirs in tandem. + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { msgBytes := wire.BinaryBytes(authSigMessage{pubKey.Wrap(), signature.Wrap()}) - _, err1 = sc.Write(msgBytes) + var _, err1 = sc.Write(msgBytes) + if err1 != nil { + return nil, err1, true // abort + } else { + return nil, nil, false + } }, - func() { + func(_ int) (val interface{}, err error, abort bool) { readBuffer := make([]byte, authSigMsgSize) - _, err2 = io.ReadFull(sc, readBuffer) + var _, err2 = io.ReadFull(sc, readBuffer) if err2 != nil { - return + return nil, err2, true // abort } n := int(0) // not used. - recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) - }) + var _recvMsg = wire.ReadBinary(authSigMessage{}, bytes.NewBuffer(readBuffer), authSigMsgSize, &n, &err2).(authSigMessage) + if err2 != nil { + return nil, err2, true // abort + } else { + return _recvMsg, nil, false + } + }, + ) - if err1 != nil { - return nil, err1 - } - if err2 != nil { - return nil, err2 + // If error: + if trs.FirstError() != nil { + err = trs.FirstError() + return } - return &recvMsg, nil + var _recvMsg = trs.FirstValue().(authSigMessage) + return &_recvMsg, nil } //-------------------------------------------------------------------------------- @@ -328,7 +346,7 @@ func incr2Nonce(nonce *[24]byte) { // increment nonce big-endian by 1 with wraparound. func incrNonce(nonce *[24]byte) { for i := 23; 0 <= i; i-- { - nonce[i] += 1 + nonce[i]++ if nonce[i] != 0 { return } diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index c81b3b285..edb0de04d 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -1,9 +1,12 @@ package conn import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" crypto "github.com/tendermint/go-crypto" cmn "github.com/tendermint/tmlibs/common" ) @@ -36,33 +39,41 @@ func makeSecretConnPair(tb testing.TB) (fooSecConn, barSecConn *SecretConnection barPrvKey := crypto.GenPrivKeyEd25519().Wrap() barPubKey := barPrvKey.PubKey() - cmn.Parallel( - func() { - var err error + var trs, ok = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { fooSecConn, err = MakeSecretConnection(fooConn, fooPrvKey) if err != nil { tb.Errorf("Failed to establish SecretConnection for foo: %v", err) - return + return nil, err, true } remotePubBytes := fooSecConn.RemotePubKey() if !remotePubBytes.Equals(barPubKey) { - tb.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected fooSecConn.RemotePubKey. Expected %v, got %v", barPubKey, fooSecConn.RemotePubKey()) + tb.Error(err) + return nil, err, false } + return nil, nil, false }, - func() { - var err error + func(_ int) (val interface{}, err error, abort bool) { barSecConn, err = MakeSecretConnection(barConn, barPrvKey) if barSecConn == nil { tb.Errorf("Failed to establish SecretConnection for bar: %v", err) - return + return nil, err, true } remotePubBytes := barSecConn.RemotePubKey() if !remotePubBytes.Equals(fooPubKey) { - tb.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", + err = fmt.Errorf("Unexpected barSecConn.RemotePubKey. Expected %v, got %v", fooPubKey, barSecConn.RemotePubKey()) + tb.Error(err) + return nil, nil, false } - }) + return nil, nil, false + }, + ) + + require.Nil(tb, trs.FirstError()) + require.True(tb, ok, "Unexpected task abortion") return } @@ -89,59 +100,76 @@ func TestSecretConnectionReadWrite(t *testing.T) { } // A helper that will run with (fooConn, fooWrites, fooReads) and vice versa - genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) func() { - return func() { + genNodeRunner := func(nodeConn kvstoreConn, nodeWrites []string, nodeReads *[]string) cmn.Task { + return func(_ int) (interface{}, error, bool) { // Node handskae nodePrvKey := crypto.GenPrivKeyEd25519().Wrap() nodeSecretConn, err := MakeSecretConnection(nodeConn, nodePrvKey) if err != nil { t.Errorf("Failed to establish SecretConnection for node: %v", err) - return + return nil, err, true } - // In parallel, handle reads and writes - cmn.Parallel( - func() { + // In parallel, handle some reads and writes. + var trs, ok = cmn.Parallel( + func(_ int) (interface{}, error, bool) { // Node writes for _, nodeWrite := range nodeWrites { n, err := nodeSecretConn.Write([]byte(nodeWrite)) if err != nil { t.Errorf("Failed to write to nodeSecretConn: %v", err) - return + return nil, err, true } if n != len(nodeWrite) { - t.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) - return + err = fmt.Errorf("Failed to write all bytes. Expected %v, wrote %v", len(nodeWrite), n) + t.Error(err) + return nil, err, true } } if err := nodeConn.PipeWriter.Close(); err != nil { t.Error(err) + return nil, err, true } + return nil, nil, false }, - func() { + func(_ int) (interface{}, error, bool) { // Node reads readBuffer := make([]byte, dataMaxSize) for { n, err := nodeSecretConn.Read(readBuffer) if err == io.EOF { - return + return nil, nil, false } else if err != nil { t.Errorf("Failed to read from nodeSecretConn: %v", err) - return + return nil, err, true } *nodeReads = append(*nodeReads, string(readBuffer[:n])) } if err := nodeConn.PipeReader.Close(); err != nil { t.Error(err) + return nil, err, true } - }) + return nil, nil, false + }, + ) + assert.True(t, ok, "Unexpected task abortion") + + // If error: + if trs.FirstError() != nil { + return nil, trs.FirstError(), true + } + + // Otherwise: + return nil, nil, false } } // Run foo & bar in parallel - cmn.Parallel( + var trs, ok = cmn.Parallel( genNodeRunner(fooConn, fooWrites, &fooReads), genNodeRunner(barConn, barWrites, &barReads), ) + require.Nil(t, trs.FirstError()) + require.True(t, ok, "unexpected task abortion") // A helper to ensure that the writes and reads match. // Additionally, small writes (<= dataMaxSize) must be atomically read. @@ -152,7 +180,7 @@ func TestSecretConnectionReadWrite(t *testing.T) { var readCount = 0 for _, readChunk := range reads { read += readChunk - readCount += 1 + readCount++ if len(write) <= len(read) { break } diff --git a/p2p/key.go b/p2p/key.go index ea0f0b071..6d0f28586 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -53,9 +53,8 @@ func LoadOrGenNodeKey(filePath string) (*NodeKey, error) { return nil, err } return nodeKey, nil - } else { - return genNodeKey(filePath) } + return genNodeKey(filePath) } func loadNodeKey(filePath string) (*NodeKey, error) { @@ -66,7 +65,7 @@ func loadNodeKey(filePath string) (*NodeKey, error) { nodeKey := new(NodeKey) err = json.Unmarshal(jsonBytes, nodeKey) if err != nil { - return nil, fmt.Errorf("Error reading NodeKey from %v: %v\n", filePath, err) + return nil, fmt.Errorf("Error reading NodeKey from %v: %v", filePath, err) } return nodeKey, nil } diff --git a/p2p/listener.go b/p2p/listener.go index 884c45ee8..e698765cd 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -72,7 +72,7 @@ func NewDefaultListener(protocol string, lAddr string, skipUPNP bool, logger log // Determine internal address... var intAddr *NetAddress - intAddr, err = NewNetAddressString(lAddr) + intAddr, err = NewNetAddressStringWithOptionalID(lAddr) if err != nil { panic(err) } diff --git a/p2p/netaddress.go b/p2p/netaddress.go index 333d16e5d..a77090a78 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -49,33 +49,45 @@ func NewNetAddress(id ID, addr net.Addr) *NetAddress { } ip := tcpAddr.IP port := uint16(tcpAddr.Port) - netAddr := NewNetAddressIPPort(ip, port) - netAddr.ID = id - return netAddr + na := NewNetAddressIPPort(ip, port) + na.ID = id + return na } -// NewNetAddressString returns a new NetAddress using the provided -// address in the form of "ID@IP:Port", where the ID is optional. +// NewNetAddressString returns a new NetAddress using the provided address in +// the form of "ID@IP:Port". // Also resolves the host if host is not an IP. func NewNetAddressString(addr string) (*NetAddress, error) { - addr = removeProtocolIfDefined(addr) + spl := strings.Split(addr, "@") + if len(spl) < 2 { + return nil, fmt.Errorf("Address (%s) does not contain ID", addr) + } + return NewNetAddressStringWithOptionalID(addr) +} + +// NewNetAddressStringWithOptionalID returns a new NetAddress using the +// provided address in the form of "ID@IP:Port", where the ID is optional. +// Also resolves the host if host is not an IP. +func NewNetAddressStringWithOptionalID(addr string) (*NetAddress, error) { + addrWithoutProtocol := removeProtocolIfDefined(addr) var id ID - spl := strings.Split(addr, "@") + spl := strings.Split(addrWithoutProtocol, "@") if len(spl) == 2 { idStr := spl[0] idBytes, err := hex.DecodeString(idStr) if err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Address (%s) contains invalid ID", addr)) + return nil, errors.Wrapf(err, "Address (%s) contains invalid ID", addrWithoutProtocol) } if len(idBytes) != IDByteLength { return nil, fmt.Errorf("Address (%s) contains ID of invalid length (%d). Should be %d hex-encoded bytes", - addr, len(idBytes), IDByteLength) + addrWithoutProtocol, len(idBytes), IDByteLength) } - id, addr = ID(idStr), spl[1] + + id, addrWithoutProtocol = ID(idStr), spl[1] } - host, portStr, err := net.SplitHostPort(addr) + host, portStr, err := net.SplitHostPort(addrWithoutProtocol) if err != nil { return nil, err } @@ -120,11 +132,10 @@ func NewNetAddressStrings(addrs []string) ([]*NetAddress, []error) { // NewNetAddressIPPort returns a new NetAddress using the provided IP // and port number. func NewNetAddressIPPort(ip net.IP, port uint16) *NetAddress { - na := &NetAddress{ + return &NetAddress{ IP: ip, Port: port, } - return na } // Equals reports whether na and other are the same addresses, @@ -294,7 +305,7 @@ func (na *NetAddress) RFC6145() bool { return rfc6145.Contains(na.IP) } func removeProtocolIfDefined(addr string) string { if strings.Contains(addr, "://") { return strings.Split(addr, "://")[1] - } else { - return addr } + return addr + } diff --git a/p2p/netaddress_test.go b/p2p/netaddress_test.go index 6c1930a2f..653b436a6 100644 --- a/p2p/netaddress_test.go +++ b/p2p/netaddress_test.go @@ -9,20 +9,18 @@ import ( ) func TestNewNetAddress(t *testing.T) { - assert, require := assert.New(t), require.New(t) - tcpAddr, err := net.ResolveTCPAddr("tcp", "127.0.0.1:8080") - require.Nil(err) + require.Nil(t, err) addr := NewNetAddress("", tcpAddr) - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) - assert.NotPanics(func() { + assert.NotPanics(t, func() { NewNetAddress("", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8000}) }, "Calling NewNetAddress with UDPAddr should not panic in testing") } -func TestNewNetAddressString(t *testing.T) { +func TestNewNetAddressStringWithOptionalID(t *testing.T) { testCases := []struct { addr string expected string @@ -57,6 +55,28 @@ func TestNewNetAddressString(t *testing.T) { {" @ ", "", false}, } + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + if tc.correct { + if assert.Nil(t, err, tc.addr) { + assert.Equal(t, tc.expected, addr.String()) + } + } else { + assert.NotNil(t, err, tc.addr) + } + } +} + +func TestNewNetAddressString(t *testing.T) { + testCases := []struct { + addr string + expected string + correct bool + }{ + {"127.0.0.1:8080", "127.0.0.1:8080", false}, + {"deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", true}, + } + for _, tc := range testCases { addr, err := NewNetAddressString(tc.addr) if tc.correct { @@ -70,23 +90,22 @@ func TestNewNetAddressString(t *testing.T) { } func TestNewNetAddressStrings(t *testing.T) { - addrs, errs := NewNetAddressStrings([]string{"127.0.0.1:8080", "127.0.0.2:8080"}) - assert.Len(t, errs, 0) + addrs, errs := NewNetAddressStrings([]string{ + "127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef@127.0.0.1:8080", + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeed@127.0.0.2:8080"}) + assert.Len(t, errs, 1) assert.Equal(t, 2, len(addrs)) } func TestNewNetAddressIPPort(t *testing.T) { - assert := assert.New(t) addr := NewNetAddressIPPort(net.ParseIP("127.0.0.1"), 8080) - - assert.Equal("127.0.0.1:8080", addr.String()) + assert.Equal(t, "127.0.0.1:8080", addr.String()) } func TestNetAddressProperties(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string valid bool local bool @@ -96,21 +115,19 @@ func TestNetAddressProperties(t *testing.T) { {"ya.ru:80", true, false, true}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - assert.Equal(t.valid, addr.Valid()) - assert.Equal(t.local, addr.Local()) - assert.Equal(t.routable, addr.Routable()) + assert.Equal(t, tc.valid, addr.Valid()) + assert.Equal(t, tc.local, addr.Local()) + assert.Equal(t, tc.routable, addr.Routable()) } } func TestNetAddressReachabilityTo(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // TODO add more test cases - tests := []struct { + testCases := []struct { addr string other string reachability int @@ -119,13 +136,13 @@ func TestNetAddressReachabilityTo(t *testing.T) { {"ya.ru:80", "127.0.0.1:8080", 1}, } - for _, t := range tests { - addr, err := NewNetAddressString(t.addr) - require.Nil(err) + for _, tc := range testCases { + addr, err := NewNetAddressStringWithOptionalID(tc.addr) + require.Nil(t, err) - other, err := NewNetAddressString(t.other) - require.Nil(err) + other, err := NewNetAddressStringWithOptionalID(tc.other) + require.Nil(t, err) - assert.Equal(t.reachability, addr.ReachabilityTo(other)) + assert.Equal(t, tc.reachability, addr.ReachabilityTo(other)) } } diff --git a/p2p/node_info.go b/p2p/node_info.go index 6f44b49cd..346de37d3 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -107,6 +107,7 @@ OUTER_LOOP: return nil } +// ID returns node's ID. func (info NodeInfo) ID() ID { return PubKeyToID(info.PubKey) } diff --git a/p2p/peer.go b/p2p/peer.go index 4af6eeaae..0fa7ca034 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -293,22 +293,20 @@ func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration return peerNodeInfo, errors.Wrap(err, "Error setting deadline") } - var err1 error - var err2 error - cmn.Parallel( - func() { + var trs, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err1) + wire.WriteBinary(&ourNodeInfo, pc.conn, &n, &err) + return }, - func() { + func(_ int) (val interface{}, err error, abort bool) { var n int - wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err2) - }) - if err1 != nil { - return peerNodeInfo, errors.Wrap(err1, "Error during handshake/write") - } - if err2 != nil { - return peerNodeInfo, errors.Wrap(err2, "Error during handshake/read") + wire.ReadBinary(&peerNodeInfo, pc.conn, MaxNodeInfoSize(), &n, &err) + return + }, + ) + if err := trs.FirstError(); err != nil { + return peerNodeInfo, errors.Wrap(err, "Error during handshake") } // Remove deadline diff --git a/p2p/peer_set.go b/p2p/peer_set.go index dc53174a1..a4565ea1d 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -68,9 +68,8 @@ func (ps *PeerSet) Get(peerKey ID) Peer { item, ok := ps.lookup[peerKey] if ok { return item.peer - } else { - return nil } + return nil } // Remove discards peer by its Key, if the peer was previously memoized. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index 174e73c9c..c8d1d64d2 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -115,7 +115,7 @@ func TestPeerSetAddDuplicate(t *testing.T) { errsTally := make(map[error]int) for i := 0; i < n; i++ { err := <-errsChan - errsTally[err] += 1 + errsTally[err]++ } // Our next procedure is to ensure that only one addition diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 3a3920486..a8462f375 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -33,24 +33,33 @@ type AddrBook interface { // Add our own addresses so we don't later add ourselves AddOurAddress(*p2p.NetAddress) + // Check if it is our address + OurAddress(*p2p.NetAddress) bool // Add and remove an address AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error - RemoveAddress(addr *p2p.NetAddress) + RemoveAddress(*p2p.NetAddress) + + // Check if the address is in the book + HasAddress(*p2p.NetAddress) bool // Do we need more peers? NeedMoreAddrs() bool // Pick an address to dial - PickAddress(newBias int) *p2p.NetAddress + PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress // Mark address MarkGood(*p2p.NetAddress) MarkAttempt(*p2p.NetAddress) MarkBad(*p2p.NetAddress) + IsGood(*p2p.NetAddress) bool + // Send a selection of addresses to peers GetSelection() []*p2p.NetAddress + // Send a selection of addresses with bias + GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress // TODO: remove ListOfKnownAddresses() []*knownAddress @@ -74,7 +83,7 @@ type addrBook struct { // accessed concurrently mtx sync.Mutex rand *rand.Rand - ourAddrs map[string]*p2p.NetAddress + ourAddrs map[string]struct{} addrLookup map[p2p.ID]*knownAddress // new & old bucketsOld []map[string]*knownAddress bucketsNew []map[string]*knownAddress @@ -89,7 +98,7 @@ type addrBook struct { func NewAddrBook(filePath string, routabilityStrict bool) *addrBook { am := &addrBook{ rand: rand.New(rand.NewSource(time.Now().UnixNano())), // TODO: seed from outside - ourAddrs: make(map[string]*p2p.NetAddress), + ourAddrs: make(map[string]struct{}), addrLookup: make(map[p2p.ID]*knownAddress), filePath: filePath, routabilityStrict: routabilityStrict, @@ -150,7 +159,15 @@ func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() a.Logger.Info("Add our address to book", "addr", addr) - a.ourAddrs[addr.String()] = addr + a.ourAddrs[addr.String()] = struct{}{} +} + +// OurAddress returns true if it is our address. +func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + _, ok := a.ourAddrs[addr.String()] + a.mtx.Unlock() + return ok } // AddAddress implements AddrBook - adds the given address as received from the given source. @@ -173,6 +190,22 @@ func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) { a.removeFromAllBuckets(ka) } +// IsGood returns true if peer was ever marked as good and haven't +// done anything wrong since then. +func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + return a.addrLookup[addr.ID].isOld() +} + +// HasAddress returns true if the address is in the book. +func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool { + a.mtx.Lock() + defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] + return ka != nil +} + // NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book. func (a *addrBook) NeedMoreAddrs() bool { return a.Size() < needAddressThreshold @@ -180,27 +213,27 @@ func (a *addrBook) NeedMoreAddrs() bool { // PickAddress implements AddrBook. It picks an address to connect to. // The address is picked randomly from an old or new bucket according -// to the newBias argument, which must be between [0, 100] (or else is truncated to that range) +// to the biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to that range) // and determines how biased we are to pick an address from a new bucket. // PickAddress returns nil if the AddrBook is empty or if we try to pick // from an empty bucket. -func (a *addrBook) PickAddress(newBias int) *p2p.NetAddress { +func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress { a.mtx.Lock() defer a.mtx.Unlock() if a.size() == 0 { return nil } - if newBias > 100 { - newBias = 100 + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 } - if newBias < 0 { - newBias = 0 + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 } // Bias between new and old addresses. - oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(newBias)) - newCorrelation := math.Sqrt(float64(a.nNew)) * float64(newBias) + oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(biasTowardsNewAddrs)) + newCorrelation := math.Sqrt(float64(a.nNew)) * float64(biasTowardsNewAddrs) // pick a random peer from a random bucket var bucket map[string]*knownAddress @@ -295,6 +328,100 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { return allAddr[:numAddresses] } +// GetSelectionWithBias implements AddrBook. +// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols. +// +// Each address is picked randomly from an old or new bucket according to the +// biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to +// that range) and determines how biased we are to pick an address from a new +// bucket. +func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress { + a.mtx.Lock() + defer a.mtx.Unlock() + + if a.size() == 0 { + return nil + } + + if biasTowardsNewAddrs > 100 { + biasTowardsNewAddrs = 100 + } + if biasTowardsNewAddrs < 0 { + biasTowardsNewAddrs = 0 + } + + numAddresses := cmn.MaxInt( + cmn.MinInt(minGetSelection, a.size()), + a.size()*getSelectionPercent/100) + numAddresses = cmn.MinInt(maxGetSelection, numAddresses) + + selection := make([]*p2p.NetAddress, numAddresses) + + oldBucketToAddrsMap := make(map[int]map[string]struct{}) + var oldIndex int + newBucketToAddrsMap := make(map[int]map[string]struct{}) + var newIndex int + + selectionIndex := 0 +ADDRS_LOOP: + for selectionIndex < numAddresses { + pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs + pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0 + bucket := make(map[string]*knownAddress) + + // loop until we pick a random non-empty bucket + for len(bucket) == 0 { + if pickFromOldBucket { + oldIndex = a.rand.Intn(len(a.bucketsOld)) + bucket = a.bucketsOld[oldIndex] + } else { + newIndex = a.rand.Intn(len(a.bucketsNew)) + bucket = a.bucketsNew[newIndex] + } + } + + // pick a random index + randIndex := a.rand.Intn(len(bucket)) + + // loop over the map to return that index + var selectedAddr *p2p.NetAddress + for _, ka := range bucket { + if randIndex == 0 { + selectedAddr = ka.Addr + break + } + randIndex-- + } + + // if we have selected the address before, restart the loop + // otherwise, record it and continue + if pickFromOldBucket { + if addrsMap, ok := oldBucketToAddrsMap[oldIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + oldBucketToAddrsMap[oldIndex] = make(map[string]struct{}) + } + oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{} + } else { + if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok { + if _, ok = addrsMap[selectedAddr.String()]; ok { + continue ADDRS_LOOP + } + } else { + newBucketToAddrsMap[newIndex] = make(map[string]struct{}) + } + newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{} + } + + selection[selectionIndex] = selectedAddr + selectionIndex++ + } + + return selection +} + // ListOfKnownAddresses returns the new and old addresses. func (a *addrBook) ListOfKnownAddresses() []*knownAddress { a.mtx.Lock() diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index 4a8df7166..2e2604286 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -157,6 +157,13 @@ func TestAddrBookPromoteToOld(t *testing.T) { t.Errorf("selection could not be bigger than the book") } + selection = book.GetSelectionWithBias(30) + t.Logf("selection: %v", selection) + + if len(selection) > book.Size() { + t.Errorf("selection with bias could not be bigger than the book") + } + assert.Equal(t, book.Size(), 100, "expecting book size to be 100") } @@ -229,3 +236,121 @@ func TestAddrBookRemoveAddress(t *testing.T) { book.RemoveAddress(nonExistingAddr) assert.Equal(t, 0, book.Size()) } + +func TestAddrBookGetSelection(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + assert.Empty(t, book.GetSelection()) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.Equal(t, 1, len(book.GetSelection())) + assert.Equal(t, addr, book.GetSelection()[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection := book.GetSelection() + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Errorf("selection %v could not be bigger than the book", selection) + } +} + +func TestAddrBookGetSelectionWithBias(t *testing.T) { + const biasTowardsNewAddrs = 30 + + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + + // 1) empty book + selection := book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Empty(t, selection) + + // 2) add one address + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + assert.Equal(t, 1, len(selection)) + assert.Equal(t, addr, selection[0]) + + // 3) add a bunch of addresses + randAddrs := randNetAddressPairs(t, 100) + for _, addrSrc := range randAddrs { + book.AddAddress(addrSrc.addr, addrSrc.src) + } + + // check there is no duplicates + addrs := make(map[string]*p2p.NetAddress) + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + for _, addr := range selection { + if dup, ok := addrs[addr.String()]; ok { + t.Fatalf("selection %v contains duplicates %v", selection, dup) + } + addrs[addr.String()] = addr + } + + if len(selection) > book.Size() { + t.Fatalf("selection %v could not be bigger than the book", selection) + } + + // 4) mark 80% of the addresses as good + randAddrsLen := len(randAddrs) + for i, addrSrc := range randAddrs { + if int((float64(i)/float64(randAddrsLen))*100) >= 20 { + book.MarkGood(addrSrc.addr) + } + } + + selection = book.GetSelectionWithBias(biasTowardsNewAddrs) + + // check that ~70% of addresses returned are good + good := 0 + for _, addr := range selection { + if book.IsGood(addr) { + good++ + } + } + got, expected := int((float64(good)/float64(len(selection)))*100), (100 - biasTowardsNewAddrs) + if got >= expected { + t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + } +} + +func TestAddrBookHasAddress(t *testing.T) { + fname := createTempFileName("addrbook_test") + defer deleteTempFile(fname) + + book := NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + addr := randIPv4Address(t) + book.AddAddress(addr, addr) + + assert.True(t, book.HasAddress(addr)) + + book.RemoveAddress(addr) + + assert.False(t, book.HasAddress(addr)) +} diff --git a/p2p/pex/known_address.go b/p2p/pex/known_address.go index 085eb10fa..0261e4902 100644 --- a/p2p/pex/known_address.go +++ b/p2p/pex/known_address.go @@ -56,7 +56,7 @@ func (ka *knownAddress) isNew() bool { func (ka *knownAddress) markAttempt() { now := time.Now() ka.LastAttempt = now - ka.Attempts += 1 + ka.Attempts++ } func (ka *knownAddress) markGood() { diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index 45a05bdb7..1bcc493dd 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -30,17 +30,24 @@ const ( defaultMinNumOutboundPeers = 10 // Seed/Crawler constants - // TODO: - // We want seeds to only advertise good peers. - // Peers are marked by external mechanisms. - // We need a config value that can be set to be - // on the order of how long it would take before a good - // peer is marked good. - defaultSeedDisconnectWaitPeriod = 2 * time.Minute // disconnect after this - defaultCrawlPeerInterval = 2 * time.Minute // dont redial for this. TODO: back-off - defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this + + // We want seeds to only advertise good peers. Therefore they should wait at + // least as long as we expect it to take for a peer to become good before + // disconnecting. + // see consensus/reactor.go: blocksToContributeToBecomeGoodPeer + // 10000 blocks assuming 1s blocks ~ 2.7 hours. + defaultSeedDisconnectWaitPeriod = 3 * time.Hour + + defaultCrawlPeerInterval = 2 * time.Minute // don't redial for this. TODO: back-off. what for? + + defaultCrawlPeersPeriod = 30 * time.Second // check some peers every this maxAttemptsToDial = 16 // ~ 35h in total (last attempt - 18h) + + // if node connects to seed, it does not have any trusted peers. + // Especially in the beginning, node should have more trusted peers than + // untrusted. + biasToSelectNewPeers = 30 // 70 to select good peers ) // PEXReactor handles PEX (peer exchange) and ensures that an @@ -157,7 +164,10 @@ func (r *PEXReactor) AddPeer(p Peer) { // peers when we need - we don't trust inbound peers as much. addr := p.NodeInfo().NetAddress() if !isAddrPrivate(addr, r.config.PrivatePeerIDs) { - r.book.AddAddress(addr, addr) + err := r.book.AddAddress(addr, addr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } } @@ -189,8 +199,7 @@ func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { // Seeds disconnect after sending a batch of addrs if r.config.SeedMode { - // TODO: should we be more selective ? - r.SendAddrs(src, r.book.GetSelection()) + r.SendAddrs(src, r.book.GetSelectionWithBias(biasToSelectNewPeers)) r.Switch.StopPeerGracefully(src) } else { r.SendAddrs(src, r.book.GetSelection()) @@ -259,7 +268,10 @@ func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { srcAddr := src.NodeInfo().NetAddress() for _, netAddr := range addrs { if netAddr != nil && !isAddrPrivate(netAddr, r.config.PrivatePeerIDs) { - r.book.AddAddress(netAddr, srcAddr) + err := r.book.AddAddress(netAddr, srcAddr) + if err != nil { + r.Logger.Error("Failed to add new address", "err", err) + } } } return nil @@ -467,9 +479,8 @@ func (r *PEXReactor) AttemptsToDial(addr *p2p.NetAddress) int { lAttempts, attempted := r.attemptsToDial.Load(addr.DialString()) if attempted { return lAttempts.(_attemptsToDial).number - } else { - return 0 } + return 0 } //---------------------------------------------------------- @@ -563,24 +574,16 @@ func (r *PEXReactor) crawlPeers() { r.book.MarkAttempt(pi.Addr) continue } - } - // Crawl the connected peers asking for more addresses - for _, pi := range peerInfos { - // We will wait a minimum period of time before crawling peers again - if now.Sub(pi.LastAttempt) >= defaultCrawlPeerInterval { - peer := r.Switch.Peers().Get(pi.Addr.ID) - if peer != nil { - r.RequestAddrs(peer) - } - } + // Ask for more addresses + peer := r.Switch.Peers().Get(pi.Addr.ID) + r.RequestAddrs(peer) } } // attemptDisconnects checks if we've been with each peer long enough to disconnect func (r *PEXReactor) attemptDisconnects() { for _, peer := range r.Switch.Peers().List() { - status := peer.Status() - if status.Duration < defaultSeedDisconnectWaitPeriod { + if peer.Status().Duration < defaultSeedDisconnectWaitPeriod { continue } if peer.IsPersistent() { diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index 20276ec86..329d17f37 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -63,35 +63,45 @@ func TestPEXReactorRunning(t *testing.T) { N := 3 switches := make([]*p2p.Switch, N) + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) + + books := make([]*addrBook, N) + logger := log.TestingLogger() // create switches for i := 0; i < N; i++ { switches[i] = p2p.MakeSwitch(config, i, "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { - sw.SetLogger(log.TestingLogger().With("switch", i)) + books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) + books[i].SetLogger(logger.With("pex", i)) + sw.SetAddrBook(books[i]) - r := NewPEXReactor(book, &PEXReactorConfig{}) - r.SetLogger(log.TestingLogger()) + sw.SetLogger(logger.With("pex", i)) + + r := NewPEXReactor(books[i], &PEXReactorConfig{}) + r.SetLogger(logger.With("pex", i)) r.SetEnsurePeersPeriod(250 * time.Millisecond) sw.AddReactor("pex", r) + return sw }) } - // fill the address book and add listeners - for _, s := range switches { - addr := s.NodeInfo().NetAddress() - book.AddAddress(addr, addr) - s.AddListener(p2p.NewDefaultListener("tcp", s.NodeInfo().ListenAddr, true, log.TestingLogger())) + addOtherNodeAddrToAddrBook := func(switchIndex, otherSwitchIndex int) { + addr := switches[otherSwitchIndex].NodeInfo().NetAddress() + books[switchIndex].AddAddress(addr, addr) } - // start switches - for _, s := range switches { - err := s.Start() // start switch and reactors + addOtherNodeAddrToAddrBook(0, 1) + addOtherNodeAddrToAddrBook(1, 0) + addOtherNodeAddrToAddrBook(2, 1) + + for i, sw := range switches { + sw.AddListener(p2p.NewDefaultListener("tcp", sw.NodeInfo().ListenAddr, true, logger.With("pex", i))) + + err := sw.Start() // start switch and reactors require.Nil(t, err) } @@ -127,6 +137,7 @@ func TestPEXReactorRequestMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -156,6 +167,7 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { defer teardownReactor(book) sw := createSwitchAndAddReactors(r) + sw.SetAddrBook(book) peer := newMockPeer() p2p.AddPeerToSwitch(sw, peer) @@ -182,13 +194,11 @@ func TestPEXReactorAddrsMessageAbuse(t *testing.T) { } func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { + // directory to store address books dir, err := ioutil.TempDir("", "pex_reactor") require.Nil(t, err) defer os.RemoveAll(dir) // nolint: errcheck - book := NewAddrBook(filepath.Join(dir, "addrbook.json"), false) - book.SetLogger(log.TestingLogger()) - // 1. create seed seed := p2p.MakeSwitch( config, @@ -196,6 +206,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook0.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor(book, &PEXReactorConfig{}) @@ -222,6 +236,10 @@ func TestPEXReactorUsesSeedsIfNeeded(t *testing.T) { "127.0.0.1", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + book := NewAddrBook(filepath.Join(dir, "addrbook1.json"), false) + book.SetLogger(log.TestingLogger()) + sw.SetAddrBook(book) + sw.SetLogger(log.TestingLogger()) r := NewPEXReactor( @@ -247,7 +265,8 @@ func TestPEXReactorCrawlStatus(t *testing.T) { defer teardownReactor(book) // Seed/Crawler mode uses data from the Switch - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) // Create a peer, add it to the peer set and the addrbook. peer := p2p.CreateRandomPeer(false) @@ -291,7 +310,8 @@ func TestPEXReactorDialPeer(t *testing.T) { pexR, book := createReactor(&PEXReactorConfig{}) defer teardownReactor(book) - _ = createSwitchAndAddReactors(pexR) + sw := createSwitchAndAddReactors(pexR) + sw.SetAddrBook(book) peer := newMockPeer() addr := peer.NodeInfo().NetAddress() @@ -397,6 +417,7 @@ func assertPeersWithTimeout( } func createReactor(config *PEXReactorConfig) (r *PEXReactor, book *addrBook) { + // directory to store address book dir, err := ioutil.TempDir("", "pex_reactor") if err != nil { panic(err) diff --git a/p2p/switch.go b/p2p/switch.go index fa037a9b8..e412d3cc9 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -33,15 +33,21 @@ const ( //----------------------------------------------------------------------------- +// An AddrBook represents an address book from the pex package, which is used +// to store peer addresses. type AddrBook interface { AddAddress(addr *NetAddress, src *NetAddress) error + AddOurAddress(*NetAddress) + OurAddress(*NetAddress) bool MarkGood(*NetAddress) + RemoveAddress(*NetAddress) + HasAddress(*NetAddress) bool Save() } //----------------------------------------------------------------------------- -// `Switch` handles peer connections and exposes an API to receive incoming messages +// Switch handles peer connections and exposes an API to receive incoming messages // on `Reactors`. Each `Reactor` is responsible for handling incoming messages of one // or more `Channels`. So while sending outgoing messages is typically performed on the peer, // incoming messages are received on the reactor. @@ -66,6 +72,7 @@ type Switch struct { rng *rand.Rand // seed for randomizing dial times and orders } +// NewSwitch creates a new Switch with the given config. func NewSwitch(config *cfg.P2PConfig) *Switch { sw := &Switch{ config: config, @@ -343,20 +350,21 @@ func (sw *Switch) IsDialing(id ID) bool { // DialPeersAsync dials a list of peers asynchronously in random order (optionally, making them persistent). func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent bool) error { netAddrs, errs := NewNetAddressStrings(peers) + // only log errors, dial correct addresses for _, err := range errs { sw.Logger.Error("Error in peer's address", "err", err) } + ourAddr := sw.nodeInfo.NetAddress() + + // TODO: move this out of here ? if addrBook != nil { // add peers to `addrBook` - ourAddr := sw.nodeInfo.NetAddress() for _, netAddr := range netAddrs { // do not add our address or ID - if netAddr.Same(ourAddr) { - continue + if !netAddr.Same(ourAddr) { + addrBook.AddAddress(netAddr, ourAddr) } - // TODO: move this out of here ? - addrBook.AddAddress(netAddr, ourAddr) } // Persist some peers to disk right away. // NOTE: integration tests depend on this @@ -367,8 +375,14 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b perm := sw.rng.Perm(len(netAddrs)) for i := 0; i < len(perm); i++ { go func(i int) { - sw.randomSleep(0) j := perm[i] + + // do not dial ourselves + if netAddrs[j].Same(ourAddr) { + return + } + + sw.randomSleep(0) err := sw.DialPeerWithAddress(netAddrs[j], persistent) if err != nil { sw.Logger.Error("Error dialing peer", "err", err) @@ -522,6 +536,15 @@ func (sw *Switch) addPeer(pc peerConn) error { // Avoid self if sw.nodeKey.ID() == peerID { + addr := peerNodeInfo.NetAddress() + + // remove the given address from the address book if we're added it earlier + sw.addrBook.RemoveAddress(addr) + + // add the given address to the address book to avoid dialing ourselves + // again this is our public address + sw.addrBook.AddOurAddress(addr) + return ErrSwitchConnectToSelf } diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 06e8b642e..c02eb26d3 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -39,8 +39,6 @@ type TestReactor struct { mtx sync.Mutex channels []*conn.ChannelDescriptor - peersAdded []Peer - peersRemoved []Peer logMessages bool msgsCounter int msgsReceived map[byte][]PeerMessage @@ -61,17 +59,9 @@ func (tr *TestReactor) GetChannels() []*conn.ChannelDescriptor { return tr.channels } -func (tr *TestReactor) AddPeer(peer Peer) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersAdded = append(tr.peersAdded, peer) -} +func (tr *TestReactor) AddPeer(peer Peer) {} -func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) { - tr.mtx.Lock() - defer tr.mtx.Unlock() - tr.peersRemoved = append(tr.peersRemoved, peer) -} +func (tr *TestReactor) RemovePeer(peer Peer, reason interface{}) {} func (tr *TestReactor) Receive(chID byte, peer Peer, msgBytes []byte) { if tr.logMessages { @@ -100,6 +90,10 @@ func MakeSwitchPair(t testing.TB, initSwitch func(int, *Switch) *Switch) (*Switc } func initSwitchFunc(i int, sw *Switch) *Switch { + sw.SetAddrBook(&addrBookMock{ + addrs: make(map[string]struct{}), + ourAddrs: make(map[string]struct{})}) + // Make two reactors of two channels each sw.AddReactor("foo", NewTestReactor([]*conn.ChannelDescriptor{ {ID: byte(0x00), Priority: 10}, @@ -109,6 +103,7 @@ func initSwitchFunc(i int, sw *Switch) *Switch { {ID: byte(0x02), Priority: 10}, {ID: byte(0x03), Priority: 10}, }, true)) + return sw } @@ -185,6 +180,32 @@ func TestConnAddrFilter(t *testing.T) { assertNoPeersAfterTimeout(t, s2, 400*time.Millisecond) } +func TestSwitchFiltersOutItself(t *testing.T) { + s1 := MakeSwitch(config, 1, "127.0.0.2", "123.123.123", initSwitchFunc) + // addr := s1.NodeInfo().NetAddress() + + // // add ourselves like we do in node.go#427 + // s1.addrBook.AddOurAddress(addr) + + // simulate s1 having a public IP by creating a remote peer with the same ID + rp := &remotePeer{PrivKey: s1.nodeKey.PrivKey, Config: DefaultPeerConfig()} + rp.Start() + + // addr should be rejected in addPeer based on the same ID + err := s1.DialPeerWithAddress(rp.Addr(), false) + if assert.Error(t, err) { + assert.Equal(t, ErrSwitchConnectToSelf, err) + } + + assert.True(t, s1.addrBook.OurAddress(rp.Addr())) + + assert.False(t, s1.addrBook.HasAddress(rp.Addr())) + + rp.Stop() + + assertNoPeersAfterTimeout(t, s1, 100*time.Millisecond) +} + func assertNoPeersAfterTimeout(t *testing.T, sw *Switch, timeout time.Duration) { time.Sleep(timeout) if sw.Peers().Size() != 0 { @@ -350,3 +371,29 @@ func BenchmarkSwitchBroadcast(b *testing.B) { b.Logf("success: %v, failure: %v", numSuccess, numFailure) } + +type addrBookMock struct { + addrs map[string]struct{} + ourAddrs map[string]struct{} +} + +var _ AddrBook = (*addrBookMock)(nil) + +func (book *addrBookMock) AddAddress(addr *NetAddress, src *NetAddress) error { + book.addrs[addr.String()] = struct{}{} + return nil +} +func (book *addrBookMock) AddOurAddress(addr *NetAddress) { book.ourAddrs[addr.String()] = struct{}{} } +func (book *addrBookMock) OurAddress(addr *NetAddress) bool { + _, ok := book.ourAddrs[addr.String()] + return ok +} +func (book *addrBookMock) MarkGood(*NetAddress) {} +func (book *addrBookMock) HasAddress(addr *NetAddress) bool { + _, ok := book.addrs[addr.String()] + return ok +} +func (book *addrBookMock) RemoveAddress(addr *NetAddress) { + delete(book.addrs, addr.String()) +} +func (book *addrBookMock) Save() {} diff --git a/p2p/test_util.go b/p2p/test_util.go index 0d6427f43..535b0bd0b 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -143,7 +143,7 @@ func MakeSwitch(cfg *cfg.P2PConfig, i int, network, version string, initSwitch f Version: version, ListenAddr: cmn.Fmt("%v:%v", network, rand.Intn(64512)+1023), } - for ch, _ := range sw.reactorsByCh { + for ch := range sw.reactorsByCh { ni.Channels = append(ni.Channels, ch) } sw.SetNodeInfo(ni) diff --git a/p2p/upnp/upnp.go b/p2p/upnp/upnp.go index cac67a730..e98538aae 100644 --- a/p2p/upnp/upnp.go +++ b/p2p/upnp/upnp.go @@ -103,7 +103,7 @@ func Discover() (nat NAT, err error) { return } } - err = errors.New("UPnP port discovery failed.") + err = errors.New("UPnP port discovery failed") return } diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index f23ba6160..c5c5822f9 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -253,13 +253,11 @@ func TestBroadcastTxCommit(t *testing.T) { } func TestTx(t *testing.T) { - assert, require := assert.New(t), require.New(t) - // first we broadcast a tx c := getHTTPClient() _, _, tx := MakeTxKV() bres, err := c.BroadcastTxCommit(tx) - require.Nil(err, "%+v", err) + require.Nil(t, err, "%+v", err) txHeight := bres.Height txHash := bres.Hash @@ -289,18 +287,19 @@ func TestTx(t *testing.T) { ptx, err := c.Tx(tc.hash, tc.prove) if !tc.valid { - require.NotNil(err) + require.NotNil(t, err) } else { - require.Nil(err, "%+v", err) - assert.EqualValues(txHeight, ptx.Height) - assert.EqualValues(tx, ptx.Tx) - assert.Zero(ptx.Index) - assert.True(ptx.TxResult.IsOK()) + require.Nil(t, err, "%+v", err) + assert.EqualValues(t, txHeight, ptx.Height) + assert.EqualValues(t, tx, ptx.Tx) + assert.Zero(t, ptx.Index) + assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof - if tc.prove && assert.EqualValues(tx, proof.Data) { - assert.True(proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) + if tc.prove && assert.EqualValues(t, tx, proof.Data) { + assert.True(t, proof.Proof.Verify(proof.Index, proof.Total, txHash, proof.RootHash)) } } } @@ -333,6 +332,7 @@ func TestTxSearch(t *testing.T) { assert.EqualValues(t, tx, ptx.Tx) assert.Zero(t, ptx.Index) assert.True(t, ptx.TxResult.IsOK()) + assert.EqualValues(t, txHash, ptx.Hash) // time to verify the proof proof := ptx.Proof diff --git a/rpc/core/net.go b/rpc/core/net.go index 9b04926ab..1918abf11 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -43,6 +43,7 @@ func NetInfo() (*ctypes.ResultNetInfo, error) { for _, peer := range p2pSwitch.Peers().List() { peers = append(peers, ctypes.Peer{ NodeInfo: peer.NodeInfo(), + ID: peer.ID(), IsOutbound: peer.IsOutbound(), ConnectionStatus: peer.Status(), }) diff --git a/rpc/core/status.go b/rpc/core/status.go index a8771c8f7..b4543a61c 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -1,9 +1,11 @@ package core import ( + "bytes" "time" ctypes "github.com/tendermint/tendermint/rpc/core/types" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" ) @@ -48,7 +50,10 @@ import ( // "remote_addr": "", // "network": "test-chain-qhVCa2", // "moniker": "vagrant-ubuntu-trusty-64", -// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6" +// "pub_key": "844981FE99ABB19F7816F2D5E94E8A74276AB1153760A7799E925C75401856C6", +// "validator_status": { +// "voting_power": 10 +// } // } // }, // "id": "", @@ -72,12 +77,50 @@ func Status() (*ctypes.ResultStatus, error) { latestBlockTime := time.Unix(0, latestBlockTimeNano) - return &ctypes.ResultStatus{ + result := &ctypes.ResultStatus{ NodeInfo: p2pSwitch.NodeInfo(), PubKey: pubKey, LatestBlockHash: latestBlockHash, LatestAppHash: latestAppHash, LatestBlockHeight: latestHeight, LatestBlockTime: latestBlockTime, - Syncing: consensusReactor.FastSync()}, nil + Syncing: consensusReactor.FastSync(), + } + + // add ValidatorStatus if node is a validator + if val := validatorAtHeight(latestHeight); val != nil { + result.ValidatorStatus = ctypes.ValidatorStatus{ + VotingPower: val.VotingPower, + } + } + + return result, nil +} + +func validatorAtHeight(h int64) *types.Validator { + lastBlockHeight, vals := consensusState.GetValidators() + + privValAddress := pubKey.Address() + + // if we're still at height h, search in the current validator set + if lastBlockHeight == h { + for _, val := range vals { + if bytes.Equal(val.Address, privValAddress) { + return val + } + } + } + + // if we've moved to the next height, retrieve the validator set from DB + if lastBlockHeight > h { + vals, err := sm.LoadValidators(stateDB, h) + if err != nil { + // should not happen + return nil + } + _, val := vals.GetByAddress(privValAddress) + return val + } + + return nil } diff --git a/rpc/core/tx.go b/rpc/core/tx.go index f592326b1..7ddc70809 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -44,7 +44,8 @@ import ( // "code": 0 // }, // "index": 0, -// "height": 52 +// "height": 52, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // }, // "id": "", // "jsonrpc": "2.0" @@ -67,11 +68,12 @@ import ( // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } r, err := txIndexer.Get(hash) @@ -93,6 +95,7 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { } return &ctypes.ResultTx{ + Hash: hash, Height: height, Index: uint32(index), TxResult: r.Result, @@ -137,7 +140,8 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // "tx": "mvZHHa7HhZ4aRT0xMDA=", // "tx_result": {}, // "index": 31, -// "height": 12 +// "height": 12, +// "hash": "2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF" // } // ], // "id": "", @@ -161,10 +165,11 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // - `tx_result`: the `abci.Result` object // - `index`: `int` - index of the transaction // - `height`: `int` - height of the block where this transaction was in +// - `hash`: `[]byte` - hash of the transaction func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { // if index is disabled, return error if _, ok := txIndexer.(*null.TxIndex); ok { - return nil, fmt.Errorf("Transaction indexing is disabled.") + return nil, fmt.Errorf("Transaction indexing is disabled") } q, err := tmquery.New(query) @@ -191,6 +196,7 @@ func TxSearch(query string, prove bool) ([]*ctypes.ResultTx, error) { } apiResults[i] = &ctypes.ResultTx{ + Hash: r.Tx.Hash(), Height: height, Index: index, TxResult: r.Result, diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index 7567fe650..8a6fff63c 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -54,14 +54,19 @@ func NewResultCommit(header *types.Header, commit *types.Commit, } } +type ValidatorStatus struct { + VotingPower int64 `json:"voting_power"` +} + type ResultStatus struct { - NodeInfo p2p.NodeInfo `json:"node_info"` - PubKey crypto.PubKey `json:"pub_key"` - LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` - LatestAppHash cmn.HexBytes `json:"latest_app_hash"` - LatestBlockHeight int64 `json:"latest_block_height"` - LatestBlockTime time.Time `json:"latest_block_time"` - Syncing bool `json:"syncing"` + NodeInfo p2p.NodeInfo `json:"node_info"` + PubKey crypto.PubKey `json:"pub_key"` + LatestBlockHash cmn.HexBytes `json:"latest_block_hash"` + LatestAppHash cmn.HexBytes `json:"latest_app_hash"` + LatestBlockHeight int64 `json:"latest_block_height"` + LatestBlockTime time.Time `json:"latest_block_time"` + Syncing bool `json:"syncing"` + ValidatorStatus ValidatorStatus `json:"validator_status,omitempty"` } func (s *ResultStatus) TxIndexEnabled() bool { @@ -93,6 +98,7 @@ type ResultDialPeers struct { type Peer struct { p2p.NodeInfo `json:"node_info"` + p2p.ID `json:"node_id"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` } @@ -123,6 +129,7 @@ type ResultBroadcastTxCommit struct { } type ResultTx struct { + Hash cmn.HexBytes `json:"hash"` Height int64 `json:"height"` Index uint32 `json:"index"` TxResult abci.ResponseDeliverTx `json:"tx_result"` diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index fe15cda21..ab2e94d0b 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -318,21 +318,21 @@ func (c *WSClient) reconnectRoutine() { c.Logger.Error("failed to reconnect", "err", err, "original_err", originalError) c.Stop() return - } else { - // drain reconnectAfter - LOOP: - for { - select { - case <-c.reconnectAfter: - default: - break LOOP - } - } - err = c.processBacklog() - if err == nil { - c.startReadWriteRoutines() + } + // drain reconnectAfter + LOOP: + for { + select { + case <-c.reconnectAfter: + default: + break LOOP } } + err := c.processBacklog() + if err == nil { + c.startReadWriteRoutines() + } + case <-c.Quit(): return } diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 515baf5dd..3f54c61ef 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -18,32 +18,51 @@ import ( ) func StartHTTPServer(listenAddr string, handler http.Handler, logger log.Logger) (listener net.Listener, err error) { - // listenAddr should be fully formed including tcp:// or unix:// prefix var proto, addr string parts := strings.SplitN(listenAddr, "://", 2) if len(parts) != 2 { - logger.Error("WARNING (tendermint/rpc/lib): Please use fully formed listening addresses, including the tcp:// or unix:// prefix") - // we used to allow addrs without tcp/unix prefix by checking for a colon - // TODO: Deprecate - proto = types.SocketType(listenAddr) - addr = listenAddr - // return nil, errors.Errorf("Invalid listener address %s", lisenAddr) - } else { - proto, addr = parts[0], parts[1] + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) } + proto, addr = parts[0], parts[1] - logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s socket %v", proto, addr)) + logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr)) listener, err = net.Listen(proto, addr) if err != nil { - return nil, errors.Errorf("Failed to listen to %v: %v", listenAddr, err) + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) } go func() { - res := http.Serve( + err := http.Serve( listener, RecoverAndLogHandler(handler, logger), ) - logger.Error("RPC HTTP server stopped", "result", res) + logger.Error("RPC HTTP server stopped", "err", err) + }() + return listener, nil +} + +func StartHTTPAndTLSServer(listenAddr string, handler http.Handler, certFile, keyFile string, logger log.Logger) (listener net.Listener, err error) { + var proto, addr string + parts := strings.SplitN(listenAddr, "://", 2) + if len(parts) != 2 { + return nil, errors.Errorf("Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", listenAddr) + } + proto, addr = parts[0], parts[1] + + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", listenAddr, certFile, keyFile)) + listener, err = net.Listen(proto, addr) + if err != nil { + return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) + } + + go func() { + err := http.ServeTLS( + listener, + RecoverAndLogHandler(handler, logger), + certFile, + keyFile, + ) + logger.Error("RPC HTTPS server stopped", "err", err) }() return listener, nil } diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index 37d451457..e4b02c58f 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -101,9 +101,8 @@ func NewRPCErrorResponse(id string, code int, msg string, data string) RPCRespon func (resp RPCResponse) String() string { if resp.Error == nil { return fmt.Sprintf("[%s %v]", resp.ID, resp.Result) - } else { - return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } + return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } func RPCParseError(id string, err error) RPCResponse { diff --git a/scripts/dep_utils/checkout.sh b/scripts/dep_utils/checkout.sh deleted file mode 100644 index 1d35e97ab..000000000 --- a/scripts/dep_utils/checkout.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -set -ex - -set +u -if [[ "$DEP" == "" ]]; then - DEP=$GOPATH/src/github.com/tendermint/tendermint/Gopkg.lock -fi -set -u - - -set -u - -function getVendoredVersion() { - grep -A100 "$LIB" "$DEP" | grep revision | head -n1 | grep -o '"[^"]\+"' | cut -d '"' -f 2 -} - - -# fetch and checkout vendored dep - -lib=$1 - -echo "----------------------------------" -echo "Getting $lib ..." -go get -t "github.com/tendermint/$lib/..." - -VENDORED=$(getVendoredVersion "$lib") -cd "$GOPATH/src/github.com/tendermint/$lib" || exit -MASTER=$(git rev-parse origin/master) - -if [[ "$VENDORED" != "$MASTER" ]]; then - echo "... VENDORED != MASTER ($VENDORED != $MASTER)" - echo "... Checking out commit $VENDORED" - git checkout "$VENDORED" &> /dev/null -fi diff --git a/scripts/wal2json/main.go b/scripts/wal2json/main.go index e44ed4b17..f6ffea431 100644 --- a/scripts/wal2json/main.go +++ b/scripts/wal2json/main.go @@ -4,6 +4,7 @@ Usage: wal2json */ + package main import ( diff --git a/state/state_test.go b/state/state_test.go index 1c5a860d5..31f2065cf 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -222,7 +222,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { changeIndex++ - power += 1 + power++ } header, blockID, responses := makeHeaderPartsResponsesValPowerChange(state, i, power) state, err = updateState(state, blockID, header, responses) @@ -240,7 +240,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { // use the next pubkey (note our counter starts at 0 this time) if changeIndex < len(changeHeights) && i == changeHeights[changeIndex]+1 { changeIndex++ - power += 1 + power++ } testCases[i-1] = power } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 24e982725..74bf4843b 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "fmt" + "sort" "strconv" "strings" "time" @@ -147,9 +148,8 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { res, err := txi.Get(hash) if res == nil { return []*types.TxResult{}, nil - } else { - return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } + return []*types.TxResult{res}, errors.Wrap(err, "error while retrieving the result") } // conditions to skip because they're handled before "everything else" @@ -170,10 +170,10 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { for _, r := range ranges { if !hashesInitialized { - hashes = txi.matchRange(r, startKeyForRange(r, height)) + hashes = txi.matchRange(r, []byte(r.key)) hashesInitialized = true } else { - hashes = intersect(hashes, txi.matchRange(r, startKeyForRange(r, height))) + hashes = intersect(hashes, txi.matchRange(r, []byte(r.key))) } } } @@ -202,6 +202,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { i++ } + // sort by height by default + sort.Slice(results, func(i, j int) bool { + return results[i].Height < results[j].Height + }) + return results, nil } @@ -236,6 +241,52 @@ type queryRange struct { includeUpperBound bool } +func (r queryRange) lowerBoundValue() interface{} { + if r.lowerBound == nil { + return nil + } + + if r.includeLowerBound { + return r.lowerBound + } else { + switch t := r.lowerBound.(type) { + case int64: + return t + 1 + case time.Time: + return t.Unix() + 1 + default: + panic("not implemented") + } + } +} + +func (r queryRange) AnyBound() interface{} { + if r.lowerBound != nil { + return r.lowerBound + } else { + return r.upperBound + } +} + +func (r queryRange) upperBoundValue() interface{} { + if r.upperBound == nil { + return nil + } + + if r.includeUpperBound { + return r.upperBound + } else { + switch t := r.upperBound.(type) { + case int64: + return t - 1 + case time.Time: + return t.Unix() - 1 + default: + panic("not implemented") + } + } +} + func lookForRanges(conditions []query.Condition) (ranges queryRanges, indexes []int) { ranges = make(queryRanges) for i, c := range conditions { @@ -299,34 +350,49 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) return } -func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { - it := dbm.IteratePrefix(txi.store, startKey) +func (txi *TxIndex) matchRange(r queryRange, prefix []byte) (hashes [][]byte) { + // create a map to prevent duplicates + hashesMap := make(map[string][]byte) + + lowerBound := r.lowerBoundValue() + upperBound := r.upperBoundValue() + + it := dbm.IteratePrefix(txi.store, prefix) defer it.Close() LOOP: for ; it.Valid(); it.Next() { if !isTagKey(it.Key()) { continue } - if r.upperBound != nil { - // no other way to stop iterator other than checking for upperBound - switch (r.upperBound).(type) { - case int64: - v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - if err == nil && v == r.upperBound { - if r.includeUpperBound { - hashes = append(hashes, it.Value()) - } - break LOOP - } - // XXX: passing time in a ABCI Tags is not yet implemented - // case time.Time: - // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) - // if v == r.upperBound { - // break - // } + switch r.AnyBound().(type) { + case int64: + v, err := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + if err != nil { + continue LOOP } + include := true + if lowerBound != nil && v < lowerBound.(int64) { + include = false + } + if upperBound != nil && v > upperBound.(int64) { + include = false + } + if include { + hashesMap[fmt.Sprintf("%X", it.Value())] = it.Value() + } + // XXX: passing time in a ABCI Tags is not yet implemented + // case time.Time: + // v := strconv.ParseInt(extractValueFromKey(it.Key()), 10, 64) + // if v == r.upperBound { + // break + // } } - hashes = append(hashes, it.Value()) + } + hashes = make([][]byte, len(hashesMap)) + i := 0 + for _, h := range hashesMap { + hashes[i] = h + i++ } return } @@ -344,33 +410,6 @@ func startKey(c query.Condition, height int64) []byte { return []byte(key) } -func startKeyForRange(r queryRange, height int64) []byte { - if r.lowerBound == nil { - return []byte(r.key) - } - - var lowerBound interface{} - if r.includeLowerBound { - lowerBound = r.lowerBound - } else { - switch t := r.lowerBound.(type) { - case int64: - lowerBound = t + 1 - case time.Time: - lowerBound = t.Unix() + 1 - default: - panic("not implemented") - } - } - var key string - if height > 0 { - key = fmt.Sprintf("%s/%v/%d", r.key, lowerBound, height) - } else { - key = fmt.Sprintf("%s/%v", r.key, lowerBound) - } - return []byte(key) -} - func isTagKey(key []byte) bool { return strings.Count(string(key), tagKeySeparator) == 3 } diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 7ffc36e50..97024b30b 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -20,7 +20,7 @@ func TestTxIndex(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash := tx.Hash() batch := txindex.NewBatch(1) @@ -35,7 +35,7 @@ func TestTxIndex(t *testing.T) { assert.Equal(t, txResult, loadedTxResult) tx2 := types.Tx("BYE BYE WORLD") - txResult2 := &types.TxResult{1, 0, tx2, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult2 := &types.TxResult{Height: 1, Index: 0, Tx: tx2, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} hash2 := tx2.Hash() err = indexer.Index(txResult2) @@ -122,6 +122,35 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { assert.Equal(t, []*types.TxResult{txResult}, results) } +func TestTxSearchMultipleTxs(t *testing.T) { + allowedTags := []string{"account.number"} + indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) + + // indexed first, but bigger height (to test the order of transactions) + txResult := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("1")}, + }) + txResult.Tx = types.Tx("Bob's account") + txResult.Height = 2 + err := indexer.Index(txResult) + require.NoError(t, err) + + // indexed second, but smaller height (to test the order of transactions) + txResult2 := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("2")}, + }) + txResult2.Tx = types.Tx("Alice's account") + txResult2.Height = 1 + err = indexer.Index(txResult2) + require.NoError(t, err) + + results, err := indexer.Search(query.MustParse("account.number >= 1")) + assert.NoError(t, err) + + require.Len(t, results, 2) + assert.Equal(t, []*types.TxResult{txResult2, txResult}, results) +} + func TestIndexAllTags(t *testing.T) { indexer := NewTxIndex(db.NewMemDB(), IndexAllTags()) @@ -146,12 +175,12 @@ func TestIndexAllTags(t *testing.T) { func txResultWithTags(tags []cmn.KVPair) *types.TxResult { tx := types.Tx("HELLO WORLD") - return &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + return &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: tags, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} } func benchmarkTxIndex(txsCount int, b *testing.B) { tx := types.Tx("HELLO WORLD") - txResult := &types.TxResult{1, 0, tx, abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} + txResult := &types.TxResult{Height: 1, Index: 0, Tx: tx, Result: abci.ResponseDeliverTx{Data: []byte{0}, Code: abci.CodeTypeOK, Log: "", Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}}} dir, err := ioutil.TempDir("", "tx_index_db") if err != nil { @@ -167,7 +196,7 @@ func benchmarkTxIndex(txsCount int, b *testing.B) { if err := batch.Add(txResult); err != nil { b.Fatal(err) } - txResult.Index += 1 + txResult.Index++ } b.ResetTimer() diff --git a/test/README.md b/test/README.md index d9f09fef7..ee53905bf 100644 --- a/test/README.md +++ b/test/README.md @@ -19,7 +19,3 @@ and run the following tests in docker containers: - send a tx on each node and ensure the state root is updated on all of them - crash and restart nodes one at a time and ensure they can sync back up (via fastsync) - crash and restart all nodes at once and ensure they can sync back up - -If on a `release-x.x.x` branch, we also run - -- `go test` for all our dependency libs (test/test_libs.sh) diff --git a/test/test_libs.sh b/test/test_libs.sh deleted file mode 100644 index 893bf35dc..000000000 --- a/test/test_libs.sh +++ /dev/null @@ -1,34 +0,0 @@ -#! /bin/bash -set -ex - -export PATH="$GOBIN:$PATH" - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -#################### -# libs we depend on -#################### - -# All libs should define `make test` and `make get_vendor_deps` -LIBS=(tmlibs go-wire go-crypto abci) -for lib in "${LIBS[@]}"; do - # checkout vendored version of lib - bash scripts/dep_utils/checkout.sh "$lib" - - echo "Testing $lib ..." - cd "$GOPATH/src/github.com/tendermint/$lib" - make get_tools - make get_vendor_deps - make test - if [[ "$?" != 0 ]]; then - echo "FAIL" - exit 1 - fi - cd "$DIR" -done - -echo "" -echo "PASS" diff --git a/types/block.go b/types/block.go index 970ca36f4..22d605d66 100644 --- a/types/block.go +++ b/types/block.go @@ -141,9 +141,8 @@ func (b *Block) StringIndented(indent string) string { func (b *Block) StringShort() string { if b == nil { return "nil-Block" - } else { - return fmt.Sprintf("Block#%v", b.Hash()) } + return fmt.Sprintf("Block#%v", b.Hash()) } //----------------------------------------------------------------------------- @@ -177,7 +176,9 @@ type Header struct { } // Hash returns the hash of the header. -// Returns nil if ValidatorHash is missing. +// Returns nil if ValidatorHash is missing, +// since a Header is not valid unless there is +// a ValidaotrsHash (corresponding to the validator set). func (h *Header) Hash() cmn.HexBytes { if h == nil || len(h.ValidatorsHash) == 0 { return nil @@ -215,7 +216,7 @@ func (h *Header) StringIndented(indent string) string { %s Data: %v %s Validators: %v %s App: %v -%s Conensus: %v +%s Consensus: %v %s Results: %v %s Evidence: %v %s}#%v`, diff --git a/types/block_test.go b/types/block_test.go index e3e22743c..4aaa16a97 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -29,7 +29,7 @@ func TestValidateBlock(t *testing.T) { // tamper with NumTxs block = MakeBlock(h, txs, commit) - block.NumTxs += 1 + block.NumTxs++ err = block.ValidateBasic() require.Error(t, err) diff --git a/types/event_bus.go b/types/event_bus.go index 4edaea588..37bd5619d 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -101,7 +101,7 @@ func (b *EventBus) PublishEventTx(event EventDataTx) error { b.Logger.Info("Got tag with an empty key (skipping)", "tag", tag, "tx", event.Tx) continue } - tags[string(tag.Key)] = tag.Value + tags[string(tag.Key)] = string(tag.Value) } // add predefined tags diff --git a/types/event_bus_test.go b/types/event_bus_test.go index aa97092f6..c1b4c1b0f 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -7,9 +7,58 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + abci "github.com/tendermint/abci/types" + cmn "github.com/tendermint/tmlibs/common" tmpubsub "github.com/tendermint/tmlibs/pubsub" + tmquery "github.com/tendermint/tmlibs/pubsub/query" ) +func TestEventBusPublishEventTx(t *testing.T) { + eventBus := NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + defer eventBus.Stop() + + tx := Tx("foo") + result := abci.ResponseDeliverTx{Data: []byte("bar"), Tags: []cmn.KVPair{}, Fee: cmn.KI64Pair{Key: []uint8{}, Value: 0}} + + txEventsCh := make(chan interface{}) + + // PublishEventTx adds all these 3 tags, so the query below should work + query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X'", tx.Hash()) + err = eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query), txEventsCh) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + for e := range txEventsCh { + edt := e.(TMEventData).Unwrap().(EventDataTx) + assert.Equal(t, int64(1), edt.Height) + assert.Equal(t, uint32(0), edt.Index) + assert.Equal(t, tx, edt.Tx) + assert.Equal(t, result, edt.Result) + close(done) + } + }() + + err = eventBus.PublishEventTx(EventDataTx{TxResult{ + Height: 1, + Index: 0, + Tx: tx, + Result: result, + }}) + assert.NoError(t, err) + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("did not receive a transaction after 1 sec.") + } +} + func BenchmarkEventBus(b *testing.B) { benchmarks := []struct { name string diff --git a/types/evidence.go b/types/evidence.go index 9e1f6af0e..0b3496045 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -144,7 +144,7 @@ func (dve *DuplicateVoteEvidence) Verify(chainID string) error { // BlockIDs must be different if dve.VoteA.BlockID.Equals(dve.VoteB.BlockID) { - return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote!", dve.VoteA.BlockID) + return fmt.Errorf("DuplicateVoteEvidence Error: BlockIDs are the same (%v) - not a real duplicate vote", dve.VoteA.BlockID) } // Signatures must be valid diff --git a/types/genesis.go b/types/genesis.go index 8d885db97..21c61806a 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -37,9 +37,8 @@ type GenesisDoc struct { func (genDoc *GenesisDoc) AppState() json.RawMessage { if len(genDoc.AppOptions) > 0 { return genDoc.AppOptions - } else { - return genDoc.AppStateJSON } + return genDoc.AppStateJSON } // SaveAs is a utility method for saving GenensisDoc as a JSON file. diff --git a/types/part_set.go b/types/part_set.go index 5c43b1518..749943291 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -30,12 +30,11 @@ type Part struct { func (part *Part) Hash() []byte { if part.hash != nil { return part.hash - } else { - hasher := ripemd160.New() - hasher.Write(part.Bytes) // nolint: errcheck, gas - part.hash = hasher.Sum(nil) - return part.hash } + hasher := ripemd160.New() + hasher.Write(part.Bytes) // nolint: errcheck, gas + part.hash = hasher.Sum(nil) + return part.hash } func (part *Part) String() string { @@ -129,20 +128,18 @@ func NewPartSetFromHeader(header PartSetHeader) *PartSet { func (ps *PartSet) Header() PartSetHeader { if ps == nil { return PartSetHeader{} - } else { - return PartSetHeader{ - Total: ps.total, - Hash: ps.hash, - } + } + return PartSetHeader{ + Total: ps.total, + Hash: ps.hash, } } func (ps *PartSet) HasHeader(header PartSetHeader) bool { if ps == nil { return false - } else { - return ps.Header().Equals(header) } + return ps.Header().Equals(header) } func (ps *PartSet) BitArray() *cmn.BitArray { @@ -251,7 +248,7 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { return n1 + n2, err } - psr.i += 1 + psr.i++ if psr.i >= len(psr.parts) { return 0, io.EOF } @@ -262,9 +259,8 @@ func (psr *PartSetReader) Read(p []byte) (n int, err error) { func (ps *PartSet) StringShort() string { if ps == nil { return "nil-PartSet" - } else { - ps.mtx.Lock() - defer ps.mtx.Unlock() - return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } + ps.mtx.Lock() + defer ps.mtx.Unlock() + return fmt.Sprintf("(%v of %v)", ps.Count(), ps.Total()) } diff --git a/types/priv_validator.go b/types/priv_validator.go index daa456bc0..052ace978 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -211,18 +211,18 @@ func LoadPrivValidatorFSWithSigner(filePath string, signerFunc func(PrivValidato } // Save persists the PrivValidatorFS to disk. -func (privVal *PrivValidatorFS) Save() { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - privVal.save() +func (pv *PrivValidatorFS) Save() { + pv.mtx.Lock() + defer pv.mtx.Unlock() + pv.save() } -func (privVal *PrivValidatorFS) save() { - outFile := privVal.filePath +func (pv *PrivValidatorFS) save() { + outFile := pv.filePath if outFile == "" { panic("Cannot save PrivValidator: filePath not set") } - jsonBytes, err := json.Marshal(privVal) + jsonBytes, err := json.Marshal(pv) if err != nil { panic(err) } @@ -234,22 +234,22 @@ func (privVal *PrivValidatorFS) save() { // Reset resets all fields in the PrivValidatorFS. // NOTE: Unsafe! -func (privVal *PrivValidatorFS) Reset() { +func (pv *PrivValidatorFS) Reset() { var sig crypto.Signature - privVal.LastHeight = 0 - privVal.LastRound = 0 - privVal.LastStep = 0 - privVal.LastSignature = sig - privVal.LastSignBytes = nil - privVal.Save() + pv.LastHeight = 0 + pv.LastRound = 0 + pv.LastStep = 0 + pv.LastSignature = sig + pv.LastSignBytes = nil + pv.Save() } // SignVote signs a canonical representation of the vote, along with the // chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signVote(chainID, vote); err != nil { +func (pv *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signVote(chainID, vote); err != nil { return errors.New(cmn.Fmt("Error signing vote: %v", err)) } return nil @@ -257,32 +257,32 @@ func (privVal *PrivValidatorFS) SignVote(chainID string, vote *Vote) error { // SignProposal signs a canonical representation of the proposal, along with // the chainID. Implements PrivValidator. -func (privVal *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() - if err := privVal.signProposal(chainID, proposal); err != nil { +func (pv *PrivValidatorFS) SignProposal(chainID string, proposal *Proposal) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() + if err := pv.signProposal(chainID, proposal); err != nil { return fmt.Errorf("Error signing proposal: %v", err) } return nil } // returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged -func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { - if privVal.LastHeight > height { +func (pv *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bool, error) { + if pv.LastHeight > height { return false, errors.New("Height regression") } - if privVal.LastHeight == height { - if privVal.LastRound > round { + if pv.LastHeight == height { + if pv.LastRound > round { return false, errors.New("Round regression") } - if privVal.LastRound == round { - if privVal.LastStep > step { + if pv.LastRound == round { + if pv.LastStep > step { return false, errors.New("Step regression") - } else if privVal.LastStep == step { - if privVal.LastSignBytes != nil { - if privVal.LastSignature.Empty() { + } else if pv.LastStep == step { + if pv.LastSignBytes != nil { + if pv.LastSignature.Empty() { panic("privVal: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -297,11 +297,11 @@ func (privVal *PrivValidatorFS) checkHRS(height int64, round int, step int8) (bo // signVote checks if the vote is good to sign and sets the vote signature. // It may need to set the timestamp as well if the vote is otherwise the same as // a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). -func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { +func (pv *PrivValidatorFS) signVote(chainID string, vote *Vote) error { height, round, step := vote.Height, vote.Round, voteToStep(vote) signBytes := vote.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -312,11 +312,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - vote.Signature = privVal.LastSignature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + vote.Signature = pv.LastSignature + } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { vote.Timestamp = timestamp - vote.Signature = privVal.LastSignature + vote.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -324,11 +324,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { } // It passed the checks. Sign the vote - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) vote.Signature = sig return nil } @@ -336,11 +336,11 @@ func (privVal *PrivValidatorFS) signVote(chainID string, vote *Vote) error { // signProposal checks if the proposal is good to sign and sets the proposal signature. // It may need to set the timestamp as well if the proposal is otherwise the same as // a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). -func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { +func (pv *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) error { height, round, step := proposal.Height, proposal.Round, stepPropose signBytes := proposal.SignBytes(chainID) - sameHRS, err := privVal.checkHRS(height, round, step) + sameHRS, err := pv.checkHRS(height, round, step) if err != nil { return err } @@ -351,11 +351,11 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) // If they only differ by timestamp, use last timestamp and signature // Otherwise, return error if sameHRS { - if bytes.Equal(signBytes, privVal.LastSignBytes) { - proposal.Signature = privVal.LastSignature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(privVal.LastSignBytes, signBytes); ok { + if bytes.Equal(signBytes, pv.LastSignBytes) { + proposal.Signature = pv.LastSignature + } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { proposal.Timestamp = timestamp - proposal.Signature = privVal.LastSignature + proposal.Signature = pv.LastSignature } else { err = fmt.Errorf("Conflicting data") } @@ -363,40 +363,40 @@ func (privVal *PrivValidatorFS) signProposal(chainID string, proposal *Proposal) } // It passed the checks. Sign the proposal - sig, err := privVal.Sign(signBytes) + sig, err := pv.Sign(signBytes) if err != nil { return err } - privVal.saveSigned(height, round, step, signBytes, sig) + pv.saveSigned(height, round, step, signBytes, sig) proposal.Signature = sig return nil } // Persist height/round/step and signature -func (privVal *PrivValidatorFS) saveSigned(height int64, round int, step int8, +func (pv *PrivValidatorFS) saveSigned(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - privVal.LastHeight = height - privVal.LastRound = round - privVal.LastStep = step - privVal.LastSignature = sig - privVal.LastSignBytes = signBytes - privVal.save() + pv.LastHeight = height + pv.LastRound = round + pv.LastStep = step + pv.LastSignature = sig + pv.LastSignBytes = signBytes + pv.save() } // SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. // Implements PrivValidator. -func (privVal *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - privVal.mtx.Lock() - defer privVal.mtx.Unlock() +func (pv *PrivValidatorFS) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { + pv.mtx.Lock() + defer pv.mtx.Unlock() var err error - heartbeat.Signature, err = privVal.Sign(heartbeat.SignBytes(chainID)) + heartbeat.Signature, err = pv.Sign(heartbeat.SignBytes(chainID)) return err } // String returns a string representation of the PrivValidatorFS. -func (privVal *PrivValidatorFS) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", privVal.GetAddress(), privVal.LastHeight, privVal.LastRound, privVal.LastStep) +func (pv *PrivValidatorFS) String() string { + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastHeight, pv.LastRound, pv.LastStep) } //------------------------------------- diff --git a/types/priv_validator/sign_info.go b/types/priv_validator/sign_info.go index 746131a96..8b135df63 100644 --- a/types/priv_validator/sign_info.go +++ b/types/priv_validator/sign_info.go @@ -49,30 +49,30 @@ func NewLastSignedInfo() *LastSignedInfo { } } -func (info *LastSignedInfo) String() string { - return fmt.Sprintf("LH:%v, LR:%v, LS:%v", info.Height, info.Round, info.Step) +func (lsi *LastSignedInfo) String() string { + return fmt.Sprintf("LH:%v, LR:%v, LS:%v", lsi.Height, lsi.Round, lsi.Step) } // Verify returns an error if there is a height/round/step regression // or if the HRS matches but there are no LastSignBytes. // It returns true if HRS matches exactly and the LastSignature exists. // It panics if the HRS matches, the LastSignBytes are not empty, but the LastSignature is empty. -func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { - if info.Height > height { +func (lsi LastSignedInfo) Verify(height int64, round int, step int8) (bool, error) { + if lsi.Height > height { return false, errors.New("Height regression") } - if info.Height == height { - if info.Round > round { + if lsi.Height == height { + if lsi.Round > round { return false, errors.New("Round regression") } - if info.Round == round { - if info.Step > step { + if lsi.Round == round { + if lsi.Step > step { return false, errors.New("Step regression") - } else if info.Step == step { - if info.SignBytes != nil { - if info.Signature.Empty() { + } else if lsi.Step == step { + if lsi.SignBytes != nil { + if lsi.Signature.Empty() { panic("info: LastSignature is nil but LastSignBytes is not!") } return true, nil @@ -85,24 +85,24 @@ func (info LastSignedInfo) Verify(height int64, round int, step int8) (bool, err } // Set height/round/step and signature on the info -func (info *LastSignedInfo) Set(height int64, round int, step int8, +func (lsi *LastSignedInfo) Set(height int64, round int, step int8, signBytes []byte, sig crypto.Signature) { - info.Height = height - info.Round = round - info.Step = step - info.Signature = sig - info.SignBytes = signBytes + lsi.Height = height + lsi.Round = round + lsi.Step = step + lsi.Signature = sig + lsi.SignBytes = signBytes } // Reset resets all the values. // XXX: Unsafe. -func (info *LastSignedInfo) Reset() { - info.Height = 0 - info.Round = 0 - info.Step = 0 - info.Signature = crypto.Signature{} - info.SignBytes = nil +func (lsi *LastSignedInfo) Reset() { + lsi.Height = 0 + lsi.Round = 0 + lsi.Step = 0 + lsi.Signature = crypto.Signature{} + lsi.SignBytes = nil } // SignVote checks the height/round/step (HRS) are greater than the latest state of the LastSignedInfo. diff --git a/types/validator_set.go b/types/validator_set.go index 83d066ec1..4b84f85d7 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -83,27 +83,30 @@ func (valSet *ValidatorSet) Copy() *ValidatorSet { } } +// HasAddress returns true if address given is in the validator set, false - +// otherwise. func (valSet *ValidatorSet) HasAddress(address []byte) bool { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - return idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) + return idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) } +// GetByAddress returns an index of the validator with address and validator +// itself if found. Otherwise, -1 and nil are returned. func (valSet *ValidatorSet) GetByAddress(address []byte) (index int, val *Validator) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx != len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { + if idx < len(valSet.Validators) && bytes.Equal(valSet.Validators[idx].Address, address) { return idx, valSet.Validators[idx].Copy() - } else { - return 0, nil } + return -1, nil } -// GetByIndex returns the validator by index. -// It returns nil values if index < 0 or -// index >= len(ValidatorSet.Validators) +// GetByIndex returns the validator's address and validator itself by index. +// It returns nil values if index is less than 0 or greater or equal to +// len(ValidatorSet.Validators). func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validator) { if index < 0 || index >= len(valSet.Validators) { return nil, nil @@ -112,10 +115,12 @@ func (valSet *ValidatorSet) GetByIndex(index int) (address []byte, val *Validato return val.Address, val.Copy() } +// Size returns the length of the validator set. func (valSet *ValidatorSet) Size() int { return len(valSet.Validators) } +// TotalVotingPower returns the sum of the voting powers of all validators. func (valSet *ValidatorSet) TotalVotingPower() int64 { if valSet.totalVotingPower == 0 { for _, val := range valSet.Validators { @@ -126,6 +131,8 @@ func (valSet *ValidatorSet) TotalVotingPower() int64 { return valSet.totalVotingPower } +// GetProposer returns the current proposer. If the validator set is empty, nil +// is returned. func (valSet *ValidatorSet) GetProposer() (proposer *Validator) { if len(valSet.Validators) == 0 { return nil @@ -146,6 +153,8 @@ func (valSet *ValidatorSet) findProposer() *Validator { return proposer } +// Hash returns the Merkle root hash build using validators (as leaves) in the +// set. func (valSet *ValidatorSet) Hash() []byte { if len(valSet.Validators) == 0 { return nil @@ -157,12 +166,14 @@ func (valSet *ValidatorSet) Hash() []byte { return merkle.SimpleHashFromHashers(hashers) } +// Add adds val to the validator set and returns true. It returns false if val +// is already in the set. func (valSet *ValidatorSet) Add(val *Validator) (added bool) { val = val.Copy() idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(val.Address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) { + if idx >= len(valSet.Validators) { valSet.Validators = append(valSet.Validators, val) // Invalidate cache valSet.Proposer = nil @@ -183,39 +194,42 @@ func (valSet *ValidatorSet) Add(val *Validator) (added bool) { } } +// Update updates val and returns true. It returns false if val is not present +// in the set. func (valSet *ValidatorSet) Update(val *Validator) (updated bool) { index, sameVal := valSet.GetByAddress(val.Address) if sameVal == nil { return false - } else { - valSet.Validators[index] = val.Copy() - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return true } + valSet.Validators[index] = val.Copy() + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return true } +// Remove deletes the validator with address. It returns the validator removed +// and true. If returns nil and false if validator is not present in the set. func (valSet *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { idx := sort.Search(len(valSet.Validators), func(i int) bool { return bytes.Compare(address, valSet.Validators[i].Address) <= 0 }) - if idx == len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { + if idx >= len(valSet.Validators) || !bytes.Equal(valSet.Validators[idx].Address, address) { return nil, false - } else { - removedVal := valSet.Validators[idx] - newValidators := valSet.Validators[:idx] - if idx+1 < len(valSet.Validators) { - newValidators = append(newValidators, valSet.Validators[idx+1:]...) - } - valSet.Validators = newValidators - // Invalidate cache - valSet.Proposer = nil - valSet.totalVotingPower = 0 - return removedVal, true } + removedVal := valSet.Validators[idx] + newValidators := valSet.Validators[:idx] + if idx+1 < len(valSet.Validators) { + newValidators = append(newValidators, valSet.Validators[idx+1:]...) + } + valSet.Validators = newValidators + // Invalidate cache + valSet.Proposer = nil + valSet.totalVotingPower = 0 + return removedVal, true } +// Iterate will run the given function over the set. func (valSet *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { for i, val := range valSet.Validators { stop := fn(i, val.Copy()) @@ -266,10 +280,9 @@ func (valSet *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height if talliedVotingPower > valSet.TotalVotingPower()*2/3 { return nil - } else { - return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", - talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } + return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", + talliedVotingPower, (valSet.TotalVotingPower()*2/3 + 1)) } // VerifyCommitAny will check to see if the set would @@ -472,9 +485,8 @@ func safeMulClip(a, b int64) int64 { if overflow { if (a < 0 || b < 0) && !(a < 0 && b < 0) { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -484,9 +496,8 @@ func safeAddClip(a, b int64) int64 { if overflow { if b < 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } @@ -496,9 +507,8 @@ func safeSubClip(a, b int64) int64 { if overflow { if b > 0 { return math.MinInt64 - } else { - return math.MaxInt64 } + return math.MaxInt64 } return c } diff --git a/types/validator_set_test.go b/types/validator_set_test.go index b346be1be..8769ac5ff 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -127,7 +127,7 @@ func TestProposerSelection2(t *testing.T) { for i := 0; i < 120*N; i++ { prop := vals.GetProposer() ii := prop.Address[19] - propCount[ii] += 1 + propCount[ii]++ vals.IncrementAccum(1) } diff --git a/types/vote_set.go b/types/vote_set.go index 37f26f4a5..e255488d6 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -94,33 +94,29 @@ func (voteSet *VoteSet) ChainID() string { func (voteSet *VoteSet) Height() int64 { if voteSet == nil { return 0 - } else { - return voteSet.height } + return voteSet.height } func (voteSet *VoteSet) Round() int { if voteSet == nil { return -1 - } else { - return voteSet.round } + return voteSet.round } func (voteSet *VoteSet) Type() byte { if voteSet == nil { return 0x00 - } else { - return voteSet.type_ } + return voteSet.type_ } func (voteSet *VoteSet) Size() int { if voteSet == nil { return 0 - } else { - return voteSet.valSet.Size() } + return voteSet.valSet.Size() } // Returns added=true if vote is valid and new. @@ -185,9 +181,8 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { if existing, ok := voteSet.getVote(valIndex, blockKey); ok { if existing.Signature.Equals(vote.Signature) { return false, nil // duplicate - } else { - return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } + return false, errors.Wrapf(ErrVoteNonDeterministicSignature, "Existing vote: %v; New vote: %v", existing, vote) } // Check signature. @@ -199,13 +194,11 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { added, conflicting := voteSet.addVerifiedVote(vote, blockKey, val.VotingPower) if conflicting != nil { return added, NewConflictingVoteError(val, conflicting, vote) - } else { - if !added { - cmn.PanicSanity("Expected to add non-conflicting vote") - } - return added, nil } - + if !added { + cmn.PanicSanity("Expected to add non-conflicting vote") + } + return added, nil } // Returns (vote, true) if vote exists for valIndex and blockKey @@ -257,13 +250,12 @@ func (voteSet *VoteSet) addVerifiedVote(vote *Vote, blockKey string, votingPower // ... and there's a conflicting vote. // We're not even tracking this blockKey, so just forget it. return false, conflicting - } else { - // ... and there's no conflicting vote. - // Start tracking this blockKey - votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) - voteSet.votesByBlock[blockKey] = votesByBlock - // We'll add the vote in a bit. } + // ... and there's no conflicting vote. + // Start tracking this blockKey + votesByBlock = newBlockVotes(false, voteSet.valSet.Size()) + voteSet.votesByBlock[blockKey] = votesByBlock + // We'll add the vote in a bit. } // Before adding to votesByBlock, see if we'll exceed quorum @@ -309,10 +301,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if existing, ok := voteSet.peerMaj23s[peerID]; ok { if existing.Equals(blockID) { return nil // Nothing to do - } else { - return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", - peerID, blockID, existing) } + return fmt.Errorf("SetPeerMaj23: Received conflicting blockID from peer %v. Got %v, expected %v", + peerID, blockID, existing) } voteSet.peerMaj23s[peerID] = blockID @@ -321,10 +312,9 @@ func (voteSet *VoteSet) SetPeerMaj23(peerID P2PID, blockID BlockID) error { if ok { if votesByBlock.peerMaj23 { return nil // Nothing to do - } else { - votesByBlock.peerMaj23 = true - // No need to copy votes, already there. } + votesByBlock.peerMaj23 = true + // No need to copy votes, already there. } else { votesByBlock = newBlockVotes(true, voteSet.valSet.Size()) voteSet.votesByBlock[blockKey] = votesByBlock @@ -422,9 +412,8 @@ func (voteSet *VoteSet) TwoThirdsMajority() (blockID BlockID, ok bool) { defer voteSet.mtx.Unlock() if voteSet.maj23 != nil { return *voteSet.maj23, true - } else { - return BlockID{}, false } + return BlockID{}, false } func (voteSet *VoteSet) String() string { diff --git a/version/version.go b/version/version.go index 74662b808..a8fbfa84f 100644 --- a/version/version.go +++ b/version/version.go @@ -1,13 +1,13 @@ package version const Maj = "0" -const Min = "17" -const Fix = "1" +const Min = "18" +const Fix = "0" var ( // Version is the current version of Tendermint // Must be a string because scripts like dist.sh read this file. - Version = "0.17.1" + Version = "0.18.0" // GitCommit is the current HEAD set using ldflags. GitCommit string