From 70ba60885057c9bf351b1a92db660d96448c6967 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Tue, 10 Oct 2017 12:56:18 -0400 Subject: [PATCH 1/7] config: write all default options to config file config: test the default file docs: spiff up config config: minor fixes & comments config: simplify test config; use a seperate config directory, #556 config: update docs & parameterize file paths config: PR comments config: use the default object fix a rebase error --- CHANGELOG.md | 1 + cmd/tendermint/commands/testnet.go | 13 +- cmd/tendermint/main.go | 4 +- config/config.go | 35 +++-- config/toml.go | 205 +++++++++++++++++++++------ config/toml_test.go | 37 ++++- consensus/test_data/build.sh | 148 +++++++++++++++++++ docs/deploy-testnets.rst | 2 +- docs/specification/configuration.rst | 195 ++++++++++++++++++------- docs/specification/genesis.rst | 2 +- docs/specification/rpc.rst | 2 +- docs/using-tendermint.rst | 16 +-- rpc/core/doc.go | 2 +- scripts/debora/unsafe_reset_net.sh | 2 +- 14 files changed, 541 insertions(+), 123 deletions(-) create mode 100755 consensus/test_data/build.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f0809d9..392b7e6ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ BREAKING CHANGES: - Better support for injecting randomness - Pass evidence/voteInfo through ABCI - Upgrade consensus for more real-time use of evidence +- the files usually found in `~/.tendermint` (`config.toml`, `genesis.json`, and `priv_validator.json`) are now in `~/.tendermint/config`. The `$TMHOME/data/` directory remains unchanged. FEATURES: - Peer reputation management diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 2c859df2b..f5551a95e 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -2,11 +2,12 @@ package commands import ( "fmt" - "path" + "path/filepath" "time" "github.com/spf13/cobra" + cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" cmn "github.com/tendermint/tmlibs/common" ) @@ -35,6 +36,7 @@ var TestnetFilesCmd = &cobra.Command{ func testnetFiles(cmd *cobra.Command, args []string) { genVals := make([]types.GenesisValidator, nValidators) + defaultConfig := cfg.DefaultBaseConfig() // Initialize core dir and priv_validator.json's for i := 0; i < nValidators; i++ { @@ -44,7 +46,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { cmn.Exit(err.Error()) } // Read priv_validator.json to populate vals - privValFile := path.Join(dataDir, mach, "priv_validator.json") + privValFile := filepath.Join(dataDir, mach, defaultConfig.PrivValidator) privVal := types.LoadPrivValidatorFS(privValFile) genVals[i] = types.GenesisValidator{ PubKey: privVal.GetPubKey(), @@ -63,7 +65,7 @@ func testnetFiles(cmd *cobra.Command, args []string) { // Write genesis file. for i := 0; i < nValidators; i++ { mach := cmn.Fmt("mach%d", i) - if err := genDoc.SaveAs(path.Join(dataDir, mach, "genesis.json")); err != nil { + if err := genDoc.SaveAs(filepath.Join(dataDir, mach, defaultConfig.Genesis)); err != nil { panic(err) } } @@ -73,14 +75,15 @@ func testnetFiles(cmd *cobra.Command, args []string) { // Initialize per-machine core directory func initMachCoreDirectory(base, mach string) error { - dir := path.Join(base, mach) + dir := filepath.Join(base, mach) err := cmn.EnsureDir(dir, 0777) if err != nil { return err } // Create priv_validator.json file if not present - ensurePrivValidator(path.Join(dir, "priv_validator.json")) + defaultConfig := cfg.DefaultBaseConfig() + ensurePrivValidator(filepath.Join(dir, defaultConfig.PrivValidator)) return nil } diff --git a/cmd/tendermint/main.go b/cmd/tendermint/main.go index c24cfe197..17b5a585b 100644 --- a/cmd/tendermint/main.go +++ b/cmd/tendermint/main.go @@ -2,10 +2,12 @@ package main import ( "os" + "path/filepath" "github.com/tendermint/tmlibs/cli" cmd "github.com/tendermint/tendermint/cmd/tendermint/commands" + cfg "github.com/tendermint/tendermint/config" nm "github.com/tendermint/tendermint/node" ) @@ -37,7 +39,7 @@ func main() { // Create & start node rootCmd.AddCommand(cmd.NewRunNodeCmd(nodeFunc)) - cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv("$HOME/.tendermint")) + cmd := cli.PrepareBaseCmd(rootCmd, "TM", os.ExpandEnv(filepath.Join("$HOME", cfg.DefaultTendermintDir))) if err := cmd.Execute(); err != nil { panic(err) } diff --git a/config/config.go b/config/config.go index 5d4a8ef65..1585fcd0e 100644 --- a/config/config.go +++ b/config/config.go @@ -7,6 +7,25 @@ import ( "time" ) +// Note: Most of the structs & relevant comments + the +// default configuration options were used to manually +// generate the config.toml. Please reflect any changes +// made here in the defaultConfigTemplate constant in +// config/toml.go +var ( + DefaultTendermintDir = ".tendermint" + defaultConfigDir = "config" + defaultDataDir = "data" + + defaultConfigFileName = "config.toml" + defaultGenesisJSONName = "genesis.json" + defaultPrivValName = "priv_validator.json" + + defaultConfigFilePath = filepath.Join(defaultConfigDir, defaultConfigFileName) + defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName) + defaultPrivValPath = filepath.Join(defaultConfigDir, defaultPrivValName) +) + // Config defines the top level configuration for a Tendermint node type Config struct { // Top level options use an anonymous struct @@ -66,10 +85,10 @@ type BaseConfig struct { // The ID of the chain to join (should be signed with every transaction and vote) ChainID string `mapstructure:"chain_id"` - // A JSON file containing the initial validator set and other meta data + // Path to the JSON file containing the initial validator set and other meta data Genesis string `mapstructure:"genesis_file"` - // A JSON file containing the private key to use as a validator in the consensus protocol + // Path to the JSON file containing the private key to use as a validator in the consensus protocol PrivValidator string `mapstructure:"priv_validator_file"` // A custom human readable name for this node @@ -107,8 +126,8 @@ type BaseConfig struct { // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ - Genesis: "genesis.json", - PrivValidator: "priv_validator.json", + Genesis: defaultGenesisJSONPath, + PrivValidator: defaultPrivValPath, Moniker: defaultMoniker, ProxyApp: "tcp://127.0.0.1:46658", ABCI: "socket", @@ -279,7 +298,7 @@ func DefaultMempoolConfig() *MempoolConfig { Recheck: true, RecheckEmpty: true, Broadcast: true, - WalPath: "data/mempool.wal", + WalPath: filepath.Join(defaultDataDir, "mempool.wal"), } } @@ -299,7 +318,7 @@ type ConsensusConfig struct { WalLight bool `mapstructure:"wal_light"` walFile string // overrides WalPath if set - // All timeouts are in ms + // All timeouts are in milliseconds TimeoutPropose int `mapstructure:"timeout_propose"` TimeoutProposeDelta int `mapstructure:"timeout_propose_delta"` TimeoutPrevote int `mapstructure:"timeout_prevote"` @@ -319,7 +338,7 @@ type ConsensusConfig struct { CreateEmptyBlocks bool `mapstructure:"create_empty_blocks"` CreateEmptyBlocksInterval int `mapstructure:"create_empty_blocks_interval"` - // Reactor sleep duration parameters are in ms + // Reactor sleep duration parameters are in milliseconds PeerGossipSleepDuration int `mapstructure:"peer_gossip_sleep_duration"` PeerQueryMaj23SleepDuration int `mapstructure:"peer_query_maj23_sleep_duration"` } @@ -367,7 +386,7 @@ func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ - WalPath: "data/cs.wal/wal", + WalPath: filepath.Join(defaultDataDir, "cs.wal/wal"), WalLight: false, TimeoutPropose: 3000, TimeoutProposeDelta: 500, diff --git a/config/toml.go b/config/toml.go index 735f45c12..56a5a1304 100644 --- a/config/toml.go +++ b/config/toml.go @@ -1,25 +1,37 @@ package config import ( + "bytes" "os" - "path" "path/filepath" - "strings" + "text/template" cmn "github.com/tendermint/tmlibs/common" ) +var configTemplate *template.Template + +func init() { + var err error + if configTemplate, err = template.New("configFileTemplate").Parse(defaultConfigTemplate); err != nil { + panic(err) + } +} + /****** these are for production settings ***********/ func EnsureRoot(rootDir string) { if err := cmn.EnsureDir(rootDir, 0700); err != nil { cmn.PanicSanity(err.Error()) } - if err := cmn.EnsureDir(rootDir+"/data", 0700); err != nil { + if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil { + cmn.PanicSanity(err.Error()) + } + if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil { cmn.PanicSanity(err.Error()) } - configFilePath := path.Join(rootDir, "config.toml") + configFilePath := filepath.Join(rootDir, defaultConfigFilePath) // Write default config file if missing. if !cmn.FileExists(configFilePath) { @@ -27,26 +39,153 @@ func EnsureRoot(rootDir string) { } } -var defaultConfigTmpl = `# This is a TOML config file. +// XXX: this func should probably be called by cmd/tendermint/commands/init.go +// alongside the writing of the genesis.json and priv_validator.json +func writeConfigFile(configFilePath string) { + var buffer bytes.Buffer + + if err := configTemplate.Execute(&buffer, DefaultConfig()); err != nil { + panic(err) + } + + cmn.MustWriteFile(configFilePath, buffer.Bytes(), 0644) +} + +// Note: any changes to the comments/variables/mapstructure +// must be reflected in the appropriate struct in config/config.go +const defaultConfigTemplate = `# This is a TOML config file. # For more information, see https://github.com/toml-lang/toml -proxy_app = "tcp://127.0.0.1:46658" -moniker = "__MONIKER__" -fast_sync = true -db_backend = "leveldb" -log_level = "state:info,*:error" +##### main base config options ##### + +# TCP or UNIX socket address of the ABCI application, +# or the name of an ABCI application compiled in with the Tendermint binary +proxy_app = "{{ .BaseConfig.ProxyApp }}" + +# A custom human readable name for this node +moniker = "{{ .BaseConfig.Moniker }}" + +# If this node is many blocks behind the tip of the chain, FastSync +# allows them to catchup quickly by downloading blocks in parallel +# and verifying their commits +fast_sync = {{ .BaseConfig.FastSync }} + +# Database backend: leveldb | memdb +db_backend = "{{ .BaseConfig.DBBackend }}" + +# Database directory +db_path = "{{ .BaseConfig.DBPath }}" +# Output level for logging +log_level = "{{ .BaseConfig.LogLevel }}" + +##### additional base config options ##### + +# The ID of the chain to join (should be signed with every transaction and vote) +chain_id = "{{ .BaseConfig.ChainID }}" + +# Path to the JSON file containing the initial validator set and other meta data +genesis_file = "{{ .BaseConfig.Genesis }}" + +# Path to the JSON file containing the private key to use as a validator in the consensus protocol +priv_validator_file = "{{ .BaseConfig.PrivValidator }}" + +# Mechanism to connect to the ABCI application: socket | grpc +abci = "{{ .BaseConfig.ABCI }}" + +# TCP or UNIX socket address for the profiling server to listen on +prof_laddr = "{{ .BaseConfig.ProfListenAddress }}" + +# If true, query the ABCI app on connecting to a new peer +# so the app can decide if we should keep the connection or not +filter_peers = {{ .BaseConfig.FilterPeers }} + +# What indexer to use for transactions +tx_index = "{{ .BaseConfig.TxIndex }}" + +##### advanced configuration options ##### + +##### rpc server configuration options ##### [rpc] -laddr = "tcp://0.0.0.0:46657" +# TCP or UNIX socket address for the RPC server to listen on +laddr = "{{ .RPC.ListenAddress }}" + +# TCP or UNIX socket address for the gRPC server to listen on +# NOTE: This server only supports /broadcast_tx_commit +grpc_laddr = "{{ .RPC.GRPCListenAddress }}" + +# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool +unsafe = {{ .RPC.Unsafe }} + +##### peer to peer configuration options ##### [p2p] -laddr = "tcp://0.0.0.0:46656" + +# Address to listen for incoming connections +laddr = "{{ .P2P.ListenAddress }}" + +# Comma separated list of seed nodes to connect to seeds = "" -` -func defaultConfig(moniker string) string { - return strings.Replace(defaultConfigTmpl, "__MONIKER__", moniker, -1) -} +# Path to address book +addr_book_file = "{{ .P2P.AddrBook }}" + +# Set true for strict address routability rules +addr_book_strict = {{ .P2P.AddrBookStrict }} + +# Time to wait before flushing messages out on the connection, in ms +flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }} + +# Maximum number of peers to connect to +max_num_peers = {{ .P2P.MaxNumPeers }} + +# Maximum size of a message packet payload, in bytes +max_msg_packet_payload_size = {{ .P2P.MaxMsgPacketPayloadSize }} + +# Rate at which packets can be sent, in bytes/second +send_rate = {{ .P2P.SendRate }} + +# Rate at which packets can be received, in bytes/second +recv_rate = {{ .P2P.RecvRate }} + +##### mempool configuration options ##### +[mempool] + +recheck = {{ .Mempool.Recheck }} +recheck_empty = {{ .Mempool.RecheckEmpty }} +broadcast = {{ .Mempool.Broadcast }} +wal_dir = "{{ .Mempool.WalPath }}" + +##### consensus configuration options ##### +[consensus] + +wal_file = "{{ .Consensus.WalPath }}" +wal_light = {{ .Consensus.WalLight }} + +# All timeouts are in milliseconds +timeout_propose = {{ .Consensus.TimeoutPropose }} +timeout_propose_delta = {{ .Consensus.TimeoutProposeDelta }} +timeout_prevote = {{ .Consensus.TimeoutPrevote }} +timeout_prevote_delta = {{ .Consensus.TimeoutPrevoteDelta }} +timeout_precommit = {{ .Consensus.TimeoutPrecommit }} +timeout_precommit_delta = {{ .Consensus.TimeoutPrecommitDelta }} +timeout_commit = {{ .Consensus.TimeoutCommit }} + +# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) +skip_timeout_commit = {{ .Consensus.SkipTimeoutCommit }} + +# BlockSize +max_block_size_txs = {{ .Consensus.MaxBlockSizeTxs }} +max_block_size_bytes = {{ .Consensus.MaxBlockSizeBytes }} + +# EmptyBlocks mode and possible interval between empty blocks in seconds +create_empty_blocks = {{ .Consensus.CreateEmptyBlocks }} +create_empty_blocks_interval = {{ .Consensus.CreateEmptyBlocksInterval }} + +# Reactor sleep duration parameters are in milliseconds +peer_gossip_sleep_duration = {{ .Consensus.PeerGossipSleepDuration }} +peer_query_maj23_sleep_duration = {{ .Consensus.PeerQueryMaj23SleepDuration }} +` /****** these are for test settings ***********/ @@ -69,13 +208,17 @@ func ResetTestRoot(testName string) *Config { if err := cmn.EnsureDir(rootDir, 0700); err != nil { cmn.PanicSanity(err.Error()) } - if err := cmn.EnsureDir(rootDir+"/data", 0700); err != nil { + if err := cmn.EnsureDir(filepath.Join(rootDir, defaultConfigDir), 0700); err != nil { + cmn.PanicSanity(err.Error()) + } + if err := cmn.EnsureDir(filepath.Join(rootDir, defaultDataDir), 0700); err != nil { cmn.PanicSanity(err.Error()) } - configFilePath := path.Join(rootDir, "config.toml") - genesisFilePath := path.Join(rootDir, "genesis.json") - privFilePath := path.Join(rootDir, "priv_validator.json") + baseConfig := DefaultBaseConfig() + configFilePath := filepath.Join(rootDir, defaultConfigFilePath) + genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) + privFilePath := filepath.Join(rootDir, baseConfig.PrivValidator) // Write default config file if missing. if !cmn.FileExists(configFilePath) { @@ -91,28 +234,6 @@ func ResetTestRoot(testName string) *Config { return config } -var testConfigTmpl = `# This is a TOML config file. -# For more information, see https://github.com/toml-lang/toml - -proxy_app = "dummy" -moniker = "__MONIKER__" -fast_sync = false -db_backend = "memdb" -log_level = "info" - -[rpc] -laddr = "tcp://0.0.0.0:36657" - -[p2p] -laddr = "tcp://0.0.0.0:36656" -seeds = "" -` - -func testConfig(moniker string) (testConfig string) { - testConfig = strings.Replace(testConfigTmpl, "__MONIKER__", moniker, -1) - return -} - var testGenesis = `{ "genesis_time": "0001-01-01T00:00:00.000Z", "chain_id": "tendermint_test", diff --git a/config/toml_test.go b/config/toml_test.go index f927a14ca..e98719f6f 100644 --- a/config/toml_test.go +++ b/config/toml_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -30,7 +31,7 @@ func TestEnsureRoot(t *testing.T) { EnsureRoot(tmpDir) // make sure config is set properly - data, err := ioutil.ReadFile(filepath.Join(tmpDir, "config.toml")) + data, err := ioutil.ReadFile(filepath.Join(tmpDir, defaultConfigFilePath)) require.Nil(err) assert.Equal([]byte(defaultConfig(defaultMoniker)), data) @@ -47,11 +48,41 @@ func TestEnsureTestRoot(t *testing.T) { rootDir := cfg.RootDir // make sure config is set properly - data, err := ioutil.ReadFile(filepath.Join(rootDir, "config.toml")) + data, err := ioutil.ReadFile(filepath.Join(rootDir, defaultConfigFilePath)) require.Nil(err) assert.Equal([]byte(testConfig(defaultMoniker)), data) // TODO: make sure the cfg returned and testconfig are the same! + baseConfig := DefaultBaseConfig() + ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidator) +} + +func checkConfig(configFile string) bool { + var valid bool - ensureFiles(t, rootDir, "data", "genesis.json", "priv_validator.json") + // list of words we expect in the config + var elems = []string{ + "moniker", + "seeds", + "proxy_app", + "fast_sync", + "create_empty_blocks", + "peer", + "timeout", + "broadcast", + "send", + "addr", + "wal", + "propose", + "max", + "genesis", + } + for _, e := range elems { + if !strings.Contains(configFile, e) { + valid = false + } else { + valid = true + } + } + return valid } diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh new file mode 100755 index 000000000..b0e1a934c --- /dev/null +++ b/consensus/test_data/build.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +# Requires: killall command and jq JSON processor. + +# Get the parent directory of where this script is. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + +# Change into that dir because we expect that. +cd "$DIR" || exit 1 + +# Make sure we have a tendermint command. +if ! hash tendermint 2>/dev/null; then + make install +fi + +# Make sure we have a cutWALUntil binary. +cutWALUntil=./scripts/cutWALUntil/cutWALUntil +cutWALUntilDir=$(dirname $cutWALUntil) +if ! hash $cutWALUntil 2>/dev/null; then + cd "$cutWALUntilDir" && go build && cd - || exit 1 +fi + +TMHOME=$(mktemp -d) +export TMHOME="$TMHOME" + +if [[ ! -d "$TMHOME" ]]; then + echo "Could not create temp directory" + exit 1 +else + echo "TMHOME: ${TMHOME}" +fi + +# TODO: eventually we should replace with `tendermint init --test` +DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test +if [ ! -d "$DIR_TO_COPY" ]; then + echo "$DIR_TO_COPY does not exist. Please run: go test ./consensus" + exit 1 +fi +echo "==> Copying ${DIR_TO_COPY} to ${TMHOME} directory..." +cp -r "$DIR_TO_COPY"/* "$TMHOME" + +# preserve original genesis file because later it will be modified (see small_block2) +cp "$TMHOME/config/genesis.json" "$TMHOME/config/genesis.json.bak" + +function reset(){ + echo "==> Resetting tendermint..." + tendermint unsafe_reset_all + cp "$TMHOME/config/genesis.json.bak" "$TMHOME/config/genesis.json" +} + +reset + +# function empty_block(){ +# echo "==> Starting tendermint..." +# tendermint node --proxy_app=persistent_dummy &> /dev/null & +# sleep 5 +# echo "==> Killing tendermint..." +# killall tendermint + +# echo "==> Copying WAL log..." +# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_empty_block.cswal +# mv consensus/test_data/new_empty_block.cswal consensus/test_data/empty_block.cswal + +# reset +# } + +function many_blocks(){ + bash scripts/txs/random.sh 1000 36657 &> /dev/null & + PID=$! + echo "==> Starting tendermint..." + tendermint node --proxy_app=persistent_dummy &> /dev/null & + sleep 10 + echo "==> Killing tendermint..." + kill -9 $PID + killall tendermint + + echo "==> Copying WAL log..." + $cutWALUntil "$TMHOME/data/cs.wal/wal" 6 consensus/test_data/new_many_blocks.cswal + mv consensus/test_data/new_many_blocks.cswal consensus/test_data/many_blocks.cswal + + reset +} + + +# function small_block1(){ +# bash scripts/txs/random.sh 1000 36657 &> /dev/null & +# PID=$! +# echo "==> Starting tendermint..." +# tendermint node --proxy_app=persistent_dummy &> /dev/null & +# sleep 10 +# echo "==> Killing tendermint..." +# kill -9 $PID +# killall tendermint + +# echo "==> Copying WAL log..." +# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block1.cswal +# mv consensus/test_data/new_small_block1.cswal consensus/test_data/small_block1.cswal + +# reset +# } + + +# # block part size = 512 +# function small_block2(){ +# cat "$TMHOME/config/genesis.json" | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > "$TMHOME/config/new_genesis.json" +# mv "$TMHOME/config/new_genesis.json" "$TMHOME/config/genesis.json" +# bash scripts/txs/random.sh 1000 36657 &> /dev/null & +# PID=$! +# echo "==> Starting tendermint..." +# tendermint node --proxy_app=persistent_dummy &> /dev/null & +# sleep 5 +# echo "==> Killing tendermint..." +# kill -9 $PID +# killall tendermint + +# echo "==> Copying WAL log..." +# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block2.cswal +# mv consensus/test_data/new_small_block2.cswal consensus/test_data/small_block2.cswal + +# reset +# } + + + +case "$1" in + # "small_block1") + # small_block1 + # ;; + # "small_block2") + # small_block2 + # ;; + # "empty_block") + # empty_block + # ;; + "many_blocks") + many_blocks + ;; + *) + # small_block1 + # small_block2 + # empty_block + many_blocks +esac + +echo "==> Cleaning up..." +rm -rf "$TMHOME" diff --git a/docs/deploy-testnets.rst b/docs/deploy-testnets.rst index 89fa4b799..658330fe5 100644 --- a/docs/deploy-testnets.rst +++ b/docs/deploy-testnets.rst @@ -13,7 +13,7 @@ It's relatively easy to setup a Tendermint cluster manually. The only requirements for a particular Tendermint node are a private key for the validator, stored as ``priv_validator.json``, and a list of the public keys of all validators, stored as ``genesis.json``. These files should -be stored in ``~/.tendermint``, or wherever the ``$TMHOME`` variable +be stored in ``~/.tendermint/config``, or wherever the ``$TMHOME`` variable might be set to. Here are the steps to setting up a testnet manually: diff --git a/docs/specification/configuration.rst b/docs/specification/configuration.rst index 74b41d09d..62e35738d 100644 --- a/docs/specification/configuration.rst +++ b/docs/specification/configuration.rst @@ -1,58 +1,151 @@ Configuration ============= -TendermintCore can be configured via a TOML file in -``$TMHOME/config.toml``. Some of these parameters can be overridden by -command-line flags. +Tendermint Core can be configured via a TOML file in +``$TMHOME/config/config.toml``. Some of these parameters can be overridden by +command-line flags. For most users, the options in the ``##### main +base configuration options #####`` are intended to be modified while +config options further below are intended for advance power users. -Config parameters -~~~~~~~~~~~~~~~~~ +Config options +~~~~~~~~~~~~~~ -The main config parameters are defined -`here `__. +The default configuration file create by ``tendermint init`` has all +the parameters set with their default values. It will look something +like the file below, however, double check by inspecting the +``config.toml`` created with your version of ``tendermint`` installed: -- ``abci``: ABCI transport (socket \| grpc). *Default*: ``socket`` -- ``db_backend``: Database backend for the blockchain and - TendermintCore state. ``leveldb`` or ``memdb``. *Default*: - ``"leveldb"`` -- ``db_dir``: Database dir. *Default*: ``"$TMHOME/data"`` -- ``fast_sync``: Whether to sync faster from the block pool. *Default*: - ``true`` -- ``genesis_file``: The location of the genesis file. *Default*: - ``"$TMHOME/genesis.json"`` -- ``log_level``: *Default*: ``"state:info,*:error"`` -- ``moniker``: Name of this node. *Default*: the host name or ``"anonymous"`` - if runtime fails to get the host name -- ``priv_validator_file``: Validator private key file. *Default*: - ``"$TMHOME/priv_validator.json"`` -- ``prof_laddr``: Profile listen address. *Default*: ``""`` -- ``proxy_app``: The ABCI app endpoint. *Default*: - ``"tcp://127.0.0.1:46658"`` +:: -- ``consensus.max_block_size_txs``: Maximum number of block txs. - *Default*: ``10000`` -- ``consensus.create_empty_blocks``: Create empty blocks w/o txs. - *Default*: ``true`` -- ``consensus.create_empty_blocks_interval``: Block creation interval, even if empty. -- ``consensus.timeout_*``: Various consensus timeout parameters -- ``consensus.wal_file``: Consensus state WAL. *Default*: - ``"$TMHOME/data/cs.wal/wal"`` -- ``consensus.wal_light``: Whether to use light-mode for Consensus - state WAL. *Default*: ``false`` - -- ``mempool.*``: Various mempool parameters - -- ``p2p.addr_book_file``: Peer address book. *Default*: - ``"$TMHOME/addrbook.json"``. **NOT USED** -- ``p2p.laddr``: Node listen address. (0.0.0.0:0 means any interface, - any port). *Default*: ``"0.0.0.0:46656"`` -- ``p2p.pex``: Enable Peer-Exchange (dev feature). *Default*: ``false`` -- ``p2p.seeds``: Comma delimited host:port seed nodes. *Default*: - ``""`` -- ``p2p.skip_upnp``: Skip UPNP detection. *Default*: ``false`` - -- ``rpc.grpc_laddr``: GRPC listen address (BroadcastTx only). Port - required. *Default*: ``""`` -- ``rpc.laddr``: RPC listen address. Port required. *Default*: - ``"0.0.0.0:46657"`` -- ``rpc.unsafe``: Enabled unsafe rpc methods. *Default*: ``true`` + # This is a TOML config file. + # For more information, see https://github.com/toml-lang/toml + + ##### main base config options ##### + + # TCP or UNIX socket address of the ABCI application, + # or the name of an ABCI application compiled in with the Tendermint binary + proxy_app = "tcp://127.0.0.1:46658" + + # A custom human readable name for this node + moniker = "anonymous" + + # If this node is many blocks behind the tip of the chain, FastSync + # allows them to catchup quickly by downloading blocks in parallel + # and verifying their commits + fast_sync = true + + # Database backend: leveldb | memdb + db_backend = "leveldb" + + # Database directory + db_path = "data" + + # Output level for logging + log_level = "state:info,*:error" + + ##### additional base config options ##### + + # The ID of the chain to join (should be signed with every transaction and vote) + chain_id = "" + + # Path to the JSON file containing the initial validator set and other meta data + genesis_file = "genesis.json" + + # Path to the JSON file containing the private key to use as a validator in the consensus protocol + priv_validator_file = "priv_validator.json" + + # Mechanism to connect to the ABCI application: socket | grpc + abci = "socket" + + # TCP or UNIX socket address for the profiling server to listen on + prof_laddr = "" + + # If true, query the ABCI app on connecting to a new peer + # so the app can decide if we should keep the connection or not + filter_peers = false + + # What indexer to use for transactions + tx_index = "kv" + + ##### advanced configuration options ##### + + ##### rpc server configuration options ##### + [rpc] + + # TCP or UNIX socket address for the RPC server to listen on + laddr = "tcp://0.0.0.0:46657" + + # TCP or UNIX socket address for the gRPC server to listen on + # NOTE: This server only supports /broadcast_tx_commit + grpc_laddr = "" + + # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool + unsafe = false + + ##### peer to peer configuration options ##### + [p2p] + + # Address to listen for incoming connections + laddr = "tcp://0.0.0.0:46656" + + # Comma separated list of seed nodes to connect to + seeds = "" + + # Path to address book + addr_book_file = "addrbook.json" + + # Set true for strict address routability rules + addr_book_strict = true + + # Time to wait before flushing messages out on the connection, in ms + flush_throttle_timeout = 100 + + # Maximum number of peers to connect to + max_num_peers = 50 + + # Maximum size of a message packet payload, in bytes + max_msg_packet_payload_size = 1024 + + # Rate at which packets can be sent, in bytes/second + send_rate = 512000 + + # Rate at which packets can be received, in bytes/second + recv_rate = 512000 + + ##### mempool configuration options ##### + [mempool] + + recheck = true + recheck_empty = true + broadcast = true + wal_dir = "data/mempool.wal" + + ##### consensus configuration options ##### + [consensus] + + wal_file = "data/cs.wal/wal" + wal_light = false + + # All timeouts are in milliseconds + timeout_propose = 3000 + timeout_propose_delta = 500 + timeout_prevote = 1000 + timeout_prevote_delta = 500 + timeout_precommit = 1000 + timeout_precommit_delta = 500 + timeout_commit = 1000 + + # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) + skip_timeout_commit = false + + # BlockSize + max_block_size_txs = 10000 + max_block_size_bytes = 1 + + # EmptyBlocks mode and possible interval between empty blocks in seconds + create_empty_blocks = true + create_empty_blocks_interval = 0 + + # Reactor sleep duration parameters are in milliseconds + peer_gossip_sleep_duration = 100 + peer_query_maj23_sleep_duration = 2000 diff --git a/docs/specification/genesis.rst b/docs/specification/genesis.rst index a7ec7a268..7e36c1315 100644 --- a/docs/specification/genesis.rst +++ b/docs/specification/genesis.rst @@ -1,7 +1,7 @@ Genesis ======= -The genesis.json file in ``$TMHOME`` defines the initial TendermintCore +The genesis.json file in ``$TMHOME/config`` defines the initial TendermintCore state upon genesis of the blockchain (`see definition `__). diff --git a/docs/specification/rpc.rst b/docs/specification/rpc.rst index 33173d196..f8e9bd3a9 100644 --- a/docs/specification/rpc.rst +++ b/docs/specification/rpc.rst @@ -18,7 +18,7 @@ Configuration ~~~~~~~~~~~~~ Set the ``laddr`` config parameter under ``[rpc]`` table in the -$TMHOME/config.toml file or the ``--rpc.laddr`` command-line flag to the +$TMHOME/config/config.toml file or the ``--rpc.laddr`` command-line flag to the desired protocol://host:port setting. Default: ``tcp://0.0.0.0:46657``. Arguments diff --git a/docs/using-tendermint.rst b/docs/using-tendermint.rst index 9076230ea..75d49544a 100644 --- a/docs/using-tendermint.rst +++ b/docs/using-tendermint.rst @@ -24,7 +24,8 @@ Initialize the root directory by running: tendermint init This will create a new private key (``priv_validator.json``), and a -genesis file (``genesis.json``) containing the associated public key. +genesis file (``genesis.json``) containing the associated public key, +in ``$TMHOME/config``. This is all that's necessary to run a local testnet with one validator. For more elaborate initialization, see our `testnet deployment @@ -153,8 +154,7 @@ The block interval setting allows for a delay (in seconds) between the creation create_empty_blocks_interval = 5 With this setting, empty blocks will be produced every 5s if no block has been produced otherwise, -regardless of the value of `create_empty_blocks`. - +regardless of the value of ``create_empty_blocks``. Broadcast API ------------- @@ -196,7 +196,7 @@ Tendermint Networks ------------------- When ``tendermint init`` is run, both a ``genesis.json`` and -``priv_validator.json`` are created in ``~/.tendermint``. The +``priv_validator.json`` are created in ``~/.tendermint/config``. The ``genesis.json`` might look like: :: @@ -263,7 +263,7 @@ with the consensus protocol. Peers ~~~~~ -To connect to peers on start-up, specify them in the ``config.toml`` or +To connect to peers on start-up, specify them in the ``$TMHOME/config/config.toml`` or on the command line. For instance, @@ -289,7 +289,7 @@ Adding a Non-Validator ~~~~~~~~~~~~~~~~~~~~~~ Adding a non-validator is simple. Just copy the original -``genesis.json`` to ``~/.tendermint`` on the new machine and start the +``genesis.json`` to ``~/.tendermint/config`` on the new machine and start the node, specifying seeds as necessary. If no seeds are specified, the node won't make any blocks, because it's not a validator, and it won't hear about any blocks, because it's not connected to the other peer. @@ -358,8 +358,8 @@ then the new ``genesis.json`` will be: ] } -Update the ``genesis.json`` in ``~/.tendermint``. Copy the genesis file -and the new ``priv_validator.json`` to the ``~/.tendermint`` on a new +Update the ``genesis.json`` in ``~/.tendermint/config``. Copy the genesis file +and the new ``priv_validator.json`` to the ``~/.tendermint/config`` on a new machine. Now run ``tendermint node`` on both machines, and use either diff --git a/rpc/core/doc.go b/rpc/core/doc.go index a72cec020..d3ec5d3b6 100644 --- a/rpc/core/doc.go +++ b/rpc/core/doc.go @@ -11,7 +11,7 @@ Tendermint RPC is built using [our own RPC library](https://github.com/tendermin ## Configuration -Set the `laddr` config parameter under `[rpc]` table in the `$TMHOME/config.toml` file or the `--rpc.laddr` command-line flag to the desired protocol://host:port setting. Default: `tcp://0.0.0.0:46657`. +Set the `laddr` config parameter under `[rpc]` table in the `$TMHOME/config/config.toml` file or the `--rpc.laddr` command-line flag to the desired protocol://host:port setting. Default: `tcp://0.0.0.0:46657`. ## Arguments diff --git a/scripts/debora/unsafe_reset_net.sh b/scripts/debora/unsafe_reset_net.sh index c6767427d..3698e5ace 100755 --- a/scripts/debora/unsafe_reset_net.sh +++ b/scripts/debora/unsafe_reset_net.sh @@ -3,7 +3,7 @@ set -euo pipefail IFS=$'\n\t' debora run -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; killall tendermint; killall logjack" -debora run -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; tendermint unsafe_reset_priv_validator; rm -rf ~/.tendermint/data; rm ~/.tendermint/genesis.json; rm ~/.tendermint/logs/*" +debora run -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; tendermint unsafe_reset_priv_validator; rm -rf ~/.tendermint/data; rm ~/.tendermint/config/genesis.json; rm ~/.tendermint/logs/*" debora run -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; git pull origin develop; make" debora run -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; mkdir -p ~/.tendermint/logs" debora run --bg --label tendermint -- bash -c "cd \$GOPATH/src/github.com/tendermint/tendermint; tendermint node 2>&1 | stdinwriter -outpath ~/.tendermint/logs/tendermint.log" From 69d8c2e554df7577c2e1e02724fb9214287a9d19 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 1 Dec 2017 22:35:15 -0600 Subject: [PATCH 2/7] fixes after my own review --- config/config.go | 2 +- config/toml.go | 25 ++++++- docs/specification/configuration.rst | 103 ++++++++++++++++----------- 3 files changed, 84 insertions(+), 46 deletions(-) diff --git a/config/config.go b/config/config.go index 1585fcd0e..25ccf0f57 100644 --- a/config/config.go +++ b/config/config.go @@ -386,7 +386,7 @@ func (cfg *ConsensusConfig) PeerQueryMaj23Sleep() time.Duration { // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ - WalPath: filepath.Join(defaultDataDir, "cs.wal/wal"), + WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), WalLight: false, TimeoutPropose: 3000, TimeoutProposeDelta: 500, diff --git a/config/toml.go b/config/toml.go index 56a5a1304..e7b00dd6c 100644 --- a/config/toml.go +++ b/config/toml.go @@ -100,9 +100,6 @@ prof_laddr = "{{ .BaseConfig.ProfListenAddress }}" # so the app can decide if we should keep the connection or not filter_peers = {{ .BaseConfig.FilterPeers }} -# What indexer to use for transactions -tx_index = "{{ .BaseConfig.TxIndex }}" - ##### advanced configuration options ##### ##### rpc server configuration options ##### @@ -185,6 +182,28 @@ create_empty_blocks_interval = {{ .Consensus.CreateEmptyBlocksInterval }} # Reactor sleep duration parameters are in milliseconds peer_gossip_sleep_duration = {{ .Consensus.PeerGossipSleepDuration }} peer_query_maj23_sleep_duration = {{ .Consensus.PeerQueryMaj23SleepDuration }} + +##### transactions indexer configuration options ##### +[tx_index] + +# What indexer to use for transactions +# +# Options: +# 1) "null" (default) +# 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). +indexer = "{{ .TxIndex.Indexer }}" + +# Comma-separated list of tags to index (by default the only tag is tx hash) +# +# It's recommended to index only a subset of tags due to possible memory +# bloat. This is, of course, depends on the indexer's DB and the volume of +# transactions. +index_tags = "{{ .TxIndex.IndexTags }}" + +# When set to true, tells indexer to index all tags. Note this may be not +# desirable (see the comment above). IndexTags has a precedence over +# IndexAllTags (i.e. when given both, IndexTags will be indexed). +index_all_tags = {{ .TxIndex.IndexAllTags }} ` /****** these are for test settings ***********/ diff --git a/docs/specification/configuration.rst b/docs/specification/configuration.rst index 62e35738d..37359885c 100644 --- a/docs/specification/configuration.rst +++ b/docs/specification/configuration.rst @@ -12,120 +12,117 @@ Config options The default configuration file create by ``tendermint init`` has all the parameters set with their default values. It will look something -like the file below, however, double check by inspecting the +like the file below, however, double check by inspecting the ``config.toml`` created with your version of ``tendermint`` installed: :: # This is a TOML config file. # For more information, see https://github.com/toml-lang/toml - + ##### main base config options ##### - + # TCP or UNIX socket address of the ABCI application, # or the name of an ABCI application compiled in with the Tendermint binary proxy_app = "tcp://127.0.0.1:46658" - + # A custom human readable name for this node moniker = "anonymous" - + # If this node is many blocks behind the tip of the chain, FastSync # allows them to catchup quickly by downloading blocks in parallel # and verifying their commits fast_sync = true - + # Database backend: leveldb | memdb db_backend = "leveldb" - + # Database directory db_path = "data" - + # Output level for logging log_level = "state:info,*:error" - + ##### additional base config options ##### - + # The ID of the chain to join (should be signed with every transaction and vote) chain_id = "" - + # Path to the JSON file containing the initial validator set and other meta data genesis_file = "genesis.json" - + # Path to the JSON file containing the private key to use as a validator in the consensus protocol priv_validator_file = "priv_validator.json" - + # Mechanism to connect to the ABCI application: socket | grpc abci = "socket" - + # TCP or UNIX socket address for the profiling server to listen on prof_laddr = "" - + # If true, query the ABCI app on connecting to a new peer # so the app can decide if we should keep the connection or not filter_peers = false - - # What indexer to use for transactions - tx_index = "kv" - + ##### advanced configuration options ##### - + ##### rpc server configuration options ##### [rpc] - + # TCP or UNIX socket address for the RPC server to listen on laddr = "tcp://0.0.0.0:46657" - + # TCP or UNIX socket address for the gRPC server to listen on # NOTE: This server only supports /broadcast_tx_commit grpc_laddr = "" - + # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool unsafe = false - + ##### peer to peer configuration options ##### [p2p] - + # Address to listen for incoming connections laddr = "tcp://0.0.0.0:46656" - + # Comma separated list of seed nodes to connect to seeds = "" - + # Path to address book addr_book_file = "addrbook.json" - + # Set true for strict address routability rules addr_book_strict = true - + # Time to wait before flushing messages out on the connection, in ms flush_throttle_timeout = 100 - + # Maximum number of peers to connect to max_num_peers = 50 - + # Maximum size of a message packet payload, in bytes max_msg_packet_payload_size = 1024 - + # Rate at which packets can be sent, in bytes/second send_rate = 512000 - + # Rate at which packets can be received, in bytes/second recv_rate = 512000 - + ##### mempool configuration options ##### [mempool] - + recheck = true recheck_empty = true broadcast = true wal_dir = "data/mempool.wal" - + ##### consensus configuration options ##### [consensus] - + wal_file = "data/cs.wal/wal" wal_light = false - + # All timeouts are in milliseconds timeout_propose = 3000 timeout_propose_delta = 500 @@ -134,18 +131,40 @@ like the file below, however, double check by inspecting the timeout_precommit = 1000 timeout_precommit_delta = 500 timeout_commit = 1000 - + # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) skip_timeout_commit = false - + # BlockSize max_block_size_txs = 10000 max_block_size_bytes = 1 - + # EmptyBlocks mode and possible interval between empty blocks in seconds create_empty_blocks = true create_empty_blocks_interval = 0 - + # Reactor sleep duration parameters are in milliseconds peer_gossip_sleep_duration = 100 peer_query_maj23_sleep_duration = 2000 + + ##### transactions indexer configuration options ##### + [tx_index] + + # What indexer to use for transactions + # + # Options: + # 1) "null" (default) + # 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). + indexer = "{{ .TxIndex.Indexer }}" + + # Comma-separated list of tags to index (by default the only tag is tx hash) + # + # It's recommended to index only a subset of tags due to possible memory + # bloat. This is, of course, depends on the indexer's DB and the volume of + # transactions. + index_tags = "{{ .TxIndex.IndexTags }}" + + # When set to true, tells indexer to index all tags. Note this may be not + # desirable (see the comment above). IndexTags has a precedence over + # IndexAllTags (i.e. when given both, IndexTags will be indexed). + index_all_tags = {{ .TxIndex.IndexAllTags }} From a6f2e502e72f483ffc847230b64edbf58d8b4101 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 2 Dec 2017 11:26:51 -0600 Subject: [PATCH 3/7] move genesis.json file into config dir --- test/p2p/data/mach1/core/{ => config}/genesis.json | 0 test/p2p/data/mach2/core/{ => config}/genesis.json | 0 test/p2p/data/mach3/core/{ => config}/genesis.json | 0 test/p2p/data/mach4/core/{ => config}/genesis.json | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename test/p2p/data/mach1/core/{ => config}/genesis.json (100%) rename test/p2p/data/mach2/core/{ => config}/genesis.json (100%) rename test/p2p/data/mach3/core/{ => config}/genesis.json (100%) rename test/p2p/data/mach4/core/{ => config}/genesis.json (100%) diff --git a/test/p2p/data/mach1/core/genesis.json b/test/p2p/data/mach1/core/config/genesis.json similarity index 100% rename from test/p2p/data/mach1/core/genesis.json rename to test/p2p/data/mach1/core/config/genesis.json diff --git a/test/p2p/data/mach2/core/genesis.json b/test/p2p/data/mach2/core/config/genesis.json similarity index 100% rename from test/p2p/data/mach2/core/genesis.json rename to test/p2p/data/mach2/core/config/genesis.json diff --git a/test/p2p/data/mach3/core/genesis.json b/test/p2p/data/mach3/core/config/genesis.json similarity index 100% rename from test/p2p/data/mach3/core/genesis.json rename to test/p2p/data/mach3/core/config/genesis.json diff --git a/test/p2p/data/mach4/core/genesis.json b/test/p2p/data/mach4/core/config/genesis.json similarity index 100% rename from test/p2p/data/mach4/core/genesis.json rename to test/p2p/data/mach4/core/config/genesis.json From 9da5cd0180c44df64457ca8c1c2ea4f41902e985 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sun, 10 Dec 2017 17:32:21 +0000 Subject: [PATCH 4/7] rebase fix --- consensus/test_data/build.sh | 148 ----------------------------------- 1 file changed, 148 deletions(-) delete mode 100755 consensus/test_data/build.sh diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh deleted file mode 100755 index b0e1a934c..000000000 --- a/consensus/test_data/build.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash - -# Requires: killall command and jq JSON processor. - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" || exit 1 - -# Make sure we have a tendermint command. -if ! hash tendermint 2>/dev/null; then - make install -fi - -# Make sure we have a cutWALUntil binary. -cutWALUntil=./scripts/cutWALUntil/cutWALUntil -cutWALUntilDir=$(dirname $cutWALUntil) -if ! hash $cutWALUntil 2>/dev/null; then - cd "$cutWALUntilDir" && go build && cd - || exit 1 -fi - -TMHOME=$(mktemp -d) -export TMHOME="$TMHOME" - -if [[ ! -d "$TMHOME" ]]; then - echo "Could not create temp directory" - exit 1 -else - echo "TMHOME: ${TMHOME}" -fi - -# TODO: eventually we should replace with `tendermint init --test` -DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test -if [ ! -d "$DIR_TO_COPY" ]; then - echo "$DIR_TO_COPY does not exist. Please run: go test ./consensus" - exit 1 -fi -echo "==> Copying ${DIR_TO_COPY} to ${TMHOME} directory..." -cp -r "$DIR_TO_COPY"/* "$TMHOME" - -# preserve original genesis file because later it will be modified (see small_block2) -cp "$TMHOME/config/genesis.json" "$TMHOME/config/genesis.json.bak" - -function reset(){ - echo "==> Resetting tendermint..." - tendermint unsafe_reset_all - cp "$TMHOME/config/genesis.json.bak" "$TMHOME/config/genesis.json" -} - -reset - -# function empty_block(){ -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 5 -# echo "==> Killing tendermint..." -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_empty_block.cswal -# mv consensus/test_data/new_empty_block.cswal consensus/test_data/empty_block.cswal - -# reset -# } - -function many_blocks(){ - bash scripts/txs/random.sh 1000 36657 &> /dev/null & - PID=$! - echo "==> Starting tendermint..." - tendermint node --proxy_app=persistent_dummy &> /dev/null & - sleep 10 - echo "==> Killing tendermint..." - kill -9 $PID - killall tendermint - - echo "==> Copying WAL log..." - $cutWALUntil "$TMHOME/data/cs.wal/wal" 6 consensus/test_data/new_many_blocks.cswal - mv consensus/test_data/new_many_blocks.cswal consensus/test_data/many_blocks.cswal - - reset -} - - -# function small_block1(){ -# bash scripts/txs/random.sh 1000 36657 &> /dev/null & -# PID=$! -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 10 -# echo "==> Killing tendermint..." -# kill -9 $PID -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block1.cswal -# mv consensus/test_data/new_small_block1.cswal consensus/test_data/small_block1.cswal - -# reset -# } - - -# # block part size = 512 -# function small_block2(){ -# cat "$TMHOME/config/genesis.json" | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > "$TMHOME/config/new_genesis.json" -# mv "$TMHOME/config/new_genesis.json" "$TMHOME/config/genesis.json" -# bash scripts/txs/random.sh 1000 36657 &> /dev/null & -# PID=$! -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 5 -# echo "==> Killing tendermint..." -# kill -9 $PID -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block2.cswal -# mv consensus/test_data/new_small_block2.cswal consensus/test_data/small_block2.cswal - -# reset -# } - - - -case "$1" in - # "small_block1") - # small_block1 - # ;; - # "small_block2") - # small_block2 - # ;; - # "empty_block") - # empty_block - # ;; - "many_blocks") - many_blocks - ;; - *) - # small_block1 - # small_block2 - # empty_block - many_blocks -esac - -echo "==> Cleaning up..." -rm -rf "$TMHOME" From a92a32b862c7051253346e6402a5e8a77be46a9d Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Sun, 10 Dec 2017 18:01:19 +0000 Subject: [PATCH 5/7] config: lil fixes --- config/toml.go | 4 ++-- config/toml_test.go | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/config/toml.go b/config/toml.go index e7b00dd6c..3522170df 100644 --- a/config/toml.go +++ b/config/toml.go @@ -35,7 +35,7 @@ func EnsureRoot(rootDir string) { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - cmn.MustWriteFile(configFilePath, []byte(defaultConfig(defaultMoniker)), 0644) + writeConfigFile(configFilePath) } } @@ -241,7 +241,7 @@ func ResetTestRoot(testName string) *Config { // Write default config file if missing. if !cmn.FileExists(configFilePath) { - cmn.MustWriteFile(configFilePath, []byte(testConfig(defaultMoniker)), 0644) + writeConfigFile(configFilePath) } if !cmn.FileExists(genesisFilePath) { cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) diff --git a/config/toml_test.go b/config/toml_test.go index e98719f6f..a1637f671 100644 --- a/config/toml_test.go +++ b/config/toml_test.go @@ -20,7 +20,7 @@ func ensureFiles(t *testing.T, rootDir string, files ...string) { } func TestEnsureRoot(t *testing.T) { - assert, require := assert.New(t), require.New(t) + require := require.New(t) // setup temp dir for test tmpDir, err := ioutil.TempDir("", "config-test") @@ -33,13 +33,16 @@ func TestEnsureRoot(t *testing.T) { // make sure config is set properly data, err := ioutil.ReadFile(filepath.Join(tmpDir, defaultConfigFilePath)) require.Nil(err) - assert.Equal([]byte(defaultConfig(defaultMoniker)), data) + + if !checkConfig(string(data)) { + t.Fatalf("config file missing some information") + } ensureFiles(t, tmpDir, "data") } func TestEnsureTestRoot(t *testing.T) { - assert, require := assert.New(t), require.New(t) + require := require.New(t) testName := "ensureTestRoot" @@ -50,7 +53,10 @@ func TestEnsureTestRoot(t *testing.T) { // make sure config is set properly data, err := ioutil.ReadFile(filepath.Join(rootDir, defaultConfigFilePath)) require.Nil(err) - assert.Equal([]byte(testConfig(defaultMoniker)), data) + + if !checkConfig(string(data)) { + t.Fatalf("config file missing some information") + } // TODO: make sure the cfg returned and testconfig are the same! baseConfig := DefaultBaseConfig() From a8e625e99dcf19b8c3594665d8f3b70163ba3487 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 10 Dec 2017 20:43:58 -0500 Subject: [PATCH 6/7] config: unexpose chainID --- config/config.go | 13 +++++++++---- config/toml.go | 5 +---- consensus/common_test.go | 7 ++++--- consensus/state_test.go | 4 ++-- consensus/types/height_vote_set_test.go | 4 ++-- lite/client/provider_test.go | 4 +++- 6 files changed, 21 insertions(+), 16 deletions(-) diff --git a/config/config.go b/config/config.go index 25ccf0f57..6689adfcc 100644 --- a/config/config.go +++ b/config/config.go @@ -78,13 +78,14 @@ func (cfg *Config) SetRoot(root string) *Config { // BaseConfig defines the base configuration for a Tendermint node type BaseConfig struct { + + // chainID is unexposed and immutable but here for convenience + chainID string + // The root directory for all data. // This should be set in viper so it can unmarshal into this struct RootDir string `mapstructure:"home"` - // The ID of the chain to join (should be signed with every transaction and vote) - ChainID string `mapstructure:"chain_id"` - // Path to the JSON file containing the initial validator set and other meta data Genesis string `mapstructure:"genesis_file"` @@ -123,6 +124,10 @@ type BaseConfig struct { DBPath string `mapstructure:"db_dir"` } +func (c BaseConfig) ChainID() string { + return c.chainID +} + // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ @@ -143,7 +148,7 @@ func DefaultBaseConfig() BaseConfig { // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { conf := DefaultBaseConfig() - conf.ChainID = "tendermint_test" + conf.chainID = "tendermint_test" conf.ProxyApp = "dummy" conf.FastSync = false conf.DBBackend = "memdb" diff --git a/config/toml.go b/config/toml.go index 3522170df..92ceb7de4 100644 --- a/config/toml.go +++ b/config/toml.go @@ -76,14 +76,11 @@ db_backend = "{{ .BaseConfig.DBBackend }}" # Database directory db_path = "{{ .BaseConfig.DBPath }}" -# Output level for logging +# Output level for logging, including package level options log_level = "{{ .BaseConfig.LogLevel }}" ##### additional base config options ##### -# The ID of the chain to join (should be signed with every transaction and vote) -chain_id = "{{ .BaseConfig.ChainID }}" - # Path to the JSON file containing the initial validator set and other meta data genesis_file = "{{ .BaseConfig.Genesis }}" diff --git a/consensus/common_test.go b/consensus/common_test.go index 6598c15eb..d5514ee18 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -78,7 +78,7 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS Type: voteType, BlockID: types.BlockID{hash, header}, } - err := vs.PrivValidator.SignVote(config.ChainID, vote) + err := vs.PrivValidator.SignVote(config.ChainID(), vote) return vote, err } @@ -129,7 +129,7 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height int64, round // Make proposal polRound, polBlockID := cs1.Votes.POLInfo() proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) - if err := vs.SignProposal(config.ChainID, proposal); err != nil { + if err := vs.SignProposal(cs1.state.ChainID, proposal); err != nil { panic(err) } return @@ -430,9 +430,10 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G privValidators[i] = privVal } sort.Sort(types.PrivValidatorsByAddress(privValidators)) + return &types.GenesisDoc{ GenesisTime: time.Now(), - ChainID: config.ChainID, + ChainID: config.ChainID(), Validators: validators, }, privValidators } diff --git a/consensus/state_test.go b/consensus/state_test.go index 6beb7da54..97562febd 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -204,7 +204,7 @@ func TestBadProposal(t *testing.T) { propBlock.AppHash = stateHash propBlockParts := propBlock.MakePartSet(partSize) proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{}) - if err := vs2.SignProposal(config.ChainID, proposal); err != nil { + if err := vs2.SignProposal(config.ChainID(), proposal); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -900,7 +900,7 @@ func TestLockPOLSafety2(t *testing.T) { // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, 2, propBlockParts0.Header(), 0, propBlockID1) - if err := vs3.SignProposal(config.ChainID, newProp); err != nil { + if err := vs3.SignProposal(config.ChainID(), newProp); err != nil { t.Fatal(err) } if err := cs1.SetProposalAndBlock(newProp, propBlock0, propBlockParts0, "some peer"); err != nil { diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 306592aa2..5719d7eea 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -18,7 +18,7 @@ func init() { func TestPeerCatchupRounds(t *testing.T) { valSet, privVals := types.RandValidatorSet(10, 1) - hvs := NewHeightVoteSet(config.ChainID, 1, valSet) + hvs := NewHeightVoteSet(config.ChainID(), 1, valSet) vote999_0 := makeVoteHR(t, 1, 999, privVals, 0) added, err := hvs.AddVote(vote999_0, "peer1") @@ -59,7 +59,7 @@ func makeVoteHR(t *testing.T, height int64, round int, privVals []*types.PrivVal Type: types.VoteTypePrecommit, BlockID: types.BlockID{[]byte("fakehash"), types.PartSetHeader{}}, } - chainID := config.ChainID + chainID := config.ChainID() err := privVal.SignVote(chainID, vote) if err != nil { panic(cmn.Fmt("Error signing vote: %v", err)) diff --git a/lite/client/provider_test.go b/lite/client/provider_test.go index 0bebfced0..91d277fc2 100644 --- a/lite/client/provider_test.go +++ b/lite/client/provider_test.go @@ -10,6 +10,7 @@ import ( liteErr "github.com/tendermint/tendermint/lite/errors" rpcclient "github.com/tendermint/tendermint/rpc/client" rpctest "github.com/tendermint/tendermint/rpc/test" + "github.com/tendermint/tendermint/types" ) func TestProvider(t *testing.T) { @@ -17,7 +18,8 @@ func TestProvider(t *testing.T) { cfg := rpctest.GetConfig() rpcAddr := cfg.RPC.ListenAddress - chainID := cfg.ChainID + genDoc, _ := types.GenesisDocFromFile(cfg.GenesisFile()) + chainID := genDoc.ChainID p := NewHTTPProvider(rpcAddr) require.NotNil(t, p) From 657ad214cb95489207246755fc65b8fc44b5f384 Mon Sep 17 00:00:00 2001 From: Zach Ramsay Date: Wed, 10 Jan 2018 15:57:39 +0000 Subject: [PATCH 7/7] p2p tests: put priv_val in right place --- test/p2p/data/mach1/core/{ => config}/priv_validator.json | 0 test/p2p/data/mach2/core/{ => config}/priv_validator.json | 0 test/p2p/data/mach3/core/{ => config}/priv_validator.json | 0 test/p2p/data/mach4/core/{ => config}/priv_validator.json | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename test/p2p/data/mach1/core/{ => config}/priv_validator.json (100%) rename test/p2p/data/mach2/core/{ => config}/priv_validator.json (100%) rename test/p2p/data/mach3/core/{ => config}/priv_validator.json (100%) rename test/p2p/data/mach4/core/{ => config}/priv_validator.json (100%) diff --git a/test/p2p/data/mach1/core/priv_validator.json b/test/p2p/data/mach1/core/config/priv_validator.json similarity index 100% rename from test/p2p/data/mach1/core/priv_validator.json rename to test/p2p/data/mach1/core/config/priv_validator.json diff --git a/test/p2p/data/mach2/core/priv_validator.json b/test/p2p/data/mach2/core/config/priv_validator.json similarity index 100% rename from test/p2p/data/mach2/core/priv_validator.json rename to test/p2p/data/mach2/core/config/priv_validator.json diff --git a/test/p2p/data/mach3/core/priv_validator.json b/test/p2p/data/mach3/core/config/priv_validator.json similarity index 100% rename from test/p2p/data/mach3/core/priv_validator.json rename to test/p2p/data/mach3/core/config/priv_validator.json diff --git a/test/p2p/data/mach4/core/priv_validator.json b/test/p2p/data/mach4/core/config/priv_validator.json similarity index 100% rename from test/p2p/data/mach4/core/priv_validator.json rename to test/p2p/data/mach4/core/config/priv_validator.json