diff --git a/.gitignore b/.gitignore index 23ae616ee..22a6be0b9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ test/logs coverage.txt docs/_build docs/tools + +scripts/wal2json/wal2json +scripts/cutWALUntil/cutWALUntil diff --git a/consensus/replay.go b/consensus/replay.go index 2dcd2e056..e81512701 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -38,7 +38,7 @@ var crc32c = crc32.MakeTable(crc32.Castagnoli) // Lines that start with "#" are ignored. // NOTE: receiveRoutine should not be running func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan interface{}) error { - // Skip over meta lines + // skip meta messages if _, ok := msg.Msg.(EndHeightMessage); ok { return nil } @@ -129,8 +129,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error { msg, err = dec.Decode() if err == io.EOF { break - } - if err != nil { + } else if err != nil { return err } // NOTE: since the priv key is set when the msgs are received diff --git a/consensus/replay_file.go b/consensus/replay_file.go index fa7800b46..24df20fb3 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -63,7 +63,7 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error { msg, err = pb.dec.Decode() if err == io.EOF { return nil - } else { + } else if err != nil { return err } @@ -130,7 +130,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { msg, err = pb.dec.Decode() if err == io.EOF { return nil - } else { + } else if err != nil { return err } if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil { diff --git a/consensus/replay_test.go b/consensus/replay_test.go index aa2b9a131..9ab2efc17 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -177,9 +177,9 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo lastMsg := split[nLines] // we write those lines up to (not including) one with the signature - bytes := bytes.Join(split[:nLines], walSeparator) - bytes = append(bytes, walSeparator...) - walFile := writeWAL(bytes) + b := bytes.Join(split[:nLines], walSeparator) + b = append(b, walSeparator...) + walFile := writeWAL(b) cs := fixedConsensusStateDummy() @@ -196,6 +196,7 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo func readTimedWALMessage(t *testing.T, rawMsg []byte) TimedWALMessage { b := bytes.NewBuffer(rawMsg) + // because rawMsg does not contain a separator and WALDecoder#Decode expects it _, err := b.Write(walSeparator) if err != nil { t.Fatal(err) @@ -496,8 +497,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) { msg, err := dec.Decode() if err == io.EOF { break - } - if err != nil { + } else if err != nil { return nil, nil, err } @@ -538,7 +538,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) { } func readPieceFromWAL(msg *TimedWALMessage) interface{} { - // Skip meta lines + // skip meta messages if _, ok := msg.Msg.(EndHeightMessage); ok { return nil } diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh index 16cf96a60..dcec6f2a0 100755 --- a/consensus/test_data/build.sh +++ b/consensus/test_data/build.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash -# XXX: removes tendermint dir -# TODO: does not work on OSX +# Requires: killall command and jq JSON processor. # Get the parent directory of where this script is. SOURCE="${BASH_SOURCE[0]}" @@ -9,91 +8,115 @@ while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" # Change into that dir because we expect that. -cd "$DIR" || exit +cd "$DIR" || exit 1 # Make sure we have a tendermint command. if ! hash tendermint 2>/dev/null; then - make install + make install fi # Make sure we have a cutWALUntil binary. -if ! hash ./scripts/cutWALUntil/cutWALUntil 2>/dev/null; then - cd ./scripts/cutWALUntil/ && go build && cd - || exit +cutWALUntil=./scripts/cutWALUntil/cutWALUntil +cutWALUntilDir=$(dirname $cutWALUntil) +if ! hash $cutWALUntil 2>/dev/null; then + cd "$cutWALUntilDir" && go build && cd - || exit 1 +fi + +TMHOME=$(mktemp -d) +export TMHOME="$TMHOME" + +if [[ ! -d "$TMHOME" ]]; then + echo "Could not create temp directory" + exit 1 +else + echo "TMHOME: ${TMHOME}" fi -# specify a dir to copy # TODO: eventually we should replace with `tendermint init --test` DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test +if [ ! -d "$DIR_TO_COPY" ]; then + echo "$DIR_TO_COPY does not exist. Please run: go test ./consensus" + exit 1 +fi +echo "==> Copying ${DIR_TO_COPY} to ${TMHOME} directory..." +cp -r "$DIR_TO_COPY"/* "$TMHOME" -TMHOME="$HOME/.tendermint" -#rm -rf "$TMHOME" -#cp -r "$DIR_TO_COPY" "$TMHOME" -#mv $TMHOME/config.toml $TMHOME/config.toml.bak -cp $TMHOME/genesis.json $TMHOME/genesis.json.bak +# preserve original genesis file because later it will be modified (see small_block2) +cp "$TMHOME/genesis.json" "$TMHOME/genesis.json.bak" function reset(){ + echo "==> Resetting tendermint..." tendermint unsafe_reset_all - cp $TMHOME/genesis.json.bak $TMHOME/genesis.json + cp "$TMHOME/genesis.json.bak" "$TMHOME/genesis.json" } reset -# empty block function empty_block(){ + echo "==> Starting tendermint..." tendermint node --proxy_app=persistent_dummy &> /dev/null & sleep 5 + echo "==> Killing tendermint..." killall tendermint - ./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_empty_block.cswal + echo "==> Copying WAL log..." + $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_empty_block.cswal mv consensus/test_data/new_empty_block.cswal consensus/test_data/empty_block.cswal reset } -# many blocks function many_blocks(){ bash scripts/txs/random.sh 1000 36657 &> /dev/null & PID=$! + echo "==> Starting tendermint..." tendermint node --proxy_app=persistent_dummy &> /dev/null & sleep 10 - killall tendermint + echo "==> Killing tendermint..." kill -9 $PID + killall tendermint - ./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 6 consensus/test_data/new_many_blocks.cswal + echo "==> Copying WAL log..." + $cutWALUntil "$TMHOME/data/cs.wal/wal" 6 consensus/test_data/new_many_blocks.cswal mv consensus/test_data/new_many_blocks.cswal consensus/test_data/many_blocks.cswal reset } -# small block 1 function small_block1(){ bash scripts/txs/random.sh 1000 36657 &> /dev/null & PID=$! + echo "==> Starting tendermint..." tendermint node --proxy_app=persistent_dummy &> /dev/null & sleep 10 - killall tendermint + echo "==> Killing tendermint..." kill -9 $PID + killall tendermint - ./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_small_block1.cswal + echo "==> Copying WAL log..." + $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block1.cswal mv consensus/test_data/new_small_block1.cswal consensus/test_data/small_block1.cswal reset } -# small block 2 (part size = 64) +# block part size = 512 function small_block2(){ - cat ~/.tendermint/genesis.json | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > ~/.tendermint/new_genesis.json - mv ~/.tendermint/new_genesis.json ~/.tendermint/genesis.json + cat "$TMHOME/genesis.json" | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > "$TMHOME/new_genesis.json" + mv "$TMHOME/new_genesis.json" "$TMHOME/genesis.json" bash scripts/txs/random.sh 1000 36657 &> /dev/null & PID=$! + echo "==> Starting tendermint..." tendermint node --proxy_app=persistent_dummy &> /dev/null & sleep 5 - killall tendermint + echo "==> Killing tendermint..." kill -9 $PID + killall tendermint - ./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_small_block2.cswal + echo "==> Copying WAL log..." + $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block2.cswal mv consensus/test_data/new_small_block2.cswal consensus/test_data/small_block2.cswal reset @@ -102,21 +125,24 @@ function small_block2(){ case "$1" in - "small_block1") - small_block1 - ;; - "small_block2") - small_block2 - ;; - "empty_block") - empty_block - ;; - "many_blocks") - many_blocks - ;; - *) - small_block1 - small_block2 - empty_block - many_blocks + "small_block1") + small_block1 + ;; + "small_block2") + small_block2 + ;; + "empty_block") + empty_block + ;; + "many_blocks") + many_blocks + ;; + *) + small_block1 + small_block2 + empty_block + many_blocks esac + +echo "==> Cleaning up..." +rm -rf "$TMHOME" diff --git a/consensus/wal.go b/consensus/wal.go index 51cb31193..32cce73ff 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -18,7 +18,7 @@ import ( // types and functions for savings consensus messages var ( - walSeparator = []byte{55, 127, 6, 130} // 0x377f0682 - magic number (can only affect tests) + walSeparator = []byte{55, 127, 6, 130} // 0x377f0682 - magic number ) type TimedWALMessage struct { @@ -27,6 +27,7 @@ type TimedWALMessage struct { } // EndHeightMessage marks the end of the given height inside WAL. +// @internal used by scripts/cutWALUntil util. type EndHeightMessage struct { Height uint64 `json:"height"` } diff --git a/glide.lock b/glide.lock index 167ab77ca..8f72ddbba 100644 --- a/glide.lock +++ b/glide.lock @@ -20,31 +20,28 @@ imports: - name: github.com/go-logfmt/logfmt version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 - name: github.com/go-playground/locales - version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6 + version: 1e5f1161c6416a5ff48840eb8724a394e48cc534 subpackages: - currency - name: github.com/go-playground/universal-translator version: 71201497bace774495daed26a3874fd339e0b538 - name: github.com/go-stack/stack - version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf + version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82 - name: github.com/gogo/protobuf - version: 117892bf1866fbaa2318c03e50e40564c8845457 + version: f7f1376d9d231a646d4e62fe1075623ced6db327 subpackages: - proto - name: github.com/golang/protobuf - version: 1643683e1b54a9e88ad26d98f81400c8c9d9f4f9 + version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8 subpackages: - proto - - ptypes - ptypes/any - - ptypes/duration - - ptypes/timestamp - name: github.com/golang/snappy version: 553a641470496b2327abcac10b36396bd98e45c9 - name: github.com/gorilla/websocket - version: 71fa72d4842364bc5f74185f4161e0099ea3624a + version: a91eba7f97777409bc2c443f5534d41dd20c5720 - name: github.com/hashicorp/hcl - version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8 + version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca subpackages: - hcl/ast - hcl/parser @@ -61,31 +58,33 @@ imports: - name: github.com/kr/logfmt version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0 - name: github.com/magiconair/properties - version: 8d7837e64d3c1ee4e54a880c5a920ab4316fc90a + version: 51463bfca2576e06c62a8504b5c0f06d61312647 - name: github.com/mitchellh/mapstructure - version: 06020f85339e21b2478f756a78e295255ffa4d6a + version: cc8532a8e9a55ea36402aa21efdf403a60d34096 +- name: github.com/pelletier/go-buffruneio + version: c37440a7cf42ac63b919c752ca73a85067e05992 - name: github.com/pelletier/go-toml - version: 8c31c2ec65b208cc2ad1608bf25a3ff91adf1944 + version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a - name: github.com/pkg/errors version: 645ef00459ed84a119197bfb8d8205042c6df63d - name: github.com/rcrowley/go-metrics version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c - name: github.com/spf13/afero - version: 5660eeed305fe5f69c8fc6cf899132a459a97064 + version: 9be650865eab0c12963d8753212f4f9c66cdcf12 subpackages: - mem - name: github.com/spf13/cast version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4 - name: github.com/spf13/cobra - version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b + version: 4cdb38c072b86bf795d2c81de50784d9fdd6eb77 - name: github.com/spf13/jwalterweatherman - version: 12bd96e66386c1960ab0f74ced1362f66f552f7b + version: 8f07c835e5cc1450c082fe3a439cf87b0cbb2d99 - name: github.com/spf13/pflag - version: 97afa5e7ca8a08a383cb259e06636b5e2cc7897f + version: e57e3eeb33f795204c1ca35f56c44f83227c6e66 - name: github.com/spf13/viper - version: 8ef37cbca71638bf32f3d5e194117d4cb46da163 + version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2 - name: github.com/syndtr/goleveldb - version: b89cc31ef7977104127d34c1bd31ebd1a9db2199 + version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4 subpackages: - leveldb - leveldb/cache @@ -138,7 +137,7 @@ imports: - merkle - test - name: golang.org/x/crypto - version: edd5e9b0879d13ee6970a50153d85b8fec9f7686 + version: c7af5bf2638a1164f2eb5467c39c6cffbd13a02e subpackages: - curve25519 - nacl/box @@ -149,7 +148,7 @@ imports: - ripemd160 - salsa20/salsa - name: golang.org/x/net - version: cd69bc3fc700721b709c3a59e16e24c67b58f6ff + version: feeb485667d1fdabe727840fe00adc22431bc86e subpackages: - context - http2 @@ -159,49 +158,43 @@ imports: - lex/httplex - trace - name: golang.org/x/sys - version: 8dbc5d05d6edcc104950cc299a1ce6641235bc86 + version: e62c3de784db939836898e5c19ffd41bece347da subpackages: - unix - name: golang.org/x/text - version: c01e4764d870b77f8abe5096ee19ad20d80e8075 + version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4 subpackages: - secure/bidirule - transform - unicode/bidi - unicode/norm - name: google.golang.org/genproto - version: f676e0f3ac6395ff1a529ae59a6670878a8371a6 + version: 411e09b969b1170a9f0c467558eb4c4c110d9c77 subpackages: - googleapis/rpc/status - name: google.golang.org/grpc - version: a5986a5c88227370a9c0a82e5277167229c034cd + version: 844f573616520565fdc6fb4db242321b5456fd6d subpackages: - - balancer - - balancer/roundrobin - codes - - connectivity - credentials - - grpclb/grpc_lb_v1/messages + - grpclb/grpc_lb_v1 - grpclog - internal - keepalive - metadata - naming - peer - - resolver - - resolver/dns - - resolver/passthrough - stats - status - tap - transport - name: gopkg.in/go-playground/validator.v9 - version: 1304298bf10d085adec514b076772a79c9cadb6b + version: 6d8c18553ea1ac493d049edd6f102f52e618f085 - name: gopkg.in/yaml.v2 - version: eb3733d160e74a9c7e442f435eb3bea458e1d19f + version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b testImports: - name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 + version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: - spew - name: github.com/pmezard/go-difflib @@ -209,7 +202,7 @@ testImports: subpackages: - difflib - name: github.com/stretchr/testify - version: 2aa2c176b9dab406a6970f6a55f513e8a8c8b18f + version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require diff --git a/scripts/wal2json/wal2json b/scripts/wal2json/wal2json deleted file mode 100755 index b7cab1467..000000000 Binary files a/scripts/wal2json/wal2json and /dev/null differ