fixes per Bucky's review

.gitignore

@@ -17,3 +17,6 @@ test/logs
 coverage.txt
 docs/_build
 docs/tools
+
+scripts/wal2json/wal2json
+scripts/cutWALUntil/cutWALUntil

consensus/replay.go

@@ -38,7 +38,7 @@ var crc32c = crc32.MakeTable(crc32.Castagnoli)
 // Lines that start with "#" are ignored.
 // NOTE: receiveRoutine should not be running
 func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan interface{}) error {
-	// Skip over meta lines
+	// skip meta messages
 	if _, ok := msg.Msg.(EndHeightMessage); ok {
 		return nil
 	}
@@ -129,8 +129,7 @@ func (cs *ConsensusState) catchupReplay(csHeight int) error {
 		msg, err = dec.Decode()
 		if err == io.EOF {
 			break
-		}
-		if err != nil {
+		} else if err != nil {
 			return err
 		}
 		// NOTE: since the priv key is set when the msgs are received

consensus/replay_file.go

@@ -63,7 +63,7 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error {
 		msg, err = pb.dec.Decode()
 		if err == io.EOF {
 			return nil
-		} else {
+		} else if err != nil {
 			return err
 		}
 
@@ -130,7 +130,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error {
 		msg, err = pb.dec.Decode()
 		if err == io.EOF {
 			return nil
-		} else {
+		} else if err != nil {
 			return err
 		}
 		if err := pb.cs.readReplayMessage(msg, newStepCh); err != nil {

consensus/replay_test.go

@@ -177,9 +177,9 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo
 	lastMsg := split[nLines]
 
 	// we write those lines up to (not including) one with the signature
-	bytes := bytes.Join(split[:nLines], walSeparator)
-	bytes = append(bytes, walSeparator...)
-	walFile := writeWAL(bytes)
+	b := bytes.Join(split[:nLines], walSeparator)
+	b = append(b, walSeparator...)
+	walFile := writeWAL(b)
 
 	cs := fixedConsensusStateDummy()
 
@@ -196,6 +196,7 @@ func setupReplayTest(t *testing.T, thisCase *testCase, nLines int, crashAfter bo
 
 func readTimedWALMessage(t *testing.T, rawMsg []byte) TimedWALMessage {
 	b := bytes.NewBuffer(rawMsg)
+	// because rawMsg does not contain a separator and WALDecoder#Decode expects it
 	_, err := b.Write(walSeparator)
 	if err != nil {
 		t.Fatal(err)
@@ -496,8 +497,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
 		msg, err := dec.Decode()
 		if err == io.EOF {
 			break
-		}
-		if err != nil {
+		} else if err != nil {
 			return nil, nil, err
 		}
 
@@ -538,7 +538,7 @@ func makeBlockchainFromWAL(wal *WAL) ([]*types.Block, []*types.Commit, error) {
 }
 
 func readPieceFromWAL(msg *TimedWALMessage) interface{} {
-	// Skip meta lines
+	// skip meta messages
 	if _, ok := msg.Msg.(EndHeightMessage); ok {
 		return nil
 	}

consensus/test_data/build.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-# XXX: removes tendermint dir
-# TODO: does not work on OSX
+# Requires: killall command and jq JSON processor.
 
 # Get the parent directory of where this script is.
 SOURCE="${BASH_SOURCE[0]}"
@@ -9,91 +8,115 @@ while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
 DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
 
 # Change into that dir because we expect that.
-cd "$DIR" || exit
+cd "$DIR" || exit 1
 
 # Make sure we have a tendermint command.
 if ! hash tendermint 2>/dev/null; then
-    make install
+	make install
 fi
 
 # Make sure we have a cutWALUntil binary.
-if ! hash ./scripts/cutWALUntil/cutWALUntil 2>/dev/null; then
-	cd ./scripts/cutWALUntil/ && go build && cd - || exit
+cutWALUntil=./scripts/cutWALUntil/cutWALUntil
+cutWALUntilDir=$(dirname $cutWALUntil)
+if ! hash $cutWALUntil 2>/dev/null; then
+	cd "$cutWALUntilDir" && go build && cd - || exit 1
+fi
+
+TMHOME=$(mktemp -d)
+export TMHOME="$TMHOME"
+
+if [[ ! -d "$TMHOME" ]]; then
+	echo "Could not create temp directory"
+  exit 1
+else
+	echo "TMHOME: ${TMHOME}"
 fi
 
-# specify a dir to copy
 # TODO: eventually we should replace with `tendermint init --test`
 DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test
+if [ ! -d "$DIR_TO_COPY" ]; then
+    echo "$DIR_TO_COPY does not exist. Please run: go test ./consensus"
+    exit 1
+fi
+echo "==> Copying ${DIR_TO_COPY} to ${TMHOME} directory..."
+cp -r "$DIR_TO_COPY"/* "$TMHOME"
 
-TMHOME="$HOME/.tendermint"
-#rm -rf "$TMHOME"
-#cp -r "$DIR_TO_COPY" "$TMHOME"
-#mv $TMHOME/config.toml $TMHOME/config.toml.bak
-cp $TMHOME/genesis.json $TMHOME/genesis.json.bak
+# preserve original genesis file because later it will be modified (see small_block2)
+cp "$TMHOME/genesis.json" "$TMHOME/genesis.json.bak"
 
 function reset(){
+	echo "==> Resetting tendermint..."
 	tendermint unsafe_reset_all
-	cp $TMHOME/genesis.json.bak $TMHOME/genesis.json
+	cp "$TMHOME/genesis.json.bak" "$TMHOME/genesis.json"
 }
 
 reset
 
-# empty block
 function empty_block(){
+	echo "==> Starting tendermint..."
 	tendermint node --proxy_app=persistent_dummy &> /dev/null &
 	sleep 5
+	echo "==> Killing tendermint..."
 	killall tendermint
 
-	./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_empty_block.cswal
+	echo "==> Copying WAL log..."
+	$cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_empty_block.cswal
 	mv consensus/test_data/new_empty_block.cswal consensus/test_data/empty_block.cswal
 
 	reset
 }
 
-# many blocks
 function many_blocks(){
 	bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 	PID=$!
+	echo "==> Starting tendermint..."
 	tendermint node --proxy_app=persistent_dummy &> /dev/null &
 	sleep 10
-	killall tendermint
+	echo "==> Killing tendermint..."
 	kill -9 $PID
+	killall tendermint
 
-	./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 6 consensus/test_data/new_many_blocks.cswal
+	echo "==> Copying WAL log..."
+	$cutWALUntil "$TMHOME/data/cs.wal/wal" 6 consensus/test_data/new_many_blocks.cswal
 	mv consensus/test_data/new_many_blocks.cswal consensus/test_data/many_blocks.cswal
 
 	reset
 }
 
 
-# small block 1
 function small_block1(){
 	bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 	PID=$!
+	echo "==> Starting tendermint..."
 	tendermint node --proxy_app=persistent_dummy &> /dev/null &
 	sleep 10
-	killall tendermint
+	echo "==> Killing tendermint..."
 	kill -9 $PID
+	killall tendermint
 
-	./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_small_block1.cswal
+	echo "==> Copying WAL log..."
+	$cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block1.cswal
 	mv consensus/test_data/new_small_block1.cswal consensus/test_data/small_block1.cswal
 
 	reset
 }
 
 
-# small block 2 (part size = 64)
+# block part size = 512
 function small_block2(){
-	cat ~/.tendermint/genesis.json | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > ~/.tendermint/new_genesis.json
-	mv ~/.tendermint/new_genesis.json ~/.tendermint/genesis.json
+	cat "$TMHOME/genesis.json" | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > "$TMHOME/new_genesis.json"
+	mv "$TMHOME/new_genesis.json" "$TMHOME/genesis.json"
 	bash scripts/txs/random.sh 1000 36657 &> /dev/null &
 	PID=$!
+	echo "==> Starting tendermint..."
 	tendermint node --proxy_app=persistent_dummy &> /dev/null &
 	sleep 5
-	killall tendermint
+	echo "==> Killing tendermint..."
 	kill -9 $PID
+	killall tendermint
 
-	./scripts/cutWALUntil/cutWALUntil ~/.tendermint/data/cs.wal/wal 1 consensus/test_data/new_small_block2.cswal
+	echo "==> Copying WAL log..."
+	$cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block2.cswal
 	mv consensus/test_data/new_small_block2.cswal consensus/test_data/small_block2.cswal
 
 	reset
@@ -102,21 +125,24 @@ function small_block2(){
 
 
 case "$1" in
-    "small_block1")
-        small_block1
-        ;;
-    "small_block2")
-        small_block2
-        ;;
-    "empty_block")
-        empty_block
-        ;;
-    "many_blocks")
-        many_blocks
-        ;;
-    *)
-        small_block1
-        small_block2
-        empty_block
-        many_blocks
+	"small_block1")
+		small_block1
+		;;
+	"small_block2")
+		small_block2
+		;;
+	"empty_block")
+		empty_block
+		;;
+	"many_blocks")
+		many_blocks
+		;;
+	*)
+		small_block1
+		small_block2
+		empty_block
+		many_blocks
 esac
+
+echo "==> Cleaning up..."
+rm -rf "$TMHOME"

consensus/wal.go

@@ -18,7 +18,7 @@ import (
 // types and functions for savings consensus messages
 
 var (
-	walSeparator = []byte{55, 127, 6, 130} // 0x377f0682 - magic number (can only affect tests)
+	walSeparator = []byte{55, 127, 6, 130} // 0x377f0682 - magic number
 )
 
 type TimedWALMessage struct {
@@ -27,6 +27,7 @@ type TimedWALMessage struct {
 }
 
 // EndHeightMessage marks the end of the given height inside WAL.
+// @internal used by scripts/cutWALUntil util.
 type EndHeightMessage struct {
 	Height uint64 `json:"height"`
 }

glide.lock

@@ -20,31 +20,28 @@ imports:
 - name: github.com/go-logfmt/logfmt
   version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 - name: github.com/go-playground/locales
-  version: e4cbcb5d0652150d40ad0646651076b6bd2be4f6
+  version: 1e5f1161c6416a5ff48840eb8724a394e48cc534
   subpackages:
   - currency
 - name: github.com/go-playground/universal-translator
   version: 71201497bace774495daed26a3874fd339e0b538
 - name: github.com/go-stack/stack
-  version: 817915b46b97fd7bb80e8ab6b69f01a53ac3eebf
+  version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/gogo/protobuf
-  version: 117892bf1866fbaa2318c03e50e40564c8845457
+  version: f7f1376d9d231a646d4e62fe1075623ced6db327
   subpackages:
   - proto
 - name: github.com/golang/protobuf
-  version: 1643683e1b54a9e88ad26d98f81400c8c9d9f4f9
+  version: 18c9bb3261723cd5401db4d0c9fbc5c3b6c70fe8
   subpackages:
   - proto
-  - ptypes
   - ptypes/any
-  - ptypes/duration
-  - ptypes/timestamp
 - name: github.com/golang/snappy
   version: 553a641470496b2327abcac10b36396bd98e45c9
 - name: github.com/gorilla/websocket
-  version: 71fa72d4842364bc5f74185f4161e0099ea3624a
+  version: a91eba7f97777409bc2c443f5534d41dd20c5720
 - name: github.com/hashicorp/hcl
-  version: 23c074d0eceb2b8a5bfdbb271ab780cde70f05a8
+  version: 392dba7d905ed5d04a5794ba89f558b27e2ba1ca
   subpackages:
   - hcl/ast
   - hcl/parser
@@ -61,31 +58,33 @@ imports:
 - name: github.com/kr/logfmt
   version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0
 - name: github.com/magiconair/properties
-  version: 8d7837e64d3c1ee4e54a880c5a920ab4316fc90a
+  version: 51463bfca2576e06c62a8504b5c0f06d61312647
 - name: github.com/mitchellh/mapstructure
-  version: 06020f85339e21b2478f756a78e295255ffa4d6a
+  version: cc8532a8e9a55ea36402aa21efdf403a60d34096
+- name: github.com/pelletier/go-buffruneio
+  version: c37440a7cf42ac63b919c752ca73a85067e05992
 - name: github.com/pelletier/go-toml
-  version: 8c31c2ec65b208cc2ad1608bf25a3ff91adf1944
+  version: 13d49d4606eb801b8f01ae542b4afc4c6ee3d84a
 - name: github.com/pkg/errors
   version: 645ef00459ed84a119197bfb8d8205042c6df63d
 - name: github.com/rcrowley/go-metrics
   version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c
 - name: github.com/spf13/afero
-  version: 5660eeed305fe5f69c8fc6cf899132a459a97064
+  version: 9be650865eab0c12963d8753212f4f9c66cdcf12
   subpackages:
   - mem
 - name: github.com/spf13/cast
   version: acbeb36b902d72a7a4c18e8f3241075e7ab763e4
 - name: github.com/spf13/cobra
-  version: 7b2c5ac9fc04fc5efafb60700713d4fa609b777b
+  version: 4cdb38c072b86bf795d2c81de50784d9fdd6eb77
 - name: github.com/spf13/jwalterweatherman
-  version: 12bd96e66386c1960ab0f74ced1362f66f552f7b
+  version: 8f07c835e5cc1450c082fe3a439cf87b0cbb2d99
 - name: github.com/spf13/pflag
-  version: 97afa5e7ca8a08a383cb259e06636b5e2cc7897f
+  version: e57e3eeb33f795204c1ca35f56c44f83227c6e66
 - name: github.com/spf13/viper
-  version: 8ef37cbca71638bf32f3d5e194117d4cb46da163
+  version: 0967fc9aceab2ce9da34061253ac10fb99bba5b2
 - name: github.com/syndtr/goleveldb
-  version: b89cc31ef7977104127d34c1bd31ebd1a9db2199
+  version: 8c81ea47d4c41a385645e133e15510fc6a2a74b4
   subpackages:
   - leveldb
   - leveldb/cache
@@ -138,7 +137,7 @@ imports:
   - merkle
   - test
 - name: golang.org/x/crypto
-  version: edd5e9b0879d13ee6970a50153d85b8fec9f7686
+  version: c7af5bf2638a1164f2eb5467c39c6cffbd13a02e
   subpackages:
   - curve25519
   - nacl/box
@@ -149,7 +148,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: cd69bc3fc700721b709c3a59e16e24c67b58f6ff
+  version: feeb485667d1fdabe727840fe00adc22431bc86e
   subpackages:
   - context
   - http2
@@ -159,49 +158,43 @@ imports:
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: 8dbc5d05d6edcc104950cc299a1ce6641235bc86
+  version: e62c3de784db939836898e5c19ffd41bece347da
   subpackages:
   - unix
 - name: golang.org/x/text
-  version: c01e4764d870b77f8abe5096ee19ad20d80e8075
+  version: 470f45bf29f4147d6fbd7dfd0a02a848e49f5bf4
   subpackages:
   - secure/bidirule
   - transform
   - unicode/bidi
   - unicode/norm
 - name: google.golang.org/genproto
-  version: f676e0f3ac6395ff1a529ae59a6670878a8371a6
+  version: 411e09b969b1170a9f0c467558eb4c4c110d9c77
   subpackages:
   - googleapis/rpc/status
 - name: google.golang.org/grpc
-  version: a5986a5c88227370a9c0a82e5277167229c034cd
+  version: 844f573616520565fdc6fb4db242321b5456fd6d
   subpackages:
-  - balancer
-  - balancer/roundrobin
   - codes
-  - connectivity
   - credentials
-  - grpclb/grpc_lb_v1/messages
+  - grpclb/grpc_lb_v1
   - grpclog
   - internal
   - keepalive
   - metadata
   - naming
   - peer
-  - resolver
-  - resolver/dns
-  - resolver/passthrough
   - stats
   - status
   - tap
   - transport
 - name: gopkg.in/go-playground/validator.v9
-  version: 1304298bf10d085adec514b076772a79c9cadb6b
+  version: 6d8c18553ea1ac493d049edd6f102f52e618f085
 - name: gopkg.in/yaml.v2
-  version: eb3733d160e74a9c7e442f435eb3bea458e1d19f
+  version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b
 testImports:
 - name: github.com/davecgh/go-spew
-  version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9
+  version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
   subpackages:
   - spew
 - name: github.com/pmezard/go-difflib
@@ -209,7 +202,7 @@ testImports:
   subpackages:
   - difflib
 - name: github.com/stretchr/testify
-  version: 2aa2c176b9dab406a6970f6a55f513e8a8c8b18f
+  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
   subpackages:
   - assert
   - require