Merge pull request #31 from tendermint/develop

Develop

terraforce/README.md

@@ -0,0 +1,149 @@
+# Stack
+
+This is a stripped down version of https://github.com/segmentio/stack
+plus some shell scripts.
+
+It is responsible for the following:
+
+	- spin up a cluster of nodes
+	- copy config files for a tendermint testnet to each node
+	- copy linux binaries for tendermint and the app to each node
+	- start tendermint on every node
+
+# How it Works
+
+To use, a user must only provide a directory containing two files: `bins` and `run.sh`.
+
+The `bins` file is a list of binaries, for instance:
+
+```
+$GOPATH/bin/tendermint
+$GOPATH/bin/dummy
+```
+
+and the `run.sh` specifies how those binaries ought to be started:
+
+```
+#! /bin/bash
+
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS_FLAG="--seeds=$SEEDS"
+fi
+
+./dummy --persist .tendermint/data/dummy_data >> app.log 2>&1 &
+./tendermint node --log_level=info $SEEDS_FLAG >> tendermint.log 2>&1 &
+```
+
+This let's you specify exactly which versions of Tendermint and the application are to be used,
+and how they ought to be started.
+
+Note that these binaries *MUST* be compiled for Linux. 
+If you are not on Linux, you can compile binaries for linux using `go build` with the `GOOS` variable:
+
+```
+GOOS=linux go build -o $GOPATH/bin/tendermint-linux $GOPATH/src/github.com/tendermint/tendermint/cmd/tendermint
+```
+
+This cross-compilation must be done for each binary you want to copy over. 
+
+If you want to use an application that requires more than just a few binaries, you may need to do more manual work, 
+for instance using `terraforce` to set up the development environment on every machine.
+
+# Dependencies
+
+We use `terraform` for spinning up the machines, 
+and a custom rolled tool, `terraforce`, 
+for running commands on many machines in parallel.
+You can download terraform here: https://www.terraform.io/downloads.html
+To download terraforce, run `go get github.com/ebuchman/terraforce`
+
+We use `tendermint` itself to generate files for a testnet.
+You can install `tendermint` with 
+
+```
+cd $GOPATH/src/github.com/tendermint/tendermint
+glide install
+go install ./cmd/tendermint
+```
+
+You also need to set the `DIGITALOCEAN_TOKEN` environment variables so that terraform can 
+spin up nodes on digital ocean.
+
+This stack is currently some terraform and a bunch of shell scripts, 
+so its helpful to work out of a directory containing everything. 
+Either 	change directory to `$GOPATH/src/github.com/tendermint/tendermint/test/net`
+or make a copy of that directory and change to it. All commands are expected to be executed from there.
+
+For terraform to work, you must first run `terraform get`
+
+# Create 
+
+To create a cluster with 4 nodes, run
+
+```
+terraform apply
+```
+
+To use a different number of nodes, change the `desired_capacity` parameter in the `main.tf`.
+
+Note that terraform keeps track of the current state of your infrastructure, 
+so if you change the `desired_capacity` and run `terraform apply` again, it will add or remove nodes as necessary.
+
+If you think that's amazing, so do we.
+
+To get some info about the cluster, run `terraform output`.
+
+See the [terraform docs](https://www.terraform.io/docs/index.html) for more details. 
+
+To tear down the cluster, run `terraform destroy`.
+
+# Initialize 
+
+Now that we have a cluster up and running, let's generate the necessary files for a Tendermint node and copy them over.
+A Tendermint node needs, at the least, a `priv_validator.json` and a `genesis.json`.
+To generate files for the nodes, run
+
+```
+tendermint testnet 4 mytestnet
+```
+
+This will create the directory `mytestnet`, containing one directory for each of the 4 nodes.
+Each node directory contains a unique `priv_validator.json` and a `genesis.json`, 
+where the `genesis.json` contains the public keys of all `priv_validator.json` files.
+
+If you want to add more files to each node for your particular app, you'll have to add them to each of the node directories.
+
+Now we can copy everything over to the cluster.
+If you are on Linux, run 
+
+```
+bash scripts/init.sh 4 mytestnet examples/in-proc
+```
+
+Otherwise (if you are not on Linux), make sure you ran 
+
+```
+GOOS=linux go build -o $GOPATH/bin/tendermint-linux $GOPATH/src/github.com/tendermint/tendermint/cmd/tendermint
+```
+
+and now run 
+
+```
+bash scripts/init.sh 4 mytestnet examples/in-proc-linux
+```
+
+# Start 
+
+Finally, to start Tendermint on all the nodes, run
+
+```
+bash scripts/start.sh 4
+```
+
+# Check
+
+Query the status of all your nodes:
+
+```
+bash scripts/query.sh 4 status
+```

terraforce/cluster/main.tf

@@ -0,0 +1,77 @@
+/**
+ * Cluster on DO
+ *
+ */
+
+variable "name" {
+  description = "The cluster name, e.g cdn"
+}
+
+variable "environment" {
+  description = "Environment tag, e.g prod"
+}
+
+variable "image_id" {
+  description = "Image ID"
+}
+
+variable "regions" {
+  description = "Regions to launch in"
+  type = "list"
+}
+
+variable "key_ids" {
+  description = "SSH keys to use"
+  type = "list"
+}
+
+variable "instance_size" {
+  description = "The instance size to use, e.g 2gb"
+}
+
+variable "desired_capacity" {
+  description = "Desired instance count"
+  default     = 3
+}
+
+#-----------------------
+# Instances
+
+resource "digitalocean_droplet" "cluster" {
+  # set the image and instance type
+  name = "${var.name}${count.index}"
+  image = "${var.image_id}"
+  size = "${var.instance_size}"
+
+  # the `element` function handles modulo
+  region = "${element(var.regions, count.index)}"
+
+  ssh_keys = "${var.key_ids}"
+
+  count = "${var.desired_capacity}"
+  lifecycle = {
+	prevent_destroy = false
+  }
+}
+
+#-----------------------
+
+// The cluster name, e.g cdn
+output "name" {
+  value = "${var.name}"
+}
+
+// The list of cluster instance ids
+output "instances" {
+  value = ["${digitalocean_droplet.cluster.*.id}"]
+}
+
+// The list of cluster instance ips
+output "private_ips" {
+  value = ["${digitalocean_droplet.cluster.*.ipv4_address_private}"]
+}
+
+// The list of cluster instance ips
+output "public_ips" {
+  value = ["${digitalocean_droplet.cluster.*.ipv4_address}"]
+}

terraforce/examples/dummy/bins

@@ -0,0 +1,2 @@
+$GOPATH/bin/tendermint
+$GOPATH/bin/dummy

terraforce/examples/dummy/run.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS_FLAG="--seeds=$SEEDS"
+fi
+
+./dummy --persist .tendermint/data/dummy_data >> app.log 2>&1 &
+./tendermint node --log_level=info $SEEDS_FLAG >> tendermint.log 2>&1 &

terraforce/examples/in-proc-linux/bins

@@ -0,0 +1 @@
+$GOPATH/bin/tendermint-linux

terraforce/examples/in-proc-linux/run.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS_FLAG="--seeds=$SEEDS"
+fi
+
+./tendermint-linux node --proxy_app=dummy --log_level=note $SEEDS_FLAG >> tendermint.log 2>&1 &

terraforce/examples/in-proc/bins

@@ -0,0 +1 @@
+$GOPATH/bin/tendermint

terraforce/examples/in-proc/run.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS_FLAG="--seeds=$SEEDS"
+fi
+
+./tendermint node --proxy_app=dummy --log_level=note $SEEDS_FLAG >> tendermint.log 2>&1 &

terraforce/main.tf

@@ -0,0 +1,34 @@
+module "cluster" {
+  source      = "./cluster"
+  environment = "test"
+  name        = "tendermint-testnet"
+
+  # curl -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" "https://api.digitalocean.com/v2/account/keys"
+  key_ids = [8163311]
+
+  image_id = "ubuntu-14-04-x64"
+  desired_capacity = 4
+  instance_size = "2gb"
+
+  regions = ["AMS2", "FRA1", "LON1", "NYC2", "SFO2", "SGP1", "TOR1"]
+}
+
+
+provider "digitalocean" {
+}
+
+output "public_ips" {
+  value = "${module.cluster.public_ips}"
+}
+
+output "private_ips" {
+  value = "${join(",",module.cluster.private_ips)}"
+}
+
+output "seeds" {
+  value = "${join(":46656,",module.cluster.public_ips)}:46656"
+}
+
+output "rpcs" {
+  value = "${join(":46657,",module.cluster.public_ips)}:46657"
+}

terraforce/scripts/copy_run.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+RUN=$2 # path to run script
+
+N_=$((N-1))
+
+# stop all tendermint
+terraforce scp --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" $RUN run.sh

terraforce/scripts/init.sh

@@ -0,0 +1,43 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+TESTNET=$2 # path to folder containing testnet info
+CONFIG=$3 # path to folder containing `bins` and `run.sh` files
+
+if [[ ! -f $CONFIG/bins ]]; then
+	echo "config folder ($CONFIG) must contain bins file"
+	exit 1
+fi
+if [[ ! -f $CONFIG/run.sh ]]; then
+	echo "config folder ($CONFIG) must contain run.sh file"
+	exit 1
+fi
+
+KEY=$HOME/.ssh/id_rsa
+
+FLAGS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
+
+N_=$((N-1)) # 0-based index
+
+MACH_ROOT="$TESTNET/mach?"
+
+
+# mkdir
+terraforce ssh --user root --ssh-key $KEY  --machines "[0-$N_]" mkdir .tendermint
+
+# copy over  genesis/priv_val
+terraforce scp --user root --ssh-key $KEY  --iterative --machines "[0-$N_]" "$MACH_ROOT/priv_validator.json" .tendermint/priv_validator.json
+terraforce scp --user root --ssh-key $KEY  --iterative --machines "[0-$N_]" "$MACH_ROOT/genesis.json" .tendermint/genesis.json
+
+# copy the run script
+terraforce scp --user root --ssh-key $KEY --machines "[0-$N_]" $CONFIG/run.sh run.sh 
+
+# copy the binaries
+while read line; do
+	local_bin=$(eval echo $line)
+  	remote_bin=$(basename $local_bin)
+  	echo $local_bin
+ 	terraforce scp --user root --ssh-key $KEY --machines "[0-$N_]" $local_bin $remote_bin
+ 	terraforce ssh --user root --ssh-key $KEY --machines "[0-$N_]" chmod +x $remote_bin
+done <$CONFIG/bins

terraforce/scripts/query.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+QUERY=$2
+
+N_=$((N-1))
+
+# start all tendermint nodes
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" curl -s localhost:46657/$QUERY
+

terraforce/scripts/reset.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+
+N_=$((N-1))
+
+# stop all tendermint
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" rm -rf .tendermint/data
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" ./tendermint unsafe_reset_priv_validator

terraforce/scripts/restart.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+
+N_=$((N-1))
+
+# start
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" SEEDS=$(terraform output seeds) bash run.sh

terraforce/scripts/start.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+
+N_=$((N-1))
+
+# start all tendermint nodes
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" SEEDS=$(terraform output seeds) bash run.sh
+

terraforce/scripts/stop.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+set -u
+
+N=$1 # number of nodes
+
+N_=$((N-1))
+
+# stop all tendermint
+terraforce ssh --user root --ssh-key $HOME/.ssh/id_rsa --machines "[0-$N_]" killall tendermint

terraforce/test.sh

@@ -0,0 +1,30 @@
+#! /bin/bash
+
+cd $GOPATH/src/github.com/tendermint/tendermint
+
+TEST_PATH=./test/net/new
+
+N=4
+TESTNET_DIR=mytestnet
+
+# install deps
+# TODO: we should build a Docker image and 
+# really do everything that follows in the container
+# bash setup.sh
+
+
+# launch infra
+terraform get
+terraform apply
+
+# create testnet files
+tendermint testnet -n $N -dir $TESTNET_DIR
+
+# expects a linux tendermint binary to be built already
+bash scripts/init.sh $N $TESTNET_DIR test/net/examples/in-proc
+
+# testnet should now be running :)
+bash scripts/start.sh 4
+
+
+

terraforce/transact/transact.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/tendermint/go-rpc/client"
+	rpctypes "github.com/tendermint/go-rpc/types"
+)
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+	if len(args) < 2 {
+		fmt.Println("transact.go expects at least two arguments (ntxs, hosts)")
+		os.Exit(1)
+	}
+
+	nTxS, hostS := args[0], args[1]
+	nTxs, err := strconv.Atoi(nTxS)
+	if err != nil {
+		fmt.Println("ntxs must be an integer:", err)
+		os.Exit(1)
+	}
+
+	hosts := strings.Split(hostS, ",")
+
+	errCh := make(chan error, 1000)
+
+	wg := new(sync.WaitGroup)
+	wg.Add(len(hosts))
+	start := time.Now()
+	fmt.Printf("Sending %d txs on every host %v\n", nTxs, hosts)
+	for i, host := range hosts {
+		go broadcastTxsToHost(wg, errCh, i, host, nTxs, 0)
+	}
+	wg.Wait()
+	fmt.Println("Done broadcasting txs. Took", time.Since(start))
+
+}
+
+func broadcastTxsToHost(wg *sync.WaitGroup, errCh chan error, valI int, valHost string, nTxs int, txCount int) {
+	reconnectSleepSeconds := time.Second * 1
+
+	// thisStart := time.Now()
+	// cli := rpcclient.NewClientURI(valHost + ":46657")
+	fmt.Println("Connecting to host to broadcast txs", valI, valHost)
+	cli := rpcclient.NewWSClient(valHost, "/websocket")
+	if _, err := cli.Start(); err != nil {
+		if nTxs == 0 {
+			time.Sleep(reconnectSleepSeconds)
+			broadcastTxsToHost(wg, errCh, valI, valHost, nTxs, txCount)
+			return
+		}
+		fmt.Printf("Error starting websocket connection to val%d (%s): %v\n", valI, valHost, err)
+		os.Exit(1)
+	}
+
+	reconnect := make(chan struct{})
+	go func(count int) {
+	LOOP:
+		for {
+			ticker := time.NewTicker(reconnectSleepSeconds)
+			select {
+			case <-cli.ResultsCh:
+				count += 1
+				// nTxs == 0 means just loop forever
+				if nTxs > 0 && count == nTxs {
+					break LOOP
+				}
+			case err := <-cli.ErrorsCh:
+				fmt.Println("err: val", valI, valHost, err)
+			case <-cli.Quit:
+				broadcastTxsToHost(wg, errCh, valI, valHost, nTxs, count)
+				return
+			case <-reconnect:
+				broadcastTxsToHost(wg, errCh, valI, valHost, nTxs, count)
+				return
+			case <-ticker.C:
+				if nTxs == 0 {
+					cli.Stop()
+					broadcastTxsToHost(wg, errCh, valI, valHost, nTxs, count)
+					return
+				}
+			}
+		}
+		fmt.Printf("Received all responses from node %d (%s)\n", valI, valHost)
+		wg.Done()
+	}(txCount)
+	var i = 0
+	for {
+		/*		if i%(nTxs/4) == 0 {
+				fmt.Printf("Have sent %d txs to node %d. Total time so far: %v\n", i, valI, time.Since(thisStart))
+			}*/
+
+		if !cli.IsRunning() {
+			return
+		}
+
+		tx := generateTx(i, valI)
+		if err := cli.WriteJSON(rpctypes.RPCRequest{
+			JSONRPC: "2.0",
+			ID:      "",
+			Method:  "broadcast_tx_async",
+			Params:  []interface{}{hex.EncodeToString(tx)},
+		}); err != nil {
+			fmt.Printf("Error sending tx %d to validator %d: %v. Attempt reconnect\n", i, valI, err)
+			reconnect <- struct{}{}
+			return
+		}
+		i += 1
+		if nTxs > 0 && i >= nTxs {
+			break
+		} else if nTxs == 0 {
+			time.Sleep(time.Millisecond * 1)
+		}
+	}
+	fmt.Printf("Done sending %d txs to node s%d (%s)\n", nTxs, valI, valHost)
+}
+
+func generateTx(i, valI int) []byte {
+	// a tx encodes the validator index, the tx number, and some random junk
+	// TODO: read random bytes into more of the tx
+	tx := make([]byte, 250)
+	binary.PutUvarint(tx[:32], uint64(valI))
+	binary.PutUvarint(tx[32:64], uint64(i))
+	if _, err := rand.Read(tx[234:]); err != nil {
+		fmt.Println("err reading from crypto/rand", err)
+		os.Exit(1)
+	}
+	return tx
+}

tm-bench/glide.lock

@@ -1,37 +1,38 @@
-hash: 7b3982490e9c4cdc9467e849959fd8d6ccec8f8f9e81fc56f29e8e9c67bf3b53
-updated: 2017-03-16T15:16:05.301074208Z
+hash: 795aa94747f3d877df3ea1ec134e9a34e1c46713dd6eb59b6fdd6a33cb698234
+updated: 2017-04-20T19:19:22.26004087-04:00
 imports:
 - name: github.com/btcsuite/btcd
   version: 583684b21bfbde9b5fc4403916fd7c807feb0289
   subpackages:
   - btcec
 - name: github.com/BurntSushi/toml
-  version: e643e9ef00b049d75de26e61109c5ea01885cd21
+  version: 99064174e013895bbd9b025c31100bd1d9b590ca
 - name: github.com/go-kit/kit
   version: b6f30a2e0632f5722fb26d8765d726335b79d3e6
   subpackages:
   - log
   - log/term
+  - term
 - name: github.com/go-logfmt/logfmt
   version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 - name: github.com/go-stack/stack
   version: 100eb0c0a9c5b306ca2fb4f165df21d80ada4b82
 - name: github.com/golang/protobuf
-  version: c9c7427a2a70d2eb3bafa0ab2dc163e45f143317
+  version: 69b215d01a5606c843240eab4937eab3acee6530
   subpackages:
   - proto
 - name: github.com/golang/snappy
   version: 553a641470496b2327abcac10b36396bd98e45c9
 - name: github.com/gorilla/websocket
-  version: b258b4fadb573ac412f187b9f31974ea99d32f50
+  version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13
 - name: github.com/jmhodges/levigo
   version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/kr/logfmt
   version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0
 - name: github.com/mattn/go-colorable
-  version: a392f450ea64cee2b268dfaacdc2502b50a22b18
+  version: d898aa9fb31c91f35dd28ca75db377eff023c076
 - name: github.com/mattn/go-isatty
-  version: 57fdcb988a5c543893cc61bce354a6e24ab70022
+  version: dda3de49cbfcec471bd7a70e6cc01fcc3ff90109
 - name: github.com/pkg/errors
   version: bfd5150e4e41705ded2129ec33379de1cb90b513
 - name: github.com/rcrowley/go-metrics
@@ -52,7 +53,7 @@ imports:
   - leveldb/table
   - leveldb/util
 - name: github.com/tendermint/abci
-  version: af792eac777de757cd496349a5f6b5313738fcbc
+  version: 56e13d87f4e3ec1ea756957d6b23caa6ebcf0998
   subpackages:
   - types
 - name: github.com/tendermint/ed25519
@@ -61,17 +62,17 @@ imports:
   - edwards25519
   - extra25519
 - name: github.com/tendermint/go-common
-  version: dcb015dff6c7af21e65c8e2f3b450df19d38c777
+  version: f9e3db037330c8a8d61d3966de8473eaf01154fa
 - name: github.com/tendermint/go-config
   version: 620dcbbd7d587cf3599dedbf329b64311b0c307a
 - name: github.com/tendermint/go-crypto
-  version: 3f47cfac5fcd9e0f1727c7db980b3559913b3e3a
+  version: 0ca2c6fdb0706001ca4c4b9b80c9f428e8cf39da
 - name: github.com/tendermint/go-data
-  version: 32271140e8fd5abdbb22e268d7a02421fa382f0b
+  version: e7fcc6d081ec8518912fcdc103188275f83a3ee5
 - name: github.com/tendermint/go-db
-  version: eac3f2bc147023957c8bf69432a4e6c4dc5c3f72
+  version: 9643f60bc2578693844aacf380a7c32e4c029fee
 - name: github.com/tendermint/go-events
-  version: f8ffbfb2be3483e9e7927495590a727f51c0c11f
+  version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6
 - name: github.com/tendermint/go-flowrate
   version: a20c98e61957faa93b4014fbd902f20ab9317a6a
   subpackages:
@@ -81,34 +82,32 @@ imports:
 - name: github.com/tendermint/go-merkle
   version: 714d4d04557fd068a7c2a1748241ce8428015a96
 - name: github.com/tendermint/go-p2p
-  version: 97a5ed2d1a17eaee8717b8a32cfaf7a9a82a273d
+  version: 17124989a93774833df33107fbf17157a7f8ef31
   subpackages:
   - upnp
 - name: github.com/tendermint/go-rpc
-  version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7
+  version: 1a42f946dc6bcd88f9f58c7f2fb86f785584d793
   subpackages:
   - client
-  - server
   - types
 - name: github.com/tendermint/go-wire
-  version: f530b7af7a8b06e612c2063bff6ace49060a085e
+  version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471
 - name: github.com/tendermint/log15
   version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
   subpackages:
   - term
 - name: github.com/tendermint/tendermint
-  version: d4f625455109d88e7f55a999fdb25e208f174802
+  version: 083fe959e25421fca3d41298d9111167a3b47122
   subpackages:
   - rpc/core/types
   - types
 - name: github.com/tendermint/tools
-  version: fb28fd4c13759026de23d98bb494a022840278c1
+  version: 12ce526668e384100afd32686ec7db3749423d51
   subpackages:
-  - tm-monitor
   - tm-monitor/eventmeter
   - tm-monitor/monitor
 - name: golang.org/x/crypto
-  version: 728b753d0135da6801d45a38e6f43ff55779c5c2
+  version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
   subpackages:
   - curve25519
   - nacl/box
@@ -119,7 +118,7 @@ imports:
   - ripemd160
   - salsa20/salsa
 - name: golang.org/x/net
-  version: a6577fac2d73be281a500b310739095313165611
+  version: 906cda9512f77671ab44f8c8563b13a8e707b230
   subpackages:
   - context
   - http2
@@ -129,17 +128,16 @@ imports:
   - lex/httplex
   - trace
 - name: golang.org/x/sys
-  version: 99f16d856c9836c42d24e7ab64ea72916925fa97
+  version: 76cc09b634294339fa19ec41b5f2a0b3932cea8b
   subpackages:
   - unix
 - name: google.golang.org/grpc
-  version: cdee119ee21e61eef7093a41ba148fa83585e143
+  version: 8b2e129857480cb0f07ef7d9d10b8b252c7ac984
   subpackages:
   - codes
   - credentials
   - grpclog
   - internal
-  - keepalive
   - metadata
   - naming
   - peer

tm-bench/glide.yaml

@@ -2,10 +2,12 @@ package: github.com/tendermint/tools/tm-bench
 import:
 - package: github.com/pkg/errors
 - package: github.com/tendermint/go-rpc
+  version: develop
   subpackages:
   - client
   - types
 - package: github.com/tendermint/tools
+  version: develop
   subpackages:
   - tm-monitor/monitor
 - package: github.com/go-kit/kit

tm-monitor/eventmeter/eventmeter.go

@@ -6,12 +6,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-kit/kit/log"
 	"github.com/gorilla/websocket"
 	"github.com/pkg/errors"
 	metrics "github.com/rcrowley/go-metrics"
-	events "github.com/tendermint/go-events"
-	client "github.com/tendermint/go-rpc/client"
+	client "github.com/tendermint/tendermint/rpc/lib/client"
+	"github.com/tendermint/tmlibs/events"
+	"github.com/tendermint/tmlibs/log"
 )
 
 //------------------------------------------------------
@@ -253,30 +253,30 @@ func (em *EventMeter) receiveRoutine() {
 		select {
 		case <-pingTicker.C:
 			if pingAttempts, err = em.pingForLatency(pingAttempts); err != nil {
-				em.logger.Log("err", errors.Wrap(err, "failed to write ping message on websocket"))
+				em.logger.Error("err", errors.Wrap(err, "failed to write ping message on websocket"))
 				em.StopAndCallDisconnectCallback()
 				return
 			} else if pingAttempts >= maxPingsPerPong {
-				em.logger.Log("err", errors.Errorf("Have not received a pong in %v", time.Duration(pingAttempts)*pingTime))
+				em.logger.Error("err", errors.Errorf("Have not received a pong in %v", time.Duration(pingAttempts)*pingTime))
 				em.StopAndCallDisconnectCallback()
 				return
 			}
 		case r := <-em.wsc.ResultsCh:
 			if r == nil {
-				em.logger.Log("err", errors.New("Expected some event, received nil"))
+				em.logger.Error("err", errors.New("Expected some event, received nil"))
 				em.StopAndCallDisconnectCallback()
 				return
 			}
 			eventID, data, err := em.unmarshalEvent(r)
 			if err != nil {
-				em.logger.Log("err", errors.Wrap(err, "failed to unmarshal event"))
+				em.logger.Error("err", errors.Wrap(err, "failed to unmarshal event"))
 				continue
 			}
 			if eventID != "" {
 				em.updateMetric(eventID, data)
 			}
 		case <-em.wsc.Quit:
-			em.logger.Log("err", errors.New("WSClient closed unexpectedly"))
+			em.logger.Error("err", errors.New("WSClient closed unexpectedly"))
 			em.StopAndCallDisconnectCallback()
 			return
 		case <-em.quit:

tm-monitor/glide.lock

@@ -1,16 +1,16 @@
-hash: d21d1f12681cd4ab5b7f0efd7bf00c1d5f7021b1ae6e8700c11bca6822337079
-updated: 2017-03-16T10:01:58.079646405Z
+hash: 80c204190057df1e74d32ecd7095e8a1a865c3a06671f1a31d5240e1e3ff2c64
+updated: 2017-05-20T17:49:23.646798165-04:00
 imports:
 - name: github.com/btcsuite/btcd
   version: 583684b21bfbde9b5fc4403916fd7c807feb0289
   subpackages:
   - btcec
-- name: github.com/BurntSushi/toml
-  version: 99064174e013895bbd9b025c31100bd1d9b590ca
 - name: github.com/go-kit/kit
   version: b6f30a2e0632f5722fb26d8765d726335b79d3e6
   subpackages:
   - log
+  - log/level
+  - log/term
 - name: github.com/go-logfmt/logfmt
   version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5
 - name: github.com/go-stack/stack
@@ -19,90 +19,50 @@ imports:
   version: 69b215d01a5606c843240eab4937eab3acee6530
   subpackages:
   - proto
-- name: github.com/golang/snappy
-  version: 553a641470496b2327abcac10b36396bd98e45c9
 - name: github.com/gorilla/websocket
   version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13
-- name: github.com/jmhodges/levigo
-  version: c42d9e0ca023e2198120196f842701bb4c55d7b9
 - name: github.com/kr/logfmt
   version: b84e30acd515aadc4b783ad4ff83aff3299bdfe0
-- name: github.com/mattn/go-colorable
-  version: d898aa9fb31c91f35dd28ca75db377eff023c076
-- name: github.com/mattn/go-isatty
-  version: dda3de49cbfcec471bd7a70e6cc01fcc3ff90109
 - name: github.com/pkg/errors
   version: bfd5150e4e41705ded2129ec33379de1cb90b513
 - name: github.com/rcrowley/go-metrics
   version: 1f30fe9094a513ce4c700b9a54458bbb0c96996c
-- name: github.com/stretchr/testify
-  version: 4d4bfba8f1d1027c4fdbe371823030df51419987
-  subpackages:
-  - assert
-  - require
-- name: github.com/syndtr/goleveldb
-  version: 3c5717caf1475fd25964109a0fc640bd150fce43
-  subpackages:
-  - leveldb
-  - leveldb/cache
-  - leveldb/comparer
-  - leveldb/errors
-  - leveldb/filter
-  - leveldb/iterator
-  - leveldb/journal
-  - leveldb/memdb
-  - leveldb/opt
-  - leveldb/storage
-  - leveldb/table
-  - leveldb/util
 - name: github.com/tendermint/abci
-  version: 1e8791bc9ac2d65eaf3f315393b1312daa46a7f5
+  version: 864d1f80b36b440bde030a5c18d8ac3aa8c2949d
   subpackages:
+  - client
+  - example/dummy
   - types
 - name: github.com/tendermint/ed25519
   version: 1f52c6f8b8a5c7908aff4497c186af344b428925
   subpackages:
   - edwards25519
   - extra25519
-- name: github.com/tendermint/go-common
-  version: e289af53b6bf6af28da129d9ef64389a4cf7987f
-- name: github.com/tendermint/go-config
-  version: e64b424499acd0eb9856b88e10c0dff41628c0d6
 - name: github.com/tendermint/go-crypto
-  version: 4b11d62bdb324027ea01554e5767b71174680ba0
-- name: github.com/tendermint/go-db
-  version: 72f6dacd22a686cdf7fcd60286503e3aceda77ba
-- name: github.com/tendermint/go-events
-  version: fddee66d90305fccb6f6d84d16c34fa65ea5b7f6
-- name: github.com/tendermint/go-flowrate
-  version: a20c98e61957faa93b4014fbd902f20ab9317a6a
-  subpackages:
-  - flowrate
-- name: github.com/tendermint/go-logger
-  version: cefb3a45c0bf3c493a04e9bcd9b1540528be59f2
-- name: github.com/tendermint/go-merkle
-  version: 7a86b4486f2cd84ac885c5bbc609fdee2905f5d1
-- name: github.com/tendermint/go-p2p
-  version: 3d98f675f30dc4796546b8b890f895926152fa8d
-  subpackages:
-  - upnp
-- name: github.com/tendermint/go-rpc
-  version: fcea0cda21f64889be00a0f4b6d13266b1a76ee7
-  subpackages:
-  - client
-  - server
-  - types
+  version: 7dff40942a64cdeefefa9446b2d104750b349f8a
 - name: github.com/tendermint/go-wire
-  version: 2f3b7aafe21c80b19b6ee3210ecb3e3d07c7a471
-- name: github.com/tendermint/log15
-  version: ae0f3d6450da9eac7074b439c8e1c3cabf0d5ce6
+  version: 5f88da3dbc1a72844e6dfaf274ce87f851d488eb
   subpackages:
-  - term
+  - data
 - name: github.com/tendermint/tendermint
-  version: 764091dfbb035f1b28da4b067526e04c6a849966
+  version: 267f134d44e76efb2adef5f0c993da8a5d5bd1b8
   subpackages:
+  - config
+  - p2p
+  - p2p/upnp
   - rpc/core/types
+  - rpc/lib/client
+  - rpc/lib/server
+  - rpc/lib/types
   - types
+- name: github.com/tendermint/tmlibs
+  version: 306795ae1d8e4f4a10dcc8bdb32a00455843c9d5
+  subpackages:
+  - common
+  - events
+  - flowrate
+  - log
+  - merkle
 - name: golang.org/x/crypto
   version: 453249f01cfeb54c3d549ddb75ff152ca243f9d8
   subpackages:
@@ -124,10 +84,6 @@ imports:
   - internal/timeseries
   - lex/httplex
   - trace
-- name: golang.org/x/sys
-  version: 76cc09b634294339fa19ec41b5f2a0b3932cea8b
-  subpackages:
-  - unix
 - name: google.golang.org/grpc
   version: 8b2e129857480cb0f07ef7d9d10b8b252c7ac984
   subpackages:
@@ -150,3 +106,8 @@ testImports:
   version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
   subpackages:
   - difflib
+- name: github.com/stretchr/testify
+  version: 4d4bfba8f1d1027c4fdbe371823030df51419987
+  subpackages:
+  - assert
+  - require

tm-monitor/glide.yaml

@@ -1,22 +1,25 @@
 package: github.com/tendermint/tools/tm-monitor
 import:
-- package: github.com/tendermint/go-common
-- package: github.com/tendermint/go-events
-- package: github.com/tendermint/go-logger
-- package: github.com/tendermint/tendermint
+- package: github.com/go-kit/kit
   subpackages:
-  - types
-  - rpc/core/types
-- package: github.com/tendermint/go-wire
+  - log
+- package: github.com/gorilla/websocket
+- package: github.com/pkg/errors
 - package: github.com/rcrowley/go-metrics
-- package: github.com/stretchr/testify
 - package: github.com/tendermint/go-crypto
-- package: github.com/gorilla/websocket
-- package: github.com/tendermint/go-rpc
+- package: github.com/tendermint/tendermint
+  version: develop
   subpackages:
-  - client
-- package: github.com/go-kit/kit
+  - rpc/core/types
+  - rpc/lib/client
+  - rpc/lib/server
+  - types
+- package: github.com/tendermint/tmlibs
   subpackages:
+  - events
   - log
-  - term
-- package: github.com/pkg/errors
+testImport:
+- package: github.com/stretchr/testify
+  subpackages:
+  - assert
+  - require

tm-monitor/main.go

@@ -6,9 +6,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/go-kit/kit/log"
-	"github.com/go-kit/kit/log/term"
-	cmn "github.com/tendermint/go-common"
+	cmn "github.com/tendermint/tmlibs/common"
+	"github.com/tendermint/tmlibs/log"
 	monitor "github.com/tendermint/tools/tm-monitor/monitor"
 )
 
@@ -48,22 +47,12 @@ Examples:
 	}
 
 	if noton {
-		// Color errors red
-		colorFn := func(keyvals ...interface{}) term.FgBgColor {
-			for i := 1; i < len(keyvals); i += 2 {
-				if _, ok := keyvals[i].(error); ok {
-					return term.FgBgColor{Fg: term.White, Bg: term.Red}
-				}
-			}
-			return term.FgBgColor{}
-		}
-
-		logger = term.NewLogger(os.Stdout, log.NewLogfmtLogger, colorFn)
+		logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "tm-monitor")
 	}
 
 	m := startMonitor(flag.Arg(0))
 
-	startRPC(listenAddr, m)
+	startRPC(listenAddr, m, logger)
 
 	var ton *Ton
 	if !noton {
@@ -81,11 +70,11 @@ Examples:
 
 func startMonitor(endpoints string) *monitor.Monitor {
 	m := monitor.NewMonitor()
-	m.SetLogger(log.With(logger, "component", "monitor"))
+	m.SetLogger(logger.With("component", "monitor"))
 
 	for _, e := range strings.Split(endpoints, ",") {
 		n := monitor.NewNode(e)
-		n.SetLogger(log.With(logger, "node", e))
+		n.SetLogger(logger.With("node", e))
 		if err := m.Monitor(n); err != nil {
 			panic(err)
 		}

tm-monitor/monitor/monitor.go

@@ -5,9 +5,9 @@ import (
 	"math/rand"
 	"time"
 
-	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
 	tmtypes "github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmlibs/log"
 )
 
 // waiting more than this many seconds for a block means we're unhealthy
@@ -140,7 +140,7 @@ func (m *Monitor) Stop() {
 
 // main loop where we listen for events from the node
 func (m *Monitor) listen(nodeName string, blockCh <-chan tmtypes.Header, blockLatencyCh <-chan float64, disconnectCh <-chan bool, quit <-chan struct{}) {
-	logger := log.With(m.logger, "node", nodeName)
+	logger := m.logger.With("node", nodeName)
 
 	for {
 		select {
@@ -159,7 +159,7 @@ func (m *Monitor) listen(nodeName string, blockCh <-chan tmtypes.Header, blockLa
 				m.Network.NodeIsOnline(nodeName)
 			}
 		case <-time.After(nodeLivenessTimeout):
-			logger.Log("event", fmt.Sprintf("node was not responding for %v", nodeLivenessTimeout))
+			logger.Info("event", fmt.Sprintf("node was not responding for %v", nodeLivenessTimeout))
 			m.Network.NodeIsDown(nodeName)
 		}
 	}
@@ -203,7 +203,7 @@ func (m *Monitor) updateNumValidatorLoop() {
 				if i == randomNodeIndex {
 					height, num, err = n.NumValidators()
 					if err != nil {
-						m.logger.Log("err", errors.Wrap(err, "update num validators failed"))
+						m.logger.Info("err", errors.Wrap(err, "update num validators failed"))
 					}
 					break
 				}

tm-monitor/monitor/node.go

@@ -5,22 +5,16 @@ import (
 	"math"
 	"time"
 
-	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
 	crypto "github.com/tendermint/go-crypto"
-	events "github.com/tendermint/go-events"
-	rpc_client "github.com/tendermint/go-rpc/client"
-	wire "github.com/tendermint/go-wire"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpc_client "github.com/tendermint/tendermint/rpc/lib/client"
 	tmtypes "github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tmlibs/events"
+	"github.com/tendermint/tmlibs/log"
 	em "github.com/tendermint/tools/tm-monitor/eventmeter"
 )
 
-// remove when https://github.com/tendermint/go-rpc/issues/8 will be fixed
-type rpcClientI interface {
-	Call(method string, params map[string]interface{}, result interface{}) (interface{}, error)
-}
-
 const maxRestarts = 25
 
 type Node struct {
@@ -38,7 +32,7 @@ type Node struct {
 	em eventMeter
 
 	// rpcClient is an client for making RPC calls to TM
-	rpcClient rpcClientI
+	rpcClient rpc_client.HTTPClient
 
 	blockCh        chan<- tmtypes.Header
 	blockLatencyCh chan<- float64
@@ -53,11 +47,11 @@ type Node struct {
 
 func NewNode(rpcAddr string, options ...func(*Node)) *Node {
 	em := em.NewEventMeter(rpcAddr, UnmarshalEvent)
-	rpcClient := rpc_client.NewClientURI(rpcAddr) // HTTP client by default
+	rpcClient := rpc_client.NewURIClient(rpcAddr) // HTTP client by default
 	return NewNodeWithEventMeterAndRpcClient(rpcAddr, em, rpcClient, options...)
 }
 
-func NewNodeWithEventMeterAndRpcClient(rpcAddr string, em eventMeter, rpcClient rpcClientI, options ...func(*Node)) *Node {
+func NewNodeWithEventMeterAndRpcClient(rpcAddr string, em eventMeter, rpcClient rpc_client.HTTPClient, options ...func(*Node)) *Node {
 	n := &Node{
 		rpcAddr:   rpcAddr,
 		em:        em,
@@ -107,7 +101,10 @@ func (n *Node) Start() error {
 	}
 
 	n.em.RegisterLatencyCallback(latencyCallback(n))
-	n.em.Subscribe(tmtypes.EventStringNewBlockHeader(), newBlockCallback(n))
+	err := n.em.Subscribe(tmtypes.EventStringNewBlockHeader(), newBlockCallback(n))
+	if err != nil {
+		return err
+	}
 	n.em.RegisterDisconnectCallback(disconnectCallback(n))
 
 	n.Online = true
@@ -129,10 +126,10 @@ func (n *Node) Stop() {
 // implements eventmeter.EventCallbackFunc
 func newBlockCallback(n *Node) em.EventCallbackFunc {
 	return func(metric *em.EventMetric, data events.EventData) {
-		block := data.(tmtypes.EventDataNewBlockHeader).Header
+		block := data.(tmtypes.TMEventData).Unwrap().(tmtypes.EventDataNewBlockHeader).Header
 
 		n.Height = uint64(block.Height)
-		n.logger.Log("event", "new block", "height", block.Height, "numTxs", block.NumTxs)
+		n.logger.Info("event", "new block", "height", block.Height, "numTxs", block.NumTxs)
 
 		if n.blockCh != nil {
 			n.blockCh <- *block
@@ -144,7 +141,7 @@ func newBlockCallback(n *Node) em.EventCallbackFunc {
 func latencyCallback(n *Node) em.LatencyCallbackFunc {
 	return func(latency float64) {
 		n.BlockLatency = latency / 1000000.0 // ns to ms
-		n.logger.Log("event", "new block latency", "latency", n.BlockLatency)
+		n.logger.Info("event", "new block latency", "latency", n.BlockLatency)
 
 		if n.blockLatencyCh != nil {
 			n.blockLatencyCh <- latency
@@ -156,17 +153,17 @@ func latencyCallback(n *Node) em.LatencyCallbackFunc {
 func disconnectCallback(n *Node) em.DisconnectCallbackFunc {
 	return func() {
 		n.Online = false
-		n.logger.Log("status", "down")
+		n.logger.Info("status", "down")
 
 		if n.disconnectCh != nil {
 			n.disconnectCh <- true
 		}
 
 		if err := n.RestartEventMeterBackoff(); err != nil {
-			n.logger.Log("err", errors.Wrap(err, "restart failed"))
+			n.logger.Info("err", errors.Wrap(err, "restart failed"))
 		} else {
 			n.Online = true
-			n.logger.Log("status", "online")
+			n.logger.Info("status", "online")
 
 			if n.disconnectCh != nil {
 				n.disconnectCh <- false
@@ -183,7 +180,7 @@ func (n *Node) RestartEventMeterBackoff() error {
 		time.Sleep(d * time.Second)
 
 		if err := n.em.Start(); err != nil {
-			n.logger.Log("err", errors.Wrap(err, "restart failed"))
+			n.logger.Info("err", errors.Wrap(err, "restart failed"))
 		} else {
 			// TODO: authenticate pubkey
 			return nil
@@ -206,11 +203,10 @@ func (n *Node) NumValidators() (height uint64, num int, err error) {
 }
 
 func (n *Node) validators() (height uint64, validators []*tmtypes.Validator, err error) {
-	var result ctypes.TMResult
-	if _, err = n.rpcClient.Call("validators", nil, &result); err != nil {
+	vals := new(ctypes.ResultValidators)
+	if _, err = n.rpcClient.Call("validators", nil, vals); err != nil {
 		return 0, make([]*tmtypes.Validator, 0), err
 	}
-	vals := result.(*ctypes.ResultValidators)
 	return uint64(vals.BlockHeight), vals.Validators, nil
 }
 
@@ -235,21 +231,20 @@ func (n *Node) checkIsValidator() {
 			}
 		}
 	} else {
-		n.logger.Log("err", errors.Wrap(err, "check is validator failed"))
+		n.logger.Info("err", errors.Wrap(err, "check is validator failed"))
 	}
 }
 
 func (n *Node) getPubKey() (crypto.PubKey, error) {
-	if n.pubKey != nil {
+	if !n.pubKey.Empty() {
 		return n.pubKey, nil
 	}
 
-	var result ctypes.TMResult
-	_, err := n.rpcClient.Call("status", nil, &result)
+	status := new(ctypes.ResultStatus)
+	_, err := n.rpcClient.Call("status", nil, status)
 	if err != nil {
-		return nil, err
+		return crypto.PubKey{}, err
 	}
-	status := result.(*ctypes.ResultStatus)
 	n.pubKey = status.PubKey
 	return n.pubKey, nil
 }
@@ -266,16 +261,9 @@ type eventMeter interface {
 
 // UnmarshalEvent unmarshals a json event
 func UnmarshalEvent(b json.RawMessage) (string, events.EventData, error) {
-	var err error
-	result := new(ctypes.TMResult)
-	wire.ReadJSONPtr(result, b, &err)
-	if err != nil {
+	event := new(ctypes.ResultEvent)
+	if err := json.Unmarshal(b, event); err != nil {
 		return "", nil, err
 	}
-	event, ok := (*result).(*ctypes.ResultEvent)
-	if !ok {
-		return "", nil, nil // TODO: handle non-event messages (ie. return from subscribe/unsubscribe)
-		// fmt.Errorf("Result is not type *ctypes.ResultEvent. Got %v", reflect.TypeOf(*result))
-	}
 	return event.Name, event.Data, nil
 }

tm-monitor/rpc.go

@@ -4,19 +4,20 @@ import (
 	"errors"
 	"net/http"
 
-	rpc "github.com/tendermint/go-rpc/server"
+	rpc "github.com/tendermint/tendermint/rpc/lib/server"
+	"github.com/tendermint/tmlibs/log"
 	monitor "github.com/tendermint/tools/tm-monitor/monitor"
 )
 
-func startRPC(listenAddr string, m *monitor.Monitor) {
+func startRPC(listenAddr string, m *monitor.Monitor, logger log.Logger) {
 	routes := routes(m)
 
 	// serve http and ws
 	mux := http.NewServeMux()
 	wm := rpc.NewWebsocketManager(routes, nil) // TODO: evsw
 	mux.HandleFunc("/websocket", wm.WebsocketHandler)
-	rpc.RegisterRPCFuncs(mux, routes)
-	if _, err := rpc.StartHTTPServer(listenAddr, mux); err != nil {
+	rpc.RegisterRPCFuncs(mux, routes, logger)
+	if _, err := rpc.StartHTTPServer(listenAddr, mux, logger); err != nil {
 		panic(err)
 	}
 }