collect and add docker logs to CircleCI artifacts (Refs #387)

How: 1) we start syslog docker container 2) all other containers use
syslog logging driver to ship their logs to that container

.gitignore

@@ -12,4 +12,5 @@ remote_dump
 vendor
 .vagrant
 test/p2p/data/
+test/logs
 .glide

circle.yml

@@ -29,5 +29,5 @@ test:
   post:
     - cd "$PROJECT_PATH" && mv test_integrations.log "${CIRCLE_ARTIFACTS}"
     - cd "$PROJECT_PATH" && bash <(curl -s https://codecov.io/bash) -f coverage.txt
-    - cd "$PROJECT_PATH" && [[ -f coverage.txt ]] && mv coverage.txt "${CIRCLE_ARTIFACTS}"
-    - sudo cp /var/log/upstart/docker.log "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && mv coverage.txt "${CIRCLE_ARTIFACTS}"
+    - cd "$PROJECT_PATH" && cp test/logs/messages "${CIRCLE_ARTIFACTS}/docker_logs.txt"

test/p2p/peer.sh

@@ -21,4 +21,8 @@ docker run -d \
   --name local_testnet_$ID \
   --entrypoint tendermint \
   -e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	--log-driver=syslog \
+	--log-opt syslog-address=udp://127.0.0.1:5514 \
+	--log-opt syslog-facility=daemon \
+	--log-opt tag="{{.Name}}" \
   $DOCKER_IMAGE node $SEEDS --proxy_app=$APP_PROXY

test/test.sh

@@ -1,17 +1,30 @@
 #! /bin/bash
 set -eu
 
-# Top Level Testing Script
-# See the github.com/tendermint/tendermint/test/README.md
+# Get the directory of where this script is.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-echo ""
-echo "* building docker image"
-bash ./test/docker/build.sh
+LOGS_DIR="$DIR/logs"
+echo
+echo "* [$(date +"%T")] cleaning up $LOGS_DIR"
+rm -rf "$LOGS_DIR"
+mkdir -p "$LOGS_DIR"
 
-echo ""
-echo "* running go tests and app tests in docker container"
+echo
+echo "* [$(date +"%T")] starting rsyslog container"
+docker rm -f rsyslog || true
+docker run -d -v "$LOGS_DIR:/var/log/" -p 127.0.0.1:5514:514/udp --name rsyslog voxxit/rsyslog
+
+echo
+echo "* [$(date +"%T")] building docker image"
+bash "$DIR/docker/build.sh"
+
+echo
+echo "* [$(date +"%T")] running go tests and app tests in docker container"
 # sometimes its helpful to mount the local test folder
-# -v $GOPATH/src/github.com/tendermint/tendermint/test:/go/src/github.com/tendermint/tendermint/test
+# -v $DIR:/go/src/github.com/tendermint/tendermint/test
 docker run --name run_test -t tester bash test/run_test.sh
 
 # copy the coverage results out of docker container
@@ -19,16 +32,20 @@ docker cp run_test:/go/src/github.com/tendermint/tendermint/coverage.txt .
 
 # test basic network connectivity
 # by starting a local testnet and checking peers connect and make blocks
-echo ""
-echo "* running p2p tests on a local docker network"
-bash test/p2p/test.sh tester
+echo
+echo "* [$(date +"%T")] running p2p tests on a local docker network"
+bash "$DIR/p2p/test.sh" tester
 
 # only run the cloud benchmark for releases
-BRANCH=`git rev-parse --abbrev-ref HEAD`
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
 if [[ $(echo "$BRANCH" | grep "release-") != "" ]]; then
-	echo ""
+	echo
 	echo "TODO: run network tests"
 	#echo "* branch $BRANCH; running mintnet/netmon throughput benchmark"
 	# TODO: replace mintnet
-	#bash test/net/test.sh
+	#bash "$DIR/net/test.sh"
 fi
+
+echo
+echo "* [$(date +"%T")] stopping rsyslog container"
+docker rm -f rsyslog