From c6f025f40e120677ca0ad64510a3849124428480 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 6 Dec 2017 15:57:00 -0600 Subject: [PATCH 1/2] generate WAL on the fly (Refs #468) --- consensus/replay_test.go | 5 +- consensus/test_data/README.md | 36 ------ consensus/test_data/build.sh | 148 --------------------- consensus/test_data/many_blocks.cswal | Bin 16661 -> 0 bytes consensus/wal.go | 2 +- consensus/wal_test.go | 9 +- consensus/walgen.go | 178 ++++++++++++++++++++++++++ scripts/cutWALUntil/main.go | 65 ---------- 8 files changed, 188 insertions(+), 255 deletions(-) delete mode 100644 consensus/test_data/README.md delete mode 100755 consensus/test_data/build.sh delete mode 100644 consensus/test_data/many_blocks.cswal create mode 100644 consensus/walgen.go delete mode 100644 scripts/cutWALUntil/main.go diff --git a/consensus/replay_test.go b/consensus/replay_test.go index af0af3e78..8643427c0 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -265,7 +265,7 @@ func (w *crashingWAL) Wait() { w.next.Wait() } // Handshake Tests var ( - NUM_BLOCKS = 6 // number of blocks in the test_data/many_blocks.cswal + NUM_BLOCKS = 6 mempool = types.MockMempool{} ) @@ -324,8 +324,7 @@ func writeWAL(walMsgs []byte) string { func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { config := ResetConfig("proxy_test_") - // copy the many_blocks file - walBody, err := cmn.ReadFile(path.Join(data_dir, "many_blocks.cswal")) + walBody, err := GenWAL(NUM_BLOCKS) if err != nil { t.Fatal(err) } diff --git a/consensus/test_data/README.md b/consensus/test_data/README.md deleted file mode 100644 index 822f16c04..000000000 --- a/consensus/test_data/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Generating test data - -To generate the data, run `build.sh`. See that script for more details. - -Make sure to adjust the stepChanges in the testCases if the number of messages changes. -This sometimes happens for the `small_block2.cswal`, where the number of block parts changes between 4 and 5. - -If you need to change the signatures, you can use a script as follows: -The privBytes comes from `config/tendermint_test/...`: - -``` -package main - -import ( - "encoding/hex" - "fmt" - - "github.com/tendermint/go-crypto" -) - -func main() { - signBytes, err := hex.DecodeString("7B22636861696E5F6964223A2274656E6465726D696E745F74657374222C22766F7465223A7B22626C6F636B5F68617368223A2242453544373939433846353044354645383533364334333932464443384537423342313830373638222C22626C6F636B5F70617274735F686561646572223A506172745365747B543A31204236323237323535464632307D2C22686569676874223A312C22726F756E64223A302C2274797065223A327D7D") - if err != nil { - panic(err) - } - privBytes, err := hex.DecodeString("27F82582AEFAE7AB151CFB01C48BB6C1A0DA78F9BDDA979A9F70A84D074EB07D3B3069C422E19688B45CBFAE7BB009FC0FA1B1EA86593519318B7214853803C8") - if err != nil { - panic(err) - } - privKey := crypto.PrivKeyEd25519{} - copy(privKey[:], privBytes) - signature := privKey.Sign(signBytes) - fmt.Printf("Signature Bytes: %X\n", signature.Bytes()) -} -``` - diff --git a/consensus/test_data/build.sh b/consensus/test_data/build.sh deleted file mode 100755 index 6f410c700..000000000 --- a/consensus/test_data/build.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash - -# Requires: killall command and jq JSON processor. - -# Get the parent directory of where this script is. -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done -DIR="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" - -# Change into that dir because we expect that. -cd "$DIR" || exit 1 - -# Make sure we have a tendermint command. -if ! hash tendermint 2>/dev/null; then - make install -fi - -# Make sure we have a cutWALUntil binary. -cutWALUntil=./scripts/cutWALUntil/cutWALUntil -cutWALUntilDir=$(dirname $cutWALUntil) -if ! hash $cutWALUntil 2>/dev/null; then - cd "$cutWALUntilDir" && go build && cd - || exit 1 -fi - -TMHOME=$(mktemp -d) -export TMHOME="$TMHOME" - -if [[ ! -d "$TMHOME" ]]; then - echo "Could not create temp directory" - exit 1 -else - echo "TMHOME: ${TMHOME}" -fi - -# TODO: eventually we should replace with `tendermint init --test` -DIR_TO_COPY=$HOME/.tendermint_test/consensus_state_test -if [ ! -d "$DIR_TO_COPY" ]; then - echo "$DIR_TO_COPY does not exist. Please run: go test ./consensus" - exit 1 -fi -echo "==> Copying ${DIR_TO_COPY} to ${TMHOME} directory..." -cp -r "$DIR_TO_COPY"/* "$TMHOME" - -# preserve original genesis file because later it will be modified (see small_block2) -cp "$TMHOME/genesis.json" "$TMHOME/genesis.json.bak" - -function reset(){ - echo "==> Resetting tendermint..." - tendermint unsafe_reset_all - cp "$TMHOME/genesis.json.bak" "$TMHOME/genesis.json" -} - -reset - -# function empty_block(){ -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 5 -# echo "==> Killing tendermint..." -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_empty_block.cswal -# mv consensus/test_data/new_empty_block.cswal consensus/test_data/empty_block.cswal - -# reset -# } - -function many_blocks(){ - bash scripts/txs/random.sh 1000 36657 &> /dev/null & - PID=$! - echo "==> Starting tendermint..." - tendermint node --proxy_app=persistent_dummy &> /dev/null & - sleep 10 - echo "==> Killing tendermint..." - kill -9 $PID - killall tendermint - - echo "==> Copying WAL log..." - $cutWALUntil "$TMHOME/data/cs.wal/wal" 6 consensus/test_data/new_many_blocks.cswal - mv consensus/test_data/new_many_blocks.cswal consensus/test_data/many_blocks.cswal - - reset -} - - -# function small_block1(){ -# bash scripts/txs/random.sh 1000 36657 &> /dev/null & -# PID=$! -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 10 -# echo "==> Killing tendermint..." -# kill -9 $PID -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block1.cswal -# mv consensus/test_data/new_small_block1.cswal consensus/test_data/small_block1.cswal - -# reset -# } - - -# # block part size = 512 -# function small_block2(){ -# cat "$TMHOME/genesis.json" | jq '. + {consensus_params: {block_size_params: {max_bytes: 22020096}, block_gossip_params: {block_part_size_bytes: 512}}}' > "$TMHOME/new_genesis.json" -# mv "$TMHOME/new_genesis.json" "$TMHOME/genesis.json" -# bash scripts/txs/random.sh 1000 36657 &> /dev/null & -# PID=$! -# echo "==> Starting tendermint..." -# tendermint node --proxy_app=persistent_dummy &> /dev/null & -# sleep 5 -# echo "==> Killing tendermint..." -# kill -9 $PID -# killall tendermint - -# echo "==> Copying WAL log..." -# $cutWALUntil "$TMHOME/data/cs.wal/wal" 1 consensus/test_data/new_small_block2.cswal -# mv consensus/test_data/new_small_block2.cswal consensus/test_data/small_block2.cswal - -# reset -# } - - - -case "$1" in - # "small_block1") - # small_block1 - # ;; - # "small_block2") - # small_block2 - # ;; - # "empty_block") - # empty_block - # ;; - "many_blocks") - many_blocks - ;; - *) - # small_block1 - # small_block2 - # empty_block - many_blocks -esac - -echo "==> Cleaning up..." -rm -rf "$TMHOME" diff --git a/consensus/test_data/many_blocks.cswal b/consensus/test_data/many_blocks.cswal deleted file mode 100644 index 2af0c3cc747862ba378c18b1a157d13e589bdd1e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16661 zcmc(`c|29?-~Zn-pMy>2AskWW%CL=XLxZ9+rlgcPB}3*ADw0A`M93UTRLYb@<}^?V z6=@(slro0zYv*&S`+l5$=hHvGd;jC{Tx(x@Kd<*%uIsg~wf3I5-O~#eEZ`uxrYCp~ z%69z%7=#2F>5l zP!tIk-rVOQ&xPoEw96<;1WRkaui6tq^$7D=3YlOeaof8n1fD0DYi>6!i>)h2lcT5* z%!ARVT+bOKJRPE_5-bk9`(!OOy5ox91>Ns(6|Shv2QCygDg_13Yc$SV@8O4X|eT7726TTf8rl?YbC z#_3)-U9RLdJ&HV;VBMfC6aTSwga3OLio7zxx^pa;sA+6Crk+NTS0Pwu=#LAKHBwvM z4pZb+3D#=Wqw-E1G93@cDDr9q%eBF%n#C0Y0c$7AH_}PbDTA$C_w6wRh3m( z$p_|M3Pk}@M{0*%!&r0l9_vsPAT{yaSfxkHv}}@|q5vtOAT=faWwG`9$|wqCf?@Vd z%<6nog7wh0t45djLh5D?P!wQO-ePg5%ub;F^$CgsY~&kQzBF(yoQ_weD4<57op&6) zh(k+L)G3N92%0km1k*t)iC5xlM(GGvHpXq)7I%}&7M-Oi;_Gku-yQGQku2TZL{U^C z=%nQxv?6!G2ZC#@!A19^D(6DW%K?o}6u z6ql*FU570&q<0B zO0go|mSwa5Q=y4P6eYN9dM=)HzlwW?{uxCHr8ui!{K(uY>&UY_iV}{`ec_(ZjS6+< zD{CoAI6ht#<<~dm7?SleC`u{>|1Z!O*-)EPwS1x=?6eakiH0+SiI{K2; zFqcAx&jYKj+~+yg!gQ4svOGbz(b`F$U_Dr1dbCb#^;p(Z3K<$^M|Us>9C&7W$b>>x zA{a8>SXSQ)Cs@BW@?Uyfu3c$6NFn3CjF`QASmis;`)|>ScG7mr#^Z zm2mwpqIS0*UF~^KQHBm%GkYPgtKlJavdD3@$w8apr}A<%2Z;!SYylmHE9$TND0oe z_Fj%J*UsmlsHhUO3(xN&SO-^)OO{*_)RXa}s6wSUA+Fro+KVIfEJYP6j_Q(Wl3Q~5 zb1A9{1g#kRD8VXV;iFAstX!ORlcEZNcJ6}~EsvulE!`=q5O}_Ezq8b?-F;?^6jcaZ zSnpM7z*}R>aG#y#)`^%II31U558_d6OE zW!@L2s6n7$bH}5u`tUGiiW+Jm@!_nuZ=tj0f!7o@)WXW6OPj_);LP+giW*cZ!`l{B z8NS_kYlNZ(6=7j9FP#oS_I?S98dM@H&-#qEA2jATPEmu(pv$U;fg#c15-*Ax4v_eV zGnHhwscFSC6g4%1)N7mQdD=*ve{&%QyA=c>LXZ~bq_1}IeK<}xiA@RwX~J}<&vaD` z+q1`DMS?EfFSwN;F|I$>QX}b=#lwiLWP-H$OOyZkT_@XEPJxvP(nr3g(h2LjyzCaR z3PBPkMct}g(dIZ^wjI*4iwZYy-nc ziDAo71Fy&F=L{I>R}iFKe(%n`zF&WoBNZ%9kaSikb7_*DG#<}@;kCq!ooBj@{HZJs z46jk$lB{FvyV(PM!5H}V@6C|&#>z}{Ze!R6_ z0b6mx+)m^sNZaaX&odn~WN-L_xdq2YogkU4%zbgs(8}Wj6E-Ojj29};y;pY}BN%cx zcBFMS5G3!A$v5qy<7uX|cmocL&ceD^Y4mK}MH*lnmi5&Yk?93HRD7(!I4nn2iZ80@ z&(LDI3PusOf9etEX}3(y7y+XQ@02&p*to`js=S5?4Mo_SyMa{SU=~mD0;34STzlPJ zVl`4mmVi-&&D%4b3Wyrrv&LW)!IRg}a7)y*y}Sb$2S-P{B>(fN{cFQ;GGWLiN&DoR z<+W*egslg|FLCUe?7E2CAG6?#L6@W|Q9F3}%$w>hXTb35E@%-Otg#uEDF(w&LQ|#b zam$qR&2yMx;V0m?O>IMX$>-1LABJC2NY9C<^*1H9C1rx)7q)(tpqt{O<#tkF_>l!7 z>XSYV6%3%D7>Y^K&vsc@ktgz`mxIX!P4IPs)seo4bp;21Wj{7SNmJue&UODz*_k>p zlw7$@D&Gypj9+&FLrHe~$dnV~B4HgUQio#sH%e~zn7H1l17YM9#w#yXaMz}9DHulU zSm~dANj zeyv2`Rf^a1^T5!{FAHAh&8{kO%mxg-HsuNWQ{m>lT|2?ht4Ld2CR}%UITHd5#$*!9 zabds1o3*}JaD(Aj)qOlXzW4L;4!j@3GAVRN^!q)xDs}W!!0~6OOKS2;tuCM@x3k=+(ms(1<8mG>6wGM;f zr{7w*PVvgw!!>eX_)&WNt0R&R>@(p7!|%4{dm`xk=LT^^I#u|2U+6zRcqTo;E&#a< zeid)s8cn?-tu&Ou@Y|ZUoFP@{m4etYumZtgu|$8%ItoFOJ12jGRyZzlF$cE8Yp?SJ zt5%%dpaB7f*N5Il`xLLqCf58;J|iCtuc&DHt2?R(`|YQ|@S37k>Zz|Wu~%FP zM!g($6K=nD+jwlmAB=i+2xfeq72R~Ly&DX_;i$Z~UVYwfVG)R`@GIiUA70@?pUZ`; zia<>#p+{duka#{ewasY-wHGj8E3A0da)_1>Zfu*;2E)q#iSp8Y)0~}h6<}B?hrU$_ zvikJqNj4Z(B3_wnp$u2WMt6f@#e3}h@xtcTU3*2rss!}jP{dKx?ZUeO;#11}pYRKIc?0W7T7-F-JPc!(uJ`tgeiv{W&&>UT( zwE5;u!^DeVh`r*Dm7m~b70JI2hM3^(;MPkQhn=rc86hsgZ0U~v1Hr8lxO_m}I@cG= z%im!BV(14(-Pk%aw#Cw``L0U>Lyy*Qnf!9MDXGixVCb1_?N)MlbCYE;zK%Kte3_!r z8Wnh%S&X`43Yfn>q+VUVqqH!9<}Zf>4LY@dy0-oKa@gM6)$`ls@N1XxaWqK&b~zg8 zY+Xsig=^qu$;NmQzp&9il-+N$ZYN0#YChRA|0THW#I+>X+CVyst>!|0;VP4d8E?9K zM#n4{!iDQUUOW?ja%P>hv=!Jml^l4Z+?QCiIP1~EfSP4hH=mtr<%yDxt`HdYq+?3! z7t_kEO;uru7v>lA+qKbX>Z0Uc4&7q*KfYZz`$)+7LqTF*wDdAO=3nsP_~}C5-qzLG zd5&iqHS_nNd=iNWDCl)X}|UDN#uAwM8)U(_Rizy zMGUs@eOvv#W5vewJMjMbrIR4~3GKg^PQERP6t>nCVefpUg4`OuSBmE6N*UPuthe8@ z+woha!nS!gpvnAdOIhk2c;Tf!^+=EU7cVrWR+RJ4ZQ-E#)2pnrN&D}Efc-z8Uf)++ zRwU_ip~3uWOW9Yfc6?uLK@*%Xc?sj9JweC%Uu==ckD))bhQsoBS`U+3i6MhdM3gtdReQhrj+f}-+mB!GCo4|>aWyJl&*q<3YX~kiTlviT zcX}a*wM{KQoX)dSrT4}NyqO>+Xgn-1l}c4t6~TW#2BJ*wJfBONYTXrE!5D~2UB;Kw z`)+n@+6Tr!v_|;pkv%HsnS&pKF+%p-KXu*VP1Buy7@?_QPx!FKwL8U1Rw+DSj6@mN zKIK~1mKg2f1!E*SD4Mpkub+;4J4PyM*xM&@L+|YK#G}s=z!-@n8niA1mzE3Zvw|^D zy6F_=M)_Md-=nryCK%?($xZvK3DWM;kt%+(I}-ZvpeDYeMY9$m`^6XsYQXSPj;x+~ z8R2+d6eAoGA8EsZ1};NRhSN;z!SIrOd(F%IIP=2Ya$tCc5$!{FgMxSLqcXfaPuS*f zYKz@>Oa=@ux&$d%wZqKQZ`r}{id}T_3VV!^=J6e1)J?DLC=)qM$}H(E7_x+c#w6lWvqnYEN^e2OOraFR@ly6&~ zu<4G9M*wy~&9O*fXgzyVLI@5CHSL{Sp=mB0&i`&4A5Ber30@nkju(%L=7fWxc4_g> zc!|7pp`_bjsHusys9XB)$dZr-LoL*g@vT)pJ7eZ5Fx28=8+L8_ zz-5}ZtO2V`sNC8DmyD~H$(9#kP7QmmK1VM3%x+zjl?R5NggkdIpS9cNxes9I`A96@ zowUD~_f;`i3D-?+odl^?c(c^rbS~`?OKgNwB71Q(Ny^mhjeEFwAo3PuN7#)qGe-eZLw(=dnk9JLd1)j!wsn zamvu1+C^riN3@_zHa+C|ukBc~tTmKK?tHC^)n z43`MzK+1ME!^`4HV7OGJ)dv(Es12Bz0mJ2##!Igxo6l_To`JD&KpD)eJy$gFG#rQ` zl}#n)442I{y0%ahsj=>3nD+M5w*Qot`m^_7 z_`NLASwh4KT!xnjJD2ij#e)R3tf05N_7T~b`R^NO)J=K=#U6o|%t>6=$k}x;Tj@xtNNWraJ8%h?Y zKNbm5B+?gTGbr%S(WIN_7G#`zsNZ)o`pm)`xvl&&DhjU{za2H3*2WHq&S$l9(!|ew z`=_iHH^(oF_s?gwa=C!Yzq1+*HBdc#xTEBIrOLzq{QhaQzszd&bqlCraXzb!U;I0( z%^d!St<)EqQa`#hrjiLXpVh|4ScCu0YS{meS&iuhQy3b|XSE5gm<2zw8k*ols&86z zUH(9is-c6l6Pe`!YwV=r#rdqp5PE{#EgSb*;f;l338UbnlQ|~V4IMQ{V*GQ8|GGEW zWIgwI>r?KhL6*9Ac9%6@_wOT5?EPzY(XsY{vFW`zne^W(do(BU0J56vx2!gy$P@fM ztKsXB)i(b+tI<-k+IU;|0%GfLo)cWs$=7G-2OoG#ZjI9B7=0bzQJa7W>BT3`o5P9r#(bDQ za5c(M(bj5Hw-VFBfb(*8?bj_QTCpADdabCry-Mb)zGG5gjO!QsjZN=}M%{Ja3Z@2z z^|D8$3fTHZr@+){$Li3?V zQLD9PM|QR+hRved)F9du9e-4(eDX;iJs1b?Y^PL})r{k_>>FSlJOkl4^60c*2`v*C zdK0~uIj863Gtb`!LvPVOoBB)Z7R@}l2ZmnSs-FG5b9 zqJm8d1byo9(~XA+rnBxMk4L%=vm?i(U)8+jkvj8i3~#`4e$WwygixZx9tzaJdFqVV zy|Cks9&l@t0E)wsV5vp`vtYtf90gF8Uo99y$);^)Gf!+f?EBQ$uF+ilrxx z=IXrFe}bEuIFT2gO&3aDq^DSf{{w2sG>*7GA#1eQRsR$ihw9SjZUOa8i%%Z80>&+& zG_4i(R|Sua=L4bF(Dk+D+Gi!HH7El$X$I*wUut;U-OIZS484~%F}i}rVM@HQVCdbR z3HYLUFuQgM%7Uax%Rkpgu&&%;Xn8{8#{KuWu?i)z<7Y2buoPyl-~-DOw1u_Nh@?Y& zC-lRLV`7T15vNS9dMan>@Wn`h&tQlYEz-zg9;x%yUEhp_*#E(t{%z{HvrSy#`&>B@^dpM>uetJj6g$_Hl!*rOQEVdl zzldVgDU?fh`OIk<)vI+pM=yU^tjRnXEK|bstC^MH5>VdF7jMiH<==6>dfR=|8~PG6 zX=|QZ7(D3OXA$R|x_;?mt&O}Kf{#Dh)mwGR4U`fl*GAi}OJ42VDPPrrl^bdbSl27k z4bk~1Hu3C76obz%qF7MH>X;vWd;ecVFnqHx%ItGn@}IelKalR+O+`%)@hEt zRWUAsva6yG^4aI@>^?l;wc5orHA+~B?Sbn6(=LwMq3-aX=j z=zJ8LWGX^Rn17*f{}jd6*gT2G8|I_fq=fqSCyr>TmAYsV-G{yNm6}Z3_&qPt{amU4 zv20dy>Zv)Je2ZedN*Te3UU;bw#fww_;)SNvkM?*nBnFInj4D266Dksk1$ZfKtM%6=#v7GBMv)6{W7{Kb3}i;&q+d^&wa&aSLOayAEA zHq@@?JkOeGt*JCD6?*uB00hOPkC+iN3_Q zvnKSE&~rpFXKEDVC9nJ`is9=K#Y}!3#ptL_lv;Ev3A5ikC%8uPs|Ek8u3DRM+Ukal zYd^2A@G_LF&i$m^Kx1>-hLAPQ7Hc|_g&0^Ge-*`Kd@F=r8quo_l!_du@zzt*uhf|! zAVRvOr5~X@Cz>o1ieU>w=75WQ$f<{NS2gFr7`r|5N(lOlQt!98dL4Tv{{G(K?b#=+ z@%Re{N&)3y))R~YW(>Q)m`1Ket&{RqeAUcn2*&WuHeD4Hp>a86bUheDrtGLyo4OCL z+K#zv}y-1w+H5GeySt2kQ6%c6cD6n_lKoPXD%Kw4Ff~U zV54Be@~J%+ga^Qo^7r7}IS`hfpScALsRN(mW-otu+D=vhLu$awTyADdq+hxq7*ZOa zs_q%*(9ll41Vie~W6`Nb{)BbDUSJi1_Dygz?yGUI?9k)4daAq=8`TKLV8wN2hXXya z(veiqrO)m6@GKrLqONLw>TYI|i*$ZsNDo#Zh;f2c7es4vKz)o-ikX?ZkSP}2&01!1 zCi|Q=7y@6nKc4gBe;2eO3am^p$rTQrsnJXpB1kLt^%12R`nfJm*n{I$$F}N0-#bM` zIap8wWVBT9Y59zR3z-}62oy^5gfC5>dFX8Cl^bAWfK6*BkY!dfGUzFHab7FLM%2T} zI85S+oA9Z?crbJdQp(IYG}m|u;UH3DW1?{M3nw-I4B0j?>ap@*xj8vGSomcK7zfGD zOtjZx=EbbeEErPq`~z!`Q5v2w(t;u76gn1qC1P03QVb7bLCQ&Fk1;Fm53( zk4{V^NF|={$i|&&dn$1%QI{_HwmjD3Uy~p6N)`;O37KMVKb|l7!_Ht>1-hL~$zBsa z@hKb(EB*NRUOOhivzFdqSkakRUGl5xmJe+GzW{U>%I8wy*@_xG6h@EmQ0J`0@@H7-vz=#d`O%>jO%qs=)B$Sw+)wT%|gP zZW;`~jij7vUiE=n6)|A=*%o-1FmsnEUc}>P)P$MQ7usa}swVs;0~mgCU!Si(blCNs zCqEc|S?@ZxOWYXAmZ!Fd-_g_La&NPWhbxZYF*Stkg-z(w^*pDJ;i#jMs3i3{qhy~= zZLHolq4@7WsAOQ6@kn4sQ#!_0&7r1S7no5>YMK<(y6&}(n!_pKyHmC2 zn7~kx-*uscx#6*87yj$1>y~lK&YPM7C@yEj!T7w{b=MpyZ#8IEtKtzkl&a^G1rzyB zv|@1!SpIjj#lHoy|AQI*+r)GKt=^vRGbQ=hk0AEHX3FnDOvHm+jt1X?7%%z4e-Xr} z6DZf$eh0~|MHlvVyiQtvnQ7|%T|4j9znWKx)Cn8wZN37bng&a6$DS@Xxp&SrrfRp-ao(fEaS*tX^4J{ z7rf**KY|#1ei6iKrd*l7_f46`6GPwrDTsMi3$4fdzXdT~WrN$uN%Jo{YFetYJkbiAuat5_%lBAE|8u4O zM-W@v_u(L#%m=akO?1KYL9E&8%6~o>O{pKPSL;H3+Il{Sxi5|R`J?|ah(-7IK1GB1 zAm&cS{lo9~FY%E$k(N6ScPToZxc}#i2Un_O{u(4*ln%i%xs@$h_Q zpKuD|X>o?b--4L?X7e9Ui{tAN#CHEWh|yDn*#3U3iEa7KbArog9e>o~FWaO)zcJsG z&2GsX6EI*%aP2!P8c?6c%V+xVMxf~WKbE_nUM}^kASNR6+F+<-WsrdSf+7Rqkfh4Z znfnQdkap3WPS4evU}F!(u!RwwZ%M`I#|TN0gW_O}=*m2ShsY^YN96gz7}1@rhuN9W zap!P80b>|F#MTx3*+@7=QUt6_&@R?*L*jWtvIyy4OJ8sc8&!!vB;QB8B~~9_^&G#O9ph;Ip51$ z+bIi%6it-&j-bcIS1;9oAtf7{l*L|Wnqg-IhSZ(K+uOcg(9LwCwue-P#(>2dOS4=} zZ!qSX9hUjqs{@qE4pP9FYc5MK(YLcN(y7N|(A32F*mL)}O^-K39vue5FLu|FGhX`r zYy9T$sPzhhPMD$bCP6wWGutGkwMOO|o~5Sd%#C@O2@03jW?qp8!)lpi{PjyVMw{QG zw8)vPqmd!EjVjk{)dxaKNiV3^{JDcWlRX%fx;!X~806fL;S;({rg;%I!X$7qXiQg$ zyFtzi3=_YOeB-YD3^FI{@su|zWu`-ahoMWjJ$=7e1*}B;!5C}6P~ks(3cK#yvDEX>J+mj4`T#kKv^;3ZtBgBa zo>}9}M8f>T=w?MyAY1Y-Sh8iGAEJqs?|0jbUATuxcjII_LE=iUTDv<_r6*{FLZ*hy zGwhL4`sC@!)KV~94($$ITee}hoVz+0%T3K(>8UCSnyK7C9JJlJeWCwOoGgG;*Iop(-S(_ag zhs@u*kvw4eF(lX-3^6h7=+a3?u2An{JaP}Q3yY1Dm&J%|$vgx`{M|`ggvBV=Z99(Z zmy0~7$41y>Q(lq3t~HnrFaX15QsSJwS_d=6RnqdL@}X})#^ zZIOcD#l`?I{LZ(se000Sb)W>Dqz28iABTMJ-ek`&r32$9J-;_8|64lyKbYpfWtb-= ze$;vUKhx)fy_J72xL`8+dHVdG&K@5tv_pgWbmrc9dBMNlv82wbT-v8D3b;vjbzPDW z@-V-@L;jY>>kYq}Yze7dX=5uyaww9XroKESC<`~SldN_)lemCBG zRQ*=8G8?xNPe96tY*l?;Vq-`k>tb7uy`@(3PtFC(*ifGbn@?vRTW|jydj2V$E&Fqn zdQWmboq2e#{=Q5`PmNIOACA^z?|eG*m^=IZ9ANmlQvV~JWh)z+pvinXDaIhF<~_B+6}fKdONv5U&ubH0MC{Qt$zP>2^L1oT7d~$Hh(X$odhZMNupDbECm=%Fcp*dMHk}-|7=~r) z^>F8NrdIza4}I@CfH90(tvaxEKWlEKMJs+Vg<}w$TzJS zO_AD>i(#31k^ITA^vU~<-r=A!FoyYzE{kw`=4UsnyTI@Z;f#p$v(AjI!*9Z<7s*AG z1pmI>Zyqe{#V@$vw=H+`EX$3fZD&`5;inP6WMbMQ*>K<_7}xHBhB`ve_pEBC?uMRz z?S0N2J`bln62Q=FROq)689O5V=qVU_Lwi$WZ{CTJcd`XTFZr$cr$Vn)=MF6dL$CJc zqmMOT#6=k=!O-ht)Zo{3-dum&1`NHyhdlucFRzy_|AJqXL2vum_4q=?RSPo+F!UbU zAAUQM$$fj#axlubd~{LRD)XT%J~J@<2Nmf3;&>p@27@Q zRVSKQjfzdz7ny_MxB8jUv_$jG)Jes%<61E0EVB7MZg*&+kMHNRnC6=LCqW( zei0RdQ*&OTFVf<{@H0BMI;nO@?U3vcm`u<B8;-W(6p6zX*3pG21M54kQkoDt-$5oI$!Swv$yNw*LE;DaX$I>dCkd#?FL|& z%&6N0mW(lVXW~PtS=1~xYRwuoI*G6fFiZk&lI;swizfUQgK=+oe6^grDPjRL1IMRY zt#y((;MAqg^;Sf_-1&rzu{baa)Tnc0dGJE^b=jA|D3HI^_B+*@MSMeU_>~_DbVjYV zO*g`0BI_1do}eG9-s9^|kdCQ^IthJcdBcI@Kuw_+vgtp~iKaG5j_9E@n zp)0$9u(})<7v#{!OY5?a(FaP4j3WDMu~o zX@kOSFkG6>^u2Fh^4=l49gND{qSRi^K%gSn6#2@flIs$pQkJT4gHf5O z#`mwKyOze=Wq{$Zc}~{igrj#2Lpgroh{`PTXK9oZtZ3&hV?Ll*loeb1)7sb|J~_X1k^LEGX^5K5+|5x|PcFBaXe3tNs3X7oPG_ zdVKtzKQ%S#aqS@`Inb< diff --git a/consensus/wal.go b/consensus/wal.go index 69519c165..c6367c7d1 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -30,7 +30,7 @@ type TimedWALMessage struct { } // EndHeightMessage marks the end of the given height inside WAL. -// @internal used by scripts/cutWALUntil util. +// @internal used by scripts/wal2json util. type EndHeightMessage struct { Height int64 `json:"height"` } diff --git a/consensus/wal_test.go b/consensus/wal_test.go index 38f2ce034..d81400b11 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -3,7 +3,6 @@ package consensus import ( "bytes" "crypto/rand" - "path" "sync" "testing" "time" @@ -43,7 +42,13 @@ func TestWALEncoderDecoder(t *testing.T) { } func TestSearchForEndHeight(t *testing.T) { - wal, err := NewWAL(path.Join(data_dir, "many_blocks.cswal"), false) + walBody, err := GenWAL(6) + if err != nil { + t.Fatal(err) + } + walFile := writeWAL(walBody) + + wal, err := NewWAL(walFile, false) if err != nil { t.Fatal(err) } diff --git a/consensus/walgen.go b/consensus/walgen.go new file mode 100644 index 000000000..2f541f1ff --- /dev/null +++ b/consensus/walgen.go @@ -0,0 +1,178 @@ +// walgen provides an utility function for generating WAL on the fly. +package consensus + +import ( + "bufio" + "bytes" + "fmt" + "math/rand" + "os" + "path/filepath" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/tendermint/abci/example/dummy" + bc "github.com/tendermint/tendermint/blockchain" + cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/proxy" + sm "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/types" + auto "github.com/tendermint/tmlibs/autofile" + "github.com/tendermint/tmlibs/db" + "github.com/tendermint/tmlibs/log" +) + +// GenWAL generates a consensus WAL. It does this by spining up a new node with a +// dummy application and special consensus wal instance (byteBufferWAL) and +// waits until numBlocks are created. Then it returns a WAL body. +func GenWAL(numBlocks int) (body []byte, err error) { + config := getConfig() + + app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "genwal")) + + logger := log.TestingLogger().With("walgen", "walgen") + + ///////////////////////////////////////////////////////////////////////////// + // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS + // NOTE: we can't import node package because of circular dependency + privValidatorFile := config.PrivValidatorFile() + privValidator := types.LoadOrGenPrivValidatorFS(privValidatorFile) + genDoc, err := types.GenesisDocFromFile(config.GenesisFile()) + if err != nil { + return nil, errors.Wrap(err, "failed to read genesis file") + } + stateDB := db.NewMemDB() + blockStoreDB := db.NewMemDB() + state, err := sm.MakeGenesisState(stateDB, genDoc) + state.SetLogger(logger.With("module", "state")) + if err != nil { + return nil, errors.Wrap(err, "failed to make genesis state") + } + blockStore := bc.NewBlockStore(blockStoreDB) + handshaker := NewHandshaker(state, blockStore) + proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app), handshaker) + proxyApp.SetLogger(logger.With("module", "proxy")) + if err := proxyApp.Start(); err != nil { + return nil, errors.Wrap(err, "failed to start proxy app connections") + } + defer proxyApp.Stop() + eventBus := types.NewEventBus() + eventBus.SetLogger(logger.With("module", "events")) + if err := eventBus.Start(); err != nil { + return nil, errors.Wrap(err, "failed to start event bus") + } + mempool := types.MockMempool{} + consensusState := NewConsensusState(config.Consensus, state.Copy(), proxyApp.Consensus(), blockStore, mempool) + consensusState.SetLogger(logger) + consensusState.SetEventBus(eventBus) + if privValidator != nil { + consensusState.SetPrivValidator(privValidator) + } + // END OF COPY PASTE + ///////////////////////////////////////////////////////////////////////////// + + // set consensus wal to buffered WAL, which will write all incoming msgs to buffer + var b bytes.Buffer + wr := bufio.NewWriter(&b) + numBlocksWritten := make(chan struct{}) + wal := &byteBufferWAL{enc: NewWALEncoder(wr), heightToStop: int64(numBlocks), signalWhenStopsTo: numBlocksWritten} + wal.Save(EndHeightMessage{0}) + consensusState.wal = wal + + if err := consensusState.Start(); err != nil { + return nil, errors.Wrap(err, "failed to start consensus state") + } + defer consensusState.Stop() + + select { + case <-numBlocksWritten: + wr.Flush() + return b.Bytes(), nil + case <-time.After(time.Duration(2*numBlocks) * time.Second): + return b.Bytes(), fmt.Errorf("waited too long for tendermint to produce %d blocks", numBlocks) + } +} + +// f**ing long, but unique for each test +func makePathname() string { + // get path + p, err := os.Getwd() + if err != nil { + panic(err) + } + fmt.Println(p) + sep := string(filepath.Separator) + return strings.Replace(p, sep, "_", -1) +} + +func randPort() int { + // returns between base and base + spread + base, spread := 20000, 20000 + return base + rand.Intn(spread) +} + +func makeAddrs() (string, string, string) { + start := randPort() + return fmt.Sprintf("tcp://0.0.0.0:%d", start), + fmt.Sprintf("tcp://0.0.0.0:%d", start+1), + fmt.Sprintf("tcp://0.0.0.0:%d", start+2) +} + +// getConfig returns a config for test cases +func getConfig() *cfg.Config { + pathname := makePathname() + c := cfg.ResetTestRoot(pathname) + + // and we use random ports to run in parallel + tm, rpc, grpc := makeAddrs() + c.P2P.ListenAddress = tm + c.RPC.ListenAddress = rpc + c.RPC.GRPCListenAddress = grpc + return c +} + +// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops +// when the heightToStop is reached. Client will be notified via +// signalWhenStopsTo channel. +type byteBufferWAL struct { + enc *WALEncoder + stopped bool + heightToStop int64 + signalWhenStopsTo chan struct{} +} + +var fixedTime, err = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z") + +// Save writes message to the internal buffer except when heightToStop is +// reached, in which case it signal the caller via signalWhenStopsTo and skip +// writing. +func (w *byteBufferWAL) Save(m WALMessage) { + if w.stopped { + return + } + + if endMsg, ok := m.(EndHeightMessage); ok { + if endMsg.Height == w.heightToStop { + w.signalWhenStopsTo <- struct{}{} + w.stopped = true + return + } + } + + err := w.enc.Encode(&TimedWALMessage{fixedTime, m}) + if err != nil { + panic(fmt.Sprintf("failed to encode the msg %v", m)) + } +} + +func (w *byteBufferWAL) Group() *auto.Group { + panic("not implemented") +} +func (w *byteBufferWAL) SearchForEndHeight(height int64) (gr *auto.GroupReader, found bool, err error) { + return nil, false, nil +} + +func (w *byteBufferWAL) Start() error { return nil } +func (w *byteBufferWAL) Stop() error { return nil } +func (w *byteBufferWAL) Wait() {} diff --git a/scripts/cutWALUntil/main.go b/scripts/cutWALUntil/main.go deleted file mode 100644 index 843368952..000000000 --- a/scripts/cutWALUntil/main.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - cutWALUntil is a small utility for cutting a WAL until the given height - (inclusively). Note it does not include last cs.EndHeightMessage. - - Usage: - cutWALUntil height-to-stop -*/ -package main - -import ( - "fmt" - "io" - "os" - "strconv" - - cs "github.com/tendermint/tendermint/consensus" -) - -func main() { - if len(os.Args) < 4 { - fmt.Println("3 arguments required: ") - os.Exit(1) - } - - var heightToStop int64 - var err error - if heightToStop, err = strconv.ParseInt(os.Args[2], 10, 64); err != nil { - panic(fmt.Errorf("failed to parse height: %v", err)) - } - - in, err := os.Open(os.Args[1]) - if err != nil { - panic(fmt.Errorf("failed to open input WAL file: %v", err)) - } - defer in.Close() - - out, err := os.Create(os.Args[3]) - if err != nil { - panic(fmt.Errorf("failed to open output WAL file: %v", err)) - } - defer out.Close() - - enc := cs.NewWALEncoder(out) - dec := cs.NewWALDecoder(in) - - for { - msg, err := dec.Decode() - if err == io.EOF { - break - } else if err != nil { - panic(fmt.Errorf("failed to decode msg: %v", err)) - } - - if m, ok := msg.Msg.(cs.EndHeightMessage); ok { - if m.Height == heightToStop { - break - } - } - - err = enc.Encode(msg) - if err != nil { - panic(fmt.Errorf("failed to encode msg: %v", err)) - } - } -} From 5cb936fa004c293d2c64ac47e9648740c6bf8d90 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 6 Dec 2017 18:28:14 -0600 Subject: [PATCH 2/2] fixes after my own review --- consensus/replay_test.go | 15 +++++++++------ consensus/{walgen.go => wal_generator.go} | 23 +++++++++++++---------- consensus/wal_test.go | 4 ++-- 3 files changed, 24 insertions(+), 18 deletions(-) rename consensus/{walgen.go => wal_generator.go} (87%) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 8643427c0..a1e06c65c 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -264,9 +264,12 @@ func (w *crashingWAL) Wait() { w.next.Wait() } //------------------------------------------------------------------------------------------ // Handshake Tests -var ( +const ( NUM_BLOCKS = 6 - mempool = types.MockMempool{} +) + +var ( + mempool = types.MockMempool{} ) //--------------------------------------- @@ -305,12 +308,12 @@ func TestHandshakeReplayNone(t *testing.T) { } } -func writeWAL(walMsgs []byte) string { +func tempWALWithData(data []byte) string { walFile, err := ioutil.TempFile("", "wal") if err != nil { panic(fmt.Errorf("failed to create temp WAL file: %v", err)) } - _, err = walFile.Write(walMsgs) + _, err = walFile.Write(data) if err != nil { panic(fmt.Errorf("failed to write to temp WAL file: %v", err)) } @@ -324,11 +327,11 @@ func writeWAL(walMsgs []byte) string { func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { config := ResetConfig("proxy_test_") - walBody, err := GenWAL(NUM_BLOCKS) + walBody, err := WALWithNBlocks(NUM_BLOCKS) if err != nil { t.Fatal(err) } - walFile := writeWAL(walBody) + walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) privVal := types.LoadPrivValidatorFS(config.PrivValidatorFile()) diff --git a/consensus/walgen.go b/consensus/wal_generator.go similarity index 87% rename from consensus/walgen.go rename to consensus/wal_generator.go index 2f541f1ff..8a2e21821 100644 --- a/consensus/walgen.go +++ b/consensus/wal_generator.go @@ -1,4 +1,3 @@ -// walgen provides an utility function for generating WAL on the fly. package consensus import ( @@ -23,15 +22,17 @@ import ( "github.com/tendermint/tmlibs/log" ) -// GenWAL generates a consensus WAL. It does this by spining up a new node with a -// dummy application and special consensus wal instance (byteBufferWAL) and -// waits until numBlocks are created. Then it returns a WAL body. -func GenWAL(numBlocks int) (body []byte, err error) { +// WALWithNBlocks generates a consensus WAL. It does this by spining up a +// stripped down version of node (proxy app, event bus, consensus state) with a +// persistent dummy application and special consensus wal instance +// (byteBufferWAL) and waits until numBlocks are created. Then it returns a WAL +// content. +func WALWithNBlocks(numBlocks int) (data []byte, err error) { config := getConfig() - app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "genwal")) + app := dummy.NewPersistentDummyApplication(filepath.Join(config.DBDir(), "wal_generator")) - logger := log.TestingLogger().With("walgen", "walgen") + logger := log.NewNopLogger() // log.TestingLogger().With("wal_generator", "wal_generator") ///////////////////////////////////////////////////////////////////////////// // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS @@ -77,6 +78,7 @@ func GenWAL(numBlocks int) (body []byte, err error) { wr := bufio.NewWriter(&b) numBlocksWritten := make(chan struct{}) wal := &byteBufferWAL{enc: NewWALEncoder(wr), heightToStop: int64(numBlocks), signalWhenStopsTo: numBlocksWritten} + // see wal.go#103 wal.Save(EndHeightMessage{0}) consensusState.wal = wal @@ -142,11 +144,12 @@ type byteBufferWAL struct { signalWhenStopsTo chan struct{} } -var fixedTime, err = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z") +// needed for determinism +var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z") // Save writes message to the internal buffer except when heightToStop is -// reached, in which case it signal the caller via signalWhenStopsTo and skip -// writing. +// reached, in which case it will signal the caller via signalWhenStopsTo and +// skip writing. func (w *byteBufferWAL) Save(m WALMessage) { if w.stopped { return diff --git a/consensus/wal_test.go b/consensus/wal_test.go index d81400b11..8ec1a7c2c 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -42,11 +42,11 @@ func TestWALEncoderDecoder(t *testing.T) { } func TestSearchForEndHeight(t *testing.T) { - walBody, err := GenWAL(6) + walBody, err := WALWithNBlocks(6) if err != nil { t.Fatal(err) } - walFile := writeWAL(walBody) + walFile := tempWALWithData(walBody) wal, err := NewWAL(walFile, false) if err != nil {