You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

698 lines
18 KiB

  1. package conn
  2. import (
  3. "context"
  4. "encoding/hex"
  5. "net"
  6. "sync"
  7. "testing"
  8. "time"
  9. "github.com/fortytw2/leaktest"
  10. "github.com/gogo/protobuf/proto"
  11. "github.com/stretchr/testify/assert"
  12. "github.com/stretchr/testify/require"
  13. "github.com/tendermint/tendermint/internal/libs/protoio"
  14. "github.com/tendermint/tendermint/libs/log"
  15. "github.com/tendermint/tendermint/libs/service"
  16. tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p"
  17. "github.com/tendermint/tendermint/proto/tendermint/types"
  18. )
  19. const maxPingPongPacketSize = 1024 // bytes
  20. func createTestMConnection(logger log.Logger, conn net.Conn) *MConnection {
  21. return createMConnectionWithCallbacks(logger, conn,
  22. // onRecieve
  23. func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  24. },
  25. // onError
  26. func(ctx context.Context, r interface{}) {
  27. })
  28. }
  29. func createMConnectionWithCallbacks(
  30. logger log.Logger,
  31. conn net.Conn,
  32. onReceive func(ctx context.Context, chID ChannelID, msgBytes []byte),
  33. onError func(ctx context.Context, r interface{}),
  34. ) *MConnection {
  35. cfg := DefaultMConnConfig()
  36. cfg.PingInterval = 90 * time.Millisecond
  37. cfg.PongTimeout = 45 * time.Millisecond
  38. chDescs := []*ChannelDescriptor{{ID: 0x01, Priority: 1, SendQueueCapacity: 1}}
  39. c := NewMConnection(logger, conn, chDescs, onReceive, onError, cfg)
  40. return c
  41. }
  42. func TestMConnectionSendFlushStop(t *testing.T) {
  43. server, client := NetPipe()
  44. t.Cleanup(closeAll(t, client, server))
  45. ctx, cancel := context.WithCancel(context.Background())
  46. defer cancel()
  47. clientConn := createTestMConnection(log.TestingLogger(), client)
  48. err := clientConn.Start(ctx)
  49. require.NoError(t, err)
  50. t.Cleanup(waitAll(clientConn))
  51. msg := []byte("abc")
  52. assert.True(t, clientConn.Send(0x01, msg))
  53. msgLength := 14
  54. // start the reader in a new routine, so we can flush
  55. errCh := make(chan error)
  56. go func() {
  57. msgB := make([]byte, msgLength)
  58. _, err := server.Read(msgB)
  59. if err != nil {
  60. t.Error(err)
  61. return
  62. }
  63. errCh <- err
  64. }()
  65. timer := time.NewTimer(3 * time.Second)
  66. select {
  67. case <-errCh:
  68. case <-timer.C:
  69. t.Error("timed out waiting for msgs to be read")
  70. }
  71. }
  72. func TestMConnectionSend(t *testing.T) {
  73. server, client := NetPipe()
  74. t.Cleanup(closeAll(t, client, server))
  75. ctx, cancel := context.WithCancel(context.Background())
  76. defer cancel()
  77. mconn := createTestMConnection(log.TestingLogger(), client)
  78. err := mconn.Start(ctx)
  79. require.NoError(t, err)
  80. t.Cleanup(waitAll(mconn))
  81. msg := []byte("Ant-Man")
  82. assert.True(t, mconn.Send(0x01, msg))
  83. // Note: subsequent Send/TrySend calls could pass because we are reading from
  84. // the send queue in a separate goroutine.
  85. _, err = server.Read(make([]byte, len(msg)))
  86. if err != nil {
  87. t.Error(err)
  88. }
  89. msg = []byte("Spider-Man")
  90. assert.True(t, mconn.Send(0x01, msg))
  91. _, err = server.Read(make([]byte, len(msg)))
  92. if err != nil {
  93. t.Error(err)
  94. }
  95. assert.False(t, mconn.Send(0x05, []byte("Absorbing Man")), "Send should return false because channel is unknown")
  96. }
  97. func TestMConnectionReceive(t *testing.T) {
  98. server, client := NetPipe()
  99. t.Cleanup(closeAll(t, client, server))
  100. receivedCh := make(chan []byte)
  101. errorsCh := make(chan interface{})
  102. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  103. select {
  104. case receivedCh <- msgBytes:
  105. case <-ctx.Done():
  106. }
  107. }
  108. onError := func(ctx context.Context, r interface{}) {
  109. select {
  110. case errorsCh <- r:
  111. case <-ctx.Done():
  112. }
  113. }
  114. logger := log.TestingLogger()
  115. ctx, cancel := context.WithCancel(context.Background())
  116. defer cancel()
  117. mconn1 := createMConnectionWithCallbacks(logger, client, onReceive, onError)
  118. err := mconn1.Start(ctx)
  119. require.NoError(t, err)
  120. t.Cleanup(waitAll(mconn1))
  121. mconn2 := createTestMConnection(logger, server)
  122. err = mconn2.Start(ctx)
  123. require.NoError(t, err)
  124. t.Cleanup(waitAll(mconn2))
  125. msg := []byte("Cyclops")
  126. assert.True(t, mconn2.Send(0x01, msg))
  127. select {
  128. case receivedBytes := <-receivedCh:
  129. assert.Equal(t, msg, receivedBytes)
  130. case err := <-errorsCh:
  131. t.Fatalf("Expected %s, got %+v", msg, err)
  132. case <-time.After(500 * time.Millisecond):
  133. t.Fatalf("Did not receive %s message in 500ms", msg)
  134. }
  135. }
  136. func TestMConnectionPongTimeoutResultsInError(t *testing.T) {
  137. server, client := net.Pipe()
  138. t.Cleanup(closeAll(t, client, server))
  139. receivedCh := make(chan []byte)
  140. errorsCh := make(chan interface{})
  141. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  142. select {
  143. case receivedCh <- msgBytes:
  144. case <-ctx.Done():
  145. }
  146. }
  147. onError := func(ctx context.Context, r interface{}) {
  148. select {
  149. case errorsCh <- r:
  150. case <-ctx.Done():
  151. }
  152. }
  153. ctx, cancel := context.WithCancel(context.Background())
  154. defer cancel()
  155. mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError)
  156. err := mconn.Start(ctx)
  157. require.NoError(t, err)
  158. t.Cleanup(waitAll(mconn))
  159. serverGotPing := make(chan struct{})
  160. go func() {
  161. // read ping
  162. var pkt tmp2p.Packet
  163. _, err := protoio.NewDelimitedReader(server, maxPingPongPacketSize).ReadMsg(&pkt)
  164. require.NoError(t, err)
  165. serverGotPing <- struct{}{}
  166. }()
  167. <-serverGotPing
  168. pongTimerExpired := mconn.config.PongTimeout + 200*time.Millisecond
  169. select {
  170. case msgBytes := <-receivedCh:
  171. t.Fatalf("Expected error, but got %v", msgBytes)
  172. case err := <-errorsCh:
  173. assert.NotNil(t, err)
  174. case <-time.After(pongTimerExpired):
  175. t.Fatalf("Expected to receive error after %v", pongTimerExpired)
  176. }
  177. }
  178. func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) {
  179. server, client := net.Pipe()
  180. t.Cleanup(closeAll(t, client, server))
  181. receivedCh := make(chan []byte)
  182. errorsCh := make(chan interface{})
  183. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  184. select {
  185. case receivedCh <- msgBytes:
  186. case <-ctx.Done():
  187. }
  188. }
  189. onError := func(ctx context.Context, r interface{}) {
  190. select {
  191. case errorsCh <- r:
  192. case <-ctx.Done():
  193. }
  194. }
  195. ctx, cancel := context.WithCancel(context.Background())
  196. defer cancel()
  197. mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError)
  198. err := mconn.Start(ctx)
  199. require.NoError(t, err)
  200. t.Cleanup(waitAll(mconn))
  201. // sending 3 pongs in a row (abuse)
  202. protoWriter := protoio.NewDelimitedWriter(server)
  203. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  204. require.NoError(t, err)
  205. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  206. require.NoError(t, err)
  207. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  208. require.NoError(t, err)
  209. serverGotPing := make(chan struct{})
  210. go func() {
  211. // read ping (one byte)
  212. var packet tmp2p.Packet
  213. _, err := protoio.NewDelimitedReader(server, maxPingPongPacketSize).ReadMsg(&packet)
  214. require.NoError(t, err)
  215. serverGotPing <- struct{}{}
  216. // respond with pong
  217. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  218. require.NoError(t, err)
  219. }()
  220. <-serverGotPing
  221. pongTimerExpired := mconn.config.PongTimeout + 20*time.Millisecond
  222. select {
  223. case msgBytes := <-receivedCh:
  224. t.Fatalf("Expected no data, but got %v", msgBytes)
  225. case err := <-errorsCh:
  226. t.Fatalf("Expected no error, but got %v", err)
  227. case <-time.After(pongTimerExpired):
  228. assert.True(t, mconn.IsRunning())
  229. }
  230. }
  231. func TestMConnectionMultiplePings(t *testing.T) {
  232. server, client := net.Pipe()
  233. t.Cleanup(closeAll(t, client, server))
  234. receivedCh := make(chan []byte)
  235. errorsCh := make(chan interface{})
  236. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  237. select {
  238. case receivedCh <- msgBytes:
  239. case <-ctx.Done():
  240. }
  241. }
  242. onError := func(ctx context.Context, r interface{}) {
  243. select {
  244. case errorsCh <- r:
  245. case <-ctx.Done():
  246. }
  247. }
  248. ctx, cancel := context.WithCancel(context.Background())
  249. defer cancel()
  250. mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError)
  251. err := mconn.Start(ctx)
  252. require.NoError(t, err)
  253. t.Cleanup(waitAll(mconn))
  254. // sending 3 pings in a row (abuse)
  255. // see https://github.com/tendermint/tendermint/issues/1190
  256. protoReader := protoio.NewDelimitedReader(server, maxPingPongPacketSize)
  257. protoWriter := protoio.NewDelimitedWriter(server)
  258. var pkt tmp2p.Packet
  259. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{}))
  260. require.NoError(t, err)
  261. _, err = protoReader.ReadMsg(&pkt)
  262. require.NoError(t, err)
  263. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{}))
  264. require.NoError(t, err)
  265. _, err = protoReader.ReadMsg(&pkt)
  266. require.NoError(t, err)
  267. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPing{}))
  268. require.NoError(t, err)
  269. _, err = protoReader.ReadMsg(&pkt)
  270. require.NoError(t, err)
  271. assert.True(t, mconn.IsRunning())
  272. }
  273. func TestMConnectionPingPongs(t *testing.T) {
  274. // check that we are not leaking any go-routines
  275. t.Cleanup(leaktest.CheckTimeout(t, 10*time.Second))
  276. server, client := net.Pipe()
  277. t.Cleanup(closeAll(t, client, server))
  278. receivedCh := make(chan []byte)
  279. errorsCh := make(chan interface{})
  280. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  281. select {
  282. case receivedCh <- msgBytes:
  283. case <-ctx.Done():
  284. }
  285. }
  286. onError := func(ctx context.Context, r interface{}) {
  287. select {
  288. case errorsCh <- r:
  289. case <-ctx.Done():
  290. }
  291. }
  292. ctx, cancel := context.WithCancel(context.Background())
  293. defer cancel()
  294. mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError)
  295. err := mconn.Start(ctx)
  296. require.NoError(t, err)
  297. t.Cleanup(waitAll(mconn))
  298. serverGotPing := make(chan struct{})
  299. go func() {
  300. protoReader := protoio.NewDelimitedReader(server, maxPingPongPacketSize)
  301. protoWriter := protoio.NewDelimitedWriter(server)
  302. var pkt tmp2p.PacketPing
  303. // read ping
  304. _, err = protoReader.ReadMsg(&pkt)
  305. require.NoError(t, err)
  306. serverGotPing <- struct{}{}
  307. // respond with pong
  308. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  309. require.NoError(t, err)
  310. time.Sleep(mconn.config.PingInterval)
  311. // read ping
  312. _, err = protoReader.ReadMsg(&pkt)
  313. require.NoError(t, err)
  314. serverGotPing <- struct{}{}
  315. // respond with pong
  316. _, err = protoWriter.WriteMsg(mustWrapPacket(&tmp2p.PacketPong{}))
  317. require.NoError(t, err)
  318. }()
  319. <-serverGotPing
  320. <-serverGotPing
  321. pongTimerExpired := (mconn.config.PongTimeout + 20*time.Millisecond) * 2
  322. select {
  323. case msgBytes := <-receivedCh:
  324. t.Fatalf("Expected no data, but got %v", msgBytes)
  325. case err := <-errorsCh:
  326. t.Fatalf("Expected no error, but got %v", err)
  327. case <-time.After(2 * pongTimerExpired):
  328. assert.True(t, mconn.IsRunning())
  329. }
  330. }
  331. func TestMConnectionStopsAndReturnsError(t *testing.T) {
  332. server, client := NetPipe()
  333. t.Cleanup(closeAll(t, client, server))
  334. receivedCh := make(chan []byte)
  335. errorsCh := make(chan interface{})
  336. onReceive := func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  337. select {
  338. case receivedCh <- msgBytes:
  339. case <-ctx.Done():
  340. }
  341. }
  342. onError := func(ctx context.Context, r interface{}) {
  343. select {
  344. case errorsCh <- r:
  345. case <-ctx.Done():
  346. }
  347. }
  348. ctx, cancel := context.WithCancel(context.Background())
  349. defer cancel()
  350. mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError)
  351. err := mconn.Start(ctx)
  352. require.NoError(t, err)
  353. t.Cleanup(waitAll(mconn))
  354. if err := client.Close(); err != nil {
  355. t.Error(err)
  356. }
  357. select {
  358. case receivedBytes := <-receivedCh:
  359. t.Fatalf("Expected error, got %v", receivedBytes)
  360. case err := <-errorsCh:
  361. assert.NotNil(t, err)
  362. assert.False(t, mconn.IsRunning())
  363. case <-time.After(500 * time.Millisecond):
  364. t.Fatal("Did not receive error in 500ms")
  365. }
  366. }
  367. func newClientAndServerConnsForReadErrors(
  368. ctx context.Context,
  369. t *testing.T,
  370. chOnErr chan struct{},
  371. ) (*MConnection, *MConnection) {
  372. server, client := NetPipe()
  373. onReceive := func(context.Context, ChannelID, []byte) {}
  374. onError := func(context.Context, interface{}) {}
  375. // create client conn with two channels
  376. chDescs := []*ChannelDescriptor{
  377. {ID: 0x01, Priority: 1, SendQueueCapacity: 1},
  378. {ID: 0x02, Priority: 1, SendQueueCapacity: 1},
  379. }
  380. logger := log.TestingLogger()
  381. mconnClient := NewMConnection(logger.With("module", "client"), client, chDescs, onReceive, onError, DefaultMConnConfig())
  382. err := mconnClient.Start(ctx)
  383. require.NoError(t, err)
  384. // create server conn with 1 channel
  385. // it fires on chOnErr when there's an error
  386. serverLogger := logger.With("module", "server")
  387. onError = func(ctx context.Context, r interface{}) {
  388. select {
  389. case <-ctx.Done():
  390. case chOnErr <- struct{}{}:
  391. }
  392. }
  393. mconnServer := createMConnectionWithCallbacks(serverLogger, server, onReceive, onError)
  394. err = mconnServer.Start(ctx)
  395. require.NoError(t, err)
  396. return mconnClient, mconnServer
  397. }
  398. func expectSend(ch chan struct{}) bool {
  399. after := time.After(time.Second * 5)
  400. select {
  401. case <-ch:
  402. return true
  403. case <-after:
  404. return false
  405. }
  406. }
  407. func TestMConnectionReadErrorBadEncoding(t *testing.T) {
  408. ctx, cancel := context.WithCancel(context.Background())
  409. defer cancel()
  410. chOnErr := make(chan struct{})
  411. mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr)
  412. client := mconnClient.conn
  413. // Write it.
  414. _, err := client.Write([]byte{1, 2, 3, 4, 5})
  415. require.NoError(t, err)
  416. assert.True(t, expectSend(chOnErr), "badly encoded msgPacket")
  417. t.Cleanup(waitAll(mconnClient, mconnServer))
  418. }
  419. func TestMConnectionReadErrorUnknownChannel(t *testing.T) {
  420. ctx, cancel := context.WithCancel(context.Background())
  421. defer cancel()
  422. chOnErr := make(chan struct{})
  423. mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr)
  424. msg := []byte("Ant-Man")
  425. // fail to send msg on channel unknown by client
  426. assert.False(t, mconnClient.Send(0x03, msg))
  427. // send msg on channel unknown by the server.
  428. // should cause an error
  429. assert.True(t, mconnClient.Send(0x02, msg))
  430. assert.True(t, expectSend(chOnErr), "unknown channel")
  431. t.Cleanup(waitAll(mconnClient, mconnServer))
  432. }
  433. func TestMConnectionReadErrorLongMessage(t *testing.T) {
  434. chOnErr := make(chan struct{})
  435. chOnRcv := make(chan struct{})
  436. ctx, cancel := context.WithCancel(context.Background())
  437. defer cancel()
  438. mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr)
  439. t.Cleanup(waitAll(mconnClient, mconnServer))
  440. mconnServer.onReceive = func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  441. select {
  442. case <-ctx.Done():
  443. case chOnRcv <- struct{}{}:
  444. }
  445. }
  446. client := mconnClient.conn
  447. protoWriter := protoio.NewDelimitedWriter(client)
  448. // send msg thats just right
  449. var packet = tmp2p.PacketMsg{
  450. ChannelID: 0x01,
  451. EOF: true,
  452. Data: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize),
  453. }
  454. _, err := protoWriter.WriteMsg(mustWrapPacket(&packet))
  455. require.NoError(t, err)
  456. assert.True(t, expectSend(chOnRcv), "msg just right")
  457. // send msg thats too long
  458. packet = tmp2p.PacketMsg{
  459. ChannelID: 0x01,
  460. EOF: true,
  461. Data: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+100),
  462. }
  463. _, err = protoWriter.WriteMsg(mustWrapPacket(&packet))
  464. require.Error(t, err)
  465. assert.True(t, expectSend(chOnErr), "msg too long")
  466. }
  467. func TestMConnectionReadErrorUnknownMsgType(t *testing.T) {
  468. ctx, cancel := context.WithCancel(context.Background())
  469. defer cancel()
  470. chOnErr := make(chan struct{})
  471. mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr)
  472. t.Cleanup(waitAll(mconnClient, mconnServer))
  473. // send msg with unknown msg type
  474. _, err := protoio.NewDelimitedWriter(mconnClient.conn).WriteMsg(&types.Header{ChainID: "x"})
  475. require.NoError(t, err)
  476. assert.True(t, expectSend(chOnErr), "unknown msg type")
  477. }
  478. func TestMConnectionTrySend(t *testing.T) {
  479. server, client := NetPipe()
  480. t.Cleanup(closeAll(t, client, server))
  481. ctx, cancel := context.WithCancel(context.Background())
  482. defer cancel()
  483. mconn := createTestMConnection(log.TestingLogger(), client)
  484. err := mconn.Start(ctx)
  485. require.NoError(t, err)
  486. t.Cleanup(waitAll(mconn))
  487. msg := []byte("Semicolon-Woman")
  488. resultCh := make(chan string, 2)
  489. assert.True(t, mconn.Send(0x01, msg))
  490. _, err = server.Read(make([]byte, len(msg)))
  491. require.NoError(t, err)
  492. assert.True(t, mconn.Send(0x01, msg))
  493. go func() {
  494. mconn.Send(0x01, msg)
  495. resultCh <- "TrySend"
  496. }()
  497. assert.False(t, mconn.Send(0x01, msg))
  498. assert.Equal(t, "TrySend", <-resultCh)
  499. }
  500. func TestConnVectors(t *testing.T) {
  501. testCases := []struct {
  502. testName string
  503. msg proto.Message
  504. expBytes string
  505. }{
  506. {"PacketPing", &tmp2p.PacketPing{}, "0a00"},
  507. {"PacketPong", &tmp2p.PacketPong{}, "1200"},
  508. {"PacketMsg", &tmp2p.PacketMsg{ChannelID: 1, EOF: false, Data: []byte("data transmitted over the wire")}, "1a2208011a1e64617461207472616e736d6974746564206f766572207468652077697265"},
  509. }
  510. for _, tc := range testCases {
  511. tc := tc
  512. pm := mustWrapPacket(tc.msg)
  513. bz, err := pm.Marshal()
  514. require.NoError(t, err, tc.testName)
  515. require.Equal(t, tc.expBytes, hex.EncodeToString(bz), tc.testName)
  516. }
  517. }
  518. func TestMConnectionChannelOverflow(t *testing.T) {
  519. chOnErr := make(chan struct{})
  520. chOnRcv := make(chan struct{})
  521. ctx, cancel := context.WithCancel(context.Background())
  522. defer cancel()
  523. mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr)
  524. t.Cleanup(waitAll(mconnClient, mconnServer))
  525. mconnServer.onReceive = func(ctx context.Context, chID ChannelID, msgBytes []byte) {
  526. select {
  527. case <-ctx.Done():
  528. case chOnRcv <- struct{}{}:
  529. }
  530. }
  531. client := mconnClient.conn
  532. protoWriter := protoio.NewDelimitedWriter(client)
  533. var packet = tmp2p.PacketMsg{
  534. ChannelID: 0x01,
  535. EOF: true,
  536. Data: []byte(`42`),
  537. }
  538. _, err := protoWriter.WriteMsg(mustWrapPacket(&packet))
  539. require.NoError(t, err)
  540. assert.True(t, expectSend(chOnRcv))
  541. packet.ChannelID = int32(1025)
  542. _, err = protoWriter.WriteMsg(mustWrapPacket(&packet))
  543. require.NoError(t, err)
  544. assert.False(t, expectSend(chOnRcv))
  545. }
  546. func waitAll(waiters ...service.Service) func() {
  547. return func() {
  548. switch len(waiters) {
  549. case 0:
  550. return
  551. case 1:
  552. waiters[0].Wait()
  553. return
  554. default:
  555. wg := &sync.WaitGroup{}
  556. for _, w := range waiters {
  557. wg.Add(1)
  558. go func(s service.Service) {
  559. defer wg.Done()
  560. s.Wait()
  561. }(w)
  562. }
  563. wg.Wait()
  564. }
  565. }
  566. }
  567. type closer interface {
  568. Close() error
  569. }
  570. func closeAll(t *testing.T, closers ...closer) func() {
  571. return func() {
  572. for _, s := range closers {
  573. if err := s.Close(); err != nil {
  574. t.Log(err)
  575. }
  576. }
  577. }
  578. }