@ -3,6 +3,7 @@ package server
import ( import (
"bufio" "bufio"
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"net" "net"
@ -26,22 +27,21 @@ type SocketServer struct {
listener net . Listener listener net . Listener
connsMtx sync . Mutex connsMtx sync . Mutex
conns map [ int ] net . Conn
connsClose map [ int ] func ( )
nextConnID int nextConnID int
appMtx sync . Mutex
app types . Application
app types . Application
} }
func NewSocketServer ( logger log . Logger , protoAddr string , app types . Application ) service . Service { func NewSocketServer ( logger log . Logger , protoAddr string , app types . Application ) service . Service {
proto , addr := tmnet . ProtocolAndAddress ( protoAddr ) proto , addr := tmnet . ProtocolAndAddress ( protoAddr )
s := & SocketServer { s := & SocketServer {
logger : logger ,
proto : proto ,
addr : addr ,
listener : nil ,
app : app ,
conns : make ( map [ int ] net . Conn ) ,
logger : logger ,
proto : proto ,
addr : addr ,
listener : nil ,
app : app ,
connsClose : make ( map [ int ] func ( ) ) ,
} }
s . BaseService = * service . NewBaseService ( logger , "ABCIServer" , s ) s . BaseService = * service . NewBaseService ( logger , "ABCIServer" , s )
return s return s
@ -67,44 +67,35 @@ func (s *SocketServer) OnStop() {
s . connsMtx . Lock ( ) s . connsMtx . Lock ( )
defer s . connsMtx . Unlock ( ) defer s . connsMtx . Unlock ( )
for id , conn := range s . conns {
delete ( s . conns , id )
if err := conn . Close ( ) ; err != nil {
s . logger . Error ( "error closing connection" , "id" , id , "conn" , conn , "err" , err )
}
for _ , closer := range s . connsClose {
closer ( )
} }
} }
func ( s * SocketServer ) addConn ( conn net . Conn ) int {
func ( s * SocketServer ) addConn ( closer func ( ) ) int {
s . connsMtx . Lock ( ) s . connsMtx . Lock ( )
defer s . connsMtx . Unlock ( ) defer s . connsMtx . Unlock ( )
connID := s . nextConnID connID := s . nextConnID
s . nextConnID ++ s . nextConnID ++
s . conns [ connID ] = conn
s . connsClose [ connID ] = closer
return connID return connID
} }
// deletes conn even if close errs
// deletes conn even if close errs
func ( s * SocketServer ) rmConn ( connID int ) error {
func ( s * SocketServer ) rmConn ( connID int ) {
s . connsMtx . Lock ( ) s . connsMtx . Lock ( )
defer s . connsMtx . Unlock ( ) defer s . connsMtx . Unlock ( )
conn , ok := s . conns [ connID ]
if ! ok {
return fmt . Errorf ( "connection %d does not exist" , connID )
if closer , ok := s . connsClose [ connID ] ; ok {
closer ( )
delete ( s . connsClose , connID )
} }
delete ( s . conns , connID )
return conn . Close ( )
} }
func ( s * SocketServer ) acceptConnectionsRoutine ( ctx context . Context ) { func ( s * SocketServer ) acceptConnectionsRoutine ( ctx context . Context ) {
for { for {
if ctx . Err ( ) != nil { if ctx . Err ( ) != nil {
return return
} }
// Accept a connection
// Accept a connection
@ -118,149 +109,134 @@ func (s *SocketServer) acceptConnectionsRoutine(ctx context.Context) {
continue continue
} }
s . logger . Info ( "Accepted a new connection" )
cctx , ccancel := context . WithCancel ( ctx )
connID := s . addConn ( ccancel )
connID := s . addConn ( conn )
s . logger . Info ( "Accepted a new connection" , "id" , connID )
closeConn := make ( chan error , 2 ) // Push to signal connection closed
responses := make ( chan * types . Response , 1000 ) // A channel to buffer responses
responses := make ( chan * types . Response , 1000 ) // A channel to buffer responses
// Read requests from conn and deal with them
go s . handleRequests ( ctx , closeConn , conn , responses )
// Pull responses from 'responses' and write them to conn.
go s . handleResponses ( ctx , closeConn , conn , responses )
// Wait until signal to close connection
go s . waitForClose ( ctx , closeConn , connID )
}
}
func ( s * SocketServer ) waitForClose ( ctx context . Context , closeConn chan error , connID int ) {
defer func ( ) {
// Close the connection
if err := s . rmConn ( connID ) ; err != nil {
s . logger . Error ( "error closing connection" , "err" , err )
once := & sync . Once { }
closer := func ( err error ) {
ccancel ( )
once . Do ( func ( ) {
if cerr := conn . Close ( ) ; err != nil {
s . logger . Error ( "error closing connection" ,
"id" , connID ,
"close_err" , cerr ,
"err" , err )
}
s . rmConn ( connID )
switch {
case errors . Is ( err , context . Canceled ) :
s . logger . Error ( "Connection terminated" ,
"id" , connID ,
"err" , err )
case errors . Is ( err , context . DeadlineExceeded ) :
s . logger . Error ( "Connection encountered timeout" ,
"id" , connID ,
"err" , err )
case errors . Is ( err , io . EOF ) :
s . logger . Error ( "Connection was closed by client" ,
"id" , connID )
case err != nil :
s . logger . Error ( "Connection error" ,
"id" , connID ,
"err" , err )
default :
s . logger . Error ( "Connection was closed" ,
"id" , connID )
}
} )
} }
} ( )
select {
case <- ctx . Done ( ) :
return
case err := <- closeConn :
switch {
case err == io . EOF :
s . logger . Error ( "Connection was closed by client" )
case err != nil :
s . logger . Error ( "Connection error" , "err" , err )
default :
// never happens
s . logger . Error ( "Connection was closed" )
}
// Read requests from conn and deal with them
go s . handleRequests ( cctx , closer , conn , responses )
// Pull responses from 'responses' and write them to conn.
go s . handleResponses ( cctx , closer , conn , responses )
} }
} }
// Read requests from conn and deal with them
// Read requests from conn and deal with them
func ( s * SocketServer ) handleRequests ( func ( s * SocketServer ) handleRequests (
ctx context . Context , ctx context . Context ,
closeConn chan error ,
closer func ( error ) ,
conn io . Reader , conn io . Reader ,
responses chan <- * types . Response , responses chan <- * types . Response ,
) { ) {
var count int
var bufReader = bufio . NewReader ( conn ) var bufReader = bufio . NewReader ( conn )
defer func ( ) { defer func ( ) {
// make sure to recover from any app-related panics to allow proper socket cleanup
// make sure to recover from any app-related panics to allow proper socket cleanup
r := recover ( )
if r != nil {
if r := recover ( ) ; r != nil {
const size = 64 << 10 const size = 64 << 10
buf := make ( [ ] byte , size ) buf := make ( [ ] byte , size )
buf = buf [ : runtime . Stack ( buf , false ) ] buf = buf [ : runtime . Stack ( buf , false ) ]
err := fmt . Errorf ( "recovered from panic: %v\n%s" , r , buf )
closeConn <- err
s . appMtx . Unlock ( )
closer ( fmt . Errorf ( "recovered from panic: %v\n%s" , r , buf ) )
} }
} ( ) } ( )
for { for {
if ctx . Err ( ) != nil {
req := & types . Request { }
if err := types . ReadMessage ( bufReader , req ) ; err != nil {
closer ( fmt . Errorf ( "error reading message: %w" , err ) )
return return
} }
var req = & types . Request { }
err := types . ReadMessage ( bufReader , req )
if err != nil {
if err == io . EOF {
closeConn <- err
} else {
closeConn <- fmt . Errorf ( "error reading message: %w" , err )
}
resp := s . processRequest ( req )
select {
case <- ctx . Done ( ) :
closer ( ctx . Err ( ) )
return return
case responses <- resp :
} }
s . appMtx . Lock ( )
count ++
s . handleRequest ( req , responses )
s . appMtx . Unlock ( )
} }
} }
func ( s * SocketServer ) handle Request( req * types . Request , responses chan <- * types . Response ) {
func ( s * SocketServer ) processRequest ( req * types . Request ) * types . Response {
switch r := req . Value . ( type ) { switch r := req . Value . ( type ) {
case * types . Request_Echo : case * types . Request_Echo :
responses <- types . ToResponseEcho ( r . Echo . Message )
return types . ToResponseEcho ( r . Echo . Message )
case * types . Request_Flush : case * types . Request_Flush :
responses <- types . ToResponseFlush ( )
return types . ToResponseFlush ( )
case * types . Request_Info : case * types . Request_Info :
res := s . app . Info ( * r . Info )
responses <- types . ToResponseInfo ( res )
return types . ToResponseInfo ( s . app . Info ( * r . Info ) )
case * types . Request_CheckTx : case * types . Request_CheckTx :
res := s . app . CheckTx ( * r . CheckTx )
responses <- types . ToResponseCheckTx ( res )
return types . ToResponseCheckTx ( s . app . CheckTx ( * r . CheckTx ) )
case * types . Request_Commit : case * types . Request_Commit :
res := s . app . Commit ( )
responses <- types . ToResponseCommit ( res )
return types . ToResponseCommit ( s . app . Commit ( ) )
case * types . Request_Query : case * types . Request_Query :
res := s . app . Query ( * r . Query )
responses <- types . ToResponseQuery ( res )
return types . ToResponseQuery ( s . app . Query ( * r . Query ) )
case * types . Request_InitChain : case * types . Request_InitChain :
res := s . app . InitChain ( * r . InitChain )
responses <- types . ToResponseInitChain ( res )
return types . ToResponseInitChain ( s . app . InitChain ( * r . InitChain ) )
case * types . Request_ListSnapshots : case * types . Request_ListSnapshots :
res := s . app . ListSnapshots ( * r . ListSnapshots )
responses <- types . ToResponseListSnapshots ( res )
return types . ToResponseListSnapshots ( s . app . ListSnapshots ( * r . ListSnapshots ) )
case * types . Request_OfferSnapshot : case * types . Request_OfferSnapshot :
res := s . app . OfferSnapshot ( * r . OfferSnapshot )
responses <- types . ToResponseOfferSnapshot ( res )
return types . ToResponseOfferSnapshot ( s . app . OfferSnapshot ( * r . OfferSnapshot ) )
case * types . Request_PrepareProposal : case * types . Request_PrepareProposal :
res := s . app . PrepareProposal ( * r . PrepareProposal )
responses <- types . ToResponsePrepareProposal ( res )
return types . ToResponsePrepareProposal ( s . app . PrepareProposal ( * r . PrepareProposal ) )
case * types . Request_ProcessProposal : case * types . Request_ProcessProposal :
res := s . app . ProcessProposal ( * r . ProcessProposal )
responses <- types . ToResponseProcessProposal ( res )
return types . ToResponseProcessProposal ( s . app . ProcessProposal ( * r . ProcessProposal ) )
case * types . Request_LoadSnapshotChunk : case * types . Request_LoadSnapshotChunk :
res := s . app . LoadSnapshotChunk ( * r . LoadSnapshotChunk )
responses <- types . ToResponseLoadSnapshotChunk ( res )
return types . ToResponseLoadSnapshotChunk ( s . app . LoadSnapshotChunk ( * r . LoadSnapshotChunk ) )
case * types . Request_ApplySnapshotChunk : case * types . Request_ApplySnapshotChunk :
res := s . app . ApplySnapshotChunk ( * r . ApplySnapshotChunk )
responses <- types . ToResponseApplySnapshotChunk ( res )
return types . ToResponseApplySnapshotChunk ( s . app . ApplySnapshotChunk ( * r . ApplySnapshotChunk ) )
case * types . Request_ExtendVote : case * types . Request_ExtendVote :
res := s . app . ExtendVote ( * r . ExtendVote )
responses <- types . ToResponseExtendVote ( res )
return types . ToResponseExtendVote ( s . app . ExtendVote ( * r . ExtendVote ) )
case * types . Request_VerifyVoteExtension : case * types . Request_VerifyVoteExtension :
res := s . app . VerifyVoteExtension ( * r . VerifyVoteExtension )
responses <- types . ToResponseVerifyVoteExtension ( res )
return types . ToResponseVerifyVoteExtension ( s . app . VerifyVoteExtension ( * r . VerifyVoteExtension ) )
case * types . Request_FinalizeBlock : case * types . Request_FinalizeBlock :
res := s . app . FinalizeBlock ( * r . FinalizeBlock )
responses <- types . ToResponseFinalizeBlock ( res )
return types . ToResponseFinalizeBlock ( s . app . FinalizeBlock ( * r . FinalizeBlock ) )
default : default :
responses <- types . ToResponseException ( "Unknown request" )
return types . ToResponseException ( "Unknown request" )
} }
} }
// Pull responses from 'responses' and write them to conn.
// Pull responses from 'responses' and write them to conn.
func ( s * SocketServer ) handleResponses ( func ( s * SocketServer ) handleResponses (
ctx context . Context , ctx context . Context ,
closeConn chan error ,
closer func ( error ) ,
conn io . Writer , conn io . Writer ,
responses <- chan * types . Response , responses <- chan * types . Response ,
) { ) {
@ -268,21 +244,15 @@ func (s *SocketServer) handleResponses(
for { for {
select { select {
case <- ctx . Done ( ) : case <- ctx . Done ( ) :
closer ( ctx . Err ( ) )
return return
case res := <- responses : case res := <- responses :
if err := types . WriteMessage ( res , bw ) ; err != nil { if err := types . WriteMessage ( res , bw ) ; err != nil {
select {
case <- ctx . Done ( ) :
case closeConn <- fmt . Errorf ( "error writing message: %w" , err ) :
}
closer ( fmt . Errorf ( "error writing message: %w" , err ) )
return return
} }
if err := bw . Flush ( ) ; err != nil { if err := bw . Flush ( ) ; err != nil {
select {
case <- ctx . Done ( ) :
case closeConn <- fmt . Errorf ( "error flushing write buffer: %w" , err ) :
}
closer ( fmt . Errorf ( "error writing message: %w" , err ) )
return return
} }
} }
Sam Kleinman
3 years ago
GitHub