From bb4a58aa0a3d08e77d2fac15d98cee78280453f7 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Sun, 6 Dec 2015 18:18:13 -0500 Subject: [PATCH 1/3] add counter app in python --- cmd/tmsp/cli.go | 20 +++++ example/python/app.py | 83 +++++++++++++++++++++ example/python/tmsp/msg.py | 54 ++++++++++++++ example/python/tmsp/reader.py | 31 ++++++++ example/python/tmsp/server.py | 134 ++++++++++++++++++++++++++++++++++ example/python/tmsp/wire.py | 99 +++++++++++++++++++++++++ 6 files changed, 421 insertions(+) create mode 100644 example/python/app.py create mode 100644 example/python/tmsp/msg.py create mode 100644 example/python/tmsp/reader.py create mode 100644 example/python/tmsp/server.py create mode 100644 example/python/tmsp/wire.py diff --git a/cmd/tmsp/cli.go b/cmd/tmsp/cli.go index 5997cf5fe..9f06c3f78 100644 --- a/cmd/tmsp/cli.go +++ b/cmd/tmsp/cli.go @@ -45,6 +45,13 @@ func main() { cmdConsole(app, c) }, }, + { + Name: "echo", + Usage: "Have the application echo a message", + Action: func(c *cli.Context) { + cmdEcho(c) + }, + }, { Name: "info", Usage: "Get some info about the application", @@ -140,6 +147,19 @@ func cmdConsole(app *cli.App, c *cli.Context) { } } +// Have the application echo a message +func cmdEcho(c *cli.Context) { + args := c.Args() + if len(args) != 1 { + Exit("echo takes 1 argument") + } + res, err := makeRequest(conn, types.RequestEcho{args[0]}) + if err != nil { + Exit(err.Error()) + } + fmt.Println(res) +} + // Get some info from the application func cmdInfo(c *cli.Context) { res, err := makeRequest(conn, types.RequestInfo{}) diff --git a/example/python/app.py b/example/python/app.py new file mode 100644 index 000000000..f77976985 --- /dev/null +++ b/example/python/app.py @@ -0,0 +1,83 @@ + +import sys +sys.path.insert(0, './tmsp') + +from wire import * +from server import * + + +# tmsp application interface + +class CounterApplication(): + def __init__(self): + self.hashCount = 0 + self.txCount = 0 + self.commitCount = 0 + + def open(self): + return CounterAppContext(self) + +class CounterAppContext(): + def __init__(self, app): + self.app = app + self.hashCount = app.hashCount + self.txCount = app.txCount + self.commitCount = app.commitCount + self.serial = False + + def echo(self, msg): + return msg, 0 + + def info(self): + return ["hash, tx, commit counts:%d, %d, %d"%(self.hashCount, self.txCount, self.commitCount)], 0 + + def set_option(self, key, value): + if key == "serial" and value == "on": + self.serial = True + return 0 + + def append_tx(self, txBytes): + if self.serial: + txValue = decode_big_endian(BytesReader(txBytes), len(txBytes)) + if txValue != self.txCount: + return [], 1 + self.txCount += 1 + return None, 0 + + def get_hash(self): + self.hashCount += 1 + if self.txCount == 0: + return "", 0 + return str(encode_big_endian(self.txCount, 8)), 0 + + def commit(self): + return 0 + + def rollback(self): + return 0 + + def add_listener(self): + return 0 + + def rm_listener(self): + return 0 + + def event(self): + return + + +if __name__ == '__main__': + l = len(sys.argv) + if l == 1: + port = 46658 + elif l == 2: + port = int(sys.argv[1]) + else: + print "too many arguments" + quit() + + print 'TMSP Demo APP (Python)' + + app = CounterApplication() + server = TMSPServer(app, port) + server.main_loop() diff --git a/example/python/tmsp/msg.py b/example/python/tmsp/msg.py new file mode 100644 index 000000000..a99386a3b --- /dev/null +++ b/example/python/tmsp/msg.py @@ -0,0 +1,54 @@ +from wire import * + +# map type_byte to message name +message_types = { +0x01 : "echo", +0x02 : "flush", +0x03 : "info", +0x04 : "set_option", +0x21 : "append_tx", +0x22 : "get_hash", +0x23 : "commit", +0x24 : "rollback", +0x25 : "add_listener", +0x26 : "rm_listener", +} + +# return the decoded arguments of tmsp messages +class RequestDecoder(): + def __init__(self, reader): + self.reader = reader + + def echo(self): + return decode_string(self.reader) + + def flush(self): + return + + def info(self): + return + + def set_option(self): + return decode_string(self.reader), decode_string(self.reader) + + def append_tx(self): + return decode_string(self.reader) + + def get_hash(self): + return + + def commit(self): + return + + def rollback(self): + return + + def add_listener(self): + # TODO + return + + def rm_listener(self): + # TODO + return + + diff --git a/example/python/tmsp/reader.py b/example/python/tmsp/reader.py new file mode 100644 index 000000000..edb88e67b --- /dev/null +++ b/example/python/tmsp/reader.py @@ -0,0 +1,31 @@ + +# Simple read() method around a bytearray +class BytesReader(): + def __init__(self, b): + self.buf = b + + def read(self, n): + if len(self.buf) < n: + # TODO: exception + return + r = self.buf[:n] + self.buf = self.buf[n:] + return r + +# Buffer bytes off a tcp connection and read them off in chunks +class ConnReader(): + def __init__(self, conn): + self.conn = conn + self.buf = bytearray() + + # blocking + def read(self, n): + while n > len(self.buf): + moreBuf = self.conn.recv(1024) + if not moreBuf: + raise IOError("dead connection") + self.buf = self.buf + bytearray(moreBuf) + + r = self.buf[:n] + self.buf = self.buf[n:] + return r diff --git a/example/python/tmsp/server.py b/example/python/tmsp/server.py new file mode 100644 index 000000000..cc1970c45 --- /dev/null +++ b/example/python/tmsp/server.py @@ -0,0 +1,134 @@ +import socket +import select +import sys +import os + + +from wire import * +from reader import * +from msg import * + +# TMSP server responds to messges by calling methods on the app +class TMSPServer(): + def __init__(self, app, port=5410): + self.app = app + self.appMap = {} # map conn file descriptors to (appContext, msgDecoder) + + self.port = port + self.listen_backlog = 10 + + self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.listener.setblocking(0) + self.listener.bind(('', port)) + + self.listener.listen(self.listen_backlog) + + self.shutdown = False + + self.read_list = [self.listener] + self.write_list = [] + + def handle_new_connection(self, r): + new_fd, new_addr = r.accept() + self.read_list.append(new_fd) + self.write_list.append(new_fd) + print 'new connection to', new_addr + + appContext = self.app.open() + self.appMap[new_fd] = (appContext, RequestDecoder(ConnReader(new_fd))) + + def handle_conn_closed(self, r): + self.read_list.remove(r) + self.write_list.remove(r) + r.close() + print "connection closed" + + def handle_recv(self, r): + appCtx, conn = self.appMap[r] + response = bytearray() + while True: + try: + # first read the request type and get the msg decoder + typeByte = conn.reader.read(1) + typeByte = int(typeByte[0]) + resTypeByte = typeByte+0x10 + req_type = message_types[typeByte] + + if req_type == "flush": + response += bytearray([resTypeByte]) + sent = r.send(str(response)) + return + + decoder = getattr(conn, req_type) + + req_args = decoder() + req_f = getattr(appCtx, req_type) + if req_args == None: + res = req_f() + elif isinstance(req_args, tuple): + res = req_f(*req_args) + else: + res = req_f(req_args) + + if isinstance(res, tuple): + res, ret_code = res + else: + ret_code = res + res = None + + if ret_code != 0: + print "non-zero retcode:", ret_code + return + + if req_type in ("echo", "info"): # these dont return a ret code + response += bytearray([resTypeByte]) + encode(res) + else: + response += bytearray([resTypeByte]) + encode(ret_code) + encode(res) + except TypeError as e: + print "TypeError on reading from connection:", e + self.handle_conn_closed(r) + return + except ValueError as e: + print "ValueError on reading from connection:", e + self.handle_conn_closed(r) + return + except IOError as e: + print "IOError on reading from connection:", e + self.handle_conn_closed(r) + return + except: + print "error reading from connection", sys.exc_info()[0] # TODO better + self.handle_conn_closed(r) + return + + def main_loop(self): + while not self.shutdown: + r_list, w_list, _ = select.select(self.read_list, self.write_list, [], 2.5) + + for r in r_list: + if (r == self.listener): + try: + self.handle_new_connection(r) + + # undo adding to read list ... + except NameError as e: + print "Could not connect due to NameError:", e + except TypeError as e: + print "Could not connect due to TypeError:", e + except: + print "Could not connect due to unexpected error:", sys.exc_info()[0] + else: + self.handle_recv(r) + + + + def handle_shutdown(self): + for r in self.read_list: + r.close() + for w in self.write_list: + try: + w.close() + except: pass + self.shutdown = True + diff --git a/example/python/tmsp/wire.py b/example/python/tmsp/wire.py new file mode 100644 index 000000000..e1d37effb --- /dev/null +++ b/example/python/tmsp/wire.py @@ -0,0 +1,99 @@ + +# the decoder works off a reader +# the encoder returns bytearray + +def bytes2hex(b): + if type(b) in (str, unicode): + return "".join([hex(ord(c))[2:].zfill(2) for c in b]) + else: + return bytes2hex(b.decode()) + + +# expects uvarint64 (no crazy big nums!) +def uvarint_size(i): + if i == 0: + return 0 + for j in xrange(1, 8): + if i < 1< int(0xF0) else False + if negate: size = size -0xF0 + i = decode_big_endian(reader, size) + if negate: i = i*(-1) + return i + +def encode_string(s): + size = encode_varint(len(s)) + return size + bytearray(s) + +def decode_string(reader): + length = decode_varint(reader) + return str(reader.read(length)) + +def encode_list(s): + b = bytearray() + map(b.extend, map(encode, s)) + return encode_varint(len(s)) + b + +def encode(s): + if s == None: + return bytearray() + if isinstance(s, int): + return encode_varint(s) + elif isinstance(s, str): + return encode_string(s) + elif isinstance(s, list): + return encode_list(s) + else: + print "UNSUPPORTED TYPE!", type(s), s + + +import binascii + +if __name__ == '__main__': + ns = [100,100,1000,256] + ss = [2,5,5,2] + bs = map(encode_big_endian, ns,ss) + ds = map(decode_big_endian, bs,ss) + print ns + print [i[0] for i in ds] + + ss = ["abc", "hi there jim", "ok now what"] + e = map(encode_string, ss) + d = map(decode_string, e) + print ss + print [i[0] for i in d] From af2a1a6fc10e22513bb3e04c8dd855a9406971bc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Mon, 14 Dec 2015 18:00:18 -0500 Subject: [PATCH 2/3] python fixes, tests --- cmd/tmsp/cli.go | 2 + example/python/app.py | 11 +++-- example/python/tmsp/reader.py | 1 + example/python/tmsp/server.py | 2 +- example/python/tmsp/wire.py | 3 ++ tests/test.sh | 13 ++++++ tests/test_counter.sh | 75 ++++++++++++++++++++++++++++++++++ test.sh => tests/test_dummy.sh | 47 --------------------- 8 files changed, 103 insertions(+), 51 deletions(-) create mode 100644 tests/test.sh create mode 100644 tests/test_counter.sh rename test.sh => tests/test_dummy.sh (56%) diff --git a/cmd/tmsp/cli.go b/cmd/tmsp/cli.go index 9f06c3f78..75cd921d3 100644 --- a/cmd/tmsp/cli.go +++ b/cmd/tmsp/cli.go @@ -121,6 +121,8 @@ func cmdBatch(app *cli.App, c *cli.Context) { Exit("input line is too long") } else if err == io.EOF { break + } else if len(line) == 0 { + continue } else if err != nil { Exit(err.Error()) } diff --git a/example/python/app.py b/example/python/app.py index f77976985..a9e9d3b75 100644 --- a/example/python/app.py +++ b/example/python/app.py @@ -38,9 +38,12 @@ class CounterAppContext(): def append_tx(self, txBytes): if self.serial: - txValue = decode_big_endian(BytesReader(txBytes), len(txBytes)) + txByteArray = bytearray(txBytes) + if len(txBytes) >= 2 and txBytes[:2] == "0x": + txByteArray = hex2bytes(txBytes[2:]) + txValue = decode_big_endian(BytesReader(txByteArray), len(txBytes)) if txValue != self.txCount: - return [], 1 + return None, 1 self.txCount += 1 return None, 0 @@ -48,7 +51,9 @@ class CounterAppContext(): self.hashCount += 1 if self.txCount == 0: return "", 0 - return str(encode_big_endian(self.txCount, 8)), 0 + h = encode_big_endian(self.txCount, 8) + h.reverse() + return str(h), 0 def commit(self): return 0 diff --git a/example/python/tmsp/reader.py b/example/python/tmsp/reader.py index edb88e67b..f1b3dfae4 100644 --- a/example/python/tmsp/reader.py +++ b/example/python/tmsp/reader.py @@ -6,6 +6,7 @@ class BytesReader(): def read(self, n): if len(self.buf) < n: + print "reader err: buf less than n" # TODO: exception return r = self.buf[:n] diff --git a/example/python/tmsp/server.py b/example/python/tmsp/server.py index cc1970c45..e25e4e1fd 100644 --- a/example/python/tmsp/server.py +++ b/example/python/tmsp/server.py @@ -77,9 +77,9 @@ class TMSPServer(): ret_code = res res = None + print "called", req_type, "ret code:", ret_code if ret_code != 0: print "non-zero retcode:", ret_code - return if req_type in ("echo", "info"): # these dont return a ret code response += bytearray([resTypeByte]) + encode(res) diff --git a/example/python/tmsp/wire.py b/example/python/tmsp/wire.py index e1d37effb..fde4e1605 100644 --- a/example/python/tmsp/wire.py +++ b/example/python/tmsp/wire.py @@ -2,6 +2,9 @@ # the decoder works off a reader # the encoder returns bytearray +def hex2bytes(h): + return bytearray(h.decode('hex')) + def bytes2hex(b): if type(b) in (str, unicode): return "".join([hex(ord(c))[2:].zfill(2) for c in b]) diff --git a/tests/test.sh b/tests/test.sh new file mode 100644 index 000000000..848d14cbc --- /dev/null +++ b/tests/test.sh @@ -0,0 +1,13 @@ + +ROOT=$GOPATH/src/github.com/tendermint/tmsp +cd $ROOT + +# test golang dummy +bash tests/test_dummy.sh + +# test golang counter +bash tests/test_counter.sh + +# test python counter +cd example/python +COUNTER_APP="python app.py" bash $ROOT/tests/test_counter.sh diff --git a/tests/test_counter.sh b/tests/test_counter.sh new file mode 100644 index 000000000..41909dde8 --- /dev/null +++ b/tests/test_counter.sh @@ -0,0 +1,75 @@ + +# so we can test other languages +if [[ "$COUNTER_APP" == "" ]]; then + COUNTER_APP="counter" +fi + +echo "Testing counter app for: $COUNTER_APP" + +# run the counter app +$COUNTER_APP &> /dev/null & +PID=`echo $!` + +if [[ "$?" != 0 ]]; then + echo "Error running tmsp command" + echo $OUTPUT + exit 1 +fi + +sleep 1 +OUTPUT=`(tmsp batch) < /dev/null +if [[ "$?" == "0" ]]; then + kill -9 $PID +fi + diff --git a/test.sh b/tests/test_dummy.sh similarity index 56% rename from test.sh rename to tests/test_dummy.sh index 2f4458612..b88475dfb 100755 --- a/test.sh +++ b/tests/test_dummy.sh @@ -51,50 +51,3 @@ echo "" kill $PID sleep 1 - -# test the counter app -echo "Counter test ..." -counter &> /dev/null & -PID=`echo $!` -sleep 1 -OUTPUT=`(tmsp batch) < Date: Mon, 14 Dec 2015 18:06:56 -0500 Subject: [PATCH 3/3] example/golang --- README.md | 5 ++++- cmd/counter/main.go | 2 +- cmd/dummy/main.go | 2 +- example/{ => golang}/counter.go | 0 example/{ => golang}/dummy.go | 0 example/{ => golang}/dummy_test.go | 0 6 files changed, 6 insertions(+), 3 deletions(-) rename example/{ => golang}/counter.go (100%) rename example/{ => golang}/dummy.go (100%) rename example/{ => golang}/dummy_test.go (100%) diff --git a/README.md b/README.md index 66d5f9fb4..7e17768dd 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # Tendermint Socket Protocol (TMSP) -**TMSP** is a socket protocol, which means applications can be written in any programming language. +**TMSP** is a socket protocol enabling a consensus engine, running in one process, +to manage an application state, running in another. +Thus the applications can be written in any programming language. + TMSP is an asynchronous protocol: message responses are written back asynchronously to the platform. *Applications must be deterministic.* diff --git a/cmd/counter/main.go b/cmd/counter/main.go index f36394e7d..3e770a43d 100644 --- a/cmd/counter/main.go +++ b/cmd/counter/main.go @@ -4,7 +4,7 @@ import ( "flag" . "github.com/tendermint/go-common" - "github.com/tendermint/tmsp/example" + "github.com/tendermint/tmsp/example/golang" "github.com/tendermint/tmsp/server" ) diff --git a/cmd/dummy/main.go b/cmd/dummy/main.go index fe5077bb5..ce9bf8d0b 100644 --- a/cmd/dummy/main.go +++ b/cmd/dummy/main.go @@ -2,7 +2,7 @@ package main import ( . "github.com/tendermint/go-common" - "github.com/tendermint/tmsp/example" + "github.com/tendermint/tmsp/example/golang" "github.com/tendermint/tmsp/server" ) diff --git a/example/counter.go b/example/golang/counter.go similarity index 100% rename from example/counter.go rename to example/golang/counter.go diff --git a/example/dummy.go b/example/golang/dummy.go similarity index 100% rename from example/dummy.go rename to example/golang/dummy.go diff --git a/example/dummy_test.go b/example/golang/dummy_test.go similarity index 100% rename from example/dummy_test.go rename to example/golang/dummy_test.go