Merge pull request #70 from uber/message

Add Message with parsing/serialziation support

CHANGELOG.rst

@@ -4,17 +4,21 @@ Releases
 0.6.0 (unreleased)
 ------------------
 
+- Added support for message envelopes. This makes it possible to talk with
+  standard Apache Thrift services and clients. For more information, see
+  :ref:`calling-apache-thrift`.
 - Constant and default values may now be structs or unions, represented in the
-  thrift file as maps with string keys.
+  Thrift file as maps with string keys.
+- Significant performance improvements to the ``BinaryProtocol``
+  implementation.
+- Removed ``thriftrw.wire.TType`` in favor of the ``thriftrw.wire.ttype``
+  module.
 - ``MapValue`` now contains ``MapItem`` objects instead of key-value tuple
   pairs.
-- **Breaking** Removed ``thriftrw.wire.TType`` in favor of the
-  ``thriftrw.wire.ttype`` module.
-- Performance improvements to ``BinaryProtocol`` implementation.
-- TypeSpecs for function arguments and results now have a reference back to the
-  function spec itself.
-- ServiceSpec now provides a ``lookup`` method to look up the specs for
-  functions defined under that service.
+- Request and response ``TypeSpecs`` now have a reference back to the
+  ``FunctionSpec``.
+- ``ServiceSpec`` now provides a ``lookup`` method to look up ``FunctionSpecs``
+  by name.
 
 
 0.5.2 (2015-10-19)

README.rst

@@ -112,12 +112,65 @@ You can use the library to send and receive requests and responses like so,
 
         return blog.dumps(response)
 
+Message Envelopes
+~~~~~~~~~~~~~~~~~
+
+Note that this example sends and receives just the request/response payload. It
+does not wrap the payload in a message envelope as expected by Apache Thrift.
+If you want to send or receive standard Apache Thrift requests to talk to other
+Apache Thrift services, you have to use the ``loads.message`` and
+``dumps.message`` APIs. For example,
+
+.. code-block:: python
+
+    # client.py
+
+    def new_post():
+        post = blog.PostDetails(...)
+        request = BlogService.newPost.request(post)
+        payload = blog.dumps.message(request)
+        # ^ Instead of using blog.dumps, we use blog.dumps.message to indicate
+        # that we want the request wrapped in a message envelope.
+
+
+        response_payload = send_to_server(payload)
+
+        # Similarly, instead of using blog.loads, we use blog.loads.message to
+        # indicate that we want to parse a response stored inside a message.
+        response_message = blog.loads.message(BlogService, response_payload)
+        response = response_message.body
+
+        if response.unauthorized is not None:
+            raise response.unauthorized
+        else:
+            return response.success
+
+
+.. code-block:: python
+
+    # server.py
+
+    def handle_request(request_payload):
+        message = blog.loads.message(BlogService, request_payload)
+        if message.name == 'newPost':
+            request = message.body
+            # ...
+            response = BlogService.newPost.response(success=post_uuid)
+            return blog.dumps.message(response, seqid=message.seqid)
+            # As before, we use blog.dumps.message instead of blog.dumps.
+            # It is important that the server use the same seqid in the
+            # response as what was used in the request, otherwise the client
+            # will not be able to process out-of-order responses.
+        else:
+            raise Exception('Unknown method %s' % message.name)
+
+For more information, see `Overview
+<http://thriftrw.readthedocs.org/en/latest/overview.html>`_.
+
 Caveats
 -------
 
 * Only the Thrift Binary protocol is supported at this time.
-* Message wrappers for Thrift calls and responses are not supported at this
-  time.
 
 Related
 -------

docs/overview.rst

@@ -97,6 +97,64 @@ The generated module contains two top-level functions ``dumps`` and ``loads``.
         contain the object that was requested or if the required fields for it
         were missing.
 
+.. py:function:: dumps.message(obj, seqid=0)
+
+    Serializes the given request or response and puts it inside a message
+    envelope.
+
+    .. code-block:: python
+
+        request = keyvalue.KeyValue.getItem.request('somekey')
+        payload = keyvalue.dumps.message(request, seqid)
+
+    The message type is determined automatically based on whether ``obj`` is a
+    request or a response, and whether it is for a function that is ``oneway``
+    or not.
+
+    For more information, see :ref:`calling-apache-thrift`.
+
+    :param obj:
+        A request or response object. This is obtained by using the ``request``
+        or ``response`` attributes on a
+        :py:class:`thriftrw.spec.ServiceFunction`.
+    :param seqid:
+        If given, this is the sequence ID used for the message envelope.
+        Clients can use the ``seqid`` to match out-of-order responses up with
+        their requests. Servers **must** use the same ``seqid`` in their
+        responses as what they got in the request.
+    :returns:
+        Serialized payload representing the message.
+
+.. py:function:: loads.message(service, payload)
+
+    Deserializes a message containing a request or response for the given
+    service from the payload.
+
+    .. code-block:: python
+
+        message = keyvalue.loads.message(keyvalue.KeyValue, payload)
+        print message.name          # => 'getItem'
+        print message.message_type  # => 1  (CALL)
+        request = message.body
+
+    The service function is resolved based on the name specified in the
+    message, and a request or response is returned based on the message type.
+
+    For more information, see :ref:`calling-apache-thrift`.
+
+    :param service:
+        Service object representing a specific service.
+    :param bytes payload:
+        Payload to parse.
+    :returns thriftrw.value.Message:
+        The parsed Message.
+    :raises thriftrw.errors.UnknownExceptionError:
+        If the message type was ``EXCEPTION``, an ``UnknownExceptionError`` is
+        raised with the parsed exception struct in the body.
+    :raises thriftrw.errors.ThriftProtocolError:
+        If the method name was not recognized or any other payload parsing
+        errors.
+
 .. py:attribute:: services
 
     Collection of all classes generated for all services defined in the source
@@ -355,14 +413,63 @@ Thrift Type     Primitive Type
 ``exception``   ``dict``
 =============   ==============
 
+.. _calling-apache-thrift:
+
+Calling Apache Thrift
+----------------------
+
+The output you get from ``service.dumps(MyService.getFoo.request(..))`` and
+``service.dumps(MyService.getFoo.response(..))`` contains just the serialized
+request or response. This is not enough to talk with Apache Thrift services.
+
+Standard Apache Thrift payloads wrap the serialized request or response in a
+message envelope containing the following additional information:
+
+- The name of the method. This is ``getFoo`` in the example above.
+- Whether this message contains a request, oneway request, unexpected failure,
+  or response.
+- Sequence ID of the message. This lets clients that receive out-of-order
+  responses match them up with their corresponding requests.
+
+Use of message envelopes is **required** if you want to communicate with Apache
+Thrift services or clients generated by Apache Thrift.
+
+To wrap your a request or response in a message envelope, simply use
+``dumps.message`` instead of ``dumps``, and specify a sequence ID.
+
+.. code-block:: python
+
+    request = MyService.getFoo.request(...)
+    payload = service.dumps.message(request, seqid=10)
+
+Similarly, to parse responses wrapped inside message envelopes, use
+``loads.message`` instead of ``loads``.
+
+.. code-block:: python
+
+    def handle_request(request_payload):
+        message = service.loads.message(MyService, request_payload)
+        # It is important that we use the same seqid in the response
+        # envelope.
+        seqid = message.seqid
+        request = message.body
+
+        # ...
+
+        response_payload = service.dumps.message(response, seqid=seqid)
+
+On the server side, it's important that the response ``seqid`` be the same as
+the request ``seqid``.
+
 Differences from Apache Thrift
 ------------------------------
 
 ``thriftrw`` interprets Thrift files slightly differently from Apache Thrift.
 Here are the differences:
 
 -  For ``struct`` and ``exception`` types, every field **MUST** specify whether
-   it is ``required`` or ``optional``.
+   it is ``required`` or ``optional``. This is to ensure that there is no
+   ambiguity around how different code generators handle the default.
 
 -  ``thriftrw`` allows forward references to types and constants. That is,
    types and constants may be referenced that are defined further down in the

examples/.gitignore

@@ -0,0 +1 @@
+ping/

examples/Makefile

@@ -0,0 +1,4 @@
+.PHONY: gen
+
+gen:
+	thrift -out . --gen py ping.thrift

examples/ping.thrift

@@ -0,0 +1,7 @@
+struct Pong {
+    1: required double timestamp
+}
+
+service Ping {
+    Pong ping(1: string name);
+}

examples/ping_client_requests.py

@@ -0,0 +1,28 @@
+from __future__ import absolute_import, unicode_literals, print_function
+
+import os.path
+import requests
+
+import thriftrw
+
+ping = thriftrw.load(
+    os.path.join(os.path.dirname(__file__), 'ping.thrift'),
+)
+
+
+def main():
+    req = ping.Ping.ping.request('world')
+
+    response = requests.post(
+        'http://127.0.0.1:8888/thrift',
+        data=ping.dumps.message(req, seqid=42),
+    )
+    reply = ping.loads.message(ping.Ping, response.content)
+    assert reply.name == 'ping'
+    assert reply.seqid == 42
+
+    resp = reply.body
+    print(resp)
+
+if __name__ == "__main__":
+    main()

examples/ping_client_thrift.py

@@ -0,0 +1,13 @@
+from __future__ import absolute_import
+
+from thrift.transport.THttpClient import THttpClient
+from thrift.protocol.TBinaryProtocol import TBinaryProtocolAccelerated
+
+from ping import Ping
+
+
+trans = THttpClient('http://localhost:8888/thrift')
+proto = TBinaryProtocolAccelerated(trans)
+client = Ping.Client(proto)
+
+print client.ping('world')

examples/ping_server_tornado.py

@@ -0,0 +1,44 @@
+from __future__ import absolute_import, unicode_literals, print_function
+
+import time
+import os.path
+
+from tornado import web
+from tornado.ioloop import IOLoop
+from tornado.httpserver import HTTPServer
+
+import thriftrw
+
+ping = thriftrw.load(
+    os.path.join(os.path.dirname(__file__), 'ping.thrift'),
+)
+
+
+class ThriftRequestHandler(web.RequestHandler):
+
+    def post(self):
+        assert self.request.body
+
+        message = ping.loads.message(ping.Ping, self.request.body)
+        method, handler = self._METHODS[message.name]
+
+        args = message.body
+        resp = method.response(success=handler(self, args))
+
+        reply = ping.dumps.message(resp, seqid=message.seqid)
+        self.write(reply)
+
+    def handle_ping(self, args):
+        print('Hello, %s' % args.name)
+        return ping.Pong(time.time())
+
+    _METHODS = {'ping': (ping.Ping.ping, handle_ping)}
+
+
+if __name__ == "__main__":
+    app = web.Application([
+        (r'/thrift', ThriftRequestHandler),
+    ])
+    HTTPServer(app).listen(8888)
+    print('Listening on http://127.0.0.1:8888/thrift')
+    IOLoop.current().start()

setup.py

@@ -13,6 +13,8 @@
 cython_modules = [
     'thriftrw._buffer',
     'thriftrw._cython',
+    'thriftrw._runtime',
+    'thriftrw.protocol.core',
     'thriftrw.protocol.binary',
     'thriftrw.spec.base',
     'thriftrw.spec.common',
@@ -27,8 +29,10 @@
     'thriftrw.spec.struct',
     'thriftrw.spec.typedef',
     'thriftrw.spec.union',
-    'thriftrw.wire.value',
+    'thriftrw.wire.message',
+    'thriftrw.wire.mtype',
     'thriftrw.wire.ttype',
+    'thriftrw.wire.value',
 ]
 
 # If Cython is available we will re-cythonize the pyx files, otherwise we just

tests/compile/test_spec.py

@@ -58,6 +58,16 @@ def test_primitive_wire_conversion(args):
     assert obj == t_spec.from_wire(value)
 
 
+@pytest.mark.parametrize('t_spec, value', [
+    (spec.BoolTypeSpec, vbyte(1)),
+    (spec.ByteTypeSpec, vbool(True)),
+    (spec.ListTypeSpec(spec.I32TypeSpec), vset(ttype.I32, vi32(42))),
+])
+def test_ttype_mismatch(t_spec, value):
+    with pytest.raises(ValueError):
+        t_spec.from_wire(value)
+
+
 @pytest.mark.parametrize('t_spec, pairs, obj', [
     (spec.MapTypeSpec(spec.TextTypeSpec, spec.I32TypeSpec), [], {}),
     (

tests/conftest.py

@@ -26,6 +26,7 @@
 import pytest
 
 from thriftrw.loader import Loader
+from thriftrw.protocol import BinaryProtocol
 
 
 def unimport(*names):
@@ -74,4 +75,4 @@ def test_something():
 
 @pytest.fixture
 def loads(request):
-    return partial(Loader().loads, request.node.name)
+    return partial(Loader(BinaryProtocol()).loads, request.node.name)