Skip to content

Commit

Permalink
Improve JSONL binary serialization performance
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Sep 6, 2024
1 parent 08b58bf commit d23a8ab
Showing 1 changed file with 17 additions and 1 deletion.
18 changes: 17 additions & 1 deletion singer_sdk/_singerlib/encoding/_msgspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,22 @@ def dec_hook(type: type, obj: t.Any) -> t.Any: # noqa: ARG001, A002, ANN401

encoder = msgspec.json.Encoder(enc_hook=enc_hook, decimal_format="number")
decoder = msgspec.json.Decoder(dec_hook=dec_hook, float_hook=decimal.Decimal)
_jsonl_msg_buffer = bytearray(64)


def serialize_jsonl(obj: object, **kwargs: t.Any) -> bytes: # noqa: ARG001
"""Serialize a dictionary into a line of jsonl.
Args:
obj: A Python object usually a dict.
**kwargs: Optional key word arguments.
Returns:
A bytes of serialized json.
"""
encoder.encode_into(obj, _jsonl_msg_buffer)
_jsonl_msg_buffer.extend(b"\n")
return _jsonl_msg_buffer


class MsgSpecReader(GenericSingerReader[str]):
Expand Down Expand Up @@ -82,7 +98,7 @@ def serialize_message(self, message: Message) -> bytes: # noqa: PLR6301
Returns:
A string of serialized json.
"""
return encoder.encode(message.to_dict())
return serialize_jsonl(message.to_dict())

def write_message(self, message: Message) -> None:
"""Write a message to stdout.
Expand Down

0 comments on commit d23a8ab

Please sign in to comment.