From cd49172fb58ef729b49174e0752158359ce7d962 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 5 Jul 2024 15:00:53 +0800 Subject: [PATCH] add escapechar parameter to parse_dataframe_to_bytes (#1934) Signed-off-by: siyuan0322 --- .../vineyard/drivers/io/adaptors/parse_dataframe_to_bytes.py | 3 ++- python/vineyard/drivers/io/adaptors/read_bytes.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/vineyard/drivers/io/adaptors/parse_dataframe_to_bytes.py b/python/vineyard/drivers/io/adaptors/parse_dataframe_to_bytes.py index ae9e2551a..58661222f 100644 --- a/python/vineyard/drivers/io/adaptors/parse_dataframe_to_bytes.py +++ b/python/vineyard/drivers/io/adaptors/parse_dataframe_to_bytes.py @@ -80,6 +80,7 @@ def parse_dataframe(vineyard_socket, stream_id, write_options, proc_num, proc_in delimiter = instream.params['delimiter'] if delimiter is None: delimiter = ',' + escapechar = write_options.get('escapechar', None) stream = ByteStream.new(client, params=instream.params) client.persist(stream.id) @@ -101,7 +102,7 @@ def parse_dataframe(vineyard_socket, stream_id, write_options, proc_num, proc_in batch = chunk_hook(batch) df = batch.to_pandas() csv_content = df.to_csv( - header=first_write, index=False, sep=delimiter + header=first_write, index=False, sep=delimiter, escapechar=escapechar ).encode('utf-8') # write to byte stream diff --git a/python/vineyard/drivers/io/adaptors/read_bytes.py b/python/vineyard/drivers/io/adaptors/read_bytes.py index 9668c2fd8..c70aec8d7 100755 --- a/python/vineyard/drivers/io/adaptors/read_bytes.py +++ b/python/vineyard/drivers/io/adaptors/read_bytes.py @@ -159,7 +159,7 @@ def read_bytes( # noqa: C901, pylint: disable=too-many-statements params[k] = v try: - # files would be empty if it's a glob pattern and globbed nothing. + # files would be empty if it's a glob pattern and glob nothing. fs, _, files = infer_fsspec_paths(path, storage_options=storage_options) except Exception: # pylint: disable=broad-except report_error(