Skip to content

Commit

Permalink
add escapechar parameter to parse_dataframe_to_bytes (#1934)
Browse files Browse the repository at this point in the history
Signed-off-by: siyuan0322 <[email protected]>
  • Loading branch information
siyuan0322 authored Jul 5, 2024
1 parent 29bf10d commit cd49172
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ def parse_dataframe(vineyard_socket, stream_id, write_options, proc_num, proc_in
delimiter = instream.params['delimiter']
if delimiter is None:
delimiter = ','
escapechar = write_options.get('escapechar', None)

stream = ByteStream.new(client, params=instream.params)
client.persist(stream.id)
Expand All @@ -101,7 +102,7 @@ def parse_dataframe(vineyard_socket, stream_id, write_options, proc_num, proc_in
batch = chunk_hook(batch)
df = batch.to_pandas()
csv_content = df.to_csv(
header=first_write, index=False, sep=delimiter
header=first_write, index=False, sep=delimiter, escapechar=escapechar
).encode('utf-8')

# write to byte stream
Expand Down
2 changes: 1 addition & 1 deletion python/vineyard/drivers/io/adaptors/read_bytes.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ def read_bytes( # noqa: C901, pylint: disable=too-many-statements
params[k] = v

try:
# files would be empty if it's a glob pattern and globbed nothing.
# files would be empty if it's a glob pattern and glob nothing.
fs, _, files = infer_fsspec_paths(path, storage_options=storage_options)
except Exception: # pylint: disable=broad-except
report_error(
Expand Down

0 comments on commit cd49172

Please sign in to comment.