From 3d93424a155cb19de36b7f331795d5576564b9cc Mon Sep 17 00:00:00 2001 From: marcos314 Date: Wed, 31 Jan 2024 08:56:59 -0300 Subject: [PATCH] fix(target-s3-parquet): add a function to conver Decimal values --- target_s3_parquet/sanitizer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/target_s3_parquet/sanitizer.py b/target_s3_parquet/sanitizer.py index 8724557..70c259f 100644 --- a/target_s3_parquet/sanitizer.py +++ b/target_s3_parquet/sanitizer.py @@ -2,12 +2,18 @@ import numpy as np import json from typing import List - +from decimal import Decimal def _remove_nulls(array): return [v for v in array if v != "null"] +def _convert_decimal(value): + if isinstance(value, Decimal): + return str(value) + return value + + def get_valid_types(types): if isinstance(types, list): return _remove_nulls(types)[0] @@ -47,7 +53,7 @@ def apply_json_dump_to_df( valid_attributes = get_valid_attributes(attributes_names, df) if len(valid_attributes) > 0: for attribute in valid_attributes: - df.loc[:, attribute] = df[attribute].apply(lambda x: json.dumps(x)) + df.loc[:, attribute] = df[attribute].apply(lambda x: json.dumps(x, default=_convert_decimal)) return df