From 55fd5d2b3f5c88ecf70e09b288ae794bf3e45d43 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Tue, 19 Dec 2023 22:57:12 +0100 Subject: [PATCH 1/4] Fixes #572 - Allow appending rows after a dynamic column was inserted --- src/tablib/core.py | 9 ++++++++- tests/test_tablib.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/tablib/core.py b/src/tablib/core.py index a7ebff2b..9010c0ad 100644 --- a/src/tablib/core.py +++ b/src/tablib/core.py @@ -155,6 +155,9 @@ def __init__(self, *args, **kwargs): # (column, callback) tuples self._formatters = [] + # {col_index: col_func} + self._dyn_columns = {} + self.headers = kwargs.get('headers') self.title = kwargs.get('title') @@ -238,7 +241,7 @@ def _set_in_format(self, fmt_key, in_stream, **kwargs): def _validate(self, row=None, col=None, safety=False): """Assures size of every row in dataset is of proper proportions.""" if row: - is_valid = (len(row) == self.width) if self.width else True + is_valid = (len(row) == (self.width - len(self._dyn_columns))) if self.width else True elif col: if len(col) < 1: is_valid = True @@ -449,6 +452,9 @@ def insert(self, index, row, tags=()): """ self._validate(row) + for pos, func in self._dyn_columns.items(): + row = list(row) + row.insert(pos, func(row)) self._data.insert(index, Row(row, tags=tags)) def rpush(self, row, tags=()): @@ -547,6 +553,7 @@ def insert_col(self, index, col=None, header=None): # Callable Columns... if hasattr(col, '__call__'): + self._dyn_columns[self.width] = col col = list(map(col, self._data)) col = self._clean_col(col) diff --git a/tests/test_tablib.py b/tests/test_tablib.py index 6c61c204..7d33f11d 100755 --- a/tests/test_tablib.py +++ b/tests/test_tablib.py @@ -186,6 +186,12 @@ def new_col(x): return x[0] self.founders.append_col(new_col, header='first_again') + # A new row can still be appended, and the dynamic column value generated. + self.founders.append(('Some', 'One', 71)) + self.assertEqual( + self.founders['first_again'], + ['John', 'George', 'Thomas', 'Some'] + ) def test_header_slicing(self): """Verify slicing by headers.""" From 7d551437221cb92632f0a5a48bf658ecd2e3252c Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 20 Dec 2023 08:34:36 +0100 Subject: [PATCH 2/4] Apply suggestions from Hugo's code review Co-authored-by: Hugo van Kemenade --- src/tablib/core.py | 8 ++++---- tests/test_tablib.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tablib/core.py b/src/tablib/core.py index 9010c0ad..ea03dba6 100644 --- a/src/tablib/core.py +++ b/src/tablib/core.py @@ -156,7 +156,7 @@ def __init__(self, *args, **kwargs): self._formatters = [] # {col_index: col_func} - self._dyn_columns = {} + self._dynamic_columns = {} self.headers = kwargs.get('headers') @@ -241,7 +241,7 @@ def _set_in_format(self, fmt_key, in_stream, **kwargs): def _validate(self, row=None, col=None, safety=False): """Assures size of every row in dataset is of proper proportions.""" if row: - is_valid = (len(row) == (self.width - len(self._dyn_columns))) if self.width else True + is_valid = (len(row) == (self.width - len(self._dynamic_columns))) if self.width else True elif col: if len(col) < 1: is_valid = True @@ -452,7 +452,7 @@ def insert(self, index, row, tags=()): """ self._validate(row) - for pos, func in self._dyn_columns.items(): + for pos, func in self._dynamic_columns.items(): row = list(row) row.insert(pos, func(row)) self._data.insert(index, Row(row, tags=tags)) @@ -553,7 +553,7 @@ def insert_col(self, index, col=None, header=None): # Callable Columns... if hasattr(col, '__call__'): - self._dyn_columns[self.width] = col + self._dynamic_columns[self.width] = col col = list(map(col, self._data)) col = self._clean_col(col) diff --git a/tests/test_tablib.py b/tests/test_tablib.py index 7d33f11d..39ead84b 100755 --- a/tests/test_tablib.py +++ b/tests/test_tablib.py @@ -186,6 +186,7 @@ def new_col(x): return x[0] self.founders.append_col(new_col, header='first_again') + # A new row can still be appended, and the dynamic column value generated. self.founders.append(('Some', 'One', 71)) self.assertEqual( From 7a34f4ac66e5f63696eaf60ffbc146f8a6dce95c Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 20 Dec 2023 09:13:28 +0100 Subject: [PATCH 3/4] Allow providing full rows when dynamic columns are present --- HISTORY.md | 3 +++ docs/tutorial.rst | 9 +++++++++ src/tablib/core.py | 10 +++++++++- tests/test_tablib.py | 29 ++++++++++++++++++++++++++--- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 17551004..71350af4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,9 @@ - The html format now supports importing from HTML content (#243) - The ODS format now supports importing from .ods files (#567). The support is still a bit experimental. +- When adding rows to a dataset with dynamic columns, it's now possible to + provide only static values, and dynamic column values will be automatically + calculated and added to the row (#572). ### Changes diff --git a/docs/tutorial.rst b/docs/tutorial.rst index e5cd12bc..d21e7fcb 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -278,6 +278,15 @@ Adding this function to our dataset as a dynamic column would result in: :: - {Age: 22, First Name: Kenneth, Gender: Male, Last Name: Reitz} - {Age: 20, First Name: Bessie, Gender: Female, Last Name: Monke} +When you add new rows to a dataset that contains dynamic columns, you should +either provide all values in the row, or only the non-dynamic values and then +the dynamic values will be automatically generated using the function initially +provided for the column calculation. + +..versionchanged:: 3.6.0 + + In older versions, you could only add new rows with fully-populated rows, + including dynamic columns. .. _tags: diff --git a/src/tablib/core.py b/src/tablib/core.py index ea03dba6..06624b42 100644 --- a/src/tablib/core.py +++ b/src/tablib/core.py @@ -190,6 +190,8 @@ def __delitem__(self, key): pos = self.headers.index(key) del self.headers[pos] + if pos in self._dynamic_columns: + del self._dynamic_columns[pos] for i, row in enumerate(self._data): @@ -241,7 +243,13 @@ def _set_in_format(self, fmt_key, in_stream, **kwargs): def _validate(self, row=None, col=None, safety=False): """Assures size of every row in dataset is of proper proportions.""" if row: - is_valid = (len(row) == (self.width - len(self._dynamic_columns))) if self.width else True + if self.width: + is_valid = ( + len(row) == self.width or + len(row) == (self.width - len(self._dynamic_columns)) + ) + else: + is_valid = True elif col: if len(col) < 1: is_valid = True diff --git a/tests/test_tablib.py b/tests/test_tablib.py index 39ead84b..a77823a8 100755 --- a/tests/test_tablib.py +++ b/tests/test_tablib.py @@ -182,16 +182,39 @@ def test_add_column_with_header_and_data_exists(self): def test_add_callable_column(self): """Verify adding column with values specified as callable.""" - def new_col(x): - return x[0] + def new_col(row): + return row[0] + + def initials(row): + return f"{row[0][0]}{row[1][0]}" self.founders.append_col(new_col, header='first_again') + self.founders.append_col(initials, header='initials') # A new row can still be appended, and the dynamic column value generated. self.founders.append(('Some', 'One', 71)) + # Also acceptable when all dynamic column values are provided. + self.founders.append(('Other', 'Second', 84, 'Other', 'OS')) + self.assertEqual( self.founders['first_again'], - ['John', 'George', 'Thomas', 'Some'] + ['John', 'George', 'Thomas', 'Some', 'Other'] + ) + self.assertEqual( + self.founders['initials'], + ['JA', 'GW', 'TJ', 'SO', 'OS'] + ) + + # However only partial dynamic values provided is not accepted. + with self.assertRaises(tablib.InvalidDimensions): + self.founders.append(('Should', 'Crash', 60, 'Partial')) + + # Add a new row after dynamic column deletion + del self.founders['first_again'] + self.founders.append(('After', 'Deletion', 75)) + self.assertEqual( + self.founders['initials'], + ['JA', 'GW', 'TJ', 'SO', 'OS', 'AD'] ) def test_header_slicing(self): From ec9c861ae0331473106a25ed494f7b8e6afab215 Mon Sep 17 00:00:00 2001 From: Claude Paroz Date: Wed, 20 Dec 2023 21:38:46 +0100 Subject: [PATCH 4/4] Avoid inserting dynamic values when row is already full --- src/tablib/core.py | 11 ++++++----- tests/test_tablib.py | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/tablib/core.py b/src/tablib/core.py index 06624b42..ef3f6fe2 100644 --- a/src/tablib/core.py +++ b/src/tablib/core.py @@ -457,12 +457,13 @@ def insert(self, index, row, tags=()): The default behaviour is to insert the given row to the :class:`Dataset` object at the given index. - """ + """ self._validate(row) - for pos, func in self._dynamic_columns.items(): - row = list(row) - row.insert(pos, func(row)) + if len(row) < self.width: + for pos, func in self._dynamic_columns.items(): + row = list(row) + row.insert(pos, func(row)) self._data.insert(index, Row(row, tags=tags)) def rpush(self, row, tags=()): @@ -560,7 +561,7 @@ def insert_col(self, index, col=None, header=None): col = [] # Callable Columns... - if hasattr(col, '__call__'): + if callable(col): self._dynamic_columns[self.width] = col col = list(map(col, self._data)) diff --git a/tests/test_tablib.py b/tests/test_tablib.py index a77823a8..a335c44b 100755 --- a/tests/test_tablib.py +++ b/tests/test_tablib.py @@ -196,6 +196,8 @@ def initials(row): # Also acceptable when all dynamic column values are provided. self.founders.append(('Other', 'Second', 84, 'Other', 'OS')) + self.assertEqual(self.founders[3], ('Some', 'One', 71, 'Some', 'SO')) + self.assertEqual(self.founders[4], ('Other', 'Second', 84, 'Other', 'OS')) self.assertEqual( self.founders['first_again'], ['John', 'George', 'Thomas', 'Some', 'Other']