From 3c987721f92fd4f0431212b53af34068544e9783 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Wed, 4 Oct 2017 00:58:18 +0100 Subject: [PATCH 01/17] testing to see if it works --- jarbas/core/management/commands/companies.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 451c4c8..7074770 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -1,6 +1,7 @@ import csv import lzma - +from rows.fields import EmailField,DatetimeField,FloatField +import rows from django.core.exceptions import ValidationError from django.core.validators import validate_email @@ -64,16 +65,16 @@ def save_activities(self, row): return [main], secondaries def serialize(self, row): - row['email'] = self.to_email(row['email']) - + row['email'] = EmailField.deserialize(row['email']) dates = ('opening', 'situation_date', 'special_situation_date') for key in dates: - row[key] = self.to_date(row[key]) + row[key] = DatetimeField.deserialize(row[key]) + decimals = ('latitude', 'longitude') for key in decimals: - row[key] = self.to_number(row[key]) - + #row[key] = self.to_number(row[key]) + row[key] = FloatField.deserialize(row[key]) return row @staticmethod From d8ddaacf180e874fc6476da30aca250d0b62e39f Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Wed, 4 Oct 2017 01:05:53 +0100 Subject: [PATCH 02/17] removed unnecessary import statement --- jarbas/core/management/commands/companies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 7074770..a8c283e 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -1,7 +1,6 @@ import csv import lzma from rows.fields import EmailField,DatetimeField,FloatField -import rows from django.core.exceptions import ValidationError from django.core.validators import validate_email From 055ee8669c08fcdeda63046720c802025322a251 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 7 Oct 2017 00:19:28 +0100 Subject: [PATCH 03/17] Importing companies data with rows library --- jarbas/core/management/commands/companies.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index a8c283e..d20325b 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -1,5 +1,7 @@ import csv import lzma +import rows + from rows.fields import EmailField,DatetimeField,FloatField from django.core.exceptions import ValidationError from django.core.validators import validate_email @@ -31,7 +33,7 @@ def save_companies(self): skip = ('main_activity', 'secondary_activity') keys = tuple(f.name for f in Company._meta.fields if f not in skip) with lzma.open(self.path, mode='rt', encoding='utf-8') as file_handler: - for row in csv.DictReader(file_handler): + for row in map(lambda x: x._asdict(),[file_rows for file_rows in rows.import_from_csv(filename_or_fobj=file_handler) ]): main, secondary = self.save_activities(row) filtered = {k: v for k, v in row.items() if k in keys} From f14a1e9d050f614fe54519f330b7adea1cea14e4 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 7 Oct 2017 01:37:05 +0100 Subject: [PATCH 04/17] some final changes --- jarbas/core/management/commands/companies.py | 29 ++++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index d20325b..6b4f479 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -1,8 +1,7 @@ -import csv import lzma import rows +import rows.fields -from rows.fields import EmailField,DatetimeField,FloatField from django.core.exceptions import ValidationError from django.core.validators import validate_email @@ -25,6 +24,8 @@ def handle(self, *args, **options): self.save_companies() + + def save_companies(self): """ Receives path to the dataset file and create a Company object for @@ -33,7 +34,7 @@ def save_companies(self): skip = ('main_activity', 'secondary_activity') keys = tuple(f.name for f in Company._meta.fields if f not in skip) with lzma.open(self.path, mode='rt', encoding='utf-8') as file_handler: - for row in map(lambda x: x._asdict(),[file_rows for file_rows in rows.import_from_csv(filename_or_fobj=file_handler) ]): + for row in map(self.transform,rows.import_from_csv(filename_or_fobj=file_handler)): main, secondary = self.save_activities(row) filtered = {k: v for k, v in row.items() if k in keys} @@ -47,6 +48,10 @@ def save_companies(self): self.count += 1 self.print_count(Company, count=self.count) + @staticmethod + def transform(row): + return row._asdict() + def save_activities(self, row): data = dict( code=row['main_activity_code'], @@ -65,17 +70,29 @@ def save_activities(self, row): return [main], secondaries + class MyDateField(rows.fields.DateField): + INPUT_FORMAT = '%Y-%m-%dT%H:%M:%S' + + value = MyDateField.deserialize('2014-02-16T00:00:00') + company = {'email': 'ahoy', + 'latitude': '3.1415', + 'longitude': '-42', + 'opening': '31/12/1969', + 'situation_date': '31/12/1969', + 'special_situation_date': '31/12/1969'} + + #lastthoughts:load test message as dict,then create class that extends the field time to be convert using class newclass(extended class).deseralize(value to deserialized).When company is imported is field type inferred def serialize(self, row): - row['email'] = EmailField.deserialize(row['email']) + row['email'] = rows.fields.EmailField().deserialize(row['email']) dates = ('opening', 'situation_date', 'special_situation_date') for key in dates: - row[key] = DatetimeField.deserialize(row[key]) + row[key] = rows.fields.DatetimeField().deserialize(row[key]) decimals = ('latitude', 'longitude') for key in decimals: #row[key] = self.to_number(row[key]) - row[key] = FloatField.deserialize(row[key]) + row[key] = rows.fields.FloatField().deserialize(row[key]) return row @staticmethod From b4b65ea32b79c85e18b97bef536ca988b42b2022 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 7 Oct 2017 14:00:32 +0100 Subject: [PATCH 05/17] Finished with companies data --- jarbas/core/management/commands/companies.py | 23 +++++++------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 6b4f479..6f8490b 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -52,6 +52,8 @@ def save_companies(self): def transform(row): return row._asdict() + + def save_activities(self, row): data = dict( code=row['main_activity_code'], @@ -70,25 +72,12 @@ def save_activities(self, row): return [main], secondaries - class MyDateField(rows.fields.DateField): - INPUT_FORMAT = '%Y-%m-%dT%H:%M:%S' - value = MyDateField.deserialize('2014-02-16T00:00:00') - company = {'email': 'ahoy', - 'latitude': '3.1415', - 'longitude': '-42', - 'opening': '31/12/1969', - 'situation_date': '31/12/1969', - 'special_situation_date': '31/12/1969'} - - #lastthoughts:load test message as dict,then create class that extends the field time to be convert using class newclass(extended class).deseralize(value to deserialized).When company is imported is field type inferred def serialize(self, row): - row['email'] = rows.fields.EmailField().deserialize(row['email']) + row['email'] = self.to_email(row['email']) dates = ('opening', 'situation_date', 'special_situation_date') for key in dates: - row[key] = rows.fields.DatetimeField().deserialize(row[key]) - - + row[key] = InputDateField().deserialize(row[key]) decimals = ('latitude', 'longitude') for key in decimals: #row[key] = self.to_number(row[key]) @@ -103,3 +92,7 @@ def to_email(email): except ValidationError: return None + + +class InputDateField(rows.fields.DateField): + INPUT_FORMAT = '%d-%m-%Y' From c28bb549177aba3ef646ad30894b0040643ba6e8 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 7 Oct 2017 14:29:24 +0100 Subject: [PATCH 06/17] fixup! Finished with companies data --- jarbas/core/management/commands/companies.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 6f8490b..8a6bc5c 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -80,7 +80,6 @@ def serialize(self, row): row[key] = InputDateField().deserialize(row[key]) decimals = ('latitude', 'longitude') for key in decimals: - #row[key] = self.to_number(row[key]) row[key] = rows.fields.FloatField().deserialize(row[key]) return row @@ -95,4 +94,4 @@ def to_email(email): class InputDateField(rows.fields.DateField): - INPUT_FORMAT = '%d-%m-%Y' + INPUT_FORMAT = '%d/%m/%Y' From 8a4bc1878ea59819e21f65dfabe248d165cf0314 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 7 Oct 2017 23:58:21 +0100 Subject: [PATCH 07/17] Applied suggestions --- jarbas/core/management/commands/companies.py | 54 ++++++-------------- 1 file changed, 16 insertions(+), 38 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 8a6bc5c..89abdf1 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -1,14 +1,23 @@ import lzma import rows -import rows.fields - -from django.core.exceptions import ValidationError -from django.core.validators import validate_email from jarbas.core.management.commands import LoadCommand from jarbas.core.models import Activity, Company +class CompaniesDate(rows.fields.DateField): + INPUT_FORMAT = '%d/%m/%Y' + +companies_csv_field_types = { + 'email': rows.fields.EmailField, + 'opening': CompaniesDate, + 'situation_date': CompaniesDate, + 'special_situation_date': CompaniesDate, + 'latitude': rows.fields.FloatField, + 'longitude': rows.fields.FloatField + } + + class Command(LoadCommand): help = 'Load Serenata de Amor companies dataset into the database' @@ -24,8 +33,6 @@ def handle(self, *args, **options): self.save_companies() - - def save_companies(self): """ Receives path to the dataset file and create a Company object for @@ -34,11 +41,12 @@ def save_companies(self): skip = ('main_activity', 'secondary_activity') keys = tuple(f.name for f in Company._meta.fields if f not in skip) with lzma.open(self.path, mode='rt', encoding='utf-8') as file_handler: - for row in map(self.transform,rows.import_from_csv(filename_or_fobj=file_handler)): + for row in rows.import_from_csv(file_handler, force_types=companies_csv_field_types): + row = row._asdict() main, secondary = self.save_activities(row) filtered = {k: v for k, v in row.items() if k in keys} - obj = Company.objects.create(**self.serialize(filtered)) + obj = Company.objects.create(**filtered) for activity in main: obj.main_activity.add(activity) for activity in secondary: @@ -48,12 +56,6 @@ def save_companies(self): self.count += 1 self.print_count(Company, count=self.count) - @staticmethod - def transform(row): - return row._asdict() - - - def save_activities(self, row): data = dict( code=row['main_activity_code'], @@ -71,27 +73,3 @@ def save_activities(self, row): secondaries.append(obj) return [main], secondaries - - - def serialize(self, row): - row['email'] = self.to_email(row['email']) - dates = ('opening', 'situation_date', 'special_situation_date') - for key in dates: - row[key] = InputDateField().deserialize(row[key]) - decimals = ('latitude', 'longitude') - for key in decimals: - row[key] = rows.fields.FloatField().deserialize(row[key]) - return row - - @staticmethod - def to_email(email): - try: - validate_email(email) - return email - - except ValidationError: - return None - - -class InputDateField(rows.fields.DateField): - INPUT_FORMAT = '%d/%m/%Y' From ceaf2039741973a9d94fb0a826532cee3c9685b8 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Thu, 21 Dec 2017 10:34:08 +0100 Subject: [PATCH 08/17] Added rows to requirements.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6eb931f..1b0d4e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ python-memcached==1.58 python-twitter==3.3 reprint==0.3.0 # pyup: ignore requests==2.18.4 +rows==0.3.1 From 383e5787761dc3bda2c9cf7a2624ac7466be9a3c Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Thu, 21 Dec 2017 10:46:36 +0100 Subject: [PATCH 09/17] Removed test for the serialize method(since it does not exist any more) of the command class in jarbas.core.management.commands.companies.py --- jarbas/core/tests/test_companies_command.py | 28 --------------------- 1 file changed, 28 deletions(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index f9995d6..69222bc 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -1,4 +1,3 @@ -from datetime import date from io import StringIO from unittest.mock import patch @@ -15,33 +14,6 @@ def setUp(self): self.command = Command() -class TestSerializer(TestCommand): - - def test_to_email(self): - expected = 'jane@example.com' - self.assertEqual(self.command.to_email('abc'), None) - self.assertEqual(self.command.to_email('jane@example.com'), expected) - - def test_serializer(self): - company = { - 'email': 'ahoy', - 'opening': '31/12/1969', - 'situation_date': '31/12/1969', - 'special_situation_date': '31/12/1969', - 'latitude': '3.1415', - 'longitude': '-42' - } - expected = { - 'email': None, - 'opening': date(1969, 12, 31), - 'situation_date': date(1969, 12, 31), - 'special_situation_date': date(1969, 12, 31), - 'latitude': 3.1415, - 'longitude': -42.0 - } - self.assertEqual(self.command.serialize(company), expected) - - class TestCreate(TestCommand): @patch.object(Activity.objects, 'update_or_create') From 422dd04cfd80f834130800d7f9c16b3d4689a09a Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 23 Dec 2017 21:24:48 +0100 Subject: [PATCH 10/17] fixup! Removed test for the serialize method(since it does not exist any more) of the command class in jarbas.core.management.commands.companies.py --- jarbas/core/tests/test_companies_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index 69222bc..b15f5fc 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -36,7 +36,6 @@ def test_save_activities(self, update_or_create): @patch('jarbas.core.management.commands.companies.lzma') @patch('jarbas.core.management.commands.companies.csv.DictReader') @patch('jarbas.core.management.commands.companies.Command.save_activities') - @patch('jarbas.core.management.commands.companies.Command.serialize') @patch('jarbas.core.management.commands.companies.Command.print_count') @patch.object(Company.objects, 'create') def test_save_companies(self, create, print_count, serialize, save_activities, rows, lzma): From b3f4e9bba53c8b1e369f94e4e9788b6b17d3a7ba Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 23 Dec 2017 21:33:37 +0100 Subject: [PATCH 11/17] Removed patch decorator for csv.DictReader --- jarbas/core/tests/test_companies_command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index b15f5fc..9c58be9 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -34,7 +34,6 @@ def test_save_activities(self, update_or_create): self.assertEqual(99, len(secondaries)) @patch('jarbas.core.management.commands.companies.lzma') - @patch('jarbas.core.management.commands.companies.csv.DictReader') @patch('jarbas.core.management.commands.companies.Command.save_activities') @patch('jarbas.core.management.commands.companies.Command.print_count') @patch.object(Company.objects, 'create') From 6070f7c9231ebe9422bd63030b83d1f58dbdde6a Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sat, 23 Dec 2017 22:43:58 +0100 Subject: [PATCH 12/17] Removed serialize.return_value --- jarbas/core/tests/test_companies_command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index 9c58be9..a6c8d8f 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -37,11 +37,10 @@ def test_save_activities(self, update_or_create): @patch('jarbas.core.management.commands.companies.Command.save_activities') @patch('jarbas.core.management.commands.companies.Command.print_count') @patch.object(Company.objects, 'create') - def test_save_companies(self, create, print_count, serialize, save_activities, rows, lzma): + def test_save_companies(self, create, print_count, save_activities, rows, lzma): self.command.count = 0 lzma.return_value = StringIO() rows.return_value = [sample_company_data] - serialize.return_value = dict(ahoy=42) save_activities.return_value = ([3], [14, 15]) self.command.path = 'companies.xz' self.command.save_companies() From 662652b708b5bea2bf004ddf30727277409a09d6 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Sun, 14 Jan 2018 20:32:04 +0100 Subject: [PATCH 13/17] Trying to fix positional argument issue --- jarbas/core/tests/test_companies_command.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index a6c8d8f..25c28a5 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -37,10 +37,9 @@ def test_save_activities(self, update_or_create): @patch('jarbas.core.management.commands.companies.Command.save_activities') @patch('jarbas.core.management.commands.companies.Command.print_count') @patch.object(Company.objects, 'create') - def test_save_companies(self, create, print_count, save_activities, rows, lzma): + def test_save_companies(self, create, print_count, save_activities,lzma): self.command.count = 0 lzma.return_value = StringIO() - rows.return_value = [sample_company_data] save_activities.return_value = ([3], [14, 15]) self.command.path = 'companies.xz' self.command.save_companies() From 1050063a59349c4415809eea7b6c74da1dc97fcd Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Mon, 15 Jan 2018 23:37:41 +0100 Subject: [PATCH 14/17] Issue with reading file with rows --- jarbas/core/management/commands/companies.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 89abdf1..978b7fa 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -40,8 +40,8 @@ def save_companies(self): """ skip = ('main_activity', 'secondary_activity') keys = tuple(f.name for f in Company._meta.fields if f not in skip) - with lzma.open(self.path, mode='rt', encoding='utf-8') as file_handler: - for row in rows.import_from_csv(file_handler, force_types=companies_csv_field_types): + with lzma.open(self.path, mode='rb') as file_handler: + for row in rows.import_from_csv(file_handler, force_types=companies_csv_field_types, encoding='utf-8'): row = row._asdict() main, secondary = self.save_activities(row) From 6424c0a0c177cdf2d784672c43f654a63a6325d3 Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Wed, 21 Feb 2018 15:26:23 +0100 Subject: [PATCH 15/17] Got test to work --- jarbas/core/management/commands/companies.py | 4 +++- jarbas/core/tests/fixtures/companies.xz | Bin 0 -> 604 bytes jarbas/core/tests/test_companies_command.py | 21 ++++++++++++------- 3 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 jarbas/core/tests/fixtures/companies.xz diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 978b7fa..0ad0464 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -8,6 +8,7 @@ class CompaniesDate(rows.fields.DateField): INPUT_FORMAT = '%d/%m/%Y' + companies_csv_field_types = { 'email': rows.fields.EmailField, 'opening': CompaniesDate, @@ -42,7 +43,8 @@ def save_companies(self): keys = tuple(f.name for f in Company._meta.fields if f not in skip) with lzma.open(self.path, mode='rb') as file_handler: for row in rows.import_from_csv(file_handler, force_types=companies_csv_field_types, encoding='utf-8'): - row = row._asdict() + row = dict(row._asdict()) + main, secondary = self.save_activities(row) filtered = {k: v for k, v in row.items() if k in keys} diff --git a/jarbas/core/tests/fixtures/companies.xz b/jarbas/core/tests/fixtures/companies.xz new file mode 100644 index 0000000000000000000000000000000000000000..09025f37643acb1876fa99b1c068093633418204 GIT binary patch literal 604 zcmV-i0;B!?H+ooF000E$*0e?f03iVu0001VFXf})6v?nO7Jkl$$J1sg69PX zhtdNVw7JxFOQBlV*(!|C8tW=C1A?U0kcle>!#%!E@_+H)ppD%9a_J*=cNd$r6e}uj z{{oEc)Vw(m`=9iD?J`xySjl!+qvVzoQl3^g_;_F>>wl4%#lT$DX|)KD8f;zxC>utx z{~{Hr40i@S&c6K$^yJw$AZ0|RH$3GGO)Dr~OH_T*e~k@SUcaGo)D)Yt$ZrTHn0Rl7 z>?Ooh4Z(~0aJ|F4RNTH|q;yYocTyRyR(wxzb*b9?#lPhO_<0RVJD_i7r`ClQ&y|;N5%;3WZV}(B5!l@*s9wP4a9t& z-^{BB8nIbVW&O}tmBD){%b)DCeaQNsN|)S`6F)z66FWI zG6P0ar*TQ(M>|*Cxjkukc*8>itl^*|vEqtG)j_`ovUnc@^Bc72k< zb<(VLSwE Date: Sun, 25 Feb 2018 19:22:36 +0100 Subject: [PATCH 16/17] Using a dictionary in assertion of the create mocked object --- jarbas/core/tests/test_companies_command.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index 8d63b6b..9104eef 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -43,13 +43,17 @@ def test_save_companies(self, create, print_count, save_activities): self.command.path = os.path.join(settings.BASE_DIR, 'jarbas', 'core', 'tests', 'fixtures', 'companies.xz') self.command.save_companies() - create.assert_called_with(additional_address_details=b'', address=b'', city=b'', cnpj=b'', - email='test@test.com.br', last_updated=b'', latitude=-15.7910966, legal_entity=b'', - longitude=-47.9508743, name='Test', neighborhood=b'', number=1, opening=None, - phone=b'', responsible_federative_entity=b'', situation=b'', zip_code=b'', - situation_date=datetime.date(2005, 9, 24), situation_reason=b'', type='Book', - special_situation_date=None, state=b'', status=b'', trade_name=b'', - special_situation=b'') + expected = { + 'additional_address_details': b'', 'address': b'', + 'city': b'', 'cnpj': b'', 'email': 'test@test.com.br', 'last_updated': b'', + 'latitude': -15.7910966, 'legal_entity': b'', 'longitude': -47.9508743, + 'name': 'Test', 'neighborhood': b'', 'number': 1, 'opening': None, + 'phone': b'', 'responsible_federative_entity': b'', + 'situation': b'', 'zip_code': b'', 'situation_date': datetime.date(2005, 9, 24), + 'situation_reason': b'', 'type': 'Book', 'special_situation_date': None, + 'state': b'', 'status': b'', 'trade_name': b'', 'special_situation': b'' + } + create.assert_called_with(**expected) create.return_value.main_activity.add.assert_called_with(3) self.assertEqual(1, print_count.call_count) self.assertEqual(2, create.return_value.secondary_activity.add.call_count) From be4513c235c79716cfdd57be12bdc0d1dbe2bf1a Mon Sep 17 00:00:00 2001 From: ilozuluchris Date: Wed, 28 Feb 2018 11:38:16 +0100 Subject: [PATCH 17/17] reformatted some necessary dictionaries --- jarbas/core/management/commands/companies.py | 18 +++++----- jarbas/core/tests/test_companies_command.py | 35 +++++++++++++++----- 2 files changed, 35 insertions(+), 18 deletions(-) diff --git a/jarbas/core/management/commands/companies.py b/jarbas/core/management/commands/companies.py index 0ad0464..5ad7dbb 100644 --- a/jarbas/core/management/commands/companies.py +++ b/jarbas/core/management/commands/companies.py @@ -9,14 +9,14 @@ class CompaniesDate(rows.fields.DateField): INPUT_FORMAT = '%d/%m/%Y' -companies_csv_field_types = { - 'email': rows.fields.EmailField, - 'opening': CompaniesDate, - 'situation_date': CompaniesDate, - 'special_situation_date': CompaniesDate, - 'latitude': rows.fields.FloatField, - 'longitude': rows.fields.FloatField - } +COMPANIES_CSV_FIELD_TYPES = { + 'email': rows.fields.EmailField, + 'opening': CompaniesDate, + 'situation_date': CompaniesDate, + 'special_situation_date': CompaniesDate, + 'latitude': rows.fields.FloatField, + 'longitude': rows.fields.FloatField +} class Command(LoadCommand): @@ -42,7 +42,7 @@ def save_companies(self): skip = ('main_activity', 'secondary_activity') keys = tuple(f.name for f in Company._meta.fields if f not in skip) with lzma.open(self.path, mode='rb') as file_handler: - for row in rows.import_from_csv(file_handler, force_types=companies_csv_field_types, encoding='utf-8'): + for row in rows.import_from_csv(file_handler, force_types=COMPANIES_CSV_FIELD_TYPES, encoding='utf-8'): row = dict(row._asdict()) main, secondary = self.save_activities(row) diff --git a/jarbas/core/tests/test_companies_command.py b/jarbas/core/tests/test_companies_command.py index 9104eef..1996da2 100644 --- a/jarbas/core/tests/test_companies_command.py +++ b/jarbas/core/tests/test_companies_command.py @@ -44,15 +44,32 @@ def test_save_companies(self, create, print_count, save_activities): 'fixtures', 'companies.xz') self.command.save_companies() expected = { - 'additional_address_details': b'', 'address': b'', - 'city': b'', 'cnpj': b'', 'email': 'test@test.com.br', 'last_updated': b'', - 'latitude': -15.7910966, 'legal_entity': b'', 'longitude': -47.9508743, - 'name': 'Test', 'neighborhood': b'', 'number': 1, 'opening': None, - 'phone': b'', 'responsible_federative_entity': b'', - 'situation': b'', 'zip_code': b'', 'situation_date': datetime.date(2005, 9, 24), - 'situation_reason': b'', 'type': 'Book', 'special_situation_date': None, - 'state': b'', 'status': b'', 'trade_name': b'', 'special_situation': b'' - } + 'additional_address_details': b'', + 'address': b'', + 'city': b'', + 'cnpj': b'', + 'email': 'test@test.com.br', + 'last_updated': b'', + 'latitude': -15.7910966, + 'legal_entity': b'', + 'longitude': -47.9508743, + 'name': 'Test', + 'neighborhood': b'', + 'number': 1, + 'opening': None, + 'phone': b'', + 'responsible_federative_entity': b'', + 'situation': b'', + 'zip_code': b'', + 'situation_date': datetime.date(2005, 9, 24), + 'situation_reason': b'', + 'type': 'Book', + 'special_situation_date': None, + 'state': b'', + 'status': b'', + 'trade_name': b'', + 'special_situation': b'' + } create.assert_called_with(**expected) create.return_value.main_activity.add.assert_called_with(3) self.assertEqual(1, print_count.call_count)