From acd5a2f6917929f8c242dd8732c4a1197e8b526a Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 14:04:05 -0300 Subject: [PATCH 1/6] fix: fix how files are downloaded --- src/io/get_files_dict.py | 73 +++++++++------ src/io/get_last_ref_date.py | 26 ++---- tests/io/test_get_files_list.py | 143 ----------------------------- tests/io/test_get_last_ref_date.py | 42 --------- 4 files changed, 56 insertions(+), 228 deletions(-) delete mode 100644 tests/io/test_get_files_list.py delete mode 100644 tests/io/test_get_last_ref_date.py diff --git a/src/io/get_files_dict.py b/src/io/get_files_dict.py index 63a38f0..747181b 100644 --- a/src/io/get_files_dict.py +++ b/src/io/get_files_dict.py @@ -3,10 +3,11 @@ import requests from bs4 import BeautifulSoup -from requests.adapters import HTTPAdapter, Retry -from src import SRC_PATH, DATA_FOLDER -from src.io import CORE_URL_FILES, HEADERS +from src import DATA_FOLDER +from src import SRC_PATH +from src.io import CORE_URL_FILES +from src.io import HEADERS from src.io.get_last_ref_date import main as get_last_ref_date @@ -19,18 +20,22 @@ def main(): ref_date = get_last_ref_date() # get page content - page = requests.get(CORE_URL_FILES, headers=HEADERS) + _folder_open_date = 'dados_abertos_cnpj' + CORE_URL = f'{CORE_URL_FILES}/{_folder_open_date}/{ref_date}' + page = requests.get(CORE_URL, headers=HEADERS) # BeautifulSoup object soup = BeautifulSoup(page.text, 'html.parser') table = soup.find('table') rows = table.find_all('tr') - dict_files_url = {'SOCIOS': {}, - 'EMPRESAS': {}, - 'ESTABELECIMENTOS': {}, - 'TAX_REGIME': {}, - 'TABELAS': {}} + dict_files_url = { + 'SOCIOS': {}, + 'EMPRESAS': {}, + 'ESTABELECIMENTOS': {}, + 'TAX_REGIME': {}, + 'TABELAS': {}, + } print('creating dict files url') for row in rows: @@ -40,21 +45,25 @@ def main(): file_name = row.find_all('td')[1].find('a')['href'] # get last modified time and parse to date (ex: '2021-07-19') last_modified = datetime.strptime(row.find_all('td')[2].text.strip(), '%Y-%m-%d %H:%M').strftime( - '%Y-%m-%d') + '%Y-%m-%d', + ) # get size file_name file_size = row.find_all('td')[3].text.strip() if 'K' in file_size: - file_size_bytes = float(file_size.replace('K', '')) * 2 ** 10 + file_size_bytes = float(file_size.replace('K', '')) * 2**10 elif 'M' in file_size: - file_size_bytes = float(file_size.replace('M', '')) * 2 ** 20 + file_size_bytes = float(file_size.replace('M', '')) * 2**20 else: file_size_bytes = 0 - dict_core = {file_name: {'last_modified': last_modified, - 'file_size_bytes': file_size_bytes, - 'link_to_download': f"{CORE_URL_FILES}/{file_name}", - 'path_save_file': os.path.join(SRC_PATH, DATA_FOLDER, ref_date, file_name)} - } + dict_core = { + file_name: { + 'last_modified': last_modified, + 'file_size_bytes': file_size_bytes, + 'link_to_download': f"{CORE_URL}/{file_name}", + 'path_save_file': os.path.join(SRC_PATH, DATA_FOLDER, ref_date, file_name), + }, + } if 'Socios' in file_name: dict_files_url['SOCIOS'].update(dict_core) elif 'Empresas' in file_name: @@ -79,21 +88,31 @@ def main(): file_name = row.find_all('td')[1].find('a')['href'] # get last modified time and parse to date (ex: '2021-07-19') last_modified = datetime.strptime(row.find_all('td')[2].text.strip(), '%Y-%m-%d %H:%M').strftime( - '%Y-%m-%d') + '%Y-%m-%d', + ) # get size file_name file_size = row.find_all('td')[3].text.strip() if 'K' in file_size: - file_size_bytes = float(file_size.replace('K', '')) * 2 ** 10 + file_size_bytes = float(file_size.replace('K', '')) * 2**10 elif 'M' in file_size: - file_size_bytes = float(file_size.replace('M', '')) * 2 ** 20 + file_size_bytes = float(file_size.replace('M', '')) * 2**20 else: file_size_bytes = 0 - dict_files_url['TAX_REGIME'].update({file_name: {'last_modified': last_modified, - 'file_size_bytes': file_size_bytes, - 'link_to_download': f"{CORE_URL_FILES}/{_folder_tax_regime}/{file_name}", - 'path_save_file': os.path.join(SRC_PATH, DATA_FOLDER, - ref_date, file_name)} - }) + dict_files_url['TAX_REGIME'].update( + { + file_name: { + 'last_modified': last_modified, + 'file_size_bytes': file_size_bytes, + 'link_to_download': f"{CORE_URL_FILES}/{_folder_tax_regime}/{file_name}", + 'path_save_file': os.path.join( + SRC_PATH, + DATA_FOLDER, + ref_date, + file_name, + ), + }, + }, + ) print('Done') @@ -102,4 +121,4 @@ def main(): if __name__ == '__main__': dict_files_url = main() - print(dict_files_url) + print(dict_files_url) \ No newline at end of file diff --git a/src/io/get_last_ref_date.py b/src/io/get_last_ref_date.py index 4c4409f..b5ae95c 100644 --- a/src/io/get_last_ref_date.py +++ b/src/io/get_last_ref_date.py @@ -1,11 +1,8 @@ -from collections import Counter -from datetime import datetime - import requests from bs4 import BeautifulSoup -from requests.adapters import HTTPAdapter, Retry -from src.io import CORE_URL_FILES, HEADERS +from src.io import CORE_URL_FILES +from src.io import HEADERS def main(): @@ -14,11 +11,11 @@ def main(): :return: dict with urls from files as well as last modified date and size in bytes """ # get page content - page = requests.get(CORE_URL_FILES, headers=HEADERS) + _folder_open_date = 'dados_abertos_cnpj' + page = requests.get(f'{CORE_URL_FILES}/{_folder_open_date}', headers=HEADERS) # BeautifulSoup object soup = BeautifulSoup(page.text, 'html.parser') - table = soup.find('table') rows = table.find_all('tr') list_last_modified_at = [] @@ -26,18 +23,15 @@ def main(): print('creating dict files url') for row in rows: if row.find_all('td'): - if row.find_all('td')[1].find('a')['href'].endswith('.zip'): + if row.find_all('td')[1].find('a')['href'].replace('-', '').replace('/', '').isdigit(): # get last modified time and parse to date (ex: '2021-07-19') - list_last_modified_at.append( - datetime.strptime(row.find_all('td')[2].text.strip(), '%Y-%m-%d %H:%M').strftime( - '%Y-%m-%d')) - + list_last_modified_at.append(row.find_all('td')[1].find('a')['href'].replace('/', '')) # get the most common on 'last_modified' from source - ref_date, occurences = Counter(list_last_modified_at).most_common(1)[0] - print( - f"ref date will be: '{ref_date}' with {occurences} out of {len(list_last_modified_at)} ({occurences / len(list_last_modified_at):.1%}) ") + ref_date = max(list_last_modified_at) + print('last updated date is ', ref_date) + return ref_date if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/tests/io/test_get_files_list.py b/tests/io/test_get_files_list.py deleted file mode 100644 index 80f01a3..0000000 --- a/tests/io/test_get_files_list.py +++ /dev/null @@ -1,143 +0,0 @@ -from unittest.mock import Mock -import os - -from src.io.get_files_dict import main as get_files_dict - -DIR_NAME = os.path.dirname(os.path.abspath(__file__)) - - -class ObjectFakeText: - def __init__(self, txt): - self.txt = txt - - @property - def text(self): - return self.txt - - -def test_get_files_dict_keys(fixture_get_files_dict): - dict_files = fixture_get_files_dict - keys = ['SOCIOS', 'EMPRESAS', 'ESTABELECIMENTOS', 'TABELAS', 'TAX_REGIME', 'folder_ref_date_save_zip'] - - assert sorted(dict_files.keys()) == sorted(keys) - - -def test_get_files_dict_keys_sub_dicts(fixture_get_files_dict): - dict_files = fixture_get_files_dict - tbls = ['SOCIOS', 'EMPRESAS', 'ESTABELECIMENTOS', 'TABELAS', 'TAX_REGIME'] - - for tbl in tbls: - _dict = dict_files[tbl] - for file, dict_file in _dict.items(): - assert sorted(dict_file.keys()) == sorted( - ['last_modified', 'file_size_bytes', 'link_to_download', 'path_save_file']) - - -def test_get_files_dict_empresas(fixture_get_files_dict): - dict_files = fixture_get_files_dict - dict_files_target = dict_files['EMPRESAS'] - - assert len(dict_files_target.keys()) == 10 - - -def test_get_files_dict_estabelecimentos(fixture_get_files_dict): - dict_files = fixture_get_files_dict - dict_files_target = dict_files['ESTABELECIMENTOS'] - - assert len(dict_files_target.keys()) == 10 - - -def test_get_files_dict_socios(fixture_get_files_dict): - dict_files = fixture_get_files_dict - dict_files_target = dict_files['SOCIOS'] - - assert len(dict_files_target.keys()) == 10 - - -def test_get_files_dict_tabelas(fixture_get_files_dict): - dict_files = fixture_get_files_dict - dict_files_target = dict_files['TABELAS'] - - assert len(dict_files_target.keys()) == 7 - - -def test_get_files_dict_tax_regime(fixture_get_files_dict): - dict_files = fixture_get_files_dict - dict_files_target = dict_files['TAX_REGIME'] - - assert len(dict_files_target.keys()) == 4 - - -def test_get_last_ref_date_mock_empresas(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests.get', mock_requests) - html_file = 'test_get_last_ref_date_all_equal.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.return_value = ObjectFakeText(open(html_path).read()) - - dict_files = get_files_dict() - dict_files_target = dict_files['EMPRESAS'] - - assert len(dict_files_target.keys()) == 10 - - list_expected_files = [f'Empresas{r}.zip' for r in range(10)] - - assert sorted(dict_files_target.keys()) == sorted(list_expected_files) - - -def test_get_last_ref_date_mock_estabelecimentos(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests.get', mock_requests) - html_file = 'test_get_last_ref_date_all_equal.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.return_value = ObjectFakeText(open(html_path).read()) - - dict_files = get_files_dict() - dict_files_target = dict_files['ESTABELECIMENTOS'] - - assert len(dict_files_target.keys()) == 10 - - list_expected_files = [f'Estabelecimentos{r}.zip' for r in range(10)] - - assert sorted(dict_files_target.keys()) == sorted(list_expected_files) - - -def test_get_last_ref_date_mock_socios(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests.get', mock_requests) - html_file = 'test_get_last_ref_date_all_equal.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.return_value = ObjectFakeText(open(html_path).read()) - - dict_files = get_files_dict() - dict_files_target = dict_files['SOCIOS'] - - assert len(dict_files_target.keys()) == 10 - - list_expected_files = [f'Socios{r}.zip' for r in range(10)] - - assert sorted(dict_files_target.keys()) == sorted(list_expected_files) - - -def test_get_last_ref_date_mock_tabelas(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests.get', mock_requests) - html_file = 'test_get_last_ref_date_all_equal.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.return_value = ObjectFakeText(open(html_path).read()) - - dict_files = get_files_dict() - dict_files_target = dict_files['TABELAS'] - - assert len(dict_files_target.keys()) == 7 - - list_expected_files = ['Simples.zip', - 'Cnaes.zip', - 'Motivos.zip', - 'Municipios.zip', - 'Naturezas.zip', - 'Paises.zip', - 'Qualificacoes.zip' - ] - - assert sorted(dict_files_target.keys()) == sorted(list_expected_files) diff --git a/tests/io/test_get_last_ref_date.py b/tests/io/test_get_last_ref_date.py deleted file mode 100644 index 707e1aa..0000000 --- a/tests/io/test_get_last_ref_date.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -from unittest.mock import Mock - -from src.io.get_last_ref_date import main as get_last_ref_date - -core_url_expected = "https://www.gov.br/receitafederal/pt-br/assuntos/orientacao-tributaria/cadastros/consultas/dados-publicos-cnpj" -DIR_NAME = os.path.dirname(os.path.abspath(__file__)) - - -class ObjectFakeText: - def __init__(self, txt): - self.txt = txt - - @property - def text(self): - return self.txt - - -def test_get_last_ref_date_all_one_date(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests', mock_requests) - html_file = 'test_get_last_ref_date_all_equal.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.get.return_value = ObjectFakeText(open(html_path)) - - ref_date = get_last_ref_date() - ref_date_expected = '2022-08-15' - - assert ref_date == ref_date_expected - - -def test_get_last_ref_date_diff_dates(mocker): - mock_requests = Mock() - mocker.patch('src.io.get_last_ref_date.requests', mock_requests) - html_file = 'test_get_last_ref_date_diffs.html' - html_path = os.path.join(DIR_NAME, 'htmls', html_file) - mock_requests.get.return_value = ObjectFakeText(open(html_path)) - - ref_date = get_last_ref_date() - ref_date_expected = '2020-01-01' - - assert ref_date == ref_date_expected From 973b474274f686d53cf48a631f234dc23b6d5fa5 Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 14:13:51 -0300 Subject: [PATCH 2/6] fix: change receita federal url --- Makefile | 58 +++++++++++++++++++++++----------------------- src/io/__init__.py | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Makefile b/Makefile index 71e46aa..cff7630 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,9 @@ all: @echo "" @echo "########################################################################################################################" @echo "[SESSION] Launch" - @echo "make up ......................................... docker-compose up -d" - @echo "make stop ....................................... docker-compose stop" - @echo "make down ....................................... docker-compose down" + @echo "make up ......................................... docker compose up -d" + @echo "make stop ....................................... docker compose stop" + @echo "make down ....................................... docker compose down" @echo "make app ........................................ run container app" @echo "make rm ......................................... remove all exited containers and all dangling volumes" @echo "" @@ -47,25 +47,25 @@ build-img: up: @echo "---------------------------------------" - @echo "docker-compose up -d" - @docker-compose up -d + @echo "docker compose up -d" + @docker compose up -d @echo "" stop: @echo "---------------------------------------" - @echo "docker-compose stop" - @docker-compose stop + @echo "docker compose stop" + @docker compose stop @echo "" down: @echo "---------------------------------------" - @echo "docker-compose down" - @docker-compose down + @echo "docker compose down" + @docker compose down @echo "" app: up @echo "compose-up run app container" - @docker-compose run --rm app + @docker compose run --rm app @echo "" rm: down @@ -82,24 +82,24 @@ rm: down db-create: up @echo "PHOENIX" - @docker-compose run app python -c "from src.db_models.utils import create_db; create_db()" + @docker compose run app python -c "from src.db_models.utils import create_db; create_db()" @echo "" db-create-tables: up @echo "Creating tables" - @docker-compose run app python -c "from src.db_models.utils import create_or_drop_all_tables; create_or_drop_all_tables(cmd='create')" + @docker compose run app python -c "from src.db_models.utils import create_or_drop_all_tables; create_or_drop_all_tables(cmd='create')" @echo "" db-setup: up @echo "SETUP" @echo "sleeping 40 seconds in order to postgres start-up" @echo "Creating db" - @docker-compose run app python -c "from src.db_models.utils import create_db, create_or_drop_all_tables; create_db();create_or_drop_all_tables(cmd='create')" + @docker compose run app python -c "from src.db_models.utils import create_db, create_or_drop_all_tables; create_db();create_or_drop_all_tables(cmd='create')" @echo "" db-phoenix: up @echo "PHOENIX" - @docker-compose run app python -c "from src.db_models.utils import phoenix; phoenix()" + @docker compose run app python -c "from src.db_models.utils import phoenix; phoenix()" @echo "" db-enter: up @@ -107,29 +107,29 @@ db-enter: up tests: up @echo "compose-up run app & [PYTEST]" - @docker-compose run app python -m pytest + @docker compose run app python -m pytest @echo "" io-download: up @echo "compose-up run app container & [DOWNLOAD]" - @docker-compose run app python src/io/download.py + @docker compose run app python src/io/download.py @echo "" io-unzip: up @echo "compose-up run app container & [UNZIP]" - @docker-compose run app python src/io/unzip.py + @docker compose run app python src/io/unzip.py @echo "" @echo "[CREATE JSONS]" - @docker-compose run app python src/io/create_jsons_from_csv.py + @docker compose run app python src/io/create_jsons_from_csv.py io-create-jsons: up @echo "[CREATE JSONS]" - @docker-compose run app python src/io/create_jsons_from_csv.py + @docker compose run app python src/io/create_jsons_from_csv.py io-download-and-unzip: up @echo "compose-up run app container & [DOWNLOAD]" - @docker-compose run app python src/io/download.py + @docker compose run app python src/io/download.py @echo "" @echo "------------------------" @echo "sleep for 30 seconds to take a breath" @@ -137,42 +137,42 @@ io-download-and-unzip: up @echo "" @echo "------------------------" @echo "[UNZIP]" - @docker-compose run app python src/io/unzip.py + @docker compose run app python src/io/unzip.py @echo "" @echo "[CREATE JSONS]" - @docker-compose run app python src/io/create_jsons_from_csv.py + @docker compose run app python src/io/create_jsons_from_csv.py engine-company: up @echo "compose-up run app container & [ENGINE COMPANY]" - @docker-compose run app python src/engine/company.py + @docker compose run app python src/engine/company.py @echo "" engine-company-tax-regime: up @echo "compose-up run app container & [ENGINE COMPANY TAX REGIME]" - @docker-compose run app python src/engine/company_tax_regime.py + @docker compose run app python src/engine/company_tax_regime.py @echo "" engine-company-root: up @echo "compose-up run app container & [ENGINE COMPANY ROOT]" - @docker-compose run app python src/engine/company_root.py + @docker compose run app python src/engine/company_root.py @echo "" engine-company-root-simples: up @echo "compose-up run app container & [ENGINE COMPANY ROOT SIMPLES]" - @docker-compose run app python src/engine/company_root_simples.py + @docker compose run app python src/engine/company_root_simples.py @echo "" engine-partners: up @echo "compose-up run app container & [ENGINE PARTNERS]" - @docker-compose run app python src/engine/partners.py + @docker compose run app python src/engine/partners.py @echo "" engine-ref-date: up @echo "compose-up run app container & [ENGINE REF DATE]" - @docker-compose run app python src/engine/ref_date.py + @docker compose run app python src/engine/ref_date.py @echo "" engine-main: up @echo "compose-up run app container & engine main" - @docker-compose run app python src/engine/main.py + @docker compose run app python src/engine/main.py @echo "" diff --git a/src/io/__init__.py b/src/io/__init__.py index 6c2032f..cf5cd94 100644 --- a/src/io/__init__.py +++ b/src/io/__init__.py @@ -1,5 +1,5 @@ CORE_URL = "https://www.gov.br/receitafederal/pt-br/assuntos/orientacao-tributaria/cadastros/consultas/dados-publicos-cnpj" -CORE_URL_FILES = "http://200.152.38.155/CNPJ" +CORE_URL_FILES = "https://arquivos.receitafederal.gov.br/dados/cnpj" CNAE_JSON_NAME = 'cnaes.json' NATJU_JSON_NAME = 'natju.json' QUAL_SOCIO_JSON_NAME = 'qual_socio.json' From ad6dbf832100fe2aeb2afceac34c6227dfee6f62 Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 14:56:19 -0300 Subject: [PATCH 3/6] fix: update city names --- tests/fixtures/municipios.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/municipios.json b/tests/fixtures/municipios.json index fa1d28d..416e742 100644 --- a/tests/fixtures/municipios.json +++ b/tests/fixtures/municipios.json @@ -3530,7 +3530,7 @@ "5869": "NOVA IGUACU", "5871": "PARACAMBI", "5873": "PARAIBA DO SUL", - "5875": "PARATY", + "5875": "PARATI", "5877": "PETROPOLIS", "5879": "PIRAI", "5881": "PORCIUNCULA", @@ -5207,7 +5207,7 @@ "9263": "BARRO ALTO", "9265": "BELA VISTA DE GOIAS", "9267": "BOM JARDIM DE GOIAS", - "9269": "BOM JESUS", + "9269": "BOM JESUS DE GOIAS", "9271": "BRAZABRANTES", "9273": "BREJINHO DE NAZARE", "9275": "BRITANIA", From b2be42c2ca686793087aafed85b1f1f376a580ef Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 14:59:59 -0300 Subject: [PATCH 4/6] fix:docker compose to docker-compose --- Dockerfile | 2 +- Makefile | 58 +++++++++++++++++++++++++++--------------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d7b961..443a874 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN pip install --user --no-cache-dir -r requirements/requirements.txt \ -name '*.jpeg' -name '*.js.map' -name '*.pyc' -name '*.c' -name '*.pxc' \ -name '*.pyc' -delete \ && find /usr/local/lib/python3.8 -name '__pycache__' | xargs rm -r -ENV LANG C.UTF-8 +ENV LANG=C.UTF-8 COPY src/ src/ COPY tests/ tests/ diff --git a/Makefile b/Makefile index cff7630..71e46aa 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,9 @@ all: @echo "" @echo "########################################################################################################################" @echo "[SESSION] Launch" - @echo "make up ......................................... docker compose up -d" - @echo "make stop ....................................... docker compose stop" - @echo "make down ....................................... docker compose down" + @echo "make up ......................................... docker-compose up -d" + @echo "make stop ....................................... docker-compose stop" + @echo "make down ....................................... docker-compose down" @echo "make app ........................................ run container app" @echo "make rm ......................................... remove all exited containers and all dangling volumes" @echo "" @@ -47,25 +47,25 @@ build-img: up: @echo "---------------------------------------" - @echo "docker compose up -d" - @docker compose up -d + @echo "docker-compose up -d" + @docker-compose up -d @echo "" stop: @echo "---------------------------------------" - @echo "docker compose stop" - @docker compose stop + @echo "docker-compose stop" + @docker-compose stop @echo "" down: @echo "---------------------------------------" - @echo "docker compose down" - @docker compose down + @echo "docker-compose down" + @docker-compose down @echo "" app: up @echo "compose-up run app container" - @docker compose run --rm app + @docker-compose run --rm app @echo "" rm: down @@ -82,24 +82,24 @@ rm: down db-create: up @echo "PHOENIX" - @docker compose run app python -c "from src.db_models.utils import create_db; create_db()" + @docker-compose run app python -c "from src.db_models.utils import create_db; create_db()" @echo "" db-create-tables: up @echo "Creating tables" - @docker compose run app python -c "from src.db_models.utils import create_or_drop_all_tables; create_or_drop_all_tables(cmd='create')" + @docker-compose run app python -c "from src.db_models.utils import create_or_drop_all_tables; create_or_drop_all_tables(cmd='create')" @echo "" db-setup: up @echo "SETUP" @echo "sleeping 40 seconds in order to postgres start-up" @echo "Creating db" - @docker compose run app python -c "from src.db_models.utils import create_db, create_or_drop_all_tables; create_db();create_or_drop_all_tables(cmd='create')" + @docker-compose run app python -c "from src.db_models.utils import create_db, create_or_drop_all_tables; create_db();create_or_drop_all_tables(cmd='create')" @echo "" db-phoenix: up @echo "PHOENIX" - @docker compose run app python -c "from src.db_models.utils import phoenix; phoenix()" + @docker-compose run app python -c "from src.db_models.utils import phoenix; phoenix()" @echo "" db-enter: up @@ -107,29 +107,29 @@ db-enter: up tests: up @echo "compose-up run app & [PYTEST]" - @docker compose run app python -m pytest + @docker-compose run app python -m pytest @echo "" io-download: up @echo "compose-up run app container & [DOWNLOAD]" - @docker compose run app python src/io/download.py + @docker-compose run app python src/io/download.py @echo "" io-unzip: up @echo "compose-up run app container & [UNZIP]" - @docker compose run app python src/io/unzip.py + @docker-compose run app python src/io/unzip.py @echo "" @echo "[CREATE JSONS]" - @docker compose run app python src/io/create_jsons_from_csv.py + @docker-compose run app python src/io/create_jsons_from_csv.py io-create-jsons: up @echo "[CREATE JSONS]" - @docker compose run app python src/io/create_jsons_from_csv.py + @docker-compose run app python src/io/create_jsons_from_csv.py io-download-and-unzip: up @echo "compose-up run app container & [DOWNLOAD]" - @docker compose run app python src/io/download.py + @docker-compose run app python src/io/download.py @echo "" @echo "------------------------" @echo "sleep for 30 seconds to take a breath" @@ -137,42 +137,42 @@ io-download-and-unzip: up @echo "" @echo "------------------------" @echo "[UNZIP]" - @docker compose run app python src/io/unzip.py + @docker-compose run app python src/io/unzip.py @echo "" @echo "[CREATE JSONS]" - @docker compose run app python src/io/create_jsons_from_csv.py + @docker-compose run app python src/io/create_jsons_from_csv.py engine-company: up @echo "compose-up run app container & [ENGINE COMPANY]" - @docker compose run app python src/engine/company.py + @docker-compose run app python src/engine/company.py @echo "" engine-company-tax-regime: up @echo "compose-up run app container & [ENGINE COMPANY TAX REGIME]" - @docker compose run app python src/engine/company_tax_regime.py + @docker-compose run app python src/engine/company_tax_regime.py @echo "" engine-company-root: up @echo "compose-up run app container & [ENGINE COMPANY ROOT]" - @docker compose run app python src/engine/company_root.py + @docker-compose run app python src/engine/company_root.py @echo "" engine-company-root-simples: up @echo "compose-up run app container & [ENGINE COMPANY ROOT SIMPLES]" - @docker compose run app python src/engine/company_root_simples.py + @docker-compose run app python src/engine/company_root_simples.py @echo "" engine-partners: up @echo "compose-up run app container & [ENGINE PARTNERS]" - @docker compose run app python src/engine/partners.py + @docker-compose run app python src/engine/partners.py @echo "" engine-ref-date: up @echo "compose-up run app container & [ENGINE REF DATE]" - @docker compose run app python src/engine/ref_date.py + @docker-compose run app python src/engine/ref_date.py @echo "" engine-main: up @echo "compose-up run app container & engine main" - @docker compose run app python src/engine/main.py + @docker-compose run app python src/engine/main.py @echo "" From be336bc270f0e979fe50b9fb98f8e95ad0771dfe Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 15:11:11 -0300 Subject: [PATCH 5/6] fix: fix gha structure --- .github/workflows/tests.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index e9c4ff5..2e9be26 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,15 +11,17 @@ env: jobs: run_tests: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f - name: clone this repo + - name: Checkout repository + uses: actions/checkout@v3 + id: checkout - - name: build img - run: make build-img + - name: Set up Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose - name: create local env file run: | From b93f2a0dcb20c8f801546352248345c2db4400ec Mon Sep 17 00:00:00 2001 From: Robson Date: Fri, 22 Nov 2024 15:15:03 -0300 Subject: [PATCH 6/6] fix: fix gha structure --- .github/workflows/tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 2e9be26..6ea838a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,6 +23,9 @@ jobs: sudo apt-get update sudo apt-get install -y docker-compose + - name: build img + run: make build-img + - name: create local env file run: | touch .env