From cc45c8da3d5e85c2490fbdc37e1b80469670baee Mon Sep 17 00:00:00 2001 From: pdp <7412441+pzuldp@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:35:19 +0200 Subject: [PATCH 001/140] Add Windows specificities to short install manual --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 26031b9c..9ef2f0ec 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,12 @@ This should not display any error and end with: `Successfully installed OpenFisca-France-Data...` +### Specifics due to Windows' handling of long paths + +On a Windows machine, the installation of Openfisca-France-Data may run into problems due to long path names, which Windows, by default, does not handle. These long paths are mostly inherited from the OpenFisca-France parameters, which are stored in a sometimes deeply nested folder. + +A possible workaround on Windows >= 10 is to lift the maximum path length limitation (as [indicated here](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=registry#enable-long-paths-in-windows-10-version-1607-and-later)). + ## Execution Let's say that you would like to format `ERFS-FPR` survey data into OpenFisca formatted data. From 2faa9c8c95f2d4035e19c3767811af8d8684e736 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 28 Mar 2023 09:01:39 +0000 Subject: [PATCH 002/140] Add a test case creator --- .gitlab-ci.yml | 2 - ci-runner/build_ci.py | 1 - gitlab_ci/all_years_build_and_aggregates.yml | 1 - openfisca_france_data/__init__.py | 2 +- .../erfs_fpr/get_survey_scenario.py | 63 ++++++-- .../input_data_builder/step_05_final.py | 1 + .../erfs_fpr/test_case_creation.py | 146 ++++++++++++++++++ openfisca_france_data/model/base.py | 3 - setup.py | 9 +- tests/erfs_fpr/integration/test_aggregates.py | 4 +- 10 files changed, 208 insertions(+), 24 deletions(-) create mode 100644 openfisca_france_data/erfs_fpr/test_case_creation.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e41981c3..16d1e951 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -134,8 +134,6 @@ diagnostics: - cat ~/.config/openfisca-france-data/config.ini - compare-erfs-fpr-input -u -s -v - cp -r /mnt/data-out/openfisca-france-data/$OUT_FOLDER/figures_directory . - - ls -alrth - - ls -alrth figures_directory - cp -r ./figures_directory $ROOT_FOLDER/$OUT_FOLDER/data_output stage: diagnostics tags: diff --git a/ci-runner/build_ci.py b/ci-runner/build_ci.py index 014c0acc..6ec8fea1 100644 --- a/ci-runner/build_ci.py +++ b/ci-runner/build_ci.py @@ -111,7 +111,6 @@ def get_erfs_years(): raise KeyError - def build_gitlab_ci(erfs_years): gitlab_ci = header() # gitlab_ci += yaml.dump(make_test()) diff --git a/gitlab_ci/all_years_build_and_aggregates.yml b/gitlab_ci/all_years_build_and_aggregates.yml index 145e4557..66a4ff01 100644 --- a/gitlab_ci/all_years_build_and_aggregates.yml +++ b/gitlab_ci/all_years_build_and_aggregates.yml @@ -11,7 +11,6 @@ input_data-2019: - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - cat ~/.config/openfisca-survey-manager/config.ini - - ls $ROOT_FOLDER/$OUT_FOLDER/data_collections - build-erfs-fpr -y 2019 -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_2019.h5 - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini stage: build_input_data diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 8bb2ec7c..9b936da8 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -9,7 +9,7 @@ import openfisca_france # type: ignore # Load input variables and output variables into entities -from openfisca_france_data.model import common, survey_variables, id_variables # noqa analysis:ignore +from openfisca_france_data.model import common, survey_variables # noqa analysis:ignore from openfisca_france_data.model.base import * # noqa analysis:ignore diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index b55b379a..9c652d1b 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -2,13 +2,61 @@ from multipledispatch import dispatch # type: ignore +from openfisca_core.model_api import Variable, ADD, YEAR from openfisca_core.reforms import Reform # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore +from openfisca_france.entities import Individu from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario from openfisca_france_data import france_data_tax_benefit_system +from openfisca_france_data.model.id_variables import ( + idmen_original, + noindiv, + ) + +variables_converted_to_annual = [ + "salaire_net", + "chomage_net", + "retraite_nette", + "ppa", + ] + + +class erfs_fpr_plugin(Reform): + name = "ERFS-FPR ids plugin" + + def apply(self): + + for variable in variables_converted_to_annual: + class_name = f"{variable}_annuel" + label = f"{variable} sur l'année entière" + + def formula_creator(variable): + def formula(individu, period): + result = individu(variable, period, options = [ADD]) + return result + + formula.__name__ = 'formula' + + return formula + + variable_instance = type(class_name, (Variable,), dict( + value_type = int, + entity = self.variables[variable].entity, + label = label, + definition_period = YEAR, + formula = formula_creator(variable), + )) + + self.add_variable(variable_instance) + del variable_instance + + self.add_variable(idmen_original) + self.add_variable(noindiv) + + def get_survey_scenario( year: int = None, rebuild_input_data: bool = False, @@ -40,15 +88,6 @@ def get_survey_scenario( baseline_tax_benefit_system, ) - - from openfisca_france_data.model.id_variables import ( - idmen_original, - noindiv, - ) - - tax_benefit_system.add_variable(idmen_original) - tax_benefit_system.add_variable(noindiv) - if not use_marginal_tax_rate: survey_scenario = ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, @@ -114,7 +153,7 @@ def get_tax_benefit_system( tax_benefit_system: None, reform: Reform, ) -> TaxBenefitSystem: - return reform(france_data_tax_benefit_system) + return reform(erfs_fpr_plugin(france_data_tax_benefit_system)) # Appelé quand *tax_benefit_system* et *reform* sont `None` @@ -123,7 +162,7 @@ def get_tax_benefit_system( tax_benefit_system: None, reform: None, ) -> TaxBenefitSystem: - return france_data_tax_benefit_system + return erfs_fpr_plugin(france_data_tax_benefit_system) # Appelé quand *tax_benefit_system* est un :class:`TaxBenefitSystem` @@ -139,4 +178,4 @@ def get_baseline_tax_benefit_system( def get_baseline_tax_benefit_system( tax_benefit_system: None, ) -> TaxBenefitSystem: - return france_data_tax_benefit_system + return erfs_fpr_plugin(france_data_tax_benefit_system) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py index 1d3e0abe..dd76a35f 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py @@ -127,6 +127,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene ) log.debug("End of create_input_data_frame") + def create_collectives_foyer_variables(individus, menages): menages_revenus_fonciers = menages[['idmen', 'rev_fonciers_bruts']].copy() idmens = menages_revenus_fonciers.query('(rev_fonciers_bruts > 0)')['idmen'].tolist() diff --git a/openfisca_france_data/erfs_fpr/test_case_creation.py b/openfisca_france_data/erfs_fpr/test_case_creation.py new file mode 100644 index 00000000..b2572da1 --- /dev/null +++ b/openfisca_france_data/erfs_fpr/test_case_creation.py @@ -0,0 +1,146 @@ +import click +import ipdb as pdb +import logging +import pandas as pd +import sys +import yaml + +from pandas.api.types import is_datetime64_any_dtype as is_datetime +from openfisca_core.model_api import ETERNITY + + +from openfisca_france_data import france_data_tax_benefit_system +from openfisca_france_data.erfs_fpr import original_id_by_entity +from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario +from openfisca_france_data.erfs_fpr.comparison import ErfsFprtoInputComparator +from openfisca_france_data.erfs_fpr.get_survey_scenario import variables_converted_to_annual + + +tax_benefit_system = france_data_tax_benefit_system +openfisca_variables_by_entity = dict( + ( + entity.key, + [variable_name for variable_name, variable in tax_benefit_system.variables.items() if variable.entity.key == entity.key], + ) + for entity in tax_benefit_system.entities + ) + +id_variable_by_entity_key = ErfsFprSurveyScenario.id_variable_by_entity_key +weight_variable_by_entity = ErfsFprSurveyScenario.weight_variable_by_entity + + +renaming_variables_to_annual = { + monthly_variable: f"{monthly_variable}_annuel" + for monthly_variable in variables_converted_to_annual + } + +def build_test(period, noindiv, target_variables = None): + if target_variables is None: + target_variables = ErfsFprtoInputComparator.default_target_variables + + comparator = ErfsFprtoInputComparator() + comparator.period = period + input_dataframe_by_entity, target_dataframe_by_entity = comparator.get_test_dataframes(rebuild = True, noindivs = [noindiv]) + + def convert_date_to_sting(dataframe): + date_columns = list(dataframe.select_dtypes(include=["datetime64"])) + dataframe[date_columns] = dataframe[date_columns].astype(str) + + def remove_non_openfisca_columns(dataframe): + openfisca_variables = set(sum([list(value) for value in openfisca_variables_by_entity.values()], [])).union(set(["noindiv", "idmen_original"])) + selected_variables = list(set(dataframe.columns).intersection(openfisca_variables)) + return dataframe[selected_variables] + + def build_test_dict(dataframe_by_entity, renaming_variables_to_annual = None): + input_by_entity = dict() + for entity, dataframe in dataframe_by_entity.items(): + convert_date_to_sting(dataframe) + identifier = "noindiv" if entity == "individu" else "idmen_original" + entity_plural = "individus" if entity == "individu" else "menages" + input_by_entity[entity_plural] = input = dict() + dataframe[identifier] = "id_" + dataframe[identifier].astype(str) + df = remove_non_openfisca_columns(dataframe).set_index(identifier) + for row, series in df.iterrows(): + series.drop( + ( + list(weight_variable_by_entity.values()) + + list(id_variable_by_entity_key.values()) + + list(original_id_by_entity.values()) + ), + inplace = True, + errors = "ignore", + ) + if renaming_variables_to_annual: + series.rename(renaming_variables_to_annual, inplace = True) + input[row] = series.dropna().to_dict() + + return input_by_entity + + input_by_entity = build_test_dict(input_dataframe_by_entity) + output_by_entity = build_test_dict(target_dataframe_by_entity, renaming_variables_to_annual) + + relative_error_margin = { + "default": 5e-3, + } + test = dict( + name = f"Observation {noindiv} on {period}", + reforms = "openfisca_france_data.erfs_fpr.get_survey_scenario.erfs_fpr_plugin", + max_spiral_loops = 4, + relative_error_margin = relative_error_margin, + period = period, + input = input_by_entity, + output = output_by_entity, + ) + return test + + + +def export_test_file(period, noindiv, test_case_name = None): + """ + Export a erfs-fpr input and output to an OpenFisca test case. + + Args: + period (int): simulation year + noindiv (int): individu id number + test_case_name (str, optional): _description_. Defaults to Name of the test case file. Defaults to 'test_case_erfs_fpr_NOINDIV'. + """ + if test_case_name is None: + test_case_name = f"test_case_erfs_fpr_{noindiv}" + + test_case_file_path = f'{test_case_name}.yaml' + test = build_test(period, noindiv) + + with open(test_case_file_path, 'w') as file: + yaml.dump(test, file, sort_keys=False) + + text = get_erfs_fpr_data_as_comment(noindiv) + + with open(test_case_file_path, "a+") as file: + _ = file.read() # this auto closes the file after reading, which is a good practice + file.write(text) + + +def get_erfs_fpr_data_as_comment(noind): + return "# Blabal" + + +@click.command() +@click.option('-n', '--noindiv', type = int, help = "Individual id number", required = True) +@click.option('-v', '--verbose', is_flag = True, default = False, help = "Increase output verbosity", show_default = True) +@click.option('-d', '--debug', is_flag = True, default = False, help = "Use python debugger", show_default = True) +def create_test(noindiv = 0, verbose = False, debug = False): + """Create test case for a specific ERFS FPR individual.""" + assert noindiv != 0, "Provide valid individual" + logging.basicConfig(level = logging.DEBUG if verbose else logging.WARNING, stream = sys.stdout) + from openfisca_france_data.erfs_fpr import REFERENCE_YEAR + period = REFERENCE_YEAR + try: + export_test_file(period, noindiv) + except Exception as e: + if debug: + pdb.post_mortem(sys.exc_info()[2]) + raise e + + +if __name__ == "__main__": + sys.exit(create_test()) diff --git a/openfisca_france_data/model/base.py b/openfisca_france_data/model/base.py index a954ca0e..cc508420 100644 --- a/openfisca_france_data/model/base.py +++ b/openfisca_france_data/model/base.py @@ -1,6 +1,3 @@ -from datetime import date - - from openfisca_france.model.base import * diff --git a/setup.py b/setup.py index 162783e4..99ee1db7 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,10 @@ "click >= 8.0.0, < 9.0.0", "matplotlib >= 3.1.1, < 4.0.0", "multipledispatch >= 0.6.0, < 1.0.0", - "openFisca-france >= 145.0.0, < 146.0.0", - "openFisca-survey-manager >= 0.47.2, < 1.0.0", + # "openFisca-france >= 145.0.0, < 146.0.0", + "OpenFisca-France @ git+https://github.com/openfisca/openfisca-france.git@version_leap", + # "openFisca-survey-manager >= 0.47.2, < 1.0.0", + "OpenFisca-Survey-Manager @ git+https://github.com/openfisca/openfisca-survey-manager.git@version_leap", "wquantiles >= 0.3.0, < 1.0.0", # To compute weighted quantiles ], extras_require = { @@ -45,10 +47,11 @@ "bumpver >= 2022.1120", "dtale", "flake8 >= 3.7.0, < 3.8.0", + "ipdb >=0.13, <1.0", "ipython >= 7.5.0, < 8.0.0", "mypy >= 0.670, < 1.0.0", "pypandoc", - 'pytest >= 5.0.0, < 7.0.0', + # "pytest", # "pytest-cov >= 2.6.0, < 3.0.0", "scipy >= 1.2.1, < 2.0.0", "toolz >= 0.9.0, < 1.0.0", diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index 479888f2..3473bc70 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -10,7 +10,9 @@ import os + from openfisca_france_data import france_data_tax_benefit_system +from openfisca_france_data.erfs_fpr import REFERENCE_YEAR from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario from openfisca_france_data.aggregates import FranceAggregates as Aggregates @@ -76,7 +78,7 @@ def test_erfs_fpr_aggregates_reform(): @click.command() -@click.option('-y', '--year', 'year', default = 2018, help = "ERFS-FPR year", show_default = True, +@click.option('-y', '--year', 'year', default = REFERENCE_YEAR, help = "ERFS-FPR year", show_default = True, type = int, required = True) @click.option('-c', '--configfile', default = None, help = 'raw_data.ini path to read years to process.', show_default = True) From 65a3888baa667de5f4c66d19f96a3e2173b70ff6 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 29 Mar 2023 07:17:22 +0000 Subject: [PATCH 003/140] Fix traitement brut erfs fpr --- openfisca_france_data/common.py | 30 +-- openfisca_france_data/erfs/scenario.py | 2 + openfisca_france_data/erfs_fpr/comparison.py | 24 +-- .../erfs_fpr/get_survey_scenario.py | 3 + .../step_03_variables_individuelles.py | 199 +++--------------- .../input_data_builder/step_05_final.py | 3 +- openfisca_france_data/erfs_fpr/scenario.py | 2 +- openfisca_france_data/surveys.py | 11 +- 8 files changed, 70 insertions(+), 204 deletions(-) diff --git a/openfisca_france_data/common.py b/openfisca_france_data/common.py index de8cf677..2507df29 100644 --- a/openfisca_france_data/common.py +++ b/openfisca_france_data/common.py @@ -230,15 +230,15 @@ def create_traitement_indiciaire_brut(individus, period = None, revenu_type = 'i contrat_de_travail = individus.contrat_de_travail heures_remunerees_volume = individus.heures_remunerees_volume - legislation = tax_benefit_system.parameters(period.start) + parameters = tax_benefit_system.parameters(period.start) - salarie = legislation.cotsoc.cotisations_salarie - plafond_securite_sociale_mensuel = legislation.prelevements_sociaux.pss.plafond_securite_sociale_mensuel - legislation_csg_deductible = legislation.prelevements_sociaux.contributions_sociales.csg.activite.deductible - taux_csg = legislation_csg_deductible.taux - taux_abattement = legislation_csg_deductible.abattement.rates[0] + salarie = parameters.cotsoc.cotisations_salarie + plafond_securite_sociale_mensuel = parameters.prelevements_sociaux.pss.plafond_securite_sociale_mensuel + parameters_csg_deductible = parameters.prelevements_sociaux.contributions_sociales.csg.activite.deductible + taux_csg = parameters_csg_deductible.taux + taux_abattement = parameters_csg_deductible.abattement.rates[0] try: - seuil_abattement = legislation_csg_deductible.abattement.thresholds[1] + seuil_abattement = parameters_csg_deductible.abattement.thresholds[1] except IndexError: # Pour gérer le fait que l'abattement n'a pas toujours été limité à 4 PSS seuil_abattement = None csg_deductible = MarginalRateTaxScale(name = 'csg_deductible') @@ -250,11 +250,11 @@ def create_traitement_indiciaire_brut(individus, period = None, revenu_type = 'i # Cas des revenus nets: # comme les salariés du privé, on ajoute CSG imposable et crds qui s'appliquent à tous les revenus # 1. csg imposable - legislation_csg_imposable = legislation.prelevements_sociaux.contributions_sociales.csg.activite.imposable - taux_csg = legislation_csg_imposable.taux - taux_abattement = legislation_csg_imposable.abattement.rates[0] + parameters_csg_imposable = parameters.prelevements_sociaux.contributions_sociales.csg.activite.imposable + taux_csg = parameters_csg_imposable.taux + taux_abattement = parameters_csg_imposable.abattement.rates[0] try: - seuil_abattement = legislation_csg_imposable.abattement.thresholds[1] + seuil_abattement = parameters_csg_imposable.abattement.thresholds[1] except IndexError: # Pour gérer le fait que l'abattement n'a pas toujours été limité à 4 PSS seuil_abattement = None csg_imposable = MarginalRateTaxScale(name = 'csg_imposable') @@ -262,11 +262,11 @@ def create_traitement_indiciaire_brut(individus, period = None, revenu_type = 'i if seuil_abattement is not None: csg_imposable.add_bracket(seuil_abattement, taux_csg) # 2. crds - legislation_crds = legislation.prelevements_sociaux.contributions_sociales.crds.activite - taux_csg = legislation_crds.taux - taux_abattement = legislation_crds.abattement.rates[0] + parameters_crds = parameters.prelevements_sociaux.contributions_sociales.crds.activite + taux_csg = parameters_crds.taux + taux_abattement = parameters_crds.abattement.rates[0] try: - seuil_abattement = legislation_crds.abattement.thresholds[1] + seuil_abattement = parameters_crds.abattement.thresholds[1] except IndexError: # Pour gérer le fait que l'abattement n'a pas toujours été limité à 4 PSS seuil_abattement = None crds = MarginalRateTaxScale(name = 'crds') diff --git a/openfisca_france_data/erfs/scenario.py b/openfisca_france_data/erfs/scenario.py index 19195397..46ad488b 100644 --- a/openfisca_france_data/erfs/scenario.py +++ b/openfisca_france_data/erfs/scenario.py @@ -26,6 +26,7 @@ class ErfsSurveyScenario(AbstractErfsSurveyScenario): 'nbN', 'nbR', 'pensions_alimentaires_percues', + 'primes_fonction_publique', 'rag', 'retraite_brute', 'retraite_imposable', @@ -36,6 +37,7 @@ class ErfsSurveyScenario(AbstractErfsSurveyScenario): 'statut_marital', 'statut_occupation_logement', 'taxe_habitation', + 'traitement_indiciaire_brut', 'zone_apl', ] diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index a649f4bf..f8e0b170 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -15,15 +15,15 @@ openfisca_by_erfs_fpr_variables = { - "chomage_i": "chomage_net", + "chomage_i": "chomage_imposable", "ident": "idmen_original", "noindiv": "noindiv", - "rag_i": "rag_net", - "retraites_i": "retraite_nette", # TODO: CHECk + "rag_i": "rag", + "retraites_i": "retraite_imposable", # TODO: CHECk "rev_fonciers_bruts": "f4ba", - "ric_i": "ric_net", - "rnc_i": "rnc_net", - "salaires_i": "salaire_net", + "ric_i": "ric", + "rnc_i": "rnc", + "salaires_i": "salaire_imposable", } @@ -31,13 +31,13 @@ class ErfsFprtoInputComparator(AbstractComparator): name = "erfs_fpr" period = None default_target_variables = [ - "chomage_net", - # "rag_net", TODO: does not exist in openfisca - "retraite_nette", - # "ric_net", TODO: does not exist in openfisca - # "rnc_net", TODO: does not exist in openfisca + "chomage_imposable", + # "rag", + "retraite_imposable", + # "ric", + # "rnc", # "f4ba", - "salaire_net", + "salaire_imposable", ] def compute_test_dataframes(self): diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 9c652d1b..290d2f1a 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -17,6 +17,9 @@ ) variables_converted_to_annual = [ + "salaire_imposable", + "chomage_imposable", + "retraite_imposable", "salaire_net", "chomage_net", "retraite_nette", diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 7b36012a..7b895658 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -7,7 +7,6 @@ from openfisca_france_data.common import ( create_salaire_de_base, create_traitement_indiciaire_brut, - create_revenus_remplacement_bruts, ) from openfisca_france_data import openfisca_france_tax_benefit_system from openfisca_france_data.smic import ( @@ -30,13 +29,13 @@ def build_variables_individuelles(temporary_store = None, year = None): individus = temporary_store['individus_{}_post_01'.format(year)] openfisca_by_erfs_variable = { - 'chomage_i': 'chomage_net', + 'chomage_i': 'chomage_imposable', 'pens_alim_recue_i': 'pensions_alimentaires_percues', 'rag_i': 'rag_net', - 'retraites_i': 'retraite_nette', + 'retraites_i': 'retraite_imposable', 'ric_i': 'ric_net', 'rnc_i': 'rnc_net', - 'salaires_i': 'salaire_net', + 'salaires_i': 'salaire_imposable', } for variable in openfisca_by_erfs_variable.keys(): @@ -48,104 +47,55 @@ def build_variables_individuelles(temporary_store = None, year = None): ) create_variables_individuelles(individus, year) assert 'salaire_de_base' in individus.columns , 'salaire de base not in individus' + assert 'traitement_indiciaire_brut' in individus.columns , 'traitement indiciaire brut not in individus' + assert 'primes_fonction_publique' in individus.columns , 'primes fonction publique not in individus' temporary_store['individus_{}'.format(year)] = individus return individus # helpers -def create_variables_individuelles(individus, year, survey_year = None): +def create_variables_individuelles(individus, year, survey_year = None, revenu_type = 'imposable'): """Création des variables individuelles""" + period = periods.period(year) + tax_benefit_system = openfisca_france_tax_benefit_system + # variables démographiques create_ages(individus, year) create_date_naissance(individus, age_variable = None, annee_naissance_variable = 'naia', mois_naissance = 'naim', year = year) + # Base pour constituer les familles, foyers, etc. + create_statut_matrimonial(individus) + + # variable d'activite create_activite(individus) - revenu_type = 'net' - period = periods.period(year) - create_revenus(individus, revenu_type = revenu_type) create_contrat_de_travail(individus, period = period, salaire_type = revenu_type) create_categorie_salarie(individus, period = period, survey_year = survey_year) - # Il faut que la base d'input se fasse au millésime des données - # On fait ça car, aussi bien le TaxBenefitSystem et celui réformé peuvent être des réformes - # Par exemple : si je veux calculer le diff entre le PLF2019 et un ammendement, - # je besoin d'un droit courant comme même du droit courrant pour l'année des données - tax_benefit_system = openfisca_france_tax_benefit_system - - # On n'a pas le salaire brut mais le salaire net ou imposable, on doit l'inverser + # inversion des revenus pour retrouver le brut + # pour les revenus de remplacement on a la csg et la crds dans l'erfs-fpr donc on peut avoir le brut directement + create_revenus_remplacement_bruts(individus) + # On n'a pas le salaire et le traitement_indiciaire brut, on doit l'inverser + # comme on a la crds et la csg non déductible on recalcule l'imposable puis on inverse l'imposable pour avoir le brut + #individus['salaire_imposable'] = individus.salaire_net + individus.csg_nd_crds_sal_i create_salaire_de_base( individus, period = period, revenu_type = revenu_type, tax_benefit_system = tax_benefit_system ) - + create_traitement_indiciaire_brut( + individus, + period = period, + revenu_type = revenu_type, + tax_benefit_system = tax_benefit_system) + # Pour les cotisations patronales qui varient avec la taille de l'entreprise' create_effectif_entreprise(individus, period = period, survey_year = survey_year) - # Base pour constituer les familles, foyers, etc. - create_statut_matrimonial(individus) - assert 'salaire_de_base' in individus.columns , 'salaire de base not in individus' return individus -def create_individu_variables_brutes(individus, revenu_type = None, period = None, - tax_benefit_system = None, mass_by_categorie_salarie = None, - calibration_eec = False): - """ - Crée les variables brutes de revenus: - - salaire_de_base - - traitement_indiciaire_brut - - primes_fonction_publique - - retraite_bruite - - chomage_brut - à partir des valeurs nettes ou imposables de ces revenus - et d'autres information individuelles - """ - assert revenu_type in ['imposable', 'net'] - assert period is not None - assert tax_benefit_system is not None - - assert 'age' in individus.columns - - created_variables = [] - create_contrat_de_travail(individus, period = period, salaire_type = revenu_type) - created_variables.append('contrat_de_travail') - created_variables.append('heures_remunerees_volume') - - create_categorie_salarie(individus, period = period) - created_variables.append('categorie_salarie') - create_categorie_non_salarie(individus) - created_variables.append('categorie_non_salarie') - - # FIXME: categorie_non_salarie modifie aussi categorie_salarie !! - if (mass_by_categorie_salarie is not None) & (calibration_eec is True): - calibrate_categorie_salarie(individus, year = None, mass_by_categorie_salarie = mass_by_categorie_salarie) - - create_salaire_de_base(individus, period = period, revenu_type = revenu_type, tax_benefit_system = tax_benefit_system) - created_variables.append('salaire_de_base') - - create_effectif_entreprise(individus, period = period) - created_variables.append('effectif_entreprise') - - create_traitement_indiciaire_brut(individus, period = period, revenu_type = revenu_type, - tax_benefit_system = tax_benefit_system) - created_variables.append('traitement_indiciaire_brut') - created_variables.append('primes_fonction_publique') - - create_taux_csg_remplacement(individus, period, tax_benefit_system) - created_variables.append('taux_csg_remplacement') - created_variables.append('taux_csg_remplacement_n_1') - created_variables.append('rfr_special_csg_n') - created_variables.append('rfr_special_csg_n_1') - - create_revenus_remplacement_bruts(individus, period, tax_benefit_system) - created_variables.append('chomage_brut') - created_variables.append('retraite_brute') - return created_variables - - def create_activite(individus): """Création de la variable activite. @@ -540,31 +490,6 @@ def create_categorie_non_salarie(individus): profession_liberale, 'categorie_non_salarie' ] = 3 - # Correction fonction publique - individus.loc[ - ( - (individus.categorie_salarie == 0) - & (individus.cstot.isin([31, 33, 34, 35, 37, 38,])) - ), - 'categorie_salarie' - ] = 1 - - # Correction encadrement - individus.loc[ - ( - (individus.categorie_salarie == 0) - & (individus.cstot.isin([31, 34, 35, 37 ,38])) # Cadres hors FP - ), - 'categorie_salarie' - ] = 1 - # Correction fonction publique - individus.loc[ - ( - (individus.categorie_salarie.isin([0, 1, 7])) - & (individus.cstot == 53) # Policiers et militaires reversé dans titulaire état - ), - 'categorie_salarie' - ] = 2 def create_contrat_de_travail(individus, period, salaire_type = 'imposable'): @@ -971,74 +896,14 @@ def create_effectif_entreprise(individus, period = None, survey_year = None): individus.effectif_entreprise.value_counts(dropna = False))) -def create_revenus(individus, revenu_type = 'imposable'): - """Création des plusieurs variablesde revenu. - - Ces variables sont: - chomage_net, - pensions_alimentaires_percues, - rag_net, - retraite_nette, - ric_net, - rnc_net, - et éventuellement, si revenus_type = 'imposable' des variables: - chomage_imposable, - rag, - retraite_imposable, - ric, - rnc, - salaire_imposable, +def create_revenus_remplacement_bruts(individus): """ - individus['chomage_brut'] = individus.csgchod_i + individus.chomage_net - individus['retraite_brute'] = individus.csgrstd_i + individus.retraite_nette - - if revenu_type == 'imposable': - variables = [ - # 'pension_alimentaires_percues', - 'chomage_imposable', - 'retraite_imposable', - ] - for variable in variables: - assert variable in individus.columns.tolist(), "La variable {} n'est pas présente".format(variable) - - for variable in variables: - if (individus[variable] < 0).any(): - - negatives_values = individus[variable].value_counts().loc[individus[variable].value_counts().index < 0] - log.debug("La variable {} contient {} valeurs négatives\n {}".format( - variable, - negatives_values.sum(), - negatives_values, - ) - ) - - # csg des revenus de replacement - # 0 - Non renseigné/non pertinent - # 1 - Exonéré - # 2 - Taux réduit - # 3 - Taux plein - taux = pd.concat( - [ - individus.csgrstd_i / individus.retraite_brute, - individus.csgchod_i / individus.chomage_brut, - ], - axis = 1 - ).max(axis = 1) - - # taux.loc[(0 < taux) & (taux < .1)].hist(bins = 100) - individus['taux_csg_remplacement'] = np.select( - [ - taux.isnull(), - taux.notnull() & (taux < 0.021), - taux.notnull() & (taux > 0.021) & (taux < 0.0407), - taux.notnull() & (taux > 0.0407) - ], - [0, 1, 2, 3] - ) - for value in [0, 1, 2, 3]: - assert (individus.taux_csg_remplacement == value).any(), \ - "taux_csg_remplacement ne prend jamais la valeur {}".format(value) - assert individus.taux_csg_remplacement.isin(range(4)).all() + Reconstitution des variables de retraite et chomage brut à partir des variables nettes et des variables de csg et crds + """ + + # revenu_brut = revenu_net + csg_deductible + csg_non_deductible_crds + individus['chomage_brut'] = individus.chomage_imposable + individus.csgchod_i #+ individus. csg_nd_crds_cho_i + individus['retraite_brute'] = individus.retraite_imposable + individus.csgrstd_i #+ individus.csg_nd_crds_ret_i def create_statut_matrimonial(individus): diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py index dd76a35f..fccaa092 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py @@ -42,7 +42,8 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'statut_marital', # 'salaire_imposable', 'salaire_de_base', - 'taux_csg_remplacement', + "traitement_indiciaire_brut", + "primes_fonction_publique", ] # TODO: fix this simplistic inference diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index a72a0d9d..dca44945 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -27,8 +27,8 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "retraite_imposable", "ric", "rnc", - "statut_marital", "salaire_de_base", + "statut_marital", "statut_occupation_logement", "taxe_habitation", "traitement_indiciaire_brut", diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index b2463c60..0ccc5c58 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -146,11 +146,13 @@ def custom_initialize(self, simulation): ] computed_variables_used_as_input = [ - # 'chomage_brut', "chomage_imposable", + "primes_fonction_publique", "retraite_brute", "retraite_imposable", "salaire_de_base", + "traitement_indiciaire_brut", + # 'chomage_brut', ] three_year_span_variables = input_variables + computed_variables_used_as_input @@ -194,13 +196,6 @@ def custom_input_data_frame(self, input_data_frame, **kwargs): if "loyer" in input_data_frame: input_data_frame["loyer"] = 12 * input_data_frame.loyer - if "categorie_salarie" in input_data_frame: - log.debug("Setting categorie_salarie to 0 and 1 only") - input_data_frame.loc[ - input_data_frame.categorie_salarie.isin(range(2, 7)), - "categorie_salarie", - ] = 1 - for variable in ["quifam", "quifoy", "quimen"]: if variable in input_data_frame: log.debug(input_data_frame[variable].value_counts(dropna = False)) From 407d7ea11dc1db959c36164ed4bc79e5a840e7d3 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 31 Mar 2023 08:49:21 +0000 Subject: [PATCH 004/140] Ajoute les variables du revenu du capital --- openfisca_france_data/comparator.py | 52 ++- openfisca_france_data/debugger.py | 437 ------------------ openfisca_france_data/erfs_fpr/comparison.py | 14 + .../erfs_fpr/get_survey_scenario.py | 43 +- .../erfs_fpr/input_data_builder/__init__.py | 18 +- .../step_03_variables_individuelles.py | 17 +- .../input_data_builder/step_05_foyer.py | 61 +++ .../{step_05_final.py => step_06_final.py} | 65 ++- openfisca_france_data/erfs_fpr/scenario.py | 3 + 9 files changed, 219 insertions(+), 491 deletions(-) delete mode 100644 openfisca_france_data/debugger.py create mode 100644 openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py rename openfisca_france_data/erfs_fpr/input_data_builder/{step_05_final.py => step_06_final.py} (84%) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 2258d41e..39c27541 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -23,18 +23,20 @@ log = logging.getLogger(__name__) +def get_entity_original_id(survey_scenario, variable): + entity = survey_scenario.tax_benefit_system.variables[variable].entity.key + return "noindiv" if entity == "individu" else "idmen_original" + + def compute_result(variable, survey_scenario, target_dataframe): result = None stats = None - entity = survey_scenario.tax_benefit_system.variables[variable].entity.key - entity_original_id = "noindiv" if entity == "individu" else "ident" + entity_original_id = get_entity_original_id(survey_scenario, variable) output_variables = [entity_original_id, variable] - entity_dataframe = survey_scenario.create_data_frame_by_entity( variables = output_variables, )[entity] - target = target_dataframe[output_variables].rename(columns = {variable: f"target_{variable}"}) if f"target_{variable}" not in target: @@ -47,12 +49,12 @@ def compute_result(variable, survey_scenario, target_dataframe): ) result[f"diff_{variable}"] = result[variable] - result[f"target_{variable}"] - result_variables = ["noindiv", variable, f"diff_{variable}", f"target_{variable}"] - stats = compute_error_stats(result, variable) + result_variables = [entity_original_id, variable, f"diff_{variable}", f"target_{variable}"] + stats = compute_error_stats(result, variable, entity_original_id = entity_original_id) return result, stats -def compute_confidence_interval(data, variable, width = .9): +def compute_confidence_interval(data, variable, width = .9, entity_original_id = None): """ Compute confidence interval @@ -64,9 +66,10 @@ def compute_confidence_interval(data, variable, width = .9): Returns: [type]: [description] """ + assert entity_original_id is not None df = pd.DataFrame({ "signed_values": data[variable].values, - "noind": data["noindiv"].values + "noind": data[entity_original_id].values }) df["abs_values"] = df.signed_values.abs() in_range_obs = ceil(width * len(df)) @@ -79,7 +82,8 @@ def compute_confidence_interval(data, variable, width = .9): return left, right, largest_errors -def compute_error_stats(data, variable): +def compute_error_stats(data, variable, entity_original_id): + assert entity_original_id is not None numerical = ( isinstance(data[variable].values.flat[0], np.integer) or isinstance(data[variable].values.flat[0], np.floating) @@ -89,13 +93,13 @@ def compute_error_stats(data, variable): df = data.loc[ (data[variable].values != 0.0) | (data[f"target_{variable}"].values != 0.0), - [variable, f"target_{variable}", "noindiv"] + [variable, f"target_{variable}", entity_original_id] ].copy() df["relative_error"] = (df[variable] - df[f"target_{variable}"]) / (df[f"target_{variable}"] + (df[f"target_{variable}"] == 0.0) * df[variable]) if df.empty: return - left, right, largest_errors = compute_confidence_interval(df, "relative_error") + left, right, largest_errors = compute_confidence_interval(df, "relative_error", entity_original_id = entity_original_id) less_than_5_pc_error = (df["relative_error"].abs() <= .05).sum() / len(df) less_than_20_pc_error = (df["relative_error"].abs() <= .2).sum() / len(df) more_than_80_pc_error = (df["relative_error"].abs() >= .8).sum() / len(df) @@ -130,7 +134,8 @@ def create_output_files(markdown_sections, figures_directory, filename): ) -def create_variable_distribution_figures(variable, result, bins = None, figures_directory = None): +def create_variable_distribution_figures(variable, result, bins = None, figures_directory = None, entity_original_id = None): + assert entity_original_id is not None log.debug(f"create_variable_distribution_figures: Examining {variable}") assert figures_directory is not None if bins is None: @@ -139,11 +144,12 @@ def create_variable_distribution_figures(variable, result, bins = None, figures_ non_both_zeroes = (result[f"{variable}"].fillna(0) != 0) | (result[f"target_{variable}"].fillna(0) != 0) non_both_zeroes_count = sum(non_both_zeroes) both_zeroes_count = len(result) - non_both_zeroes_count + melted = result.loc[ non_both_zeroes, - ["noindiv", variable, f"target_{variable}"] + [entity_original_id, variable, f"target_{variable}"] ].melt( - id_vars = ["noindiv"], + id_vars = [entity_original_id], value_vars = [f"{variable}", f"target_{variable}"] ) @@ -246,8 +252,8 @@ def create_variable_markdown_summary_section(variable, stats, figures_directory) return markdown_section -def create_diff_variable_distribution_figures(variable, result, bins = None, figures_directory = None): - +def create_diff_variable_distribution_figures(variable, result, bins = None, figures_directory = None, entity_original_id = None): + assert entity_original_id is not None numerical = ( isinstance(result[f"{variable}"].values.flat[0], np.integer) or isinstance(result[f"{variable}"].values.flat[0], np.floating) @@ -475,8 +481,18 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti ) result_by_variable[variable] = result - variable_distribution_figures_created = create_variable_distribution_figures(variable, result, figures_directory = figures_directory) - diff_variable_distribution_figures_created = create_diff_variable_distribution_figures(variable, result, figures_directory = figures_directory) + variable_distribution_figures_created = create_variable_distribution_figures( + variable, + result, + figures_directory = figures_directory, + entity_original_id = get_entity_original_id(survey_scenario, variable), + ) + diff_variable_distribution_figures_created = create_diff_variable_distribution_figures( + variable, + result, + figures_directory = figures_directory, + entity_original_id = get_entity_original_id(survey_scenario, variable) + ) stats_by_variable[variable] = stats variable_markdown_section = create_variable_markdown_section( diff --git a/openfisca_france_data/debugger.py b/openfisca_france_data/debugger.py deleted file mode 100644 index 8ed43ac2..00000000 --- a/openfisca_france_data/debugger.py +++ /dev/null @@ -1,437 +0,0 @@ -#! /usr/bin/env python -import logging - - -import numpy -from pandas import merge, concat, DataFrame - - -from openfisca_france_data.erfs.input_data_builder.base import ( - year_specific_by_generic_data_frame_name) -from openfisca_france_data.erfs import get_erf2of, get_of2erf -from openfisca_plugin_aggregates.aggregates import Aggregates -from openfisca_survey_manager.statshelpers import mark_weighted_percentiles as mwp -from openfisca_survey_manager.survey_collections import SurveyCollection - - -from openfisca_parsers import input_variables_extractors - -log = logging.getLogger(__name__) - - -def clean(parameter): - return parameter[:-len('_holder')] if parameter.endswith('_holder') else parameter - - -class Debugger(object): - def __init__(self): - super(Debugger, self).__init__() - self.erf_menage = None - self.erf_eec_indivi = None - self.of_menages_data_frame = None - self.of_individus_data_frame = None - self.variable = None - self.survey_scenario = None - - def set_survey_scenario(self, survey_scenario = None): - assert survey_scenario is not None - self.survey_scenario = survey_scenario - self.variables = self.survey_scenario.simulation.tax_benefit_system.variables - self.simulation = self.survey_scenario.simulation - assert survey_scenario.simulation is not None, "The simulation attibute of survey_scenario is None" - - def set_variable(self, variable): - if isinstance(variable, list): - self.variable = variable[0] - else: - self.variable = variable - - def show_aggregates(self): - from openfisca_france_data.erf.aggregates import build_erf_aggregates - - assert self.survey_scenario is not None, 'simulation attribute is None' - assert self.variable is not None, 'variable attribute is None' - variable = self.variable - openfisca_aggregates = Aggregates() - openfisca_aggregates.set_survey_scenario(self.survey_scenario) - openfisca_aggregates.compute() - - variables = self.variables - temp = (build_erf_aggregates(variables=[variable], year= self.survey_scenario.year)) - selection = openfisca_aggregates.aggr_frame["Mesure"] == variables[variable].label - print(openfisca_aggregates.aggr_frame[selection]) - print(temp) - # TODO: clean this - return - - def extract(self, data_frame, entities = "men"): - variables = self.variables - filtered_data_frame_columns = list(set(variables.keys()).intersection(set(data_frame.columns))) - extracted_columns = [column_name for column_name in filtered_data_frame_columns - if variables[column_name].entity in entities] - extracted_columns = list(set(extracted_columns).union(set(['idmen']))) - return data_frame[extracted_columns].copy() - - def get_all_parameters(self, column_list): - global x - print([column.name for column in column_list]) - x = x + 1 - if x == 20: - boum - variables = self.variables - tax_benefit_system = self.survey_scenario.simulation.tax_benefit_system - - extractor = input_variables_extractors.setup(tax_benefit_system) - - if len(column_list) == 0: - return [] - else: - column_name = column_list[0].name - print(column_name) - if extractor.get_input_variables(variables[column_name]) is None: - return column_list - else: - first_column = [column_list[0]] - input_columns = self.get_all_parameters([ - variables[clean(parameter)] - for parameter in list(extractor.get_input_variables(variables[column_name])) - ]) - other_columns = list( - set(self.get_all_parameters(column_list[1:])) - set(first_column + input_columns) - ) - print('input_variables: ', [column.name for column in input_columns]) - - print('new_variables: ', [column.name for column in other_columns]) - - new_column_list = first_column + input_columns + other_columns - print('final list: ', [column.name for column in new_column_list]) - return new_column_list - - def build_columns_to_fetch(self): - variables = self.variables -# parameters_column = self.get_all_parameters([variables.get(x) for x in [self.variable]]) -# parameters = [x.name for x in parameters_column] - parameters = [self.variable] - # We want to get all parameters and consumers that we're going to encounter -# consumers = [] -# for variable in [self.variable]: -# column = variables.get(variable) -# consumers = list(set(consumers).union(set(column.consumers))) -# column_names = list(set(parameters).union(set(consumers))) - - # self.columns_to_fetch = column_names - # self.variable_consumers = list(set(consumers)) - self.variable_parameters = list(set(parameters)) - self.columns_to_fetch = list(set(parameters)) - - def build_openfisca_data_frames(self): - column_names = self.columns_to_fetch - for column in column_names: - assert column in survey_scenario.tax_benefit_system.variables.keys() - data_frame_by_entity_key_plural = survey_scenario.create_data_frame_by_entity( - variables = column_names + ['idmen_original'], - indices = True, - roles = True, - ) - self.data_frame_by_entity_key_plural = data_frame_by_entity_key_plural - - projected = self.project_on(data_frame_by_entity_key_plural = data_frame_by_entity_key_plural) - idmen_original_by_idmen = dict( - zip( - data_frame_by_entity_key_plural['menages'].index.values, - data_frame_by_entity_key_plural['menages']["idmen_original"].values - ) - ) - self.idmen_original_by_idmen = idmen_original_by_idmen - - idmen_by_idmen_original = dict( - zip( - data_frame_by_entity_key_plural['menages']["idmen_original"].values, - data_frame_by_entity_key_plural['menages'].index.values, - ) - ) - self.idmen_by_idmen_original = idmen_by_idmen_original - - data_frame_by_entity_key_plural['menages'] = projected.rename( - columns = {"idmen_original": "idmen"}) - data_frame_by_entity_key_plural['individus'].replace( - {'idmen': idmen_original_by_idmen}, inplace = True) - self.data_frame_by_entity_key_plural = data_frame_by_entity_key_plural - - def project_on(self, receiving_entity_key_plural = 'menages', data_frame_by_entity_key_plural = None): - tax_benefit_system = self.survey_scenario.tax_benefit_system - assert data_frame_by_entity_key_plural is not None - assert receiving_entity_key_plural is not tax_benefit_system.person_key_plural - - entity_data_frame = data_frame_by_entity_key_plural[receiving_entity_key_plural] - person_data_frame = data_frame_by_entity_key_plural[tax_benefit_system.person_key_plural] - - entity_keys_plural = list( - set(tax_benefit_system.entity_class_by_key_plural.keys()).difference(set( - [tax_benefit_system.person_key_plural, receiving_entity_key_plural] - )) - ) - - for entity_key_plural in entity_keys_plural: - entity = tax_benefit_system.entity_class_by_key_plural[entity_key_plural] - # Getting only heads of other entities prenent in the projected on entity - boolean_index = person_data_frame[entity.role_for_person_variable_name] == 0 # Heads - index_entity = person_data_frame.loc[boolean_index, entity.index_for_person_variable_name].values # Ent. - for column_name, column_series in self.data_frame_by_entity_key_plural[entity_key_plural].items(): - person_data_frame.loc[boolean_index, column_name] \ - = column_series.iloc[index_entity].values - person_data_frame[column_name].fillna(0) - - receiving_entity = tax_benefit_system.entity_class_by_key_plural[receiving_entity_key_plural] - grouped_data_frame = person_data_frame.groupby(by = receiving_entity.index_for_person_variable_name).agg(sum) - grouped_data_frame.drop(receiving_entity.role_for_person_variable_name, axis = 1, inplace = True) - data_frame = concat([entity_data_frame, grouped_data_frame], axis = 1) - - assert data_frame.notnull().all().all() - return data_frame - - def build_erf_data_frames(self): - # TODO: remove this - self.columns_to_fetch = ['af'] - variables = self.columns_to_fetch - erf_survey_collection = SurveyCollection.load( - collection = "erfs", config_files_directory = config_files_directory) - erf_survey = erf_survey_collection.get_survey("erfs_{}".format(year)) - year_specific_by_generic = year_specific_by_generic_data_frame_name(year) - generic_by_year_specific = dict(zip(year_specific_by_generic.values(), year_specific_by_generic.keys())) - - erf_variables = list(set(variables + ["ident", "wprm", "quelfic", "noi"])) - of2erf = get_of2erf() - for index, variable in enumerate(erf_variables): - if variable in of2erf: - erf_variables[index] = of2erf[variable] - data_frame_by_table = dict(eec_indivi = None, erf_indivi = None, erf_menage = None) - erf_variables_by_generic_table = dict(eec_indivi = [], erf_indivi = [], erf_menage = []) - - year_specific_tables_by_erf_variable = dict( - [ - ( - erf_variable, - set( - erf_survey.find_tables(variable = erf_variable) - ).intersection( - set([year_specific_by_generic[key] for key in erf_variables_by_generic_table.keys()]) - ) - ) for erf_variable in erf_variables - ] - ) - for variable, year_specific_tables in year_specific_tables_by_erf_variable.items(): - if len(year_specific_tables) < 1: - log.info("No tables are present for variable {}".format(variable)) - continue - else: - log.info("Variable {} is present in multiple tables : {}".format(variable, year_specific_tables)) - for table in year_specific_tables: - log.info("Variable {} is retrieved from table {}".format(variable, table)) - erf_variables_by_generic_table[generic_by_year_specific[table]].append(variable) - - erf2of = get_erf2of() - - for table, erf_variables in erf_variables_by_generic_table.items(): - if erf_variables: - data_frame_by_table[table] = erf_survey.get_values( - variables = erf_variables, table = year_specific_by_generic[table] - ) - data_frame_by_table[table].rename(columns = erf2of, inplace = True) - data_frame_by_table[table].rename(columns = {'ident': 'idmen'}, inplace = True) - - assert not data_frame_by_table["erf_menage"].duplicated().any(), "Duplicated idmen in erf_menage" - self.erf_data_frame_by_entity_key_plural = dict( - menages = data_frame_by_table["erf_menage"], - individus = data_frame_by_table["erf_indivi"].merge(data_frame_by_table["eec_indivi"]) - ) - # TODO: fichier foyer - - def get_major_differences(self): - variable = self.variable - - of_menages_data_frame = self.data_frame_by_entity_key_plural['menages'] - erf_menages_data_frame = self.erf_data_frame_by_entity_key_plural['menages'] - - merged_menage_data_frame = merge( - erf_menages_data_frame[[variable, 'idmen']], - of_menages_data_frame[[variable, 'idmen']], - on = 'idmen', - how = 'inner', - suffixes = ('_erf', '_of') - ) - - log.info('Length of merged_menage_data_frame is {}'.format(len(merged_menage_data_frame))) - merged_menage_data_frame.set_index('idmen', drop = False, inplace = True) - table = merged_menage_data_frame[ - numpy.logical_and( - merged_menage_data_frame[variable + '_erf'] != 0, - merged_menage_data_frame[variable + '_of'] != 0 - ) - ] - table[variable + "_rel_diff"] = (table[variable + '_of'] - table[variable + '_erf']) \ - / table[variable + '_erf'] # Difference relative - log.info( - "Minimum difference between the two tables for {} is {}".format( - variable, str(table[variable + "_rel_diff"].min()) - ) - ) - log.info( - "Maximum difference between the two tables for {} is {}".format( - variable, str(table[variable + "_rel_diff"].max()) - ) - ) - table[variable + '_ratio'] = ( - table[variable + '_of'] / table[variable + '_erf'] - ) - log.info(table[variable + "_rel_diff"].describe()) - try: - assert len(table[variable + "_rel_diff"]) == len(table['wprm_of']), "PINAGS" - dec, values = mwp( - table[variable + "_rel_diff"], - numpy.arange(1, 11), table['wprm_of'], - 2, - return_quantiles = True - ) - log.info(sorted(values)) - dec, values = mwp( - table[variable + "_rel_diff"], - numpy.arange(1, 101), - table['wprm_erf'], - 2, - return_quantiles = True - ) - log.info(sorted(values)[90:]) - del dec, values - except Exception: - log.info('Weighted percentile method did not work for {}'.format(variable + "_rel_diff")) - pass - table.sort(columns = variable + "_rel_diff", ascending = False, inplace = True) - - print(table[:10].to_string()) - return table - - def describe_discrepancies(self, fov = 10, consumers = False, parameters = True, descending = True, to_men = False): - variable = self.variable - major_differences_data_frame = self.get_major_differences() - major_differences_data_frame.sort( - columns = self.variable + "_rel_diff", - ascending = not descending, - inplace = True - ) - debug_data_frame = major_differences_data_frame[0:fov].copy() - del major_differences_data_frame - - of_menages_data_frame = self.data_frame_by_entity_key_plural['menages'] - of_individus_data_frame = self.data_frame_by_entity_key_plural['individus'] - erf_individus_data_frame = self.erf_data_frame_by_entity_key_plural['individus'] - erf_menages_data_frame = self.erf_data_frame_by_entity_key_plural['menages'] - return debug_data_frame - - kept_columns = set() - if parameters: - kept_columns.update(set(self.variable_parameters)) - if consumers: - kept_columns.update(set(self.variable_consumers)) - kept_columns = list(kept_columns) - kept_columns = list(set(kept_columns).union( - set(['idmen', 'idfam', 'idfoy', 'quimen', 'quifam', 'quifoy'] + list(major_differences_data_frame.columns))) - ) - - if to_men: - entities_ind = ['ind'] - entities_men = ['men', 'fam', 'foy'] - else: - entities_ind = ['ind', 'fam', 'foy'] - entities_men = ['men'] - - debug_data_frame = debug_data_frame.merge( - self.extract(of_menages_data_frame, entities = entities_men), - how = 'inner', - on = 'idmen', - ) - - print(debug_data_frame.to_string()) - - debug_data_frame = debug_data_frame.merge( - self.extract(of_individus_data_frame, entities = entities_ind), - how = 'inner', - on = 'idmen', - ) - - debug_data_frame = debug_data_frame.merge( - erf_individus_data_frame, - how = 'inner', - on = 'idmen', - ) - - suffixes = ["_erf", "_of", "_rel_diff", "_ratio"] - reordered_columns = [variable + suffixe for suffixe in suffixes] \ - + ["idmen", "quimen", "idfam", "quifam", "idfoy", "quifoy"] - reordered_columns = reordered_columns + list(set(kept_columns) - set(reordered_columns)) - debug_data_frame = debug_data_frame[reordered_columns].copy() - return debug_data_frame - - def generate_test_case(self): - entity_class_by_key_plural = self.survey_scenario.tax_benefit_system.entity_class_by_key_plural - menages_entity = entity_class_by_key_plural['menages'] - idmen_by_idmen_original = self.idmen_by_idmen_original - idmen_original = self.describe_discrepancies(descending = False)[ - menages_entity.index_for_person_variable_name].iloc[0] - idmen = idmen_by_idmen_original[idmen_original] - input_data_frame = self.survey_scenario.input_data_frame - individus_index = input_data_frame.index[ - input_data_frame[menages_entity.index_for_person_variable_name] == idmen] - index_by_entity = { - entity_class_by_key_plural['individus']: individus_index, - } - for entity in list(entity_class_by_key_plural.values()): - if entity.key_plural != 'individus': - index_by_entity[entity] = input_data_frame.loc[ - individus_index, entity.index_for_person_variable_name].unique() - - extracted_indices = individus_index - for entity, entity_index in index_by_entity.items(): - if entity.key_plural in ['menages', 'individus']: - continue - extracted_indices = extracted_indices + \ - input_data_frame.index[input_data_frame[entity.index_for_person_variable_name].isin(entity_index)] - - extracted_input_data_frame = input_data_frame.loc[extracted_indices] - return extracted_input_data_frame - - -if __name__ == '__main__': - import sys - from openfisca_plugin_aggregates.tests.test_aggregates import create_survey_scenario - logging.basicConfig(level = logging.INFO, stream = sys.stdout) - restart = True - if restart: - year = 2009 - survey_scenario = create_survey_scenario(year) - survey_scenario.simulation = survey_scenario.new_simulation() - - debugger = Debugger() - debugger.set_survey_scenario(survey_scenario = survey_scenario) - debugger.set_variable('af') - debugger.build_columns_to_fetch() - debugger.build_openfisca_data_frames() - debugger.build_erf_data_frames() - - # df_menage = debugger.data_frame_by_entity_key_plural['menages'] - # df_famille = debugger.data_frame_by_entity_key_plural['familles'] - # df_individus = debugger.data_frame_by_entity_key_plural['individus'] - - # df = debugger.get_major_differences() - - # debugger.show_aggregates() - df = debugger.describe_discrepancies(descending = False) - df = debugger.generate_test_case() - - boum - entity_class_by_key_plural = debugger.survey_scenario.tax_benefit_system.entity_class_by_key_plural - menages_entity = entity_class_by_key_plural['menages'] - - idmen = debugger.describe_discrepancies(descending = False)[menages_entity.index_for_person_variable_name].iloc[0] - - input_data_frame = debugger.survey_scenario.input_data_frame diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index f8e0b170..fc4bfb45 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -24,6 +24,9 @@ "ric_i": "ric", "rnc_i": "rnc", "salaires_i": "salaire_imposable", + "rev_fonciers_bruts": "revenu_categoriel_foncier_menage", + "rev_valeurs_mobilieres_bruts": "revenus_capitaux_prelevement_forfaitaire_unique_ir_menage", + "rev_financier_prelev_lib_imputes": "rev_financier_prelev_lib_imputes_menage", } @@ -40,6 +43,17 @@ class ErfsFprtoInputComparator(AbstractComparator): "salaire_imposable", ] + from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables + + target_menage_projected_variables = [ + f"{menage_projected_variable}_menage" + for menage_projected_variable + in menage_projected_variables + ] + + default_target_variables += target_menage_projected_variables + + def compute_test_dataframes(self): erfs_fpr_survey_collection = SurveyCollection.load(collection = "erfs_fpr") # infer names of the survey and data tables diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 290d2f1a..4f8e065c 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -6,7 +6,7 @@ from openfisca_core.reforms import Reform # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore -from openfisca_france.entities import Individu +from openfisca_france.entities import Individu, FoyerFiscal, Menage from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario from openfisca_france_data import france_data_tax_benefit_system @@ -27,6 +27,13 @@ ] +menage_projected_variables = [ + # "rev_financier_prelev_lib_imputes", + "revenu_categoriel_foncier", + "revenus_capitaux_prelevement_forfaitaire_unique_ir", + ] + + class erfs_fpr_plugin(Reform): name = "ERFS-FPR ids plugin" @@ -36,7 +43,7 @@ def apply(self): class_name = f"{variable}_annuel" label = f"{variable} sur l'année entière" - def formula_creator(variable): + def annual_formula_creator(variable): def formula(individu, period): result = individu(variable, period, options = [ADD]) return result @@ -46,16 +53,42 @@ def formula(individu, period): return formula variable_instance = type(class_name, (Variable,), dict( - value_type = int, + value_type = float, entity = self.variables[variable].entity, label = label, definition_period = YEAR, - formula = formula_creator(variable), + formula = annual_formula_creator(variable), )) self.add_variable(variable_instance) del variable_instance + for variable in menage_projected_variables: + class_name = f"{variable}_menage" + label = f"{variable} agrégée à l'échelle du ménage" + + def projection_formula_creator(variable): + def formula(menage, period): + result_i = menage.members.foyer_fiscal(variable, period, options = [ADD]) + result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL) + return result + + formula.__name__ = 'formula' + + return formula + + variable_instance = type(class_name, (Variable,), dict( + value_type = float, + entity = Menage, + label = label, + definition_period = YEAR, + formula = projection_formula_creator(variable), + )) + + self.add_variable(variable_instance) + del variable_instance + + self.add_variable(idmen_original) self.add_variable(noindiv) @@ -111,6 +144,7 @@ def get_survey_scenario( # S'il n'y a pas de données, on sait où les trouver. if data is None: input_data_table_by_entity = dict( + foyer_fiscal = f"foyer_fiscal_{year}", individu = f"individu_{year}", menage = f"menage_{year}", ) @@ -120,7 +154,6 @@ def get_survey_scenario( data = dict( input_data_table_by_entity_by_period = input_data_table_by_entity_by_period, - # input_data_survey_prefix = "openfisca_erfs_fpr_data", survey = survey_name ) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py index f2037474..6844c59f 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py @@ -18,7 +18,8 @@ # step_02_imputation_loyer as imputation_loyer, step_03_variables_individuelles as variables_individuelles, step_04_famille as famille, - step_05_final as final, + step_05_foyer as foyer, + step_06_final as final, ) log = logging.getLogger(__name__) @@ -48,7 +49,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # - On merge les tables individus / menages # # Note : c'est ici où on objectivise les hypothèses, step 1 - log.info('\n [[[ Year {} - Step 1 / 5 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 1 / 6 ]]] \n'.format(year)) preprocessing.build_merged_dataframes(year = year) # Step 02 : Si on veut calculer les allocations logement, il faut faire le matching avec une autre enquête (ENL) @@ -57,10 +58,10 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # stata_directory = openfisca_survey_collection.config.get('data', 'stata_directory') # stata_file = os.path.join(stata_directory, 'log_men_ERFS.dta') # imputation_loyer.merge_imputation_loyer(stata_file = stata_file, year = year) - log.info('\n [[[ Year {} - Step 2 / 5 SKIPPED ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 2 / 6 SKIPPED ]]] \n'.format(year)) # Step 03 : on commence par les variables indivuelles - log.info('\n [[[ Year {} - Step 3 / 5 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 3 / 6 ]]] \n'.format(year)) variables_individuelles.build_variables_individuelles(year = year) # Step 04 : ici on va constituer foyer et famille à partir d'invididu et ménage @@ -69,15 +70,18 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # - On va faire des suppositions pour faire les familles # - On va faire les foyers fiscaux à partir des familles # - On va faire de suppositions pour faire les foyers fiscaux - log.info('\n [[[ Year {} - Step 4 / 5 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 4 / 6 ]]] \n'.format(year)) famille.build_famille(year = year) - + + log.info('\n [[[ Year {} - Step 5 / 6 ]]] \n'.format(year)) + foyer.build_variables_foyers_fiscal(year = year) + # Affreux ! On injectait tout dans un même DataFrame !!! # C'est très moche ! # # On crée une df par entité par période. # Elles sont stockées dans un fichier h5 - log.info('\n [[[ Year {} - Step 5 / 5 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 6 / 6 ]]] \n'.format(year)) final.create_input_data_frame(year = year, export_flattened_df_filepath = export_flattened_df_filepath) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 7b895658..294e3779 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -56,11 +56,11 @@ def build_variables_individuelles(temporary_store = None, year = None): # helpers def create_variables_individuelles(individus, year, survey_year = None, revenu_type = 'imposable'): - """Création des variables individuelles""" + """Création des variables individuelles.""" period = periods.period(year) tax_benefit_system = openfisca_france_tax_benefit_system - # variables démographiques + # variables démographiques create_ages(individus, year) create_date_naissance(individus, age_variable = None, annee_naissance_variable = 'naia', mois_naissance = 'naim', year = year) @@ -82,14 +82,15 @@ def create_variables_individuelles(individus, year, survey_year = None, revenu_t individus, period = period, revenu_type = revenu_type, - tax_benefit_system = tax_benefit_system + tax_benefit_system = tax_benefit_system, ) create_traitement_indiciaire_brut( - individus, - period = period, + individus, + period = period, revenu_type = revenu_type, - tax_benefit_system = tax_benefit_system) - + tax_benefit_system = tax_benefit_system, + ) + # Pour les cotisations patronales qui varient avec la taille de l'entreprise' create_effectif_entreprise(individus, period = period, survey_year = survey_year) @@ -900,7 +901,7 @@ def create_revenus_remplacement_bruts(individus): """ Reconstitution des variables de retraite et chomage brut à partir des variables nettes et des variables de csg et crds """ - + # revenu_brut = revenu_net + csg_deductible + csg_non_deductible_crds individus['chomage_brut'] = individus.chomage_imposable + individus.csgchod_i #+ individus. csg_nd_crds_cho_i individus['retraite_brute'] = individus.retraite_imposable + individus.csgrstd_i #+ individus.csg_nd_crds_ret_i diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py new file mode 100644 index 00000000..4fc912ae --- /dev/null +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py @@ -0,0 +1,61 @@ +import logging +import pandas as pd + + +from openfisca_survey_manager.temporary import temporary_store_decorator # type: ignore + + +log = logging.getLogger(__name__) + + +@temporary_store_decorator(file_name = 'erfs_fpr') +def build_variables_foyers_fiscal(temporary_store = None, year = None): + + assert temporary_store is not None + assert year is not None + + individus = temporary_store['individus_{}'.format(year)] + menages = temporary_store['menages_{}'.format(year)] + + individus['idfoy'] = individus['idfam'].copy() + individus['quifoy'] = individus['quifam'].copy() + + foyers_fiscaux = individus[['idfoy','ident',]].drop_duplicates() + foyers_fiscaux = pd.merge( + menages[[ + 'ident', + 'rev_financier_prelev_lib_imputes', + 'rev_fonciers_bruts', + 'rev_valeurs_mobilieres_bruts', + 'wprm', + ]], + foyers_fiscaux, + how = 'inner', + on = 'ident' + ) + # première version pour splitter les revenus du capital du ménage dans les foyers fiscaux + # on attribue l'ensemble des revenus du capital du ménage au foyer avec la personne ayant les plus hauts revenus + # procédure à améliorer + idfoy = (individus + .sort_values( + [ + 'ident', + 'salaire_de_base', + 'traitement_indiciaire_brut', + 'retraite_brute' + ], + ascending = False + ) + .groupby('ident') + .first() + .idfoy + ) + foyers_fiscaux['revenu_categoriel_foncier'] = foyers_fiscaux['rev_fonciers_bruts'] * foyers_fiscaux.idfoy.isin(idfoy) + foyers_fiscaux['revenus_capitaux_prelevement_forfaitaire_unique_ir'] = foyers_fiscaux['rev_valeurs_mobilieres_bruts'] * foyers_fiscaux.idfoy.isin(idfoy) + foyers_fiscaux['rev_financier_prelev_lib_imputes'] = foyers_fiscaux['rev_financier_prelev_lib_imputes'] * foyers_fiscaux.idfoy.isin(idfoy) + + #assert sum(menages.wprm * menages.rev_fonciers_bruts) == sum(foyers_fiscaux.rev_fonciers_bruts * foyers_fiscaux.wprm) + #assert sum(menages.wprm * menages.rev_valeurs_mobilieres_bruts) == sum(foyers_fiscaux.rev_financier_prelev_lib_imputes * foyers_fiscaux.wprm) + #assert sum(menages.wprm * menages.rev_financier_prelev_lib_imputes) == sum(foyers_fiscaux.rev_valeurs_mobilieres_bruts * foyers_fiscaux.wprm) + + temporary_store[f"foyers_fiscaux_{year}"] = foyers_fiscaux diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py similarity index 84% rename from openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py rename to openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index fccaa092..c95f79d8 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -16,9 +16,10 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene individus = temporary_store['individus_{}'.format(year)] menages = temporary_store['menages_{}'.format(year)] + foyers_fiscaux = temporary_store['foyers_fiscaux_{}'.format(year)] # ici : variables à garder - variables = [ + var_individus = [ 'activite', 'age', 'categorie_salarie', @@ -39,13 +40,27 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'retraite_brute', 'ric', 'rnc', - 'statut_marital', - # 'salaire_imposable', 'salaire_de_base', - "traitement_indiciaire_brut", + 'statut_marital', "primes_fonction_publique", + "traitement_indiciaire_brut", + ] + var_foyers_fiscaux = [ + 'idfoy', + 'rev_financier_prelev_lib_imputes', + 'revenu_categoriel_foncier', + 'revenus_capitaux_prelevement_forfaitaire_unique_ir', ] + var_menages = [ + 'idmen', + 'loyer', + 'statut_occupation_logement', + 'taxe_habitation', + 'wprm', + 'zone_apl', + ] + # TODO: fix this simplistic inference individus.rename(columns = { 'ric_net': 'ric', @@ -56,7 +71,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene ) individus = create_ids_and_roles(individus) - individus = individus[variables].copy() + individus = individus[var_individus].copy() gc.collect() # This looks like it could have a sizeable impact @@ -71,7 +86,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene menages = extract_menages_variables(menages) - individus = create_collectives_foyer_variables(individus, menages) + # individus = create_collectives_foyer_variables(individus, menages) idmens = individus.idmen.unique() menages = menages.loc[ @@ -92,11 +107,20 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene menages = menages.rename(columns = {'idmen':'idmen_original'}) unique_idmen = individus[['idmen','idmen_original']].drop_duplicates() assert len(unique_idmen) == len(menages), "Number of idmen should be the same individus and menages tables." - menages = menages.merge( - unique_idmen, - how = 'inner', - on = 'idmen_original' - ) + + menages = menages.merge(unique_idmen, + how = 'inner', + on = 'idmen_original') + + foyers_fiscaux = foyers_fiscaux.rename(columns = {'idfoy':'idfoy_original'}) + unique_idfoy = individus[['idfoy','idfoy_original']].drop_duplicates() + assert len(unique_idmen) == len(menages), "Number of idfoy should be the same individus and foyers tables." + + foyers_fiscaux = foyers_fiscaux.merge(unique_idfoy, + how = 'inner', + on = 'idfoy_original') + + foyers_fiscaux = foyers_fiscaux[var_foyers_fiscaux] if export_flattened_df_filepath: supermerge = individus.merge( @@ -117,6 +141,18 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene collection = "openfisca_erfs_fpr", survey_name = survey_name, ) + + foyers_fiscaux = foyers_fiscaux.sort_values(by = ['idfoy']) + log.debug(f"Saving entity 'foyers fiscaux' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") + set_table_in_survey( + foyers_fiscaux, + entity = "foyer_fiscal", + period = year, + collection = "openfisca_erfs_fpr", + survey_name = survey_name, + ) + log.debug("End of create_input_data_frame") + menages = menages.sort_values(by = ['idmen']) log.debug(f"Saving entity 'menage' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") set_table_in_survey( @@ -184,7 +220,7 @@ def create_collectives_foyer_variables(individus, menages): assert set(foyers_revenus_fonciers.columns) == set(['idfoy', 'rev_fonciers_bruts', 'quifoy']) individus = individus.merge(foyers_revenus_fonciers, how = 'outer', on = ['idfoy', 'quifoy']) assert set(idmens) == set(individus .query('(rev_fonciers_bruts > 0)')['idmen'].tolist()) - individus.rename(columns = {'rev_fonciers_bruts': 'f4ba'}, inplace = True) + individus.rename(columns = {'rev_fonciers_bruts': 'revenu_categoriel_foncier'}, inplace = True) return individus @@ -229,16 +265,13 @@ def extract_menages_variables_from_store(temporary_store = None, year = None): def extract_menages_variables(menages): - variables = ['ident', 'wprm', 'taxe_habitation', 'rev_fonciers_bruts'] + variables = ['ident', 'wprm', 'taxe_habitation'] external_variables = ['loyer', 'zone_apl', 'statut_occupation_logement'] for external_variable in external_variables: if external_variable in menages.columns: log.debug("Found {} in menages table: we keep it".format(external_variable)) variables.append(external_variable) - # TODO: 2007-2010 ont la variable rev_fonciers et non pas rev_fonciers_bruts. Est-ce la même? - menages = menages.rename(columns={'rev_fonciers': 'rev_fonciers_bruts'}) menages = menages[variables].copy() - menages.taxe_habitation = - menages.taxe_habitation # taxes should be negative menages.rename(columns = dict(ident = 'idmen'), inplace = True) return menages diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index dca44945..7f81adea 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -25,6 +25,9 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "rag", "retraite_brute", "retraite_imposable", + # "rev_financier_prelev_lib_imputes", + "revenu_categoriel_foncier", + "revenus_capitaux_prelevement_forfaitaire_unique_ir", "ric", "rnc", "salaire_de_base", From 8eec6de1d8a9184939fd4d95e47dd082f3f6d638 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 31 Mar 2023 09:54:15 +0000 Subject: [PATCH 005/140] Ajout variable apl --- openfisca_france_data/comparator.py | 6 ++++-- openfisca_france_data/erfs_fpr/comparison.py | 7 ++++++- .../erfs_fpr/input_data_builder/__init__.py | 6 ++++-- .../input_data_builder/step_02_menage.py | 18 ++++++++++++++++++ .../step_03_variables_individuelles.py | 5 +++++ .../input_data_builder/step_05_foyer.py | 4 ---- .../input_data_builder/step_06_final.py | 6 +++--- 7 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 39c27541..d788bfe2 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -159,11 +159,13 @@ def create_variable_distribution_figures(variable, result, bins = None, figures_ unique_values_count = melted["value"].nunique() - bins == unique_values_count if unique_values_count < bins else bins + bins = unique_values_count if unique_values_count < bins else bins print(f"create_variable_distribution_figures (total): variable = {variable}, bins = {bins}") melted["value"] = melted["value"].clip(1, melted["value"].max()) + log_scale = bins > 10 + sns_plot = sns.histplot( data = melted, # palette = "crest", @@ -175,7 +177,7 @@ def create_variable_distribution_figures(variable, result, bins = None, figures_ hue = "variable", linewidth = 0, x = "value", - log_scale = True, + log_scale = log_scale, ) sns_plot.annotate( diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index fc4bfb45..cfb4e87b 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -24,6 +24,7 @@ "ric_i": "ric", "rnc_i": "rnc", "salaires_i": "salaire_imposable", + "logt": "statut_occupation_logement", "rev_fonciers_bruts": "revenu_categoriel_foncier_menage", "rev_valeurs_mobilieres_bruts": "revenus_capitaux_prelevement_forfaitaire_unique_ir_menage", "rev_financier_prelev_lib_imputes": "rev_financier_prelev_lib_imputes_menage", @@ -35,12 +36,13 @@ class ErfsFprtoInputComparator(AbstractComparator): period = None default_target_variables = [ "chomage_imposable", + "loyer", # "rag", "retraite_imposable", # "ric", # "rnc", - # "f4ba", "salaire_imposable", + "statut_occupation_logement", ] from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables @@ -78,6 +80,9 @@ def compute_test_dataframes(self): "individu": openfisca_individu, "menage": openfisca_menage, } + + fpr_menage.loyer = 12 * fpr_menage.loyer + target_dataframe_by_entity = { "individu": fpr_individu.rename(columns = openfisca_by_erfs_fpr_variables), "menage": fpr_menage.rename(columns = openfisca_by_erfs_fpr_variables), diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py index 6844c59f..5dc8df4b 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py @@ -16,6 +16,7 @@ from openfisca_france_data.erfs_fpr.input_data_builder import ( step_01_preprocessing as preprocessing, # step_02_imputation_loyer as imputation_loyer, + step_02_menage as menage, step_03_variables_individuelles as variables_individuelles, step_04_famille as famille, step_05_foyer as foyer, @@ -59,6 +60,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # stata_file = os.path.join(stata_directory, 'log_men_ERFS.dta') # imputation_loyer.merge_imputation_loyer(stata_file = stata_file, year = year) log.info('\n [[[ Year {} - Step 2 / 6 SKIPPED ]]] \n'.format(year)) + menage.build_variables_menage(year = year) # Step 03 : on commence par les variables indivuelles log.info('\n [[[ Year {} - Step 3 / 6 ]]] \n'.format(year)) @@ -72,10 +74,10 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # - On va faire de suppositions pour faire les foyers fiscaux log.info('\n [[[ Year {} - Step 4 / 6 ]]] \n'.format(year)) famille.build_famille(year = year) - + log.info('\n [[[ Year {} - Step 5 / 6 ]]] \n'.format(year)) foyer.build_variables_foyers_fiscal(year = year) - + # Affreux ! On injectait tout dans un même DataFrame !!! # C'est très moche ! # diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py new file mode 100644 index 00000000..538f1fc3 --- /dev/null +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py @@ -0,0 +1,18 @@ +import logging +import pandas as pd + + +from openfisca_survey_manager.temporary import temporary_store_decorator # type: ignore + +log = logging.getLogger(__name__) + + +@temporary_store_decorator(file_name = 'erfs_fpr') +def build_variables_menage(temporary_store = None, year = None): + if year >= 2018: + menages = temporary_store['menages_{}'.format(year)] + menages['zone_apl'] = 2 + # pour l'instant on met tout le monde à 2 mais à améliorer, peut être en fonction de la taille de l'aire urbaine ? + menages['statut_occupation_logement'] = menages['so'].copy() + menages.loc[(menages.statut_occupation_logement == 7),'statut_occupation_logement'] = 2 + temporary_store['menages_{}'.format(year)] = menages diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 294e3779..92364169 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -90,6 +90,11 @@ def create_variables_individuelles(individus, year, survey_year = None, revenu_t revenu_type = revenu_type, tax_benefit_system = tax_benefit_system, ) + create_traitement_indiciaire_brut( + individus, + period = period, + revenu_type = revenu_type, + tax_benefit_system = tax_benefit_system) # Pour les cotisations patronales qui varient avec la taille de l'entreprise' create_effectif_entreprise(individus, period = period, survey_year = survey_year) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py index 4fc912ae..21683d06 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py @@ -54,8 +54,4 @@ def build_variables_foyers_fiscal(temporary_store = None, year = None): foyers_fiscaux['revenus_capitaux_prelevement_forfaitaire_unique_ir'] = foyers_fiscaux['rev_valeurs_mobilieres_bruts'] * foyers_fiscaux.idfoy.isin(idfoy) foyers_fiscaux['rev_financier_prelev_lib_imputes'] = foyers_fiscaux['rev_financier_prelev_lib_imputes'] * foyers_fiscaux.idfoy.isin(idfoy) - #assert sum(menages.wprm * menages.rev_fonciers_bruts) == sum(foyers_fiscaux.rev_fonciers_bruts * foyers_fiscaux.wprm) - #assert sum(menages.wprm * menages.rev_valeurs_mobilieres_bruts) == sum(foyers_fiscaux.rev_financier_prelev_lib_imputes * foyers_fiscaux.wprm) - #assert sum(menages.wprm * menages.rev_financier_prelev_lib_imputes) == sum(foyers_fiscaux.rev_valeurs_mobilieres_bruts * foyers_fiscaux.wprm) - temporary_store[f"foyers_fiscaux_{year}"] = foyers_fiscaux diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index c95f79d8..582e296f 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -80,9 +80,9 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene menages[k] = 0 # Again artificially putting missing variables in their default state - menages["loyer"] = 0 - menages["zone_apl"] = 2 - menages["statut_occupation_logement"] = 0 + #menages["loyer"] = 0 + #menages["zone_apl"] = 2 + #menages["statut_occupation_logement"] = 0 menages = extract_menages_variables(menages) From 48ad827b730227d7c416d33a1dc55cf3ac2aaa7a Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:19:32 +0200 Subject: [PATCH 006/140] Add check-version-and-changelog --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 16d1e951..e87029b8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -139,6 +139,26 @@ diagnostics: tags: - openfisca +check-version-and-changelog: + stage: diagnostics + before_script: + - '' + needs: + - input_data-2019 + script: + - .github/is-version-number-acceptable.sh + +check-for-functional-changes: + stage: diagnostics + needs: + - check-version-and-changelog + before_script: + - '' + script: + - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi + when: + - master + run_on_all_years: stage: run_on_all_years From 3a8e8449014e67fc67c134ab679e887a8efa7204 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Tue, 28 Mar 2023 11:22:18 +0200 Subject: [PATCH 007/140] Add script for GitLab CI wip wip : adapt script wip wip wip wip wip wip Add emoji --- .github/has-functional-changes.sh | 2 +- .gitlab-ci.yml | 30 ++++++++------- ci-runner/has-functional-changes.sh | 18 +++++++++ ci-runner/is-version-number-acceptable.sh | 47 +++++++++++++++++++++++ 4 files changed, 82 insertions(+), 15 deletions(-) create mode 100755 ci-runner/has-functional-changes.sh create mode 100755 ci-runner/is-version-number-acceptable.sh diff --git a/.github/has-functional-changes.sh b/.github/has-functional-changes.sh index 48f97805..b010f0d1 100755 --- a/.github/has-functional-changes.sh +++ b/.github/has-functional-changes.sh @@ -3,7 +3,7 @@ IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*" last_tagged_commit=`git describe --tags --abbrev=0 --first-parent` # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit - +echo ".github/has-functional-changes.sh : last_tagged_commit=$last_tagged_commit" if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'` # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. then echo "No functional changes detected." diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e87029b8..b376ac35 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,6 +7,7 @@ variables: # OUT_FOLDER: "$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA" # For branch-commit_id OUT_FOLDER: "$CI_COMMIT_REF_NAME" # For just branch ROOT_FOLDER: "/mnt/data-out/openfisca-france-data" +# GIT_DEPTH: 1000 # To be able to get last tag (default 50) cache: paths: @@ -140,24 +141,25 @@ diagnostics: - openfisca check-version-and-changelog: - stage: diagnostics + # stage: diagnostics TODO: Put it back + stage: test before_script: - '' needs: - - input_data-2019 - script: - - .github/is-version-number-acceptable.sh - -check-for-functional-changes: - stage: diagnostics - needs: - - check-version-and-changelog - before_script: - - '' + # - input_data-2019 + - test script: - - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi - when: - - master + - ci-runner/is-version-number-acceptable.sh + +# check-for-functional-changes: +# stage: diagnostics +# needs: +# - check-version-and-changelog +# before_script: +# - '' +# script: +# - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi +# when: master run_on_all_years: diff --git a/ci-runner/has-functional-changes.sh b/ci-runner/has-functional-changes.sh new file mode 100755 index 00000000..f53acff4 --- /dev/null +++ b/ci-runner/has-functional-changes.sh @@ -0,0 +1,18 @@ +#! /usr/bin/env bash + +IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*" + +# Fetch all tags +git fetch --tags + +last_tagged_commit=`git tag --list | sort -V | grep -v ipp | tail -1` +echo "ci-runner/has-functional-changes.sh : avec git tag --list last_tagged_commit=$last_tagged_commit" + +# Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. +if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'` +then + echo "No functional changes detected." + exit 1 +else + echo "The functional files above were changed." +fi diff --git a/ci-runner/is-version-number-acceptable.sh b/ci-runner/is-version-number-acceptable.sh new file mode 100755 index 00000000..ad7850a2 --- /dev/null +++ b/ci-runner/is-version-number-acceptable.sh @@ -0,0 +1,47 @@ +#! /usr/bin/env bash + +if [[ ${CI_COMMIT_REF_NAME} == master ]] +then + echo "No need for a version check on master.👍" + exit 0 +else + echo "Not on master." +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh +then + echo "No need for a version update.👍" + exit 0 +else + echo "Need for a version update." +fi + +current_version=`python setup.py --version` + +if git rev-parse --verify --quiet $current_version +then + echo "Version $current_version already exists in commit:" + git --no-pager log -1 $current_version + echo + echo "Update the version number in setup.py before merging this branch into master.😒" + echo "Look at the CONTRIBUTING.md file to learn how the version number should be updated." + exit 1 +else + echo "Version $current_version don't exists in commit history 👍" +fi + +if ! $(dirname "$BASH_SOURCE")/has-functional-changes.sh | grep --quiet CHANGELOG.md +then + echo "CHANGELOG.md has not been modified, while functional changes were made. 😒" + echo "Explain what you changed before merging this branch into master." + echo "Look at the CONTRIBUTING.md file to learn how to write the changelog." + exit 2 +else + if ! grep --quiet $current_version CHANGELOG.md + then + echo "CHANGELOG.md has been modified. BUT $current_version don't exists in it. 😒" + exit 3 + else + echo "CHANGELOG.md has been modified and $current_version exists in it. 👍" + fi +fi From 9d502d761b244d1a55a7d60494c48d5e23813a94 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Fri, 31 Mar 2023 10:32:49 +0200 Subject: [PATCH 008/140] Change path --- .gitlab-ci.yml | 12 ++++++------ {ci-runner => .gitlab-ci}/README.md | 6 +++--- .../all_years_build_and_aggregates.yml | 6 +++--- {ci-runner => .gitlab-ci}/build_ci.py | 8 ++++---- .../empty_openfisca_erfs_fpr.json | 0 {ci-runner => .gitlab-ci}/has-functional-changes.sh | 2 +- .../is-version-number-acceptable.sh | 0 .../openfisca_france_data_config.ini | 0 .../openfisca_survey_manager_config.ini | 2 +- .../openfisca_survey_manager_raw_data.ini | 2 +- 10 files changed, 19 insertions(+), 19 deletions(-) rename {ci-runner => .gitlab-ci}/README.md (93%) rename {gitlab_ci => .gitlab-ci}/all_years_build_and_aggregates.yml (99%) rename {ci-runner => .gitlab-ci}/build_ci.py (95%) rename {ci-runner => .gitlab-ci}/empty_openfisca_erfs_fpr.json (100%) rename {ci-runner => .gitlab-ci}/has-functional-changes.sh (83%) rename {ci-runner => .gitlab-ci}/is-version-number-acceptable.sh (100%) rename {ci-runner => .gitlab-ci}/openfisca_france_data_config.ini (100%) rename {ci-runner => .gitlab-ci}/openfisca_survey_manager_config.ini (89%) rename {ci-runner => .gitlab-ci}/openfisca_survey_manager_raw_data.ini (95%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b376ac35..a8a3f227 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -include: 'gitlab_ci/all_years_build_and_aggregates.yml' +include: '.gitlab-ci/all_years_build_and_aggregates.yml' variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -30,7 +30,7 @@ stages: before_script: # To be sure we are up to date even if we do not rebuild docker image - make install - - cp ./ci-runner/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini + - cp ./.gitlab-ci/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini - echo "End of before_script" @@ -79,7 +79,7 @@ build_collection: - rm -rf $ROOT_FOLDER/$OUT_FOLDER || true - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_collections/ - mkdir -p $ROOT_FOLDER/$OUT_FOLDER/data_output/ - - cp ./ci-runner/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini + - cp ./.gitlab-ci/openfisca_survey_manager_config.ini ~/.config/openfisca-survey-manager/config.ini - echo "Custom output folder" - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-survey-manager/config.ini - 'echo "{\"name\": \"erfs_fpr\", \"surveys\": {}}" > $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json' @@ -109,7 +109,7 @@ copy_previous_build_collections: cp $ROOT_FOLDER/master/openfisca_survey_manager_config-after-build-collection.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini sed -i "s/master/$OUT_FOLDER/" $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini cp $ROOT_FOLDER/master/data_collections/erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/erfs_fpr.json - cp ./ci-runner/empty_openfisca_erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json + cp ./.gitlab-ci/empty_openfisca_erfs_fpr.json $ROOT_FOLDER/$OUT_FOLDER/data_collections/openfisca_erfs_fpr.json fi stage: build_collection tags: @@ -129,7 +129,7 @@ diagnostics: - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini ~/.config/openfisca-survey-manager/config.ini - mkdir -p ~/.config/openfisca-france-data - - cp ./ci-runner/openfisca_france_data_config.ini + - cp ./.gitlab-ci/openfisca_france_data_config.ini ~/.config/openfisca-france-data/config.ini - sed -i "s/BRANCH_NAME/$OUT_FOLDER/" ~/.config/openfisca-france-data/config.ini - cat ~/.config/openfisca-france-data/config.ini @@ -149,7 +149,7 @@ check-version-and-changelog: # - input_data-2019 - test script: - - ci-runner/is-version-number-acceptable.sh + - .gitlab-ci/is-version-number-acceptable.sh # check-for-functional-changes: # stage: diagnostics diff --git a/ci-runner/README.md b/.gitlab-ci/README.md similarity index 93% rename from ci-runner/README.md rename to .gitlab-ci/README.md index e11492ca..094679db 100644 --- a/ci-runner/README.md +++ b/.gitlab-ci/README.md @@ -7,7 +7,7 @@ This folder contains files needed for the CI. To separate the different years and survey files we have a script that build the CI script. ``` -python ci-runner/build_ci.py +python .gitlab-ci/build_ci.py ``` It will create the file `.gitlab-ci.yml` that is read by Gitlab Runner to execute the CI. @@ -31,8 +31,8 @@ All following steps is run with this docker image. It is a manual step because it does not to be build each time and took a very long time : between 2 and 4 hours. It use the `build-collection` command from [OpenFisca-Survey-Manager](https://github.com/openfisca/openfisca-survey-manager). Input : -- [../ci-runner/openfisca_survey_manager_config.ini](ci-runner/openfisca_survey_manager_config.ini) -- [../ci-runner/openfisca_survey_manager_raw_data.ini](ci-runner/openfisca_survey_manager_raw_data.ini) +- [../.gitlab-ci/openfisca_survey_manager_config.ini](.gitlab-ci/openfisca_survey_manager_config.ini) +- [../.gitlab-ci/openfisca_survey_manager_raw_data.ini](.gitlab-ci/openfisca_survey_manager_raw_data.ini) - All survey's files located in `/mnt/data-in/erfs-fpr/` folder that is accessible to the CI Runner. Output : diff --git a/gitlab_ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml similarity index 99% rename from gitlab_ci/all_years_build_and_aggregates.yml rename to .gitlab-ci/all_years_build_and_aggregates.yml index 66a4ff01..36c088d0 100644 --- a/gitlab_ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -1,7 +1,7 @@ ################################################ # GENERATED FILE, DO NOT EDIT -# Please visit ci-runner/README.md +# Please visit .gitlab-ci/README.md ################################################ input_data-2019: @@ -9,8 +9,8 @@ input_data-2019: script: - echo "build_input_data-2019" - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini - - cat ~/.config/openfisca-survey-manager/config.ini + - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini + ~/.config/openfisca-survey-manager/config.ini - build-erfs-fpr -y 2019 -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_2019.h5 - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2019.ini stage: build_input_data diff --git a/ci-runner/build_ci.py b/.gitlab-ci/build_ci.py similarity index 95% rename from ci-runner/build_ci.py rename to .gitlab-ci/build_ci.py index 6ec8fea1..3553e39e 100644 --- a/ci-runner/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -3,20 +3,20 @@ Run in project root folder: -python ci-runner/build_ci.py +python .gitlab-ci/build_ci.py """ import configparser import yaml # Config file use to get the available years -CONFIG = "./ci-runner/openfisca_survey_manager_raw_data.ini" +CONFIG = "./.gitlab-ci/openfisca_survey_manager_raw_data.ini" def header(): return """ ################################################ # GENERATED FILE, DO NOT EDIT -# Please visit ci-runner/README.md +# Please visit .gitlab-ci/README.md ################################################ """ @@ -133,7 +133,7 @@ def main(): # For testing only some years # erfs_years = ["2016", "2017", "2018"] gitlab_ci = build_gitlab_ci(erfs_years) - with open(r"./gitlab_ci/all_years_build_and_aggregates.yml", mode="w") as file: + with open(r"./.gitlab-ci/all_years_build_and_aggregates.yml", mode="w") as file: file.write(gitlab_ci) print("Done with success!") diff --git a/ci-runner/empty_openfisca_erfs_fpr.json b/.gitlab-ci/empty_openfisca_erfs_fpr.json similarity index 100% rename from ci-runner/empty_openfisca_erfs_fpr.json rename to .gitlab-ci/empty_openfisca_erfs_fpr.json diff --git a/ci-runner/has-functional-changes.sh b/.gitlab-ci/has-functional-changes.sh similarity index 83% rename from ci-runner/has-functional-changes.sh rename to .gitlab-ci/has-functional-changes.sh index f53acff4..0784a74b 100755 --- a/ci-runner/has-functional-changes.sh +++ b/.gitlab-ci/has-functional-changes.sh @@ -6,7 +6,7 @@ IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .github/*" git fetch --tags last_tagged_commit=`git tag --list | sort -V | grep -v ipp | tail -1` -echo "ci-runner/has-functional-changes.sh : avec git tag --list last_tagged_commit=$last_tagged_commit" +echo ".gitlab-ci/has-functional-changes.sh : avec git tag --list last_tagged_commit=$last_tagged_commit" # Check if any file that has not be listed in IGNORE_DIFF_ON has changed since the last tag was published. if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNORE_DIFF_ON" | sed 's/ / :(exclude)/g'` diff --git a/ci-runner/is-version-number-acceptable.sh b/.gitlab-ci/is-version-number-acceptable.sh similarity index 100% rename from ci-runner/is-version-number-acceptable.sh rename to .gitlab-ci/is-version-number-acceptable.sh diff --git a/ci-runner/openfisca_france_data_config.ini b/.gitlab-ci/openfisca_france_data_config.ini similarity index 100% rename from ci-runner/openfisca_france_data_config.ini rename to .gitlab-ci/openfisca_france_data_config.ini diff --git a/ci-runner/openfisca_survey_manager_config.ini b/.gitlab-ci/openfisca_survey_manager_config.ini similarity index 89% rename from ci-runner/openfisca_survey_manager_config.ini rename to .gitlab-ci/openfisca_survey_manager_config.ini index a75a227a..0507bcb9 100644 --- a/ci-runner/openfisca_survey_manager_config.ini +++ b/.gitlab-ci/openfisca_survey_manager_config.ini @@ -1,6 +1,6 @@ # Template du fichier config.ini de openfisca-survey-manager # pour qu'il fonctionne avec oepnfisca-france-data -# sur le ci-runner gitlab piloté par ipp/openfisca-france/data +# sur le runner gitlab CI piloté par ipp/openfisca-france/data [collections] collections_directory = /mnt/data-out/openfisca-france-data/BRANCH_NAME/data_collections diff --git a/ci-runner/openfisca_survey_manager_raw_data.ini b/.gitlab-ci/openfisca_survey_manager_raw_data.ini similarity index 95% rename from ci-runner/openfisca_survey_manager_raw_data.ini rename to .gitlab-ci/openfisca_survey_manager_raw_data.ini index 00de83c8..6231dda7 100644 --- a/ci-runner/openfisca_survey_manager_raw_data.ini +++ b/.gitlab-ci/openfisca_survey_manager_raw_data.ini @@ -1,6 +1,6 @@ # Template du fichier raw_data.ini de openfisca-survey-manager # pour qu'il fonctionne avec oepnfisca-france-data -# sur le ci-runner gitlab piloté par ipp/openfisca-france/data +# sur le runner GitLab CI piloté par ipp/openfisca-france/data ; [enquete_logement] ; 2006 = /home/ipp/data/enquete_logement/2006/stata From 20dc36376670504309143eb2b58b23c3913564e4 Mon Sep 17 00:00:00 2001 From: benoit-cty <4-benoit-cty@users.noreply.git.leximpact.dev> Date: Fri, 31 Mar 2023 10:42:26 +0200 Subject: [PATCH 009/140] Clean CI Fix typo Fix typo --- .gitlab-ci.yml | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8a3f227..ce4dc8ae 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,14 +26,12 @@ stages: - aggregates_all - anaconda - before_script: # To be sure we are up to date even if we do not rebuild docker image - make install - cp ./.gitlab-ci/openfisca_survey_manager_raw_data.ini ~/.config/openfisca-survey-manager/raw_data.ini - echo "End of before_script" - build docker image: stage: docker tags: @@ -52,7 +50,6 @@ build docker image: # Build Docker is needed only if code as changed. when: manual - test: image: $CI_REGISTRY_IMAGE:latest script: @@ -61,7 +58,6 @@ test: tags: - openfisca - clean_folder: before_script: - '' @@ -71,7 +67,6 @@ clean_folder: - openfisca when: manual - build_collection: image: $CI_REGISTRY_IMAGE:latest script: @@ -95,7 +90,6 @@ build_collection: - openfisca when: manual - copy_previous_build_collections: before_script: - '' @@ -141,27 +135,14 @@ diagnostics: - openfisca check-version-and-changelog: - # stage: diagnostics TODO: Put it back - stage: test + stage: diagnostics before_script: - '' needs: - # - input_data-2019 - - test + - input_data-2019 script: - .gitlab-ci/is-version-number-acceptable.sh -# check-for-functional-changes: -# stage: diagnostics -# needs: -# - check-version-and-changelog -# before_script: -# - '' -# script: -# - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi -# when: master - - run_on_all_years: stage: run_on_all_years # Prevent call of before_script because it will fail in this context @@ -171,15 +152,26 @@ run_on_all_years: - echo "On ne fait rien" when: manual +check-for-functional-changes: + stage: anaconda + needs: + - check-version-and-changelog + before_script: + - '' + script: + - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi + only: + - master build_conda_package: + stage: anaconda + needs: + - check-for-functional-changes before_script: - '' - except: + only: - master image: continuumio/miniconda3 script: - conda install -y conda-build anaconda-client - - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca - .conda - stage: anaconda + - conda build -c conda-forge -c openfisca --token $ANACONDA_TOKEN --user OpenFisca .conda From 986bfcb14366efe4b30e38ce27058238b93a4e62 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 31 May 2023 08:59:53 +0000 Subject: [PATCH 010/140] Add ability to customize survey_scenario in comparator --- CHANGELOG.md | 12 +++++++----- openfisca_france_data/comparator.py | 11 +++++++++-- setup.py | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 331a0987..4ad6b4b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,18 @@ # Changelog + +### 0.25 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) + +* Technical changes + - Amélioration du comparateur pour permettre de définir le survey_scenario + ### 0.24 [#215](https://github.com/openfisca/openfisca-france-data/pull/215) * Technical changes - - Ajout d'un comparatuer ERFS-FPR vs simulation oepnfisca + - Ajout d'un comparateur ERFS-FPR vs simulation oepnfisca - produit des graphes diagnostics - produit de un tableur ouvert dans dtale pour rechercher les cas les plus problématiques - - Amélioration de la CI GitLab : - - Ajout d'une étape manuelle pour initialiser les bases de la branche à partir de la dernière CI de master. - - Ajout d'une étape manuelle pour faire tourner sur toutes les branches. - ### 0.23.1 [#213](https://github.com/openfisca/openfisca-france-data/pull/213) * Technical changes diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index d788bfe2..95f01523 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -456,8 +456,7 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti else None ) - survey_scenario = get_survey_scenario( - year = str(self.period), + survey_scenario = self.get_survey_scenario( data = data, survey_name = f'openfisca_erfs_fpr_{period}' ) @@ -563,6 +562,14 @@ def filter(self, data_frame): log.info(log_message) self.messages.append(log_message + "\n") + def get_survey_scenario(self, data = None, survey_name = None): + + return get_survey_scenario( + year = str(self.period), + data = data, + survey_name = f'openfisca_erfs_fpr_{self.period}' if survey_name is None else survey_name, + ) + def _load_test_dataframes(self, noindivs = None, idents = None): name = self.get_name() backup_directory = PurePath.joinpath(Path(config.get("paths", "backup"))) diff --git a/setup.py b/setup.py index 99ee1db7..f1277112 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.24", + version = "0.25", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 9747b6124a56173d490424dd9c2a5f7d58c38276 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 1 Jun 2023 15:26:51 +0000 Subject: [PATCH 011/140] Bump flake8 version --- CHANGELOG.md | 4 ++++ openfisca_france_data/comparator.py | 1 - setup.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ad6b4b7..e8d615d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 0.25.1 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) + +* Technical changes + - Change la version de flake8 ### 0.25 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 95f01523..3339128e 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -563,7 +563,6 @@ def filter(self, data_frame): self.messages.append(log_message + "\n") def get_survey_scenario(self, data = None, survey_name = None): - return get_survey_scenario( year = str(self.period), data = data, diff --git a/setup.py b/setup.py index f1277112..2871cec4 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.25", + version = "0.25.1", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -46,7 +46,7 @@ "autopep8 >= 1.4.0, < 1.5.0", "bumpver >= 2022.1120", "dtale", - "flake8 >= 3.7.0, < 3.8.0", + "flake8 >= 3.7.0, < 4.0.0", "ipdb >=0.13, <1.0", "ipython >= 7.5.0, < 8.0.0", "mypy >= 0.670, < 1.0.0", From abd4decaf14b1ae5e582cac1d230bf4fcf124443 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 5 Jun 2023 09:04:13 +0000 Subject: [PATCH 012/140] Aggregats --- .gitlab-ci/all_years_build_and_aggregates.yml | 34 ++++++ CHANGELOG.md | 12 +- openfisca_france_data/__init__.py | 17 ++- openfisca_france_data/aggregates.py | 109 +++++++----------- .../assets/agregats_tests_taxipp_2_0.xlsx | Bin 0 -> 12991 bytes openfisca_france_data/erfs_fpr/comparison.py | 2 +- .../step_01_preprocessing.py | 15 ++- .../input_data_builder/step_02_menage.py | 17 ++- .../step_03_variables_individuelles.py | 11 +- .../input_data_builder/step_06_final.py | 15 +-- openfisca_france_data/erfs_fpr/scenario.py | 2 + openfisca_france_data/model/common.py | 94 ++++++++++++++- setup.py | 2 +- tests/erfs_fpr/integration/test_aggregates.py | 3 +- 14 files changed, 233 insertions(+), 100 deletions(-) create mode 100644 openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index 36c088d0..099787fe 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -35,6 +35,40 @@ aggregates-2019: stage: aggregates tags: - openfisca + +input_data-2018: + image: $CI_REGISTRY_IMAGE:latest + script: + - echo "build_input_data-2018" + - mkdir -p $ROOT_FOLDER/$OUT_FOLDER + - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini + ~/.config/openfisca-survey-manager/config.ini + - build-erfs-fpr -y 2018 -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_2018.h5 + - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2018.ini + stage: build_input_data + tags: + - openfisca + +aggregates-2018: + artifacts: + paths: + - ./*.html + - ./*.csv + image: $CI_REGISTRY_IMAGE:latest + needs: + - input_data-2018 + script: + - echo "aggregates-2018" + - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2018.ini + ~/.config/openfisca-survey-manager/config.ini + - python tests/erfs_fpr/integration/test_aggregates.py --year 2018 + - mkdir -p $ROOT_FOLDER/$OUT_FOLDER + - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output + - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output + stage: aggregates + tags: + - openfisca + in_dt-1996: image: $CI_REGISTRY_IMAGE:latest needs: diff --git a/CHANGELOG.md b/CHANGELOG.md index e8d615d5..709b6b72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,16 +1,22 @@ # Changelog -### 0.25.1 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) +## 0.26 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) * Technical changes + - Utilise les cibles de taxipp dans les agrégats + + +### 0.25.1 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) + +* Minor change - Change la version de flake8 -### 0.25 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) +## 0.25 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) * Technical changes - Amélioration du comparateur pour permettre de définir le survey_scenario -### 0.24 [#215](https://github.com/openfisca/openfisca-france-data/pull/215) +## 0.24 [#215](https://github.com/openfisca/openfisca-france-data/pull/215) * Technical changes - Ajout d'un comparateur ERFS-FPR vs simulation oepnfisca diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 9b936da8..f6b7be5c 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -153,9 +153,20 @@ def apply(self): AGGREGATES_DEFAULT_VARS = [ 'cotisations_non_contributives', + "salaire_brut", + "retraite_brute", + "chomage_brut", + "salaire_imposable", + "retraite_imposable", + "chomage_imposable", + "csg_salaire", + "csg_non_salarie", + "csg_remplacement", + "csg_revenus_capital", 'csg', 'crds', - 'irpp', + 'impot_revenu', + "taxe_habitation", 'ppe', 'ppe_brute', 'af', @@ -170,8 +181,8 @@ def apply(self): 'ars', 'aeeh', 'asf', - # 'aspa', - 'minimum_vieillesse', + 'aspa', + # 'minimum_vieillesse', 'aah', 'caah', 'rsa', diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 5daac9fb..6106651d 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -2,14 +2,12 @@ from datetime import datetime import logging import os +from pathlib import Path import numpy as np import pandas as pd +import pkg_resources -try: - from ipp_macro_series_parser.config import Config # type: ignore -except ImportError: - Config = None from openfisca_survey_manager.aggregates import AbstractAggregates from openfisca_france_data import AGGREGATES_DEFAULT_VARS # type: ignore @@ -37,73 +35,46 @@ class FranceAggregates(AbstractAggregates): def load_actual_data(self, year = None): assert year is not None - - if not Config: - log.info("No actual data available") - return - - parser = Config() - - # Cotisations CSG -CRDS - try: - directory = os.path.join( - parser.get('data', 'prelevements_sociaux_directory'), - 'clean', + taxipp_aggregates_file = Path( + pkg_resources.get_distribution("openfisca-france_data").location, + "openfisca_france_data", + "assets", + "agregats_tests_taxipp_2_0.xlsx" + ) + df = ( + pd.read_excel( + taxipp_aggregates_file, + # "https://gitlab.com/ipp/partage-public-ipp/taxipp/-/blob/master/simulation/assets/agregats_tests_taxipp_2_0.xlsx", + # engine = 'openpyxl' ) - csg_crds_amounts = pd.read_csv( - os.path.join(directory, 'recette_csg_crds.csv'), - index_col = 0 - ).rename( - dict( - recette_csg = 'csg', - recette_crds = 'crds', - ) - ) / 1e6 - csg_by_type_amounts = pd.read_csv( - os.path.join(directory, 'recette_csg_by_type.csv'), - index_col = 0, - ).drop( - ['source'] - ).astype(float) / 1e6 - assiette_csg_by_type_amounts = pd.read_csv( - os.path.join(directory, 'assiette_csg_by_type.csv'), - index_col = 0, - ) / 1e6 - except Exception: - assiette_csg_by_type_amounts = None - csg_by_type_amounts = None - csg_crds_amounts = None - pass - # Prestations sociales - directory = os.path.join( - parser.get('data', 'prestations_sociales_directory'), - 'clean', + .rename(columns = str.lower) + .rename(columns = {"unnamed: 0": "description"}) + .dropna(subset = ["annee 2019", "annee 2018", "annee 2017", "annee 2016"], how = "all") ) - amounts_csv = os.path.join(directory, 'historique_depenses.csv') - beneficiaries_csv = os.path.join(directory, 'historique_beneficiaires.csv') - prestations_sociales_amounts = pd.read_csv(amounts_csv, index_col = 0) - prestations_sociales_beneficiaries = pd.read_csv(beneficiaries_csv, index_col = 0) - # Minimum vieillesses - minimum_vieillesse_beneficiaries_csv = os.path.join( - directory, 'historique_beneficiaires_minimum_vieillesse.csv') - if os.path.exists(minimum_vieillesse_beneficiaries_csv): - minimum_vieillesse_beneficiaries = pd.read_csv(minimum_vieillesse_beneficiaries_csv, index_col = 0) + if f"annee {year}" not in df: + return - amounts = pd.concat( - [ - assiette_csg_by_type_amounts, - csg_by_type_amounts, - csg_crds_amounts, - prestations_sociales_amounts, - ], - sort = True, - ) - beneficiaries = pd.concat( - [minimum_vieillesse_beneficiaries, prestations_sociales_beneficiaries], - sort = True, + df = ( + df[["variable_openfisca", f"annee {year}"]] + .dropna() + .rename(columns = { + "variable_openfisca": "variable", + f"annee {year}": year, + }) ) - return pd.DataFrame(data = { - "actual_amount": amounts[str(year)], - "actual_beneficiaries": beneficiaries[str(year)], - }) + beneficiaries = ( + df.loc[df.variable.str.startswith("nombre")] + .set_index("variable") + .rename(index = lambda x : x.replace("nombre_", "")) + .rename(columns = {year: "actual_beneficiaries"}) + ) / self.beneficiaries_unit + + amounts = ( + df.loc[~df.variable.str.startswith("nombre")] + .set_index("variable") + .rename(columns = {year: "actual_amount"}) + ) / self.amount_unit + + result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") + return result diff --git a/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx b/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..5b2d733518687700dfb85ff57001f10c45a5ddb2 GIT binary patch literal 12991 zcmeHugL@^*_V$i#+fF8!SQFc}ZF2_`+s?$A*tV^iWMXTgiLEc^;&<+yd%nNm-rCR8 zwR?BJYj;<@t5#R7w-uzpz|jGa0B8UJKmsuOm1Us^0szE;0|00MXi#lodpj3XI~RQw zPX|+H-47nNHpKbhpwzhl(D(EISN@Azpdx8puAd26^iJYgc$YzHwN3znqpD zO7Ete+Byp7Sm}4Xwi5 z2F9T4Od)Zdh-*_DknLR@gtDx-b)^y4Wm-dV)^y!K9iUe?bxh>7!ebjdhJ`+ZcDBH0 zl7MfE00ryF2FywpyJ??+PZ$L<)Bcm;xJ@Ti<_&~ejS*Trb(fc>_Ta?4Ls~#cb(H;_VpU~6IkwYHulbJp3z+V?B zhkh)tkO08j8yG<0-`uiMm5J>7ooh1hUI+K?mikVnHqMM6{@DJ{9si4c@Lz7dB0*NZ zp9wzXOyW6oA(a{x(meMIj%&}r?0L5+oX8tx^T*{N$tuvOkyya`2^ymhe z*!K&OTIA^u>bNjyMff3@eCeOHhGaF@jqa;J=7p3`D?@6UIdYE@r!)N)Qi=}|K810~ zoP0~i7;!c-U#bF**pS>j;HoNHa9P$EWjk?`dg`0l^jwK#cB8!j8RWi$W*jDPnN zcY7ymBYS)6Kcd$^eFo$`guPq&?=BTeveE-g$enPnp-k?XZWt&_E{vons>evsqYX4G zlq4U0uQ%{$+jKR4NHc=jhj^Y%4!hm4A#Q`A-SpCZjX(wWLbEyJfv}&vngoM3I;|uO z5QjiRJvllqKZQfga7Jt8PZ*)kM(5nWr>4LOEhr`xxo-Uy!fG|eve&w^remTgaRMPomi?92iMn1W&;1&p8sh`(Dvo=*y00Gg;3m z>)wJ#_&xk`X~Hr#K46Foap;IRksnQh^Im>>yXx7Ni_p^;P4~(r{rL5n+3x7@{Kk7v z|0iYKtybJY-|2D>1^^(vpYcwae-GFRgHVFQ^U8VulI;oGNK~>Qf1BdO$?9#+M52$XQkx5tg_S zwJd|u=|F5I!*4Tnz@VdL+y)JFg&2-0Ika$eDCYVg)e@%iBQ=Ms@5AXU zlsvZmAdEs5e<1&>cB#$skEa6vf~QJP9*PpGJUC|=8n@|jP2fpNj>uMd($Egmz>DuG zTA0tC^5fRpO=0^J{F*DY-fXmpdvY!KwT@8GSs3@wIhlEaF++o} zy926x{_DU@!_psQ_)oe-3|KKdy$9$_Gynko?{smtFmy6CQFd{%v@>`9V}ME0m~>iU zNA7YjKJ%Xyy&XuAd7yF|auX+eBPZaw(2%Rf4GSPQk?^OGh;|<5xt`OGo#ji-exUSB z@P&&@5R!26LZ$#2ANVu%tvud@iB7af zVRDNF5VQDVWv{yX3tl2~NBM-Pafaz=rrS!8GK%x}Z{Z4yILEB}IIqQtO_EBz;%2lW znBu2Xy}gn=`$}vYU!9Y;R;MHb_oQCaBeS)+cx-jD@jN}Vb@?OG6*$kNSjmL#T9(Ldm<>Uz{O@4Jtb)9Qtf^`MPx6?amd z$dap7m~o_+9l0`Rr`;!6-^7VB);QSwS|8ploS&9jW*<+2p*cp8v@V1u7irIIz#;>f zY)G^B9pymO>A5Gq&TEr-xyg<}GZ5Pp|I*o9CiH#N2B}L>dy$zQk{=jA#PhOg;U3$| zz1RvHz)i>pJwX-c0lH9+ufz#9VIO5;B4;zmrIQb;N?q)cQj$$WH<+aMG-X#K@zNp= zRa~tGlZGAl-K*nFA}*K!+JsJP5dP7zNSW{oHKly``1qBOZl6rD3IODWqPHY)7g#z1 zzvktAHT=Z+WC+zzeO$MZm_^^_fD7wq9i&)hIwZ!Vns!=yNwk?zZ`3tx^a|soiGvCb z;1Q|ACY$?GAkw-NY*{0Bl%(^yQFLtt1&DWjLb*qaE^|7#_1G;$|lh$L*}DATXP7fNw<|#nj_I7?KIv8&XnRxFG=A<=cLQI zcUVYnuNPP=+p?`$okC2XZ%{O-PL5RemhG$_(&V{8S%bvU>~7MwdEWTKeM!3o7E!4T zZm0L~WOYTph^*R}7E2LesB5VgJ;5AnYGOQecf%R3phDBCuHu>z`s-|=$#Ew#NluS% zcS!mtoRR=^PA+V(VqOum3hOek5UmV)9j4S@Ql#B|;ke10==^%OqkB!qz$U794wZ7H zZ~zr?frj*T6iw!e;lsgCadIfxyqUw#PajMK%m@Ja26I ztEz+6rTRYQ!D;L$d_l&u2^jBca$wxBf2w%<^}4gxNR@CwzFC?|rQCnaNEzNd@Zhr6 zt3LVbbmzwQuC(irGPrlVsF564%bMcCKY;-zzu;s{_Otv6Z?-b#Ca=N`C_=a9V^B`! z1EjO#pO~Lg!=ee-JBjh~VeB=qWLC2KLZTMlgosxj4)YjEy)$|l$r_4*My~({%miD=)_U(0J=|FKwlN3l427U)xPl%;!=kD()>|E` zyoE!9ZATu8WDUWp6)|4{nu1CZQ-8jEPmNV!r*xQOHH(|=C)h65cx%=Zm$nrno-Ey= zY*z4sb(N@LR#t!h#5w{E-GM~H(=aODu~lBjjq z7F%UQG*MzER>4fSc7qLzU<6|Q$RS^?NZr_NR;4MJiB_f{IVf(k`|Le%Z}L2$7|@FM zr-!51$m><1sj<=e>MpL<(Sn7<9 zDQNy`bZm8dAo|N-NxfKjlKmbmV{!jjO8VQgm5YU`ttsQ*w#|q2x+sc6ke;oS2PF zg5^plKYmh1RX>E(P*@#!^BTndui#Vn4X)TGks#v}ftMirye2#t<`vr12j~ctPs3)M zpY737h9r8dxfx=WUEtk}wW?HU1-Zn?9kaj(4cxF_o)NIyS<>TB+?rE=JfUYufTu!` zW&Dy};jGR#91c|t--ILlHpeB@a4IBh?8ZCL9DJ-W00FdU$GAC2X`W7?qU>vvj77$$rCi6iE6rfzQ?&Pmrw^!OtPuJp7 zdp9ipV)MEw`@C?nu)ygKeE4_{o3wQI<2+ChT95Vp`<73nJZ|I6A3tvevH=Ue9eN^w zRSTd{jBeq!ZA0BSEUjRe^!<#%SikHPEJ(p)RER?;KS_|4(I92t(~7dIbpZfi3MJwO zP$F@%b5RN%!>G9=!t14rQOMWfDWZfOi_~ybj?VR8d2b4N1Dcq=SxM{l1XW{5@*@%z znNTkOdNP=wC!Wp$-aU_!9Xuq4Qr!{etUg`rzP==Se==!`ir}G6Kf=iIc^S>o_x<^N z@w40)C4(gY?Z$%@~`+XiflSu_^6WtYsMcv~KRmU%HNAvcKPxrI90h%5{}3D8|M%l$u#+C1+YdGknqn+c0onH3Ic> zF=uaI{Kg{5P|Erm`M~)xAx=?LeYk@iYDr|+JcY260lAW*W#|BMYH&1h(DM1@Izq|# z*UAMxpb4p=0I?Oxr-3<#9Z^;+>ZWx3{Iny)4N=m%_*MNvGJ=HzP*T3ZB(4|c{Z+LC zDcG0!0AHAaLBdMC>DJ;SKtzjOxejPWjL_KThF77pI>xNnDco&& z*aVr`x`Xogc|@7V@#s`}qh$3xEJ+kD`TJp*rWfg4Bge9Y(#?*}5mSPeAW}H4B(Le4 zg2^U@P}|MpLigcGS3%TW7RLwXW(p1^!VtpnooIz$yWGOU=2!;bl$X688E7U>lB_<> z-Y;_HUR~n`pkLvs9`-PAIRI29JGdwv@>@~sNZ;gSWm->;#3N`!>zqW)QO3;CbtgWy zpw40t+%#Q&zD>Eq!EgK37g+km;O(a3X}eU*OAc+)Xo^)ubSlG**~cfbW2h`m1VJs` zrQ8;bv7xGQj$1QS9=mgO}^u3 z0_jH1yz&ci%CS`P{n0Ps?koeSW&vI=2SnbIfO3VtHQ@+@+0-m@^EwdW-KLR+u=Cvs zYfcSejL-@zs&&AWm^ly{C22E-Eo%cx})=xj= zBx$(0_+5<{f@MMX=d?oM5*E>YX&is+4e2ZjZawozK_|T8S zUYCc+!9#HS%!V#j+zU;X21oV&GpBRUS4+pyGGv%xkklsV@{?pSlk)XMerW^S*xMxp zLwx`g*6a)(l3Vu_j3*xay!(t(=HPUM9Te2_(Pp)d18j&|lL(=5`xRk2p`Z+{KHXgo z6jYK|wxv5D=4m{!H7cs3N}UYZURb_Kt&X%cB>c3>t zZy`B_4L%ybME+{lk&9f&nXqn7vIDtMRZ@JgbQIr@E~=5p7ss~02_Y6uC1N~iVkx;H zzK%P0fSoK$U<9O_R+iV!+!)iQFGSm*ISZJqWJOv(-hxS?q^P(YIcSZRHa=}DV=DIO2U!FNZvU$ zCjt(@Bcz=ql)Y|!Ximc+E4Ke!68GV**doUClG{cqiy5E6CFAXwimn_!8QPG=ZP(P zg^8eQEmmKN=|OUb`k@R4X@?OQSh8RL&8 z*xAL)jWhQl+Gpn2V~8U5wG_cj>Mu?}HWet+Y~UIiCJ%@NmDe3a|H;D9AXWvdpuBlF zI8p?vaAC)HDds!>^8tz;y5#H^=QFXSOty*jZ)^TiR5wY!qkk8Ex)pLfX7F~kLn z%1L}sCCD9P-XF(JWDd|9t?#h5(Nid?l#bQsT*6*A6DXZqN8ir~Br&d8z**0phxdvpPufEYhQFtb8s%xHw1e zV{v~}2Vsy}8~Y^uwC;M6?QYD|*^3 zwhn`82>ohPS%*Oz+eg}afJ0VcLs7AfmAmxzm&3E|w5i!9900)12moOJeR%$3CD+-) z)YQe9@o&}NrW{dP`p!Eo2!3ZW-GReMK0Mk<5>iq-s{?BJt1>G~hk7gKgNmrr%PH1$ z?cPs^NW?VZ{$Fiw4HGxtS7;@kNd}MgyNPb=t8X6atYgYsHAJp7eOB4!+jqvTUUN3i z{6_cdrL_ri%6yzF)UQ4rjXy7BcE9dDYCFF=c~%~~L88W_&ObNW%Bzp>nLcR|m5t|f z&5H|UXI|x5ZB1W|UpjFw%qQZwEHds;e;XHW{)f%BFW|?lIgOKtL^saurFY)g_~j2L;@|Q4?dj&&15Hofp0PfFFp;3B zpVnG-<=$@yD1b-kL}{oLE;qGTH)6EPIqbdJyYz1e)|ytQzj>GJb!r~UA1oQWmU2wK z`n8wbVyFab<{TXy*>1eHKin#u>FH_bjzN5dXK9`!1n+sP??>byzVCdr^YwM!VK}yV zajQhBNAODm-Lr%Zfkp#MwF!`{G0D+%r%xnQ*97dXY^)s%foeoWfx8)(SAEJ5h{aR zSE^7)te7-+s!+w6NkcSO$)65t+y;;rKY$4Pb60RZ zu`Gd)FNS;@k`yXYq;!T{cg;-*m%&(3AV?!ulH6bamgC$fX<*ZXya6$BWxY+vZXCaC zF;;{b!Pqp|5LQT7*@%~FxkzDiocbDsre=VE4RKGPq<*zXZWwj%El15ja>zx<3#w^9 zB6v1{na|pEkE?|!kMVw>S9y>OS+zzfI+_8&lkE=t9-WpX4aputQqJZ{DGuTRQt{kB zT0;YsAY=#Z6dAO1?YPM$9@pF^I2$R zxCXn4)RkDavC3=}mKe!cQ&)rRvG;^jAY4F&)H80no=H@bk1k5WqDlrx>6m>8{xYEc zGvUHVY`+y z92H6-c#5&m`Rk7r1~6xN2ff8})pG$UNP1-mGFOtkiWR{*4@}`1`QDMcz9j?nc~vxH zx0Z$?`KCWf!sOH8P)cQ@jpp{H+6x+@awS4_B_CA2$au^xc#=-TeM0iy2YtxusZYNq zGv&K=J|-p_4dn6MR7`@(Mp=T=1BW2g!GPxN!#bF`HW74Z;Sbw2@J|sbhjAbBB{`p< zn52wcVnf9J((APds(_}6I*005sI(5@0#BP(WHc+t1wh749lbKhJ+ z!*d;;Y=YD#|9k&;nVd*IqK8Iuvj%+@Zg95=uUwlVA<_oA-81Mufn+s{913X)PVvFPlz`MtQxGQt&;RnhkH&sH88&k|O zrH|B0aw^5~E>59=iE0Kf10tH>aP|ZzBVz;w$KZ>jVg$)0Kq}={MH@&C@LAQEHWNp!I6s->6cvT&(hd- zH^P~YLpOb}1^cS$&RGiqXE^)OWGU!WPAG}%AiAlcdcDx{8aRO_Y=qQr5-39va!$f> zL1)Jz1<#*Rze#OSAqh{KT+{3b^6A9(h{CnxvlwtR2sEmuBc6gKksvEalH!?NebuSpz*R+ z{GPWbn)n|a4h-Ri-p-99zO=xSyMLzdtaMa33r!MzfmA>vz(wWg{z>aK^F{W-sy*H^ z32t&AMNHW|fD4509vv5J%Uba8&N;!hkxlSnZyr*CRm$qC8eHC%{>+W&aHwXM_JK%` zsCb>f(>jLB`cpnBa?|UPC%TozeT-6qw4g7mNRqx%lv_C;?vN!Z_$S^z59LMK>OtF+ zq26!xYNak$k8YZ>pzSO=w&bmpyYrBAn;X2^+&Lu4wFJTSv-^uDIqiv#(n{&Ol+gb zc29CsR{R_iRxI-fj=&z>B3h~db%}>R_=JGq1GB`hMxvWN@OfSi5G!L|pzC6R2rr-%;syjoEF=EKtLmCAw6kL3{1k^8qRdl`cC-kFNhqakP zyL0r&pYm--pdK zVg)aj%k8Cm?(D z8HvvDbS<4ZlB8~!1Gh*q{8v_KU3tGL)@a~u#|dguHybU0&3m|^Ld-9ln?TrVJ;lJm z`79Q;TJd9p`zD{wIZCp=hxjQC;!)+!1}B_=tRV(V#|=Tu8K+RW^G0-RZaH+CK5p{p z(LK60&Iym!Q{1IOj{ZdC%DBx+2Z5ktg$&<(7eYwp54t4yyD)f9zyKef9%8;-mHfQ) z%khdsfabb-{_O$dyssPA!23WrfsNt)IGX%`@I?C z&R1D0UoevTmiQM=S1jGicLZ{*?jPP-`C3h# zwk|iGOXB?egbTPGv^3xCdAfFkV$5vsZl1JM7w*7~)hPNE&Cjtg`Tcv~?B%}w{MPkd z^uyb&7uQVupyPW|EB9sM`Em5zH9tk=vd-_q_loK-X9eG7cL#OfWj;ag8V{6rNsx)X zv4WGmgEOPCy_4xb#H{~Q0=+jKFhNm1kO@6_L-rY|@Js=AA`MdGOB+=x_Mio@{B-RH zbZ~>K&D%OCUY!NdX1Y~>x;1XFkXjF}lpV*5CgqjwGRjn{rcNy` z4-A`invqdyT|v*1cGuI3qnmNU3o?}*w zV~I^Kl%va8G=8-03Z(7ou)_n4uQng=ovKf1o35jPCoSh5Zd#>l*GxiSvk4Kd5^w97 zn+A)qy4{*JjFBU!rah+o9nC9)I~uNA-IJCl%Q3^w65kRB{yM}~>x7GihmzjTO1n|` zC3RF75o!)Ogro&fcJ!n((v5w?75hV(anDD+*Y3ShaKT5uH@gL<&!nobaANpd5OC-=cW(p1l*gzwN-Ouz| zbN;o6nrl+Bg|Q zxE)Al8~IeN6Ia!!NI_jgk8M0K5t5Gc-^E3b4IsO&l?Zqsf0V(3SJg2bnK~Bia;a{# zPicY;+%0^n3?~T=sS6DhEZOZSP4f;kBVr(Mt=i>P#REe;^C+_Gp-{QhQILlY%(AXw zww(W7CN4~3Hu`ERoJ#tH00T9nbdAj5VYF#zBQ><>8lLYv(CHdOP|ds(Y&IRn%gHwE z;A=*>f~Ok=hOdK&2#SWx@j;PJy5mH>K1AXy4$Jv6-f+9iAggH#W_bLn>TS+X@El>!@KP1uU{4TXRrOU{)-m}6r}&%z`u){{|Wq~PJMTce~~!<4*Yka=f46w z-aYRBFZ=x6&hP5KKP{cTzYX!57VvlQ?^>`w!G!P2Q19UXP=@_(;r9gjpBAK1|1w?v zyMft>~{p0^0&uN(c*!zDVacY(T literal 0 HcmV?d00001 diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index cfb4e87b..d9630d20 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -63,7 +63,7 @@ def compute_test_dataframes(self): year = int(self.period) table_by_name = build_table_by_name(year, erfs_fpr_survey_collection) - log.debug("Loading tables for year {} [{}]".format(year, table_by_name)) + log.debug(f"Loading tables for year {year} [{table_by_name.values()}]") # load survey and tables survey = erfs_fpr_survey_collection.get_survey(table_by_name['survey']) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_01_preprocessing.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_01_preprocessing.py index 8f254a89..bbd7fc9f 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_01_preprocessing.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_01_preprocessing.py @@ -28,6 +28,11 @@ def build_table_by_name(year, erfs_fpr_survey_collection): "fpr_menage": f"fpr_menage_{year}_retropole" if year in add_suffix_retropole_years else f"fpr_menage_{year}" } + capitalized_table_by_name_stata = dict( + (name, table.upper()) + for name, table in table_by_name_stata.items() + ) + table_by_name_sas = { "eec_individu": "IRF", "eec_menage": "MRF", @@ -38,11 +43,19 @@ def build_table_by_name(year, erfs_fpr_survey_collection): if tables == set(table_by_name_stata.values()): table_by_name = table_by_name_stata.copy() + elif tables == set(capitalized_table_by_name_stata.values()): + table_by_name = capitalized_table_by_name_stata.copy() + elif tables == set(table_by_name_sas.values()): table_by_name = table_by_name_sas.copy() else: - raise ValueError("No incorrect table pattern: {tables}") + raise ValueError(f""" +Incorrect table pattern: {tables} differs from: + - stata: {set(table_by_name_stata.values())} + - capitalized_stata: {set(capitalized_table_by_name_stata.values())} + - sas: {set(table_by_name_sas.values())} +""") table_by_name["survey"] = f"erfs_fpr_{year}" diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py index 538f1fc3..9bb038a6 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py @@ -1,18 +1,27 @@ +"""Constructin des variables ménages.""" + + import logging import pandas as pd +from openfisca_survey_manager.temporary import temporary_store_decorator -from openfisca_survey_manager.temporary import temporary_store_decorator # type: ignore log = logging.getLogger(__name__) @temporary_store_decorator(file_name = 'erfs_fpr') def build_variables_menage(temporary_store = None, year = None): + if year == 2018: + menages = temporary_store[f'menages_{year}'] + menages['statut_occupation_logement'] = menages['so'].copy() + + if year == 2019: # SO ne fonctionne pas en 2019 bcp de 0 = non-renseigné + menages = temporary_store[f'menages_{year}'] + menages['statut_occupation_logement'] = menages['logt'].copy() + if year >= 2018: - menages = temporary_store['menages_{}'.format(year)] menages['zone_apl'] = 2 # pour l'instant on met tout le monde à 2 mais à améliorer, peut être en fonction de la taille de l'aire urbaine ? - menages['statut_occupation_logement'] = menages['so'].copy() - menages.loc[(menages.statut_occupation_logement == 7),'statut_occupation_logement'] = 2 + menages.loc[(menages.statut_occupation_logement == 7), 'statut_occupation_logement'] = 2 temporary_store['menages_{}'.format(year)] = menages diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 92364169..659997c8 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -31,10 +31,11 @@ def build_variables_individuelles(temporary_store = None, year = None): openfisca_by_erfs_variable = { 'chomage_i': 'chomage_imposable', 'pens_alim_recue_i': 'pensions_alimentaires_percues', - 'rag_i': 'rag_net', + 'pens_invalidite_i': 'pensions_invalidite', + 'rag_i': 'rag', 'retraites_i': 'retraite_imposable', - 'ric_i': 'ric_net', - 'rnc_i': 'rnc_net', + 'ric_i': 'ric', + 'rnc_i': 'rnc', 'salaires_i': 'salaire_imposable', } @@ -45,6 +46,10 @@ def build_variables_individuelles(temporary_store = None, year = None): columns = openfisca_by_erfs_variable, inplace = True, ) + + + individus['rpns_imposables'] = individus['rag'] + individus['ric'] + individus['rnc'] + create_variables_individuelles(individus, year) assert 'salaire_de_base' in individus.columns , 'salaire de base not in individus' assert 'traitement_indiciaire_brut' in individus.columns , 'traitement indiciaire brut not in individus' diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index 582e296f..db72b2a0 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -33,6 +33,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'idmen', 'noindiv', 'pensions_alimentaires_percues', + 'pensions_invalidite', 'quifam', 'quifoy', 'quimen', @@ -40,6 +41,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'retraite_brute', 'ric', 'rnc', + 'rpns_imposables', 'salaire_de_base', 'statut_marital', "primes_fonction_publique", @@ -61,15 +63,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'zone_apl', ] - # TODO: fix this simplistic inference - individus.rename(columns = { - 'ric_net': 'ric', - 'rag_net': 'rag', - 'rnc_net': 'rnc', - }, - inplace = True - ) - individus = create_ids_and_roles(individus) individus = individus[var_individus].copy() gc.collect() @@ -79,10 +72,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene for k in missingvariablesmenages: menages[k] = 0 - # Again artificially putting missing variables in their default state - #menages["loyer"] = 0 - #menages["zone_apl"] = 2 - #menages["statut_occupation_logement"] = 0 menages = extract_menages_variables(menages) diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index 7f81adea..80c8d852 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -21,6 +21,7 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "heures_remunerees_volume", "loyer", "pensions_alimentaires_percues", + "pensions_invalidite", "primes_fonction_publique", "rag", "retraite_brute", @@ -30,6 +31,7 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "revenus_capitaux_prelevement_forfaitaire_unique_ir", "ric", "rnc", + "rpns_imposables", "salaire_de_base", "statut_marital", "statut_occupation_logement", diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 590768b2..c6e17cbe 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -51,12 +51,68 @@ class assiette_csg_chomage(Variable): label = "Assiette CSG chomage" definition_period = MONTH - def formula(individu, period, parameters): + def formula(individu, period): chomage_brut = individu('chomage_brut', period) taux_csg_remplacement = individu('taux_csg_remplacement', period) return chomage_brut * (taux_csg_remplacement >= 2) +class csg_salaire(Variable): + value_type = float + entity = Individu + label = "CSG salaire" + definition_period = MONTH + + def formula(individu, period): + return ( + individu('csg_deductible_salaire', period) + + individu('csg_imposable_salaire', period) + ) + + +class csg_non_salarie(Variable): + value_type = float + entity = Individu + label = "CSG non salarié" + definition_period = YEAR + + def formula(individu, period): + return ( + individu('csg_deductible_non_salarie', period) + + individu('csg_imposable_non_salarie', period) + ) + + +class csg_remplacement(Variable): + value_type = float + entity = Individu + label = "CSG sur les revenus de remplacement" + definition_period = MONTH + + def formula(individu, period): + return ( + individu('csg_imposable_retraite', period) + + individu('csg_deductible_retraite', period) + + individu('csg_imposable_chomage', period) + + individu('csg_deductible_chomage', period) + ) + + +class impot_revenu(Variable): + value_type = float + entity = FoyerFiscal + label = "Impôt sur le revenu" + definition_period = YEAR + + def formula(foyer_fiscal, period): + return ( + foyer_fiscal('irpp_economique', period) + + foyer_fiscal('prelevement_forfaitaire_liberatoire', period) + + foyer_fiscal('prelevement_forfaitaire_unique_ir', period) + + foyer_fiscal('ir_pv_immo', period) + ) + + class decile(Variable): value_type = Enum possible_values = Deciles @@ -237,3 +293,39 @@ def formula(menage, period): percentile, values = mark_weighted_percentiles(nivvie, labels, wprm * menage_ordinaire, method, return_quantiles = True) threshold = .6 * values[1] return (nivvie <= threshold) * menage_ordinaire + + +class salaire_brut(Variable): + value_type = int + entity = Individu + label = u"Salaire brut (y compris fonction publique)" + set_input = set_input_divide_by_period + definition_period = MONTH + + def formula(individu, period): + salaire_de_base = individu('salaire_de_base', period) + primes_salaires = individu('primes_salaires', period) + primes_fonction_publique = individu('primes_fonction_publique', period) + indemnite_residence = individu('indemnite_residence', period) + supplement_familial_traitement = individu('supplement_familial_traitement', period) + remuneration_principale = individu('remuneration_principale', period) + indemnite_fin_contrat = individu('indemnite_fin_contrat', period) + complementaire_sante_salarie = individu('complementaire_sante_salarie', period) + indemnite_compensatrice_csg = individu('indemnite_compensatrice_csg', period) + + # Revenu du foyer fiscal projeté sur le demandeur + rev_microsocial = individu.foyer_fiscal('rev_microsocial', period, options = [DIVIDE]) + rev_microsocial_declarant1 = rev_microsocial * individu.has_role(FoyerFiscal.DECLARANT_PRINCIPAL) + + return ( + salaire_de_base + + primes_salaires + + remuneration_principale + + primes_fonction_publique + + indemnite_residence + + supplement_familial_traitement + + rev_microsocial_declarant1 + + indemnite_fin_contrat + + complementaire_sante_salarie + + indemnite_compensatrice_csg + ) diff --git a/setup.py b/setup.py index 2871cec4..478692c6 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.25.1", + version = "0.26", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index 3473bc70..9e3e56c2 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -24,7 +24,7 @@ ) -def test_erfs_fpr_survey_simulation_aggregates(year = 2014, rebuild_input_data = False, use_marginal_tax_rate = True, variation_factor = 0.03, varying_variable = 'salaire_de_base'): +def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_input_data = False, use_marginal_tax_rate = True, variation_factor = 0.03, varying_variable = 'salaire_de_base'): log.info(f'test_erfs_fpr_survey_simulation_aggregates for {year}...') np.seterr(all = 'raise') tax_benefit_system = france_data_tax_benefit_system @@ -123,6 +123,7 @@ def main(year, configfile = None, verbose = False): print(aggregates.to_markdown()) aggregates.to_html(f'aggregates_erfs_fpr_{year}.html') + continue survey_scenario._set_used_as_input_variables_by_entity() mtr_rd = survey_scenario.compute_marginal_tax_rate(target_variable = target_variable, period = year, use_baseline = True) From 4fa0f379b16c0b13aec8380103a873489fcccf08 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 8 Jun 2023 18:18:10 +0200 Subject: [PATCH 013/140] Update france dep --- setup.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 478692c6..bd2f0b2d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.26", + version = "0.27", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -35,10 +35,8 @@ "click >= 8.0.0, < 9.0.0", "matplotlib >= 3.1.1, < 4.0.0", "multipledispatch >= 0.6.0, < 1.0.0", - # "openFisca-france >= 145.0.0, < 146.0.0", - "OpenFisca-France @ git+https://github.com/openfisca/openfisca-france.git@version_leap", - # "openFisca-survey-manager >= 0.47.2, < 1.0.0", - "OpenFisca-Survey-Manager @ git+https://github.com/openfisca/openfisca-survey-manager.git@version_leap", + "openFisca-france >= 148.0.0, < 149.0.0", + "openFisca-survey-manager >= 1, < 2.0.0", "wquantiles >= 0.3.0, < 1.0.0", # To compute weighted quantiles ], extras_require = { From 47437c3af494c2552bad0d98e1191eb334e4c7d5 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 07:00:18 +0200 Subject: [PATCH 014/140] Update workflow --- .github/workflows/workflow.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d765aac7..94daabbc 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -11,8 +11,8 @@ jobs: strategy: fail-fast: true matrix: - os: ["ubuntu-20.04"] - python-version: ["3.8.9", "3.9.9"] + os: ["ubuntu-20.04"] # On peut ajouter "macos-latest" si besoin + python-version: ["3.9.9", "3.10.6"] steps: - name: Checkout uses: actions/checkout@v3 @@ -22,7 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache build id: restore-build - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} @@ -33,7 +33,7 @@ jobs: run: make build - name: Cache release id: restore-release - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: dist key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} @@ -51,7 +51,7 @@ jobs: python-version: 3.9.9 - name: Cache build id: restore-build - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 @@ -116,7 +116,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8.9 + python-version: 3.9.9 - name: Check version number has been properly updated run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh" @@ -157,13 +157,13 @@ jobs: python-version: 3.9.9 - name: Cache build id: restore-build - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ env.pythonLocation }} key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 - name: Cache release id: restore-release - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: dist key: release-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04 From 181dafed241fe34d1abd684598c4041263443391 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 07:09:50 +0200 Subject: [PATCH 015/140] Do not run test on Pyhton 3.8. Add 3.10 --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 94daabbc..01d016bd 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -91,7 +91,7 @@ jobs: fail-fast: true matrix: os: [ "ubuntu-20.04" ] # On peut ajouter "macos-latest" si besoin - python-version: ["3.8.9", "3.9.9"] + python-version: ["3.9.9", "3.10.6"] steps: - uses: actions/checkout@v3 - name: Set up Python From 585bc1f90cc7533cab1693f5e467bc2b20db32cf Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 07:27:08 +0200 Subject: [PATCH 016/140] Add pytest as dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd2f0b2d..7d99a627 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ "ipython >= 7.5.0, < 8.0.0", "mypy >= 0.670, < 1.0.0", "pypandoc", - # "pytest", + 'pytest >= 7.2.2, < 8.0', # "pytest-cov >= 2.6.0, < 3.0.0", "scipy >= 1.2.1, < 2.0.0", "toolz >= 0.9.0, < 1.0.0", From d51110f883f0a94adb2152f8cffe86790007d8c6 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 07:38:40 +0200 Subject: [PATCH 017/140] Install test extras to workflow test-python --- .github/workflows/workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 01d016bd..2f159826 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -104,6 +104,7 @@ jobs: with: path: ${{ env.pythonLocation }} key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-${{ matrix.os }} + - run: pip install .[test] - run: make test check-version-and-changelog: From 7937634e1955106b6402366a2d0ba21e516b9d87 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 08:20:58 +0200 Subject: [PATCH 018/140] Bump openfisca-france-dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7d99a627..73bccd76 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ "click >= 8.0.0, < 9.0.0", "matplotlib >= 3.1.1, < 4.0.0", "multipledispatch >= 0.6.0, < 1.0.0", - "openFisca-france >= 148.0.0, < 149.0.0", + "openFisca-france >= 149.0.0, < 150.0.0", "openFisca-survey-manager >= 1, < 2.0.0", "wquantiles >= 0.3.0, < 1.0.0", # To compute weighted quantiles ], From af778da337180ae91a085f67a6af70062dcbc7ab Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 19:53:17 +0200 Subject: [PATCH 019/140] Add data_files entry to setup.py --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index 73bccd76..a23938f8 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,12 @@ "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering :: Information Analysis", ], + data_files = [ + ( + 'openfisca_france_data/assets', + ['openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'], + ), + ], entry_points = { 'console_scripts': [ 'build-erfs-fpr=openfisca_france_data.erfs_fpr.input_data_builder:main', From 2234ebdfde7162520fc23508fa093d0167664bc7 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 22:36:52 +0200 Subject: [PATCH 020/140] Improve finding files --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a23938f8..2d4cfa6b 100644 --- a/setup.py +++ b/setup.py @@ -61,5 +61,5 @@ "toolz >= 0.9.0, < 1.0.0", ], }, - packages = find_packages(exclude = ("docs", "tests")), + packages = find_namespace_packages(exclude = ("docs", "tests")), ) From 7186ae97840c856f63168ca4ebdc9774026e6004 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 22:46:57 +0200 Subject: [PATCH 021/140] Fix import --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2d4cfa6b..7be8abfe 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -from setuptools import setup, find_packages +from setuptools import setup, find_namespace_packages with open('README.md') as file: From 667521ef5ee8abdceab32754784042a3fc3b2011 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sun, 11 Jun 2023 17:20:19 +0200 Subject: [PATCH 022/140] Use pacakge_data instead of data_files --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7be8abfe..fb5d3248 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering :: Information Analysis", ], - data_files = [ + package_data = [ ( 'openfisca_france_data/assets', ['openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'], From b59e56cf5eec7a66b66d39e43256d65bdfbd9790 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sun, 11 Jun 2023 17:30:42 +0200 Subject: [PATCH 023/140] Fic package_data format --- setup.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index fb5d3248..039b605a 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,9 @@ "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering :: Information Analysis", ], - package_data = [ - ( - 'openfisca_france_data/assets', - ['openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'], - ), - ], + package_data = { + 'openfisca_france_data/assets': ['openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'], + }, entry_points = { 'console_scripts': [ 'build-erfs-fpr=openfisca_france_data.erfs_fpr.input_data_builder:main', From b3dfb5d15facae68d48901f3ec908b1d726a91d8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sun, 11 Jun 2023 17:43:21 +0200 Subject: [PATCH 024/140] Add __init__.py --- openfisca_france_data/assets/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 openfisca_france_data/assets/__init__.py diff --git a/openfisca_france_data/assets/__init__.py b/openfisca_france_data/assets/__init__.py new file mode 100644 index 00000000..e69de29b From 124cc6e69199a5daace3a0e69d4e22df11ecaf04 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 13:52:53 +0200 Subject: [PATCH 025/140] Fix path typo --- openfisca_france_data/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 6106651d..2f81aa60 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -36,7 +36,7 @@ class FranceAggregates(AbstractAggregates): def load_actual_data(self, year = None): assert year is not None taxipp_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + pkg_resources.get_distribution("openfisca-france-data").location, "openfisca_france_data", "assets", "agregats_tests_taxipp_2_0.xlsx" From 006e99cd6ed20543a76df15e716469ec63bdeec1 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 14:20:13 +0200 Subject: [PATCH 026/140] Ignore error --- tests/test_aggregate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 468fdbd2..faaca97c 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -8,6 +8,9 @@ ) +@pytest.mark.skip( + reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", + ) def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 2009): # On ititialise le survey scenario survey_scenario = survey_scenario(year) From c82b42ff8fbf45221492301ffe7e1de3118209a6 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 14:28:00 +0200 Subject: [PATCH 027/140] Skip another test usng taxipp xlsx file --- tests/test_aggregate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index faaca97c..3b9af3ef 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -27,6 +27,9 @@ def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 20 return aggregates.base_data_frame +@pytest.mark.skip( + reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", + ) def test_erfs_fpr_aggregates_reform(fake_input_data, year:int = 2013): survey_scenario = ErfsFprSurveyScenario.create( year = year, From fc41ff60b31056f5590d066ea9d3ae626ac665c2 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 15:17:13 +0200 Subject: [PATCH 028/140] Update changelog --- CHANGELOG.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 709b6b72..7995e213 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,17 @@ # Changelog -## 0.26 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) +## 0.27 [#223](https://github.com/openfisca/openfisca-france-data/pull/223) * Technical changes - - Utilise les cibles de taxipp dans les agrégats + - Utilise les version Python >= 3.9 des dépendances liées à openfisca + - Contient les versions précédentes non concernées par une PR sur github + +## 0.26 +* Technical changes + - Utilise les cibles de taxipp dans les agrégats -### 0.25.1 [#XXX](https://github.com/openfisca/openfisca-france-data/pull/XXX) +### 0.25.1 * Minor change - Change la version de flake8 From 82afb65e4b124d5caf6737c811b75aeec29f18fe Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 18:06:07 +0200 Subject: [PATCH 029/140] Update .gitlab-ci.yml Co-authored-by: Benoit Courty <6603048+benoit-cty@users.noreply.github.com> --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce4dc8ae..2a5b89b6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -159,7 +159,7 @@ check-for-functional-changes: before_script: - '' script: - - if `.github/has-functional-changes.sh` ; then echo "OK to build package" ; fi + - if `.gitlab-ci/has-functional-changes.sh` ; then echo "OK to build package" ; fi only: - master From 0440c39d2df01a9a793314c362f6d985b4162cca Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 18:06:23 +0200 Subject: [PATCH 030/140] Update .github/workflows/workflow.yml Co-authored-by: Benoit Courty <6603048+benoit-cty@users.noreply.github.com> --- .github/workflows/workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 2f159826..d956bdbf 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: true matrix: - os: ["ubuntu-20.04"] # On peut ajouter "macos-latest" si besoin + os: ["ubuntu-20.04"] python-version: ["3.9.9", "3.10.6"] steps: - name: Checkout From 8af5e9d12bc1038b52d61f153fdb145d53cb71ad Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 14 Jun 2023 11:28:02 +0200 Subject: [PATCH 031/140] Supprime rev_microsocial de salaire_brut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ce n'est pas un salaire, ni brut, et supprimé d'OF-france --- openfisca_france_data/model/common.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index c6e17cbe..f9730a7f 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -313,10 +313,6 @@ def formula(individu, period): complementaire_sante_salarie = individu('complementaire_sante_salarie', period) indemnite_compensatrice_csg = individu('indemnite_compensatrice_csg', period) - # Revenu du foyer fiscal projeté sur le demandeur - rev_microsocial = individu.foyer_fiscal('rev_microsocial', period, options = [DIVIDE]) - rev_microsocial_declarant1 = rev_microsocial * individu.has_role(FoyerFiscal.DECLARANT_PRINCIPAL) - return ( salaire_de_base + primes_salaires @@ -324,7 +320,6 @@ def formula(individu, period): + primes_fonction_publique + indemnite_residence + supplement_familial_traitement - + rev_microsocial_declarant1 + indemnite_fin_contrat + complementaire_sante_salarie + indemnite_compensatrice_csg From 7f46ba9798964c9732d050bef35400bdb58d1889 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 14 Jun 2023 14:47:39 +0200 Subject: [PATCH 032/140] Bump --- CHANGELOG.md | 9 +++++++++ setup.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7995e213..a5652549 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +# 1.0.0 [#224](https://github.com/openfisca/openfisca-france-data/pull/224) + +* Breaking changes +- dans model/common.py/salaire_brut: + - Retire rev_microsocial + +- Détails + - rev_microsocial n'était pas un salaire mais un CA net des cotisations sociales pour les micro-entrepreneurs optant pour le versement libératoire de l'IR. Elle était donc à tort dans salaire_brut, d'autant plus qu'il n'y a pas d'autres rpns qui y sont répertoriés. + ## 0.27 [#223](https://github.com/openfisca/openfisca-france-data/pull/223) * Technical changes diff --git a/setup.py b/setup.py index 039b605a..84ee5e0a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "0.27", + version = "1.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From a0707f0d940c2f35787a5466ba086fc900692ffc Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 2 Jun 2023 17:09:25 +0200 Subject: [PATCH 033/140] wip --- .../input_data_builder/step_03_variables_individuelles.py | 2 +- openfisca_france_data/erfs_fpr/scenario.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 659997c8..693e9f16 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -76,7 +76,7 @@ def create_variables_individuelles(individus, year, survey_year = None, revenu_t create_activite(individus) create_contrat_de_travail(individus, period = period, salaire_type = revenu_type) create_categorie_salarie(individus, period = period, survey_year = survey_year) - + create_categorie_non_salarie(individus) # inversion des revenus pour retrouver le brut # pour les revenus de remplacement on a la csg et la crds dans l'erfs-fpr donc on peut avoir le brut directement create_revenus_remplacement_bruts(individus) diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index 80c8d852..e32b1d69 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -11,6 +11,7 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "activite", "autonomie_financiere", "categorie_salarie", + "categorie_non_salarie", "chomage_brut", "chomage_imposable", "contrat_de_travail", From 457707868725c8f64a24b7f9015ee6b6dc3c4d6b Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 6 Jun 2023 12:17:09 +0200 Subject: [PATCH 034/140] fix categorie_non_salarie --- .../input_data_builder/step_03_variables_individuelles.py | 7 +++++++ .../erfs_fpr/input_data_builder/step_06_final.py | 1 + 2 files changed, 8 insertions(+) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py index 693e9f16..ff20b639 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py @@ -489,6 +489,7 @@ def create_categorie_non_salarie(individus): commercant = individus.cstot.isin([22]) chef_entreprise = individus.cstot.isin([23]) profession_liberale = individus.cstot.isin([31]) + individus['categorie_non_salarie'] = 0 individus.loc[ agriculteur | artisan, 'categorie_non_salarie' @@ -501,6 +502,12 @@ def create_categorie_non_salarie(individus): profession_liberale, 'categorie_non_salarie' ] = 3 + #fix un peu crade : cstot ne semble pas recouvrir tout le champ des personnes qui ont du rpns + # on met par défaut ces gens là en chef d'entreprise + individus.loc[ + ((individus['rpns_imposables'] != 0) & (individus['categorie_non_salarie'] == 0)), + 'categorie_non_salarie' + ] = 2 def create_contrat_de_travail(individus, period, salaire_type = 'imposable'): diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index db72b2a0..c47e203c 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -23,6 +23,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'activite', 'age', 'categorie_salarie', + 'categorie_non_salarie', 'chomage_brut', 'contrat_de_travail', 'date_naissance', From 77b24067beec82e37f14e9af5a1796532ef7f4b0 Mon Sep 17 00:00:00 2001 From: cgl Date: Wed, 7 Jun 2023 09:25:03 +0200 Subject: [PATCH 035/140] Ameliore la variable zone_apl + ajoute logement_conventionne --- .../erfs_fpr/input_data_builder/step_02_menage.py | 8 +++++++- .../erfs_fpr/input_data_builder/step_06_final.py | 12 +++--------- openfisca_france_data/erfs_fpr/scenario.py | 1 + 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py index 9bb038a6..320a33ac 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py @@ -21,7 +21,13 @@ def build_variables_menage(temporary_store = None, year = None): menages['statut_occupation_logement'] = menages['logt'].copy() if year >= 2018: - menages['zone_apl'] = 2 + menages['zone_apl'] = 3 + menages.loc[menages['tau2010'] == 10,'zone_apl'] = 1 + menages.loc[menages['tau2010'] == 9,'zone_apl'] = 2 + #menages.loc[menages['tau2010'] == 8,'zone_apl'] = 2 # pour l'instant on met tout le monde à 2 mais à améliorer, peut être en fonction de la taille de l'aire urbaine ? menages.loc[(menages.statut_occupation_logement == 7), 'statut_occupation_logement'] = 2 + menages['logement_conventionne'] = False + menages.loc[menages['statut_occupation_logement'] == 3 ,'logement_conventionne'] = True temporary_store['menages_{}'.format(year)] = menages + diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index c47e203c..03401de3 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -62,6 +62,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'taxe_habitation', 'wprm', 'zone_apl', + 'logement_conventionne', ] individus = create_ids_and_roles(individus) @@ -81,14 +82,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene idmens = individus.idmen.unique() menages = menages.loc[ menages.idmen.isin(idmens), - [ - 'idmen', - 'loyer', - 'statut_occupation_logement', - 'taxe_habitation', - 'wprm', - 'zone_apl', - ] + var_menages ].copy() survey_name = 'openfisca_erfs_fpr_' + str(year) @@ -256,7 +250,7 @@ def extract_menages_variables_from_store(temporary_store = None, year = None): def extract_menages_variables(menages): variables = ['ident', 'wprm', 'taxe_habitation'] - external_variables = ['loyer', 'zone_apl', 'statut_occupation_logement'] + external_variables = ['loyer', 'zone_apl', 'statut_occupation_logement','logement_conventionne'] for external_variable in external_variables: if external_variable in menages.columns: log.debug("Found {} in menages table: we keep it".format(external_variable)) diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index e32b1d69..949f9f6f 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -20,6 +20,7 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "effectif_entreprise", "f4ba", "heures_remunerees_volume", + "logement_conventionne", "loyer", "pensions_alimentaires_percues", "pensions_invalidite", From 572123c51ec0eadc781a204ec7e28227ed76646f Mon Sep 17 00:00:00 2001 From: cgl Date: Wed, 7 Jun 2023 13:34:13 +0200 Subject: [PATCH 036/140] =?UTF-8?q?d=C3=A9place=20l'annualisation=20des=20?= =?UTF-8?q?loyers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../erfs_fpr/input_data_builder/step_02_menage.py | 7 +++++-- openfisca_france_data/surveys.py | 3 --- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py index 320a33ac..9b7a2f0e 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py @@ -12,12 +12,15 @@ @temporary_store_decorator(file_name = 'erfs_fpr') def build_variables_menage(temporary_store = None, year = None): + menages = temporary_store[f'menages_{year}'] + + if "loyer" in menages.columns: + menages['loyer'] = menages['loyer'] * 12 + if year == 2018: - menages = temporary_store[f'menages_{year}'] menages['statut_occupation_logement'] = menages['so'].copy() if year == 2019: # SO ne fonctionne pas en 2019 bcp de 0 = non-renseigné - menages = temporary_store[f'menages_{year}'] menages['statut_occupation_logement'] = menages['logt'].copy() if year >= 2018: diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index 0ccc5c58..a595570d 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -193,9 +193,6 @@ def custom_initialize(self, simulation): def custom_input_data_frame(self, input_data_frame, **kwargs): - if "loyer" in input_data_frame: - input_data_frame["loyer"] = 12 * input_data_frame.loyer - for variable in ["quifam", "quifoy", "quimen"]: if variable in input_data_frame: log.debug(input_data_frame[variable].value_counts(dropna = False)) From 63af349a66cff3689bc655795501319b1136d51b Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 10:55:43 +0200 Subject: [PATCH 037/140] Supprime les variables foyer fiscal du builder de l'erfs-fpr --- openfisca_france_data/erfs_fpr/input_data_builder/__init__.py | 2 +- openfisca_france_data/erfs_fpr/scenario.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py index 5dc8df4b..c22ed521 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py @@ -76,7 +76,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: famille.build_famille(year = year) log.info('\n [[[ Year {} - Step 5 / 6 ]]] \n'.format(year)) - foyer.build_variables_foyers_fiscal(year = year) + # foyer.build_variables_foyers_fiscal(year = year) # Affreux ! On injectait tout dans un même DataFrame !!! # C'est très moche ! diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index 949f9f6f..7e9711d3 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -28,9 +28,6 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "rag", "retraite_brute", "retraite_imposable", - # "rev_financier_prelev_lib_imputes", - "revenu_categoriel_foncier", - "revenus_capitaux_prelevement_forfaitaire_unique_ir", "ric", "rnc", "rpns_imposables", From 63a7064c3ea693f5dd356b48a104f5580b31cfc2 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:18:32 +0200 Subject: [PATCH 038/140] Ajoute la variable aah / caah dans la table de sortie --- .../erfs_fpr/input_data_builder/step_06_final.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index 03401de3..7c8db463 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -63,6 +63,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene 'wprm', 'zone_apl', 'logement_conventionne', + 'prest_precarite_hand' # on récupère la variable de montant de aah / caah pour pouvoir faire une imputation du handicap ] individus = create_ids_and_roles(individus) @@ -250,7 +251,7 @@ def extract_menages_variables_from_store(temporary_store = None, year = None): def extract_menages_variables(menages): variables = ['ident', 'wprm', 'taxe_habitation'] - external_variables = ['loyer', 'zone_apl', 'statut_occupation_logement','logement_conventionne'] + external_variables = ['loyer', 'zone_apl', 'statut_occupation_logement','logement_conventionne', 'prest_precarite_hand'] for external_variable in external_variables: if external_variable in menages.columns: log.debug("Found {} in menages table: we keep it".format(external_variable)) From fc2d187b92c432cc6dc2f6796fcb3ff2b10b2c67 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:22:29 +0200 Subject: [PATCH 039/140] =?UTF-8?q?enl=C3=A8ve=20les=20variables=20foyers?= =?UTF-8?q?=5Ffiscaux=20(suite)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../input_data_builder/step_06_final.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index 7c8db463..f65e0430 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -16,7 +16,7 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene individus = temporary_store['individus_{}'.format(year)] menages = temporary_store['menages_{}'.format(year)] - foyers_fiscaux = temporary_store['foyers_fiscaux_{}'.format(year)] + # foyers_fiscaux = temporary_store['foyers_fiscaux_{}'.format(year)] # ici : variables à garder var_individus = [ @@ -48,12 +48,12 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene "primes_fonction_publique", "traitement_indiciaire_brut", ] - var_foyers_fiscaux = [ - 'idfoy', - 'rev_financier_prelev_lib_imputes', - 'revenu_categoriel_foncier', - 'revenus_capitaux_prelevement_forfaitaire_unique_ir', - ] + # var_foyers_fiscaux = [ + # 'idfoy', + # 'rev_financier_prelev_lib_imputes', + # 'revenu_categoriel_foncier', + # 'revenus_capitaux_prelevement_forfaitaire_unique_ir', + # ] var_menages = [ 'idmen', @@ -97,15 +97,15 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene how = 'inner', on = 'idmen_original') - foyers_fiscaux = foyers_fiscaux.rename(columns = {'idfoy':'idfoy_original'}) - unique_idfoy = individus[['idfoy','idfoy_original']].drop_duplicates() + # foyers_fiscaux = foyers_fiscaux.rename(columns = {'idfoy':'idfoy_original'}) + # unique_idfoy = individus[['idfoy','idfoy_original']].drop_duplicates() assert len(unique_idmen) == len(menages), "Number of idfoy should be the same individus and foyers tables." - foyers_fiscaux = foyers_fiscaux.merge(unique_idfoy, - how = 'inner', - on = 'idfoy_original') + # foyers_fiscaux = foyers_fiscaux.merge(unique_idfoy, + # how = 'inner', + # on = 'idfoy_original') - foyers_fiscaux = foyers_fiscaux[var_foyers_fiscaux] + # foyers_fiscaux = foyers_fiscaux[var_foyers_fiscaux] if export_flattened_df_filepath: supermerge = individus.merge( @@ -127,16 +127,16 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene survey_name = survey_name, ) - foyers_fiscaux = foyers_fiscaux.sort_values(by = ['idfoy']) - log.debug(f"Saving entity 'foyers fiscaux' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") - set_table_in_survey( - foyers_fiscaux, - entity = "foyer_fiscal", - period = year, - collection = "openfisca_erfs_fpr", - survey_name = survey_name, - ) - log.debug("End of create_input_data_frame") + # foyers_fiscaux = foyers_fiscaux.sort_values(by = ['idfoy']) + # log.debug(f"Saving entity 'foyers fiscaux' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") + # set_table_in_survey( + # foyers_fiscaux, + # entity = "foyer_fiscal", + # period = year, + # collection = "openfisca_erfs_fpr", + # survey_name = survey_name, + # ) + # log.debug("End of create_input_data_frame") menages = menages.sort_values(by = ['idmen']) log.debug(f"Saving entity 'menage' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") From 855344f56d5fc59d1f575c0800ae84bb97546e67 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:39:13 +0200 Subject: [PATCH 040/140] fix --- .../erfs_fpr/get_survey_scenario.py | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 4f8e065c..c24e6adb 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -27,11 +27,11 @@ ] -menage_projected_variables = [ - # "rev_financier_prelev_lib_imputes", - "revenu_categoriel_foncier", - "revenus_capitaux_prelevement_forfaitaire_unique_ir", - ] +# menage_projected_variables = [ +# # "rev_financier_prelev_lib_imputes", +# "revenu_categoriel_foncier", +# "revenus_capitaux_prelevement_forfaitaire_unique_ir", +# ] class erfs_fpr_plugin(Reform): @@ -63,30 +63,30 @@ def formula(individu, period): self.add_variable(variable_instance) del variable_instance - for variable in menage_projected_variables: - class_name = f"{variable}_menage" - label = f"{variable} agrégée à l'échelle du ménage" + # for variable in menage_projected_variables: + # class_name = f"{variable}_menage" + # label = f"{variable} agrégée à l'échelle du ménage" - def projection_formula_creator(variable): - def formula(menage, period): - result_i = menage.members.foyer_fiscal(variable, period, options = [ADD]) - result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL) - return result + # def projection_formula_creator(variable): + # def formula(menage, period): + # result_i = menage.members.foyer_fiscal(variable, period, options = [ADD]) + # result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL) + # return result - formula.__name__ = 'formula' + # formula.__name__ = 'formula' - return formula + # return formula - variable_instance = type(class_name, (Variable,), dict( - value_type = float, - entity = Menage, - label = label, - definition_period = YEAR, - formula = projection_formula_creator(variable), - )) + # variable_instance = type(class_name, (Variable,), dict( + # value_type = float, + # entity = Menage, + # label = label, + # definition_period = YEAR, + # formula = projection_formula_creator(variable), + # )) - self.add_variable(variable_instance) - del variable_instance + # self.add_variable(variable_instance) + # del variable_instance self.add_variable(idmen_original) @@ -144,7 +144,7 @@ def get_survey_scenario( # S'il n'y a pas de données, on sait où les trouver. if data is None: input_data_table_by_entity = dict( - foyer_fiscal = f"foyer_fiscal_{year}", + #foyer_fiscal = f"foyer_fiscal_{year}", individu = f"individu_{year}", menage = f"menage_{year}", ) From 6b7d899aa30c603b53a37242daacd052d7d596c1 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:48:10 +0200 Subject: [PATCH 041/140] fix --- openfisca_france_data/erfs_fpr/comparison.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index d9630d20..1c2a2a49 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -45,15 +45,15 @@ class ErfsFprtoInputComparator(AbstractComparator): "statut_occupation_logement", ] - from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables + # from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables - target_menage_projected_variables = [ - f"{menage_projected_variable}_menage" - for menage_projected_variable - in menage_projected_variables - ] + # target_menage_projected_variables = [ + # f"{menage_projected_variable}_menage" + # for menage_projected_variable + # in menage_projected_variables + # ] - default_target_variables += target_menage_projected_variables + # default_target_variables += target_menage_projected_variables def compute_test_dataframes(self): From 3536ac25b583e61b77022441fe196a6eede0c9eb Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 14:54:00 +0200 Subject: [PATCH 042/140] Bump --- CHANGELOG.md | 8 ++++++++ setup.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5652549..7d9e1f29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +# 1.1.0 [#225](https://github.com/openfisca/openfisca-france-data/pull/225) + + +* Technical changes +- Ajoute des variables concernant le calcul des aides au logement, des non salariés et du handicap dans le builder de openfisca-france-data + + + # 1.0.0 [#224](https://github.com/openfisca/openfisca-france-data/pull/224) * Breaking changes diff --git a/setup.py b/setup.py index 84ee5e0a..02e93474 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "1.0", + version = "1.1.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From ecfc05749581f1dd4374d4b4634f282ec1b3d2bf Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 17:10:22 +0200 Subject: [PATCH 043/140] fix --- openfisca_france_data/erfs_fpr/comparison.py | 11 ---- .../erfs_fpr/get_survey_scenario.py | 34 ----------- .../erfs_fpr/input_data_builder/__init__.py | 13 ++--- .../input_data_builder/step_05_foyer.py | 57 ------------------- .../input_data_builder/step_06_final.py | 28 --------- 5 files changed, 5 insertions(+), 138 deletions(-) delete mode 100644 openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index 1c2a2a49..b542e47c 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -45,17 +45,6 @@ class ErfsFprtoInputComparator(AbstractComparator): "statut_occupation_logement", ] - # from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables - - # target_menage_projected_variables = [ - # f"{menage_projected_variable}_menage" - # for menage_projected_variable - # in menage_projected_variables - # ] - - # default_target_variables += target_menage_projected_variables - - def compute_test_dataframes(self): erfs_fpr_survey_collection = SurveyCollection.load(collection = "erfs_fpr") # infer names of the survey and data tables diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index c24e6adb..cd33c01a 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -27,13 +27,6 @@ ] -# menage_projected_variables = [ -# # "rev_financier_prelev_lib_imputes", -# "revenu_categoriel_foncier", -# "revenus_capitaux_prelevement_forfaitaire_unique_ir", -# ] - - class erfs_fpr_plugin(Reform): name = "ERFS-FPR ids plugin" @@ -63,32 +56,6 @@ def formula(individu, period): self.add_variable(variable_instance) del variable_instance - # for variable in menage_projected_variables: - # class_name = f"{variable}_menage" - # label = f"{variable} agrégée à l'échelle du ménage" - - # def projection_formula_creator(variable): - # def formula(menage, period): - # result_i = menage.members.foyer_fiscal(variable, period, options = [ADD]) - # result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL) - # return result - - # formula.__name__ = 'formula' - - # return formula - - # variable_instance = type(class_name, (Variable,), dict( - # value_type = float, - # entity = Menage, - # label = label, - # definition_period = YEAR, - # formula = projection_formula_creator(variable), - # )) - - # self.add_variable(variable_instance) - # del variable_instance - - self.add_variable(idmen_original) self.add_variable(noindiv) @@ -144,7 +111,6 @@ def get_survey_scenario( # S'il n'y a pas de données, on sait où les trouver. if data is None: input_data_table_by_entity = dict( - #foyer_fiscal = f"foyer_fiscal_{year}", individu = f"individu_{year}", menage = f"menage_{year}", ) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py index c22ed521..5c85de8d 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py @@ -50,7 +50,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # - On merge les tables individus / menages # # Note : c'est ici où on objectivise les hypothèses, step 1 - log.info('\n [[[ Year {} - Step 1 / 6 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 1 / 5 ]]] \n'.format(year)) preprocessing.build_merged_dataframes(year = year) # Step 02 : Si on veut calculer les allocations logement, il faut faire le matching avec une autre enquête (ENL) @@ -59,11 +59,11 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # stata_directory = openfisca_survey_collection.config.get('data', 'stata_directory') # stata_file = os.path.join(stata_directory, 'log_men_ERFS.dta') # imputation_loyer.merge_imputation_loyer(stata_file = stata_file, year = year) - log.info('\n [[[ Year {} - Step 2 / 6 SKIPPED ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 2 / 5 SKIPPED ]]] \n'.format(year)) menage.build_variables_menage(year = year) # Step 03 : on commence par les variables indivuelles - log.info('\n [[[ Year {} - Step 3 / 6 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 3 / 5 ]]] \n'.format(year)) variables_individuelles.build_variables_individuelles(year = year) # Step 04 : ici on va constituer foyer et famille à partir d'invididu et ménage @@ -72,18 +72,15 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None: # - On va faire des suppositions pour faire les familles # - On va faire les foyers fiscaux à partir des familles # - On va faire de suppositions pour faire les foyers fiscaux - log.info('\n [[[ Year {} - Step 4 / 6 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 4 / 5 ]]] \n'.format(year)) famille.build_famille(year = year) - log.info('\n [[[ Year {} - Step 5 / 6 ]]] \n'.format(year)) - # foyer.build_variables_foyers_fiscal(year = year) - # Affreux ! On injectait tout dans un même DataFrame !!! # C'est très moche ! # # On crée une df par entité par période. # Elles sont stockées dans un fichier h5 - log.info('\n [[[ Year {} - Step 6 / 6 ]]] \n'.format(year)) + log.info('\n [[[ Year {} - Step 5 / 5 ]]] \n'.format(year)) final.create_input_data_frame(year = year, export_flattened_df_filepath = export_flattened_df_filepath) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py deleted file mode 100644 index 21683d06..00000000 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import pandas as pd - - -from openfisca_survey_manager.temporary import temporary_store_decorator # type: ignore - - -log = logging.getLogger(__name__) - - -@temporary_store_decorator(file_name = 'erfs_fpr') -def build_variables_foyers_fiscal(temporary_store = None, year = None): - - assert temporary_store is not None - assert year is not None - - individus = temporary_store['individus_{}'.format(year)] - menages = temporary_store['menages_{}'.format(year)] - - individus['idfoy'] = individus['idfam'].copy() - individus['quifoy'] = individus['quifam'].copy() - - foyers_fiscaux = individus[['idfoy','ident',]].drop_duplicates() - foyers_fiscaux = pd.merge( - menages[[ - 'ident', - 'rev_financier_prelev_lib_imputes', - 'rev_fonciers_bruts', - 'rev_valeurs_mobilieres_bruts', - 'wprm', - ]], - foyers_fiscaux, - how = 'inner', - on = 'ident' - ) - # première version pour splitter les revenus du capital du ménage dans les foyers fiscaux - # on attribue l'ensemble des revenus du capital du ménage au foyer avec la personne ayant les plus hauts revenus - # procédure à améliorer - idfoy = (individus - .sort_values( - [ - 'ident', - 'salaire_de_base', - 'traitement_indiciaire_brut', - 'retraite_brute' - ], - ascending = False - ) - .groupby('ident') - .first() - .idfoy - ) - foyers_fiscaux['revenu_categoriel_foncier'] = foyers_fiscaux['rev_fonciers_bruts'] * foyers_fiscaux.idfoy.isin(idfoy) - foyers_fiscaux['revenus_capitaux_prelevement_forfaitaire_unique_ir'] = foyers_fiscaux['rev_valeurs_mobilieres_bruts'] * foyers_fiscaux.idfoy.isin(idfoy) - foyers_fiscaux['rev_financier_prelev_lib_imputes'] = foyers_fiscaux['rev_financier_prelev_lib_imputes'] * foyers_fiscaux.idfoy.isin(idfoy) - - temporary_store[f"foyers_fiscaux_{year}"] = foyers_fiscaux diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index f65e0430..eaa28055 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -16,7 +16,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene individus = temporary_store['individus_{}'.format(year)] menages = temporary_store['menages_{}'.format(year)] - # foyers_fiscaux = temporary_store['foyers_fiscaux_{}'.format(year)] # ici : variables à garder var_individus = [ @@ -48,12 +47,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene "primes_fonction_publique", "traitement_indiciaire_brut", ] - # var_foyers_fiscaux = [ - # 'idfoy', - # 'rev_financier_prelev_lib_imputes', - # 'revenu_categoriel_foncier', - # 'revenus_capitaux_prelevement_forfaitaire_unique_ir', - # ] var_menages = [ 'idmen', @@ -97,16 +90,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene how = 'inner', on = 'idmen_original') - # foyers_fiscaux = foyers_fiscaux.rename(columns = {'idfoy':'idfoy_original'}) - # unique_idfoy = individus[['idfoy','idfoy_original']].drop_duplicates() - assert len(unique_idmen) == len(menages), "Number of idfoy should be the same individus and foyers tables." - - # foyers_fiscaux = foyers_fiscaux.merge(unique_idfoy, - # how = 'inner', - # on = 'idfoy_original') - - # foyers_fiscaux = foyers_fiscaux[var_foyers_fiscaux] - if export_flattened_df_filepath: supermerge = individus.merge( menages, @@ -127,17 +110,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene survey_name = survey_name, ) - # foyers_fiscaux = foyers_fiscaux.sort_values(by = ['idfoy']) - # log.debug(f"Saving entity 'foyers fiscaux' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") - # set_table_in_survey( - # foyers_fiscaux, - # entity = "foyer_fiscal", - # period = year, - # collection = "openfisca_erfs_fpr", - # survey_name = survey_name, - # ) - # log.debug("End of create_input_data_frame") - menages = menages.sort_values(by = ['idmen']) log.debug(f"Saving entity 'menage' in collection 'openfisca_erfs_fpr' and survey name '{survey_name}' with set_table_in_survey") set_table_in_survey( From e89ff571cf8a29d08ba0eac98a9a4c2f3d4e61ce Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 17:19:09 +0200 Subject: [PATCH 044/140] fix --- openfisca_france_data/erfs_fpr/input_data_builder/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py index 5c85de8d..6fa75695 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/__init__.py @@ -19,7 +19,6 @@ step_02_menage as menage, step_03_variables_individuelles as variables_individuelles, step_04_famille as famille, - step_05_foyer as foyer, step_06_final as final, ) From 7c3cda24aed5565f758193250c88b6d8541316b4 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 27 Jun 2023 10:24:50 +0200 Subject: [PATCH 045/140] =?UTF-8?q?r=C3=A9introduit=20la=20projection=20de?= =?UTF-8?q?=20variables=20=C3=A0=20l'=C3=A9chelle=20du=20m=C3=A9nage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../erfs_fpr/get_survey_scenario.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index cd33c01a..448467fc 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -27,6 +27,11 @@ ] +menage_projected_variables = [ + +] + + class erfs_fpr_plugin(Reform): name = "ERFS-FPR ids plugin" @@ -56,6 +61,32 @@ def formula(individu, period): self.add_variable(variable_instance) del variable_instance + for variable in menage_projected_variables: + class_name = f"{variable}_menage" + label = f"{variable} agrégée à l'échelle du ménage" + + def projection_formula_creator(variable): + def formula(menage, period): + result_i = menage.members.foyer_fiscal(variable, period, options = [ADD]) + result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL) + return result + + formula.__name__ = 'formula' + + return formula + + variable_instance = type(class_name, (Variable,), dict( + value_type = float, + entity = Menage, + label = label, + definition_period = YEAR, + formula = projection_formula_creator(variable), + )) + + self.add_variable(variable_instance) + del variable_instance + + self.add_variable(idmen_original) self.add_variable(noindiv) From 16b6c27ee58ee351caf585fb8d0819e86719d977 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 8 Jun 2023 11:30:14 +0200 Subject: [PATCH 046/140] =?UTF-8?q?Cr=C3=A9e=20des=20agr=C3=A9gats=20d'ine?= =?UTF-8?q?s=20et=20adapte=20la=20comparaison=20d'agr=C3=A9gats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit supprime temp code --- openfisca_france_data/__init__.py | 17 +- openfisca_france_data/aggregates.py | 112 ++++++++----- .../aggregats/ines/cibles_ines_2019.json | 158 ++++++++++++++++++ .../taxipp}/agregats_tests_taxipp_2_0.xlsx | Bin 4 files changed, 237 insertions(+), 50 deletions(-) create mode 100644 openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json rename openfisca_france_data/assets/{ => aggregats/taxipp}/agregats_tests_taxipp_2_0.xlsx (100%) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index f6b7be5c..059f0167 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -152,7 +152,10 @@ def apply(self): CountryTaxBenefitSystem = lambda: france_data_tax_benefit_system # noqa analysis:ignore AGGREGATES_DEFAULT_VARS = [ - 'cotisations_non_contributives', + "cotisation_salariales", + "cotisations_employeur", + "cotisation_non_salarie", + "allegement_fillon", "salaire_brut", "retraite_brute", "chomage_brut", @@ -166,7 +169,10 @@ def apply(self): 'csg', 'crds', 'impot_revenu', - "taxe_habitation", + 'irpp', + 'prelevement_forfaitaire_unique_ir', + 'prelevement_forfaitaire_liberatoire', + 'taxe_habitation', 'ppe', 'ppe_brute', 'af', @@ -176,25 +182,24 @@ def apply(self): 'cf', 'paje_base', 'paje_naissance', - 'paje_clca', + 'paje_prepare', 'paje_cmg', 'ars', 'aeeh', 'asf', 'aspa', - # 'minimum_vieillesse', 'aah', 'caah', 'rsa', - 'rsa_activite', + 'ppa', 'aefa', 'api', - # 'majo_rsa', 'psa', 'aides_logement', 'alf', 'als', 'apl', + 'garantie_jeunes' ] # ajouter csgd pour le calcul des agrégats erfs # ajouter rmi pour le calcul des agrégats erfs diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 2f81aa60..436b1da8 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -1,7 +1,6 @@ import collections -from datetime import datetime import logging -import os +import json from pathlib import Path import numpy as np @@ -33,48 +32,73 @@ class FranceAggregates(AbstractAggregates): )) aggregate_variables = AGGREGATES_DEFAULT_VARS - def load_actual_data(self, year = None): + def load_actual_data(self, year = None, source_cible = None): + assert source_cible in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" assert year is not None - taxipp_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france-data").location, - "openfisca_france_data", - "assets", - "agregats_tests_taxipp_2_0.xlsx" - ) - df = ( - pd.read_excel( - taxipp_aggregates_file, - # "https://gitlab.com/ipp/partage-public-ipp/taxipp/-/blob/master/simulation/assets/agregats_tests_taxipp_2_0.xlsx", - # engine = 'openpyxl' + + if source_cible == "taxipp": + taxipp_aggregates_file = Path( + pkg_resources.get_distribution("openfisca-france_data").location, + "openfisca_france_data", + "assets", + "aggregats", + "taxipp", + "agregats_tests_taxipp_2_0.xlsx" ) - .rename(columns = str.lower) - .rename(columns = {"unnamed: 0": "description"}) - .dropna(subset = ["annee 2019", "annee 2018", "annee 2017", "annee 2016"], how = "all") - ) - if f"annee {year}" not in df: - return - - df = ( - df[["variable_openfisca", f"annee {year}"]] - .dropna() - .rename(columns = { - "variable_openfisca": "variable", - f"annee {year}": year, - }) - ) - - beneficiaries = ( - df.loc[df.variable.str.startswith("nombre")] - .set_index("variable") - .rename(index = lambda x : x.replace("nombre_", "")) - .rename(columns = {year: "actual_beneficiaries"}) - ) / self.beneficiaries_unit - - amounts = ( - df.loc[~df.variable.str.startswith("nombre")] - .set_index("variable") - .rename(columns = {year: "actual_amount"}) - ) / self.amount_unit - - result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") + df = ( + pd.read_excel( + taxipp_aggregates_file, + # "https://gitlab.com/ipp/partage-public-ipp/taxipp/-/blob/master/simulation/assets/agregats_tests_taxipp_2_0.xlsx", + # engine = 'openpyxl' + ) + .rename(columns = str.lower) + .rename(columns = {"unnamed: 0": "description"}) + .dropna(subset = ["annee 2019", "annee 2018", "annee 2017", "annee 2016"], how = "all") + ) + if f"annee {year}" not in df: + return + + df = ( + df[["variable_openfisca", f"annee {year}"]] + .dropna() + .rename(columns = { + "variable_openfisca": "variable", + f"annee {year}": year, + }) + ) + + beneficiaries = ( + df.loc[df.variable.str.startswith("nombre")] + .set_index("variable") + .rename(index = lambda x : x.replace("nombre_", "")) + .rename(columns = {year: "actual_beneficiaries"}) + ) / self.beneficiaries_unit + + amounts = ( + df.loc[~df.variable.str.startswith("nombre")] + .set_index("variable") + .rename(columns = {year: "actual_amount"}) + ) / self.amount_unit + + result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") + + elif source_cible == "cibles_ines": + ines_aggregates_file = Path( + pkg_resources.get_distribution("openfisca-france_data").location, + "openfisca_france_data", + "assets", + "aggregats", + "ines", + f"cibles_ines_{year}.json" + ) + + with open(ines_aggregates_file, 'r') as f: + data = json.load(f) + + result = pd.DataFrame(data['data']).drop('notes', axis = 1) + result['actual_beneficiaries'] = result. actual_beneficiaries / self.beneficiaries_unit + result['actual_amount'] = result. actual_amount / self.amount_unit + + result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") + return result diff --git a/openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json b/openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json new file mode 100644 index 00000000..e4fcd8b7 --- /dev/null +++ b/openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json @@ -0,0 +1,158 @@ +{ + "documentation":"Pour les prestations sociales, les cotisations et prélèvements sociaux, note de validation INES 2019, pour l'ir note de validation 2019 et 2020", + "data":[ + { + "variable":"af", + "actual_beneficiaries": 5100000, + "actual_amount": 12000000000 + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 500000, + "actual_amount": 90000000 + }, + { + "variable":"aeeh", + "actual_beneficiaries": 300000, + "actual_amount": 1000000000 + }, + { + "variable":"asf", + "actual_beneficiaries": 700000, + "actual_amount": 1500000000 + }, + { + "variable":"paje_base", + "actual_beneficiaries": 2000000, + "actual_amount": 3200000000 + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 500000, + "actual_amount": 500000 + }, + { + "variable":"cf", + "actual_beneficiaries": 1000000, + "actual_amount":2200000000 + }, + { + "variable":"ars", + "actual_beneficiaries": 2900000, + "actual_amount": 1900000000 + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 1000000, + "actual_amount":6200000000 + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 6600000, + "actual_amount": 14000000000 + }, + { + "variable":"aah", + "actual_beneficiaries": 1100000, + "actual_amount":10400000000 + }, + { + "variable":"aspa", + "actual_beneficiaries":0, + "actual_amount": 2500000000 + }, + { + "variable":"asi", + "actual_beneficiaries":0, + "actual_amount":200000000 + }, + { + "variable":"garantie_jeunes", + "actual_beneficiaries": 100000, + "actual_amount":300000000 + }, + { + "variable":"rsa", + "actual_beneficiaries": 1900000, + "actual_amount":7900000000 + }, + { + "variable":"ppa", + "actual_beneficiaries": 5800000, + "actual_amount": 8900000000 + }, + { + "variable":"cotisations_salariales", + "actual_beneficiaries":0, + "actual_amount": 78600000000 + }, + { + "variable":"cotisations_non_salarie", + "actual_beneficiaries":0, + "actual_amount":22900000000 + }, + { + "variable":"cotisations_employeur", + "actual_beneficiaries":0, + "actual_amount":221800000000 + }, + { + "variable":"allegement_fillon", + "actual_beneficiaries":0, + "actual_amount":26400000000 + }, + { + "variable":"csg_salaire", + "actual_beneficiaries":0, + "actual_amount":74400000000 + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries":0, + "actual_amount":10800000000 + }, + { + "variable":"csg_remplacement", + "actual_beneficiaries":0, + "actual_amount":21100000000 + }, + { + "variable":"crds_salaire", + "actual_beneficiaries":0, + "actual_amount":4100000000 + }, + { + "variable":"crds_non_salarie", + "actual_beneficiaries":0, + "actual_amount":600000000 + }, + { + "variable":"crds_remplacement", + "actual_beneficiaries":0, + "actual_amount":17000000000 + }, + { + "variable":"irpp", + "actual_beneficiaries":0, + "actual_amount":64500000000, + "notes":"somme de prélèvement à la source réalisés en 2019 sur revenus 2019 (note 2019) 69,2 + impot résiduel sur le revenu dû au titre des revenus 2019, net de crédits et réduction (note 2020) -8.1 sans prendre en compte le crédit d'impôt de prélèvement forfaitaire non libératoire de N-1 de 3,4 soit 3.4 - 8.1 + 69.2 " + }, + { + "variable":"prelevement_forfaitaire_unique_ir", + "actual_beneficiaries":0, + "actual_amount":4800000000, + "notes":"prélèvement forfaitaire unique libératoire sur revenus 2019 (note 2020)" + }, + { + "variable":"prelevement_forfaitaire_liberatoire", + "actual_beneficiaries":0, + "actual_amount":100000000, + "notes":"prélèvement forfaitaire libératoire sur revenus 2019 (note 2019)" + }, + { + "variable":"ifi", + "actual_beneficiaries":0, + "actual_amount":1100000000 + } + ] +} \ No newline at end of file diff --git a/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx b/openfisca_france_data/assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx similarity index 100% rename from openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx rename to openfisca_france_data/assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx From e88eadebaace9a2614bb243a21c4f6f5a6edff03 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 09:59:28 +0200 Subject: [PATCH 047/140] Adapte le comparateur pour qu'il fonctionne avec plusieurs get_survey_scenario --- openfisca_france_data/__init__.py | 78 ++++++++++---------- openfisca_france_data/comparator.py | 12 ++- openfisca_france_data/erfs_fpr/comparison.py | 12 ++- 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 059f0167..2f0689ba 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -153,53 +153,53 @@ def apply(self): AGGREGATES_DEFAULT_VARS = [ "cotisation_salariales", - "cotisations_employeur", + #"cotisations_employeur", "cotisation_non_salarie", - "allegement_fillon", - "salaire_brut", - "retraite_brute", - "chomage_brut", - "salaire_imposable", - "retraite_imposable", - "chomage_imposable", + #"allegement_fillon", + #"salaire_brut", + #"retraite_brute", + #"chomage_brut", + #"salaire_imposable", + #"retraite_imposable", + #"chomage_imposable", "csg_salaire", "csg_non_salarie", "csg_remplacement", "csg_revenus_capital", - 'csg', - 'crds', - 'impot_revenu', + #'csg', + #'crds', + #'impot_revenu', 'irpp', 'prelevement_forfaitaire_unique_ir', 'prelevement_forfaitaire_liberatoire', - 'taxe_habitation', - 'ppe', - 'ppe_brute', - 'af', - 'af_base', - 'af_majoration', - 'af_allocation_forfaitaire', - 'cf', - 'paje_base', - 'paje_naissance', - 'paje_prepare', - 'paje_cmg', - 'ars', - 'aeeh', - 'asf', - 'aspa', - 'aah', - 'caah', - 'rsa', - 'ppa', - 'aefa', - 'api', - 'psa', - 'aides_logement', - 'alf', - 'als', - 'apl', - 'garantie_jeunes' + #'taxe_habitation', + #'ppe', + #'ppe_brute', + #'af', + #'af_base', + #'af_majoration', + #'af_allocation_forfaitaire', + #'cf', + #'paje_base', + #'paje_naissance', + #'paje_prepare', + #'paje_cmg', + #'ars', + #'aeeh', + #'asf', + #'aspa', + #'aah', + #'caah', + #'rsa', + #'ppa', + #'aefa', + #'api', + #'psa', + #'aides_logement', + #'alf', + #'als', + #'apl', + #'garantie_jeunes' ] # ajouter csgd pour le calcul des agrégats erfs # ajouter rmi pour le calcul des agrégats erfs diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 3339128e..a24e7ce1 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -338,7 +338,8 @@ def get_test_dataframes(self, rebuild = False, noindivs = None): } return input_dataframe_by_entity, target_dataframe_by_entity - def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False): + def compare(self, browse, load, verbose, debug, target_variables = None, period = None, + rebuild = False, summary = False, collection = "openfisca_erfs_fpr", survey_name = None): """Compare actual data with openfisca-france-data computation.""" log.setLevel(level = logging.DEBUG if verbose else logging.WARNING) @@ -394,6 +395,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period None, # To force load the data_table from hdf file target_dataframe_by_entity, specific_figures_directory, + collection = collection, + survey_name = survey_name, target_variables = target_variables, period = period, summary = summary, @@ -434,7 +437,7 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period raise error def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_entity, figures_directory, - target_variables = None, period = None, summary = False): + collection, survey_name, target_variables = None, period = None, summary = False): """ Compare openfisca-france-data computation with data targets. @@ -457,8 +460,9 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti ) survey_scenario = self.get_survey_scenario( + collection = collection, data = data, - survey_name = f'openfisca_erfs_fpr_{period}' + survey_name = f'openfisca_erfs_fpr_{self.period}' if survey_name is None else survey_name, ) tax_benefit_system = survey_scenario.tax_benefit_system @@ -562,7 +566,7 @@ def filter(self, data_frame): log.info(log_message) self.messages.append(log_message + "\n") - def get_survey_scenario(self, data = None, survey_name = None): + def get_survey_scenario(self, collection, data = None, survey_name = None): return get_survey_scenario( year = str(self.period), data = data, diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index b542e47c..00414bb4 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -20,7 +20,7 @@ "noindiv": "noindiv", "rag_i": "rag", "retraites_i": "retraite_imposable", # TODO: CHECk - "rev_fonciers_bruts": "f4ba", + #"rev_fonciers_bruts": "f4ba", "ric_i": "ric", "rnc_i": "rnc", "salaires_i": "salaire_imposable", @@ -28,6 +28,16 @@ "rev_fonciers_bruts": "revenu_categoriel_foncier_menage", "rev_valeurs_mobilieres_bruts": "revenus_capitaux_prelevement_forfaitaire_unique_ir_menage", "rev_financier_prelev_lib_imputes": "rev_financier_prelev_lib_imputes_menage", + "ppa":"ppa_menage", + "nivviem":"niveau_de_vie", + "prest_fam_autres":"prestations_familiales_autres_menage", + "prest_fam_petite_enfance":"paje_menage", + "prest_logement":"aides_logement_menage", + "prest_precarite_rsa":"rsa_menage", + "prest_precarite_hand":"aah_menage", + "prest_precarite_vieil":'aspa_menage', + "revdispm":"revenu_disponible", + "nb_uci":"unites_consommation", } From b2c8e39460cf670baa9a076beba5b1f1a68a2175 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:36:27 +0200 Subject: [PATCH 048/140] Adapte target_source --- openfisca_france_data/aggregates.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 436b1da8..99336670 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -31,12 +31,14 @@ class FranceAggregates(AbstractAggregates): ('beneficiaries_relative_difference', "Diff. relative\nBénéficiaires"), )) aggregate_variables = AGGREGATES_DEFAULT_VARS + target_source = None - def load_actual_data(self, year = None, source_cible = None): - assert source_cible in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" + def load_actual_data(self, year = None): + target_source = self.target_source + assert target_source in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" assert year is not None - if source_cible == "taxipp": + if target_source == "taxipp": taxipp_aggregates_file = Path( pkg_resources.get_distribution("openfisca-france_data").location, "openfisca_france_data", @@ -82,7 +84,7 @@ def load_actual_data(self, year = None, source_cible = None): result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") - elif source_cible == "cibles_ines": + elif target_source == "cibles_ines": ines_aggregates_file = Path( pkg_resources.get_distribution("openfisca-france_data").location, "openfisca_france_data", From d9d255c8fcef0d4209307d7a2ce272dc22a7ac2c Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 11:54:47 +0200 Subject: [PATCH 049/140] =?UTF-8?q?Adapte=20la=20CI=20=C3=A0=20target=20so?= =?UTF-8?q?urce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/erfs_fpr/integration/test_aggregates.py | 2 +- tests/erfs_fpr/integration/test_rebuild_input_data.py | 4 ---- tests/test_aggregate.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index 9e3e56c2..ef35b867 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -38,7 +38,7 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in varying_variable = varying_variable, survey_name = survey_name, ) - aggregates = Aggregates(survey_scenario = survey_scenario) + aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') if False: mtr_rd = survey_scenario.compute_marginal_tax_rate(target_variable = 'revenu_disponible', period = year, use_baseline = True) diff --git a/tests/erfs_fpr/integration/test_rebuild_input_data.py b/tests/erfs_fpr/integration/test_rebuild_input_data.py index 1b4ff7b3..645cd8db 100644 --- a/tests/erfs_fpr/integration/test_rebuild_input_data.py +++ b/tests/erfs_fpr/integration/test_rebuild_input_data.py @@ -1,11 +1,7 @@ import logging -import numpy as np - from openfisca_france_data import base_survey from openfisca_france_data.erfs_fpr.get_survey_scenario import get_survey_scenario -from openfisca_france_data.aggregates import FranceAggregates as Aggregates - log = logging.getLogger(__name__) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 3b9af3ef..12ff9c3c 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -42,6 +42,6 @@ def test_erfs_fpr_aggregates_reform(fake_input_data, year:int = 2013): # On initialise le survey scenario survey_scenario.init_from_data(data = dict(input_data_frame = input_data)) - aggregates = Aggregates(survey_scenario = survey_scenario) + aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') base_data_frame = aggregates.compute_aggregates() return aggregates, base_data_frame From 5f139a2303075ef6f59c3932263e5334eb2dc489 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 12:04:51 +0200 Subject: [PATCH 050/140] simplifie modifications comparator --- openfisca_france_data/comparator.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index a24e7ce1..2ca2c886 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -304,6 +304,7 @@ class AbstractComparator(object): filter_expr_by_label = None period = None messages = list() + survey_name = f'openfisca_erfs_fpr_{period}' def get_name(self): return self.name + "_" + str(self.period) @@ -338,8 +339,7 @@ def get_test_dataframes(self, rebuild = False, noindivs = None): } return input_dataframe_by_entity, target_dataframe_by_entity - def compare(self, browse, load, verbose, debug, target_variables = None, period = None, - rebuild = False, summary = False, collection = "openfisca_erfs_fpr", survey_name = None): + def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False,): """Compare actual data with openfisca-france-data computation.""" log.setLevel(level = logging.DEBUG if verbose else logging.WARNING) @@ -395,8 +395,6 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period None, # To force load the data_table from hdf file target_dataframe_by_entity, specific_figures_directory, - collection = collection, - survey_name = survey_name, target_variables = target_variables, period = period, summary = summary, @@ -436,8 +434,7 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period pdb.post_mortem(sys.exc_info()[2]) raise error - def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_entity, figures_directory, - collection, survey_name, target_variables = None, period = None, summary = False): + def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_entity, figures_directory, target_variables = None, period = None, summary = False): """ Compare openfisca-france-data computation with data targets. @@ -460,9 +457,7 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti ) survey_scenario = self.get_survey_scenario( - collection = collection, - data = data, - survey_name = f'openfisca_erfs_fpr_{self.period}' if survey_name is None else survey_name, + data = data ) tax_benefit_system = survey_scenario.tax_benefit_system @@ -566,7 +561,8 @@ def filter(self, data_frame): log.info(log_message) self.messages.append(log_message + "\n") - def get_survey_scenario(self, collection, data = None, survey_name = None): + def get_survey_scenario(self, data = None): + survey_name = self.survey_name return get_survey_scenario( year = str(self.period), data = data, From d3217da589bb76ffb0054cd7ebd9ca1fcba06f25 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 14:49:47 +0200 Subject: [PATCH 051/140] fix fix fix --- openfisca_france_data/aggregates.py | 4 ++++ openfisca_france_data/comparator.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 99336670..d771b577 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -33,6 +33,10 @@ class FranceAggregates(AbstractAggregates): aggregate_variables = AGGREGATES_DEFAULT_VARS target_source = None + def __init__(self, survey_scenario = None, target_source = None): + self.__init__(self, survey_scenario = survey_scenario) + self.target_source = target_source + def load_actual_data(self, year = None): target_source = self.target_source assert target_source in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 2ca2c886..36861a87 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -304,7 +304,7 @@ class AbstractComparator(object): filter_expr_by_label = None period = None messages = list() - survey_name = f'openfisca_erfs_fpr_{period}' + survey_name = None def get_name(self): return self.name + "_" + str(self.period) From e9305f6d94200972d7e4e725c2eab85663c5cb97 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 16:01:58 +0200 Subject: [PATCH 052/140] fix --- tests/test_aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 12ff9c3c..9d787df6 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -22,7 +22,7 @@ def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 20 survey_scenario.init_from_data(data = dict(input_data_frame = input_data)) # On calcule les agrégats - aggregates = Aggregates(survey_scenario = survey_scenario) + aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') aggregates.compute_aggregates(use_baseline = False) return aggregates.base_data_frame From d0bcfcc51bf6b477abd76cf0fcd8487e63d02732 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 16 Jun 2023 16:12:04 +0200 Subject: [PATCH 053/140] fix --- tests/test_aggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 9d787df6..61adb5a4 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -13,7 +13,7 @@ ) def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 2009): # On ititialise le survey scenario - survey_scenario = survey_scenario(year) + survey_scenario = ErfsFprSurveyScenario.create(year) # On charge les données input_data = fake_input_data(year) From d65a7d7f693a1f28f5aa4beecc0a84ff80acb6b4 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 22 Jun 2023 11:35:34 +0200 Subject: [PATCH 054/140] Fix aggregates init --- openfisca_france_data/aggregates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index d771b577..648729b8 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -34,14 +34,14 @@ class FranceAggregates(AbstractAggregates): target_source = None def __init__(self, survey_scenario = None, target_source = None): - self.__init__(self, survey_scenario = survey_scenario) + super().__init__(survey_scenario = survey_scenario) self.target_source = target_source def load_actual_data(self, year = None): target_source = self.target_source assert target_source in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" assert year is not None - + if target_source == "taxipp": taxipp_aggregates_file = Path( pkg_resources.get_distribution("openfisca-france_data").location, @@ -87,7 +87,7 @@ def load_actual_data(self, year = None): ) / self.amount_unit result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") - + elif target_source == "cibles_ines": ines_aggregates_file = Path( pkg_resources.get_distribution("openfisca-france_data").location, From 2b193a978fa080a7e9e9f47907365b72779cbd45 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 27 Jun 2023 11:50:09 +0200 Subject: [PATCH 055/140] bump --- CHANGELOG.md | 8 +++++--- setup.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d9e1f29..f13f957e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,13 +1,15 @@ # Changelog -# 1.1.0 [#225](https://github.com/openfisca/openfisca-france-data/pull/225) +# 1.2.0 [#226](https://github.com/openfisca/openfisca-france-data/pull/226) +* Technical changes +- Adapte le comparator pour permettre de changer les agrégats cibles. +- Ajoute les agrégats de la note de validation d'INES +# 1.1.0 [#225](https://github.com/openfisca/openfisca-france-data/pull/225) * Technical changes - Ajoute des variables concernant le calcul des aides au logement, des non salariés et du handicap dans le builder de openfisca-france-data - - # 1.0.0 [#224](https://github.com/openfisca/openfisca-france-data/pull/224) * Breaking changes diff --git a/setup.py b/setup.py index 02e93474..02833496 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "1.1.0", + version = "1.2.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From cf765df677092844f5ff43a9017baffcd1fb52ff Mon Sep 17 00:00:00 2001 From: chloelallemand <55277136+clallemand@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:50:31 +0200 Subject: [PATCH 056/140] Update openfisca_france_data/comparator.py Co-authored-by: Mahdi Ben Jelloul --- openfisca_france_data/comparator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 36861a87..49bd8901 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -339,7 +339,7 @@ def get_test_dataframes(self, rebuild = False, noindivs = None): } return input_dataframe_by_entity, target_dataframe_by_entity - def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False,): + def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False): """Compare actual data with openfisca-france-data computation.""" log.setLevel(level = logging.DEBUG if verbose else logging.WARNING) From 2325149fbcd8702e73762ac69cdce0ce8670167b Mon Sep 17 00:00:00 2001 From: chloelallemand <55277136+clallemand@users.noreply.github.com> Date: Thu, 29 Jun 2023 15:50:39 +0200 Subject: [PATCH 057/140] Update openfisca_france_data/erfs_fpr/comparison.py Co-authored-by: Mahdi Ben Jelloul --- openfisca_france_data/erfs_fpr/comparison.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/comparison.py b/openfisca_france_data/erfs_fpr/comparison.py index 00414bb4..f67c5461 100644 --- a/openfisca_france_data/erfs_fpr/comparison.py +++ b/openfisca_france_data/erfs_fpr/comparison.py @@ -28,16 +28,16 @@ "rev_fonciers_bruts": "revenu_categoriel_foncier_menage", "rev_valeurs_mobilieres_bruts": "revenus_capitaux_prelevement_forfaitaire_unique_ir_menage", "rev_financier_prelev_lib_imputes": "rev_financier_prelev_lib_imputes_menage", - "ppa":"ppa_menage", - "nivviem":"niveau_de_vie", - "prest_fam_autres":"prestations_familiales_autres_menage", - "prest_fam_petite_enfance":"paje_menage", - "prest_logement":"aides_logement_menage", - "prest_precarite_rsa":"rsa_menage", - "prest_precarite_hand":"aah_menage", - "prest_precarite_vieil":'aspa_menage', - "revdispm":"revenu_disponible", - "nb_uci":"unites_consommation", + "ppa": "ppa_menage", + "nivviem": "niveau_de_vie", + "prest_fam_autres": "prestations_familiales_autres_menage", + "prest_fam_petite_enfance": "paje_menage", + "prest_logement": "aides_logement_menage", + "prest_precarite_rsa": "rsa_menage", + "prest_precarite_hand": "aah_menage", + "prest_precarite_vieil": 'aspa_menage', + "revdispm": "revenu_disponible", + "nb_uci": "unites_consommation", } From ff1a4a017b0ec65295a24c9f7492d27a5504bb31 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 16:31:03 +0200 Subject: [PATCH 058/140] Add environment --- .gitlab-ci/all_years_build_and_aggregates.yml | 81 ++++++++++++++++++- .gitlab-ci/build_ci.py | 12 ++- 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index 099787fe..6c1e39e3 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -21,6 +21,9 @@ aggregates-2019: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2019 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.html image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2019 @@ -35,7 +38,6 @@ aggregates-2019: stage: aggregates tags: - openfisca - input_data-2018: image: $CI_REGISTRY_IMAGE:latest script: @@ -48,12 +50,14 @@ input_data-2018: stage: build_input_data tags: - openfisca - aggregates-2018: artifacts: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2018 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.html image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2018 @@ -68,7 +72,6 @@ aggregates-2018: stage: aggregates tags: - openfisca - in_dt-1996: image: $CI_REGISTRY_IMAGE:latest needs: @@ -88,6 +91,9 @@ agg-1996: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 1996 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1996.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1996 @@ -121,6 +127,9 @@ agg-1997: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 1997 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1997.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1997 @@ -154,6 +163,9 @@ agg-1998: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 1998 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1998.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1998 @@ -187,6 +199,9 @@ agg-1999: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 1999 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1999.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1999 @@ -220,6 +235,9 @@ agg-2000: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2000 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2000.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2000 @@ -253,6 +271,9 @@ agg-2001: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2001 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2001.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2001 @@ -286,6 +307,9 @@ agg-2002: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2002 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2002.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2002 @@ -319,6 +343,9 @@ agg-2003: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2003 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2003.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2003 @@ -352,6 +379,9 @@ agg-2004: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2004 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2004.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2004 @@ -385,6 +415,9 @@ agg-2005: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2005 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2005.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2005 @@ -418,6 +451,9 @@ agg-2006: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2006 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2006.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2006 @@ -451,6 +487,9 @@ agg-2007: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2007 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2007.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2007 @@ -484,6 +523,9 @@ agg-2008: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2008 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2008.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2008 @@ -517,6 +559,9 @@ agg-2009: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2009 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2009.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2009 @@ -550,6 +595,9 @@ agg-2010: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2010 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2010.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2010 @@ -583,6 +631,9 @@ agg-2011: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2011 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2011.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2011 @@ -616,6 +667,9 @@ agg-2012: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2012 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2012.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2012 @@ -649,6 +703,9 @@ agg-2013: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2013 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2013.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2013 @@ -682,6 +739,9 @@ agg-2014: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2014 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2014.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2014 @@ -715,6 +775,9 @@ agg-2015: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2015 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2015.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2015 @@ -748,6 +811,9 @@ agg-2016: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2016 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2016.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2016 @@ -781,6 +847,9 @@ agg-2017: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2017 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2017.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2017 @@ -814,6 +883,9 @@ agg-2018: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2018 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2018 @@ -847,6 +919,9 @@ agg-2019: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2019 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2019 diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index 3553e39e..cba78f77 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -40,8 +40,8 @@ def build_input_data(year: str, stage: str = "build_input_data_all"): "cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini ~/.config/openfisca-survey-manager/config.ini", f"build-erfs-fpr -y {year} -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_{year}.h5", "cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-" - + year - + ".ini", + + year + + ".ini", ], } } @@ -73,8 +73,12 @@ def aggregates(year, stage: str = "aggregates_all"): "cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output", ], "artifacts": {"paths": ["./*.html", "./*.csv"]}, + "environment": { + "name": f"Aggregates {year}", + "url": f"https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_{year}.html", + } + } } - } # Warning : not used yet : test are independant for now. @@ -118,6 +122,8 @@ def build_gitlab_ci(erfs_years): # gitlab_ci += yaml.dump(build_collections()) gitlab_ci += yaml.dump(build_input_data("2019", stage="build_input_data")) gitlab_ci += yaml.dump(aggregates("2019", stage="aggregates")) + gitlab_ci += yaml.dump(build_input_data("2018", stage="build_input_data")) + gitlab_ci += yaml.dump(aggregates("2018", stage="aggregates")) for year in erfs_years: print("\t ERFS : Building for year", year) From f6f7a8c91cfcd577719a492a7941267e478161c9 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 16:47:13 +0200 Subject: [PATCH 059/140] Improve CI --- .gitlab-ci/all_years_build_and_aggregates.yml | 84 ++----------------- .gitlab-ci/build_ci.py | 19 +++-- 2 files changed, 17 insertions(+), 86 deletions(-) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index 6c1e39e3..eae81eb4 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -21,9 +21,6 @@ aggregates-2019: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2019 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.html image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2019 @@ -38,6 +35,9 @@ aggregates-2019: stage: aggregates tags: - openfisca +environment: + name: Aggregates 2019 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.csv input_data-2018: image: $CI_REGISTRY_IMAGE:latest script: @@ -55,9 +55,6 @@ aggregates-2018: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2018 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.html image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2018 @@ -72,6 +69,9 @@ aggregates-2018: stage: aggregates tags: - openfisca +environment: + name: Aggregates 2018 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.csv in_dt-1996: image: $CI_REGISTRY_IMAGE:latest needs: @@ -91,9 +91,6 @@ agg-1996: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 1996 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1996.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1996 @@ -127,9 +124,6 @@ agg-1997: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 1997 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1997.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1997 @@ -163,9 +157,6 @@ agg-1998: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 1998 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1998.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1998 @@ -199,9 +190,6 @@ agg-1999: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 1999 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_1999.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-1999 @@ -235,9 +223,6 @@ agg-2000: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2000 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2000.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2000 @@ -271,9 +256,6 @@ agg-2001: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2001 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2001.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2001 @@ -307,9 +289,6 @@ agg-2002: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2002 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2002.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2002 @@ -343,9 +322,6 @@ agg-2003: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2003 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2003.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2003 @@ -379,9 +355,6 @@ agg-2004: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2004 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2004.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2004 @@ -415,9 +388,6 @@ agg-2005: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2005 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2005.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2005 @@ -451,9 +421,6 @@ agg-2006: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2006 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2006.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2006 @@ -487,9 +454,6 @@ agg-2007: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2007 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2007.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2007 @@ -523,9 +487,6 @@ agg-2008: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2008 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2008.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2008 @@ -559,9 +520,6 @@ agg-2009: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2009 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2009.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2009 @@ -595,9 +553,6 @@ agg-2010: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2010 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2010.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2010 @@ -631,9 +586,6 @@ agg-2011: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2011 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2011.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2011 @@ -667,9 +619,6 @@ agg-2012: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2012 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2012.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2012 @@ -703,9 +652,6 @@ agg-2013: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2013 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2013.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2013 @@ -739,9 +685,6 @@ agg-2014: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2014 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2014.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2014 @@ -775,9 +718,6 @@ agg-2015: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2015 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2015.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2015 @@ -811,9 +751,6 @@ agg-2016: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2016 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2016.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2016 @@ -847,9 +784,6 @@ agg-2017: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2017 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2017.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2017 @@ -883,9 +817,6 @@ agg-2018: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2018 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2018 @@ -919,9 +850,6 @@ agg-2019: paths: - ./*.html - ./*.csv - environment: - name: Aggregates 2019 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.html image: $CI_REGISTRY_IMAGE:latest needs: - in_dt-2019 diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index cba78f77..67f3e608 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -50,14 +50,14 @@ def build_input_data(year: str, stage: str = "build_input_data_all"): return step -def aggregates(year, stage: str = "aggregates_all"): +def aggregates(year, stage: str = "aggregates_all", env = False): if stage == "aggregates_all": prefix = "agg-" prefix_input_data = "in_dt-" else: prefix = "aggregates-" prefix_input_data = "input_data-" - return { + entry = { prefix + year: { "stage": stage, @@ -73,13 +73,16 @@ def aggregates(year, stage: str = "aggregates_all"): "cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output", ], "artifacts": {"paths": ["./*.html", "./*.csv"]}, - "environment": { - "name": f"Aggregates {year}", - "url": f"https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_{year}.html", - } } } + if env: + entry["environment"] = { + "name": f"Aggregates {year}", + "url": f"https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_{year}.csv", + } + + return entry # Warning : not used yet : test are independant for now. def make_test_by_year(year): @@ -121,9 +124,9 @@ def build_gitlab_ci(erfs_years): # gitlab_ci += yaml.dump(build_and_deploy_conda_package()) # gitlab_ci += yaml.dump(build_collections()) gitlab_ci += yaml.dump(build_input_data("2019", stage="build_input_data")) - gitlab_ci += yaml.dump(aggregates("2019", stage="aggregates")) + gitlab_ci += yaml.dump(aggregates("2019", stage="aggregates", env = True)) gitlab_ci += yaml.dump(build_input_data("2018", stage="build_input_data")) - gitlab_ci += yaml.dump(aggregates("2018", stage="aggregates")) + gitlab_ci += yaml.dump(aggregates("2018", stage="aggregates", env = True)) for year in erfs_years: print("\t ERFS : Building for year", year) From 70700a6e7f2de704e090d62ba73d43f3675152c0 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 16:53:46 +0200 Subject: [PATCH 060/140] Again --- .gitlab-ci/build_ci.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index 67f3e608..2c81063f 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -77,7 +77,7 @@ def aggregates(year, stage: str = "aggregates_all", env = False): } if env: - entry["environment"] = { + entry[prefix + year]["environment"] = { "name": f"Aggregates {year}", "url": f"https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_{year}.csv", } From e90591bc80fd75cbe9afdc1603354198f6e41078 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 18:04:01 +0200 Subject: [PATCH 061/140] Again --- .gitlab-ci/all_years_build_and_aggregates.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index eae81eb4..f88c411c 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -21,6 +21,9 @@ aggregates-2019: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2019 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.csv image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2019 @@ -35,9 +38,6 @@ aggregates-2019: stage: aggregates tags: - openfisca -environment: - name: Aggregates 2019 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2019.csv input_data-2018: image: $CI_REGISTRY_IMAGE:latest script: @@ -55,6 +55,9 @@ aggregates-2018: paths: - ./*.html - ./*.csv + environment: + name: Aggregates 2018 + url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.csv image: $CI_REGISTRY_IMAGE:latest needs: - input_data-2018 @@ -69,9 +72,6 @@ aggregates-2018: stage: aggregates tags: - openfisca -environment: - name: Aggregates 2018 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.csv in_dt-1996: image: $CI_REGISTRY_IMAGE:latest needs: From 37b934d5e1b5451d0f57951c63390d67a7dca9ce Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 19:58:12 +0200 Subject: [PATCH 062/140] Ines and taxipp --- .gitlab-ci/all_years_build_and_aggregates.yml | 26 ------------------- .gitlab-ci/build_ci.py | 2 +- tests/erfs_fpr/integration/test_aggregates.py | 15 ++++++----- 3 files changed, 10 insertions(+), 33 deletions(-) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index f88c411c..7ede64c6 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -19,7 +19,6 @@ input_data-2019: aggregates-2019: artifacts: paths: - - ./*.html - ./*.csv environment: name: Aggregates 2019 @@ -53,7 +52,6 @@ input_data-2018: aggregates-2018: artifacts: paths: - - ./*.html - ./*.csv environment: name: Aggregates 2018 @@ -89,7 +87,6 @@ in_dt-1996: agg-1996: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -122,7 +119,6 @@ in_dt-1997: agg-1997: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -155,7 +151,6 @@ in_dt-1998: agg-1998: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -188,7 +183,6 @@ in_dt-1999: agg-1999: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -221,7 +215,6 @@ in_dt-2000: agg-2000: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -254,7 +247,6 @@ in_dt-2001: agg-2001: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -287,7 +279,6 @@ in_dt-2002: agg-2002: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -320,7 +311,6 @@ in_dt-2003: agg-2003: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -353,7 +343,6 @@ in_dt-2004: agg-2004: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -386,7 +375,6 @@ in_dt-2005: agg-2005: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -419,7 +407,6 @@ in_dt-2006: agg-2006: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -452,7 +439,6 @@ in_dt-2007: agg-2007: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -485,7 +471,6 @@ in_dt-2008: agg-2008: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -518,7 +503,6 @@ in_dt-2009: agg-2009: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -551,7 +535,6 @@ in_dt-2010: agg-2010: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -584,7 +567,6 @@ in_dt-2011: agg-2011: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -617,7 +599,6 @@ in_dt-2012: agg-2012: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -650,7 +631,6 @@ in_dt-2013: agg-2013: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -683,7 +663,6 @@ in_dt-2014: agg-2014: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -716,7 +695,6 @@ in_dt-2015: agg-2015: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -749,7 +727,6 @@ in_dt-2016: agg-2016: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -782,7 +759,6 @@ in_dt-2017: agg-2017: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -815,7 +791,6 @@ in_dt-2018: agg-2018: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: @@ -848,7 +823,6 @@ in_dt-2019: agg-2019: artifacts: paths: - - ./*.html - ./*.csv image: $CI_REGISTRY_IMAGE:latest needs: diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index 2c81063f..1b2868d6 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -72,7 +72,7 @@ def aggregates(year, stage: str = "aggregates_all", env = False): "cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output", "cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output", ], - "artifacts": {"paths": ["./*.html", "./*.csv"]}, + "artifacts": {"paths": ["./*.csv"]}, } } diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index ef35b867..26c53607 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -38,7 +38,8 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in varying_variable = varying_variable, survey_name = survey_name, ) - aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') + aggregates_taxipp = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') + aggregates_ines = Aggregates(survey_scenario = survey_scenario, target_source = 'ines') if False: mtr_rd = survey_scenario.compute_marginal_tax_rate(target_variable = 'revenu_disponible', period = year, use_baseline = True) @@ -53,7 +54,7 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in np.quantile(mtr_rd, q = np.arange(0, 1.1, .1)) - return survey_scenario, aggregates + return survey_scenario, aggregates_taxipp, aggregates_ines def test_erfs_fpr_aggregates_reform(): @@ -111,7 +112,7 @@ def main(year, configfile = None, verbose = False): years = [year] for year in years: - survey_scenario, aggregates = test_erfs_fpr_survey_simulation_aggregates( + survey_scenario, aggregates_taxipp, aggregates_ines = test_erfs_fpr_survey_simulation_aggregates( year = year, rebuild_input_data = False, use_marginal_tax_rate = True, @@ -119,9 +120,11 @@ def main(year, configfile = None, verbose = False): varying_variable = varying_variable ) - aggregates.to_csv(f'aggregates_erfs_fpr_{year}.csv') - print(aggregates.to_markdown()) - aggregates.to_html(f'aggregates_erfs_fpr_{year}.html') + df = pd.concat({ + "taxipp": aggregates_taxipp, + "ines": aggregates_ines, + }) + df.to_csv(f'aggregates_erfs_fpr_{year}.csv') continue survey_scenario._set_used_as_input_variables_by_entity() From 7b0909746d63dd7112b7dd61db5e2992aacc244a Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 20:17:09 +0200 Subject: [PATCH 063/140] USe dataframes --- tests/erfs_fpr/integration/test_aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index 26c53607..aad235f9 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -54,7 +54,7 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in np.quantile(mtr_rd, q = np.arange(0, 1.1, .1)) - return survey_scenario, aggregates_taxipp, aggregates_ines + return survey_scenario, aggregates_taxipp.get_data_frame(), aggregates_ines.get_data_frame() def test_erfs_fpr_aggregates_reform(): From f38df6c951e218758be94b593c2ebfcb120b27e2 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 21:02:15 +0200 Subject: [PATCH 064/140] Fix target_source --- openfisca_france_data/aggregates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 648729b8..3d0bb7e8 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -39,7 +39,7 @@ def __init__(self, survey_scenario = None, target_source = None): def load_actual_data(self, year = None): target_source = self.target_source - assert target_source in ["cibles_ines", "taxipp"], "les options possible pour source_cible sont cibles_ines ou taxipp" + assert target_source in ["ines", "taxipp"], "les options possible pour source_cible sont ines ou taxipp" assert year is not None if target_source == "taxipp": @@ -88,14 +88,14 @@ def load_actual_data(self, year = None): result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") - elif target_source == "cibles_ines": + elif target_source == "ines": ines_aggregates_file = Path( pkg_resources.get_distribution("openfisca-france_data").location, "openfisca_france_data", "assets", "aggregats", "ines", - f"cibles_ines_{year}.json" + f"ines_{year}.json" ) with open(ines_aggregates_file, 'r') as f: From 717166909732b145ed45d451c7282daf4816877f Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 21:25:14 +0200 Subject: [PATCH 065/140] Fix --- .gitlab-ci/all_years_build_and_aggregates.yml | 33 ------------------- .gitlab-ci/build_ci.py | 2 -- .../{cibles_ines_2019.json => ines_2019.json} | 0 3 files changed, 35 deletions(-) rename openfisca_france_data/assets/aggregats/ines/{cibles_ines_2019.json => ines_2019.json} (100%) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index 7ede64c6..730053d8 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -37,39 +37,6 @@ aggregates-2019: stage: aggregates tags: - openfisca -input_data-2018: - image: $CI_REGISTRY_IMAGE:latest - script: - - echo "build_input_data-2018" - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config-after-build-collection.ini - ~/.config/openfisca-survey-manager/config.ini - - build-erfs-fpr -y 2018 -f $ROOT_FOLDER/$OUT_FOLDER/data_output/erfs_flat_2018.h5 - - cp ~/.config/openfisca-survey-manager/config.ini $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2018.ini - stage: build_input_data - tags: - - openfisca -aggregates-2018: - artifacts: - paths: - - ./*.csv - environment: - name: Aggregates 2018 - url: https://git.leximpact.dev/benjello/openfisca-france-data/-/jobs/$CI_JOB_ID/artifacts/file/aggregates_erfs_fpr_2018.csv - image: $CI_REGISTRY_IMAGE:latest - needs: - - input_data-2018 - script: - - echo "aggregates-2018" - - cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-2018.ini - ~/.config/openfisca-survey-manager/config.ini - - python tests/erfs_fpr/integration/test_aggregates.py --year 2018 - - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output - stage: aggregates - tags: - - openfisca in_dt-1996: image: $CI_REGISTRY_IMAGE:latest needs: diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index 1b2868d6..f738404c 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -125,8 +125,6 @@ def build_gitlab_ci(erfs_years): # gitlab_ci += yaml.dump(build_collections()) gitlab_ci += yaml.dump(build_input_data("2019", stage="build_input_data")) gitlab_ci += yaml.dump(aggregates("2019", stage="aggregates", env = True)) - gitlab_ci += yaml.dump(build_input_data("2018", stage="build_input_data")) - gitlab_ci += yaml.dump(aggregates("2018", stage="aggregates", env = True)) for year in erfs_years: print("\t ERFS : Building for year", year) diff --git a/openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json b/openfisca_france_data/assets/aggregats/ines/ines_2019.json similarity index 100% rename from openfisca_france_data/assets/aggregats/ines/cibles_ines_2019.json rename to openfisca_france_data/assets/aggregats/ines/ines_2019.json From a472db89bc176978b0a389fe716fbcdb58e6d775 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 29 Jun 2023 23:37:30 +0200 Subject: [PATCH 066/140] Remove html --- .gitlab-ci/all_years_build_and_aggregates.yml | 25 ------------------- .gitlab-ci/build_ci.py | 1 - 2 files changed, 26 deletions(-) diff --git a/.gitlab-ci/all_years_build_and_aggregates.yml b/.gitlab-ci/all_years_build_and_aggregates.yml index 730053d8..5176ebbb 100644 --- a/.gitlab-ci/all_years_build_and_aggregates.yml +++ b/.gitlab-ci/all_years_build_and_aggregates.yml @@ -32,7 +32,6 @@ aggregates-2019: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2019 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates tags: @@ -64,7 +63,6 @@ agg-1996: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 1996 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -96,7 +94,6 @@ agg-1997: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 1997 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -128,7 +125,6 @@ agg-1998: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 1998 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -160,7 +156,6 @@ agg-1999: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 1999 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -192,7 +187,6 @@ agg-2000: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2000 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -224,7 +218,6 @@ agg-2001: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2001 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -256,7 +249,6 @@ agg-2002: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2002 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -288,7 +280,6 @@ agg-2003: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2003 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -320,7 +311,6 @@ agg-2004: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2004 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -352,7 +342,6 @@ agg-2005: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2005 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -384,7 +373,6 @@ agg-2006: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2006 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -416,7 +404,6 @@ agg-2007: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2007 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -448,7 +435,6 @@ agg-2008: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2008 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -480,7 +466,6 @@ agg-2009: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2009 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -512,7 +497,6 @@ agg-2010: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2010 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -544,7 +528,6 @@ agg-2011: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2011 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -576,7 +559,6 @@ agg-2012: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2012 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -608,7 +590,6 @@ agg-2013: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2013 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -640,7 +621,6 @@ agg-2014: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2014 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -672,7 +652,6 @@ agg-2015: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2015 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -704,7 +683,6 @@ agg-2016: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2016 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -736,7 +714,6 @@ agg-2017: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2017 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -768,7 +745,6 @@ agg-2018: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2018 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: @@ -800,7 +776,6 @@ agg-2019: ~/.config/openfisca-survey-manager/config.ini - python tests/erfs_fpr/integration/test_aggregates.py --year 2019 - mkdir -p $ROOT_FOLDER/$OUT_FOLDER - - cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output - cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output stage: aggregates_all tags: diff --git a/.gitlab-ci/build_ci.py b/.gitlab-ci/build_ci.py index f738404c..cb16d31e 100644 --- a/.gitlab-ci/build_ci.py +++ b/.gitlab-ci/build_ci.py @@ -69,7 +69,6 @@ def aggregates(year, stage: str = "aggregates_all", env = False): f"cp $ROOT_FOLDER/$OUT_FOLDER/openfisca_survey_manager_config_input_data-after-build-erfs-fprs-{year}.ini ~/.config/openfisca-survey-manager/config.ini", f"python tests/erfs_fpr/integration/test_aggregates.py --year {year}", "mkdir -p $ROOT_FOLDER/$OUT_FOLDER", - "cp ./*.html $ROOT_FOLDER/$OUT_FOLDER/data_output", "cp ./*.csv $ROOT_FOLDER/$OUT_FOLDER/data_output", ], "artifacts": {"paths": ["./*.csv"]}, From 4b03fe89b94344d839155dc66805a4f1a0169172 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 30 Jun 2023 13:56:10 +0200 Subject: [PATCH 067/140] =?UTF-8?q?D=C3=A9commente=20variables=20aggregats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openfisca_france_data/__init__.py | 77 +++++++++++++++---------------- 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 2f0689ba..a96a9892 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -153,53 +153,52 @@ def apply(self): AGGREGATES_DEFAULT_VARS = [ "cotisation_salariales", - #"cotisations_employeur", + "cotisations_employeur", "cotisation_non_salarie", - #"allegement_fillon", - #"salaire_brut", - #"retraite_brute", - #"chomage_brut", - #"salaire_imposable", - #"retraite_imposable", - #"chomage_imposable", + "salaire_brut", + "retraite_brute", + "chomage_brut", + "salaire_imposable", + "retraite_imposable", + "chomage_imposable", "csg_salaire", "csg_non_salarie", "csg_remplacement", "csg_revenus_capital", - #'csg', - #'crds', - #'impot_revenu', + 'csg', + 'crds', + 'impot_revenu', 'irpp', 'prelevement_forfaitaire_unique_ir', 'prelevement_forfaitaire_liberatoire', - #'taxe_habitation', - #'ppe', - #'ppe_brute', - #'af', - #'af_base', - #'af_majoration', - #'af_allocation_forfaitaire', - #'cf', - #'paje_base', - #'paje_naissance', - #'paje_prepare', - #'paje_cmg', - #'ars', - #'aeeh', - #'asf', - #'aspa', - #'aah', - #'caah', - #'rsa', - #'ppa', - #'aefa', - #'api', - #'psa', - #'aides_logement', - #'alf', - #'als', - #'apl', - #'garantie_jeunes' + 'taxe_habitation', + 'ppe', + 'ppe_brute', + 'af', + 'af_base', + 'af_majoration', + 'af_allocation_forfaitaire', + 'cf', + 'paje_base', + 'paje_naissance', + 'paje_prepare', + 'paje_cmg', + 'ars', + 'aeeh', + 'asf', + 'aspa', + 'aah', + 'caah', + 'rsa', + 'ppa', + 'aefa', + 'api', + 'psa', + 'aides_logement', + 'alf', + 'als', + 'apl', + 'garantie_jeunes' ] # ajouter csgd pour le calcul des agrégats erfs # ajouter rmi pour le calcul des agrégats erfs From 35fc66396784a9640ee9413c4ed8004d183158dc Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 30 Jun 2023 15:13:18 +0200 Subject: [PATCH 068/140] wip' --- openfisca_france_data/__init__.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index a96a9892..81ef2499 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -153,7 +153,7 @@ def apply(self): AGGREGATES_DEFAULT_VARS = [ "cotisation_salariales", - "cotisations_employeur", + #"cotisations_employeur", "cotisation_non_salarie", "salaire_brut", "retraite_brute", @@ -172,8 +172,8 @@ def apply(self): 'prelevement_forfaitaire_unique_ir', 'prelevement_forfaitaire_liberatoire', 'taxe_habitation', - 'ppe', - 'ppe_brute', + #'ppe', + #'ppe_brute', 'af', 'af_base', 'af_majoration', @@ -184,21 +184,21 @@ def apply(self): 'paje_prepare', 'paje_cmg', 'ars', - 'aeeh', + #'aeeh', 'asf', 'aspa', 'aah', 'caah', 'rsa', 'ppa', - 'aefa', - 'api', - 'psa', + #'aefa', + #'api', + #'psa', 'aides_logement', 'alf', 'als', 'apl', - 'garantie_jeunes' + #'garantie_jeunes' ] # ajouter csgd pour le calcul des agrégats erfs # ajouter rmi pour le calcul des agrégats erfs From dca36777e13cbb7223fdbec822fba752307e1633 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 8 Jun 2023 18:05:52 +0200 Subject: [PATCH 069/140] WIP --- openfisca_france_data/comparator.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 49bd8901..373b22c3 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -306,6 +306,9 @@ class AbstractComparator(object): messages = list() survey_name = None + def compute_aggregates_comparison(self, input_dataframe_by_entity = None): + pass + def get_name(self): return self.name + "_" + str(self.period) @@ -352,6 +355,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period if not figures_directory.exists(): figures_directory.mkdir(parents = True, exist_ok = True) + self.figures_directory = figures_directory + if target_variables is not None and isinstance(target_variables, str): target_variables = [target_variables] @@ -386,20 +391,22 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period log.debug(f"Test data has been prepared in {datetime.datetime.now() - start_time}") - # specific_figures_directory = PurePath.joinpath(figures_directory, self.name) - specific_figures_directory = figures_directory - specific_figures_directory.mkdir(parents = True, exist_ok = True) + self.compute_aggregates_comparison( + input_dataframe_by_entity = input_dataframe_by_entity, + ) result_by_variable = self.compute_divergence( # input_dataframe_by_entity, - None, # To force load the data_table from hdf file - target_dataframe_by_entity, - specific_figures_directory, + input_dataframe_by_entity = None, # To force load the data_table from hdf file + target_dataframe_by_entity = target_dataframe_by_entity, target_variables = target_variables, period = period, summary = summary, ) + if result_by_variable is None: + return + result = pd.concat(result_by_variable, ignore_index = True) log.debug(f"Eveyrthing has been computed in {datetime.datetime.now() - start_time}") @@ -434,7 +441,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period pdb.post_mortem(sys.exc_info()[2]) raise error - def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_entity, figures_directory, target_variables = None, period = None, summary = False): + def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_entity, + target_variables = None, period = None, summary = False): """ Compare openfisca-france-data computation with data targets. @@ -443,7 +451,7 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti target_dataframe_by_period (dict): Targets to macth figures_directory (path): Where to store the figures """ - figures_directory = figures_directory.resolve() + figures_directory = self.figures_directory.resolve() assert Path.exists(figures_directory) if target_variables is None: @@ -505,7 +513,6 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti markdown_section_by_variable[variable] = variable_markdown_section if summary: - # create_stats_by_period_figure(variable, result, period, figures_directory = figures_directory) variable_markdown_summary_section = create_variable_markdown_summary_section( variable, stats, From 10783b59ceaacab17d4257f782b43b269bf09894 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 10 Jun 2023 06:39:27 +0200 Subject: [PATCH 070/140] WIP --- openfisca_france_data/comparator.py | 70 ++++++++++++++++++----------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 373b22c3..6dbaf8a6 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -395,8 +395,7 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period input_dataframe_by_entity = input_dataframe_by_entity, ) - result_by_variable = self.compute_divergence( - # input_dataframe_by_entity, + result_by_variable, markdown_section_by_variable, markdown_summary_section_by_variable = self.compute_divergence( input_dataframe_by_entity = None, # To force load the data_table from hdf file target_dataframe_by_entity = target_dataframe_by_entity, target_variables = target_variables, @@ -404,6 +403,10 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period summary = summary, ) + # Deal with markdown_section + + self.create_report(markdown_section_by_variable, markdown_summary_section_by_variable) + if result_by_variable is None: return @@ -411,6 +414,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period log.debug(f"Eveyrthing has been computed in {datetime.datetime.now() - start_time}") del input_dataframe_by_entity, target_dataframe_by_entity + + if browse: start_browsing_time = datetime.datetime.now() result = result.dropna(axis = 1, how = 'all') @@ -456,7 +461,7 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti if target_variables is None: log.info(f"No target variables. Exiting divergence computation.") - return + return None, None, None data = ( dict(input_dataframe_by_entity = input_dataframe_by_entity) @@ -521,39 +526,50 @@ def compute_divergence(self, input_dataframe_by_entity, target_dataframe_by_enti if variable_markdown_summary_section is not None: markdown_summary_section_by_variable[variable] = variable_markdown_summary_section - messages_markdown_section = """ + return result_by_variable, markdown_section_by_variable, markdown_summary_section_by_variable + + def compute_test_dataframes(self): + NotImplementedError + + def create_report(self, markdown_section_by_variable, markdown_summary_section_by_variable): + figures_directory = self.figures_directory + + if self.messages: + messages_markdown_section = """ Filtres appliqués: """ + "\n".join(f"- {message}" for message in self.messages) + """ """ - with open(figures_directory / "filters.md", "w", encoding = 'utf-8') as filters_md_file: - filters_md_file.write(messages_markdown_section) - - markdown_sections = list(filter( - lambda x: x is not None, - [messages_markdown_section] + list(markdown_section_by_variable.values()), - )) - create_output_files( - markdown_sections, - figures_directory, - "variables", - ) - if summary: - markdown_sections = list(filter( - lambda x: x is not None, - [messages_markdown_section] + list(markdown_summary_section_by_variable.values()), - )) + else: + messages_markdown_section = "" + + table_agregats_markdown = None + if PurePath.joinpath(figures_directory, "table_agregats.md").exists(): + with open(figures_directory / "table_agregats.md", "r", encoding = 'utf-8') as table_agregats_md_file: + table_agregats_markdown = table_agregats_md_file.read() + + front_sections = [messages_markdown_section, table_agregats_markdown] + sections_by_filename = { + "variables": markdown_section_by_variable, + "summary_variables": markdown_summary_section_by_variable + } + + for filename, section_by_variable in sections_by_filename.items(): + if section_by_variable is not None: + markdown_sections = list(filter( + lambda x: x is not None, + front_sections + list(section_by_variable.values()), + )) + else: + markdown_sections = list(filter( + lambda x: x is not None,front_sections + )) create_output_files( markdown_sections, figures_directory, - "summary_variables", + filename, ) - return result_by_variable - - def compute_test_dataframes(self): - NotImplementedError - def filter(self, data_frame): for label, filter_expr in self.filter_expr_by_label.items(): obs_before = data_frame.noind.nunique() From 95bca5de3107e04730cdc767ae4fb5917b727da8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 12 Jun 2023 18:54:42 +0200 Subject: [PATCH 071/140] WIP --- openfisca_france_data/comparator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 6dbaf8a6..87175e6e 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -309,6 +309,10 @@ class AbstractComparator(object): def compute_aggregates_comparison(self, input_dataframe_by_entity = None): pass + def compute_distibution_comparison(self, input_dataframe_by_entity = None): + pass + + def get_name(self): return self.name + "_" + str(self.period) From b194926cb736f60f68478fb7b3a981e2fcfedadf Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 20 Jun 2023 11:25:43 +0200 Subject: [PATCH 072/140] Improve comparator modularity --- openfisca_france_data/comparator.py | 42 +++++++++++++++++------------ 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 87175e6e..e98e4eef 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -304,7 +304,18 @@ class AbstractComparator(object): filter_expr_by_label = None period = None messages = list() - survey_name = None + survey_scenario = None + + def __init__(self): + name = self.get_name() + assert name is not None and isinstance(name, str) + + figures_directory = Path(config.get("paths", "figures_directory")) / name + + if not figures_directory.exists(): + figures_directory.mkdir(parents = True, exist_ok = True) + + self.figures_directory = figures_directory def compute_aggregates_comparison(self, input_dataframe_by_entity = None): pass @@ -312,7 +323,6 @@ def compute_aggregates_comparison(self, input_dataframe_by_entity = None): def compute_distibution_comparison(self, input_dataframe_by_entity = None): pass - def get_name(self): return self.name + "_" + str(self.period) @@ -350,17 +360,6 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period """Compare actual data with openfisca-france-data computation.""" log.setLevel(level = logging.DEBUG if verbose else logging.WARNING) - name = self.get_name() - - assert name is not None and isinstance(name, str) - - figures_directory = Path(config.get("paths", "figures_directory")) / name - - if not figures_directory.exists(): - figures_directory.mkdir(parents = True, exist_ok = True) - - self.figures_directory = figures_directory - if target_variables is not None and isinstance(target_variables, str): target_variables = [target_variables] @@ -374,7 +373,7 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period backup_directory = PurePath.joinpath(Path(config.get("paths", "backup"))) backup_directory.mkdir(parents = True, exist_ok = True) - + name = self.name backup_path = PurePath.joinpath(backup_directory, f"{name}_backup.h5") if load: @@ -399,6 +398,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period input_dataframe_by_entity = input_dataframe_by_entity, ) + self.compute_distibution_comparison(input_dataframe_by_entity = input_dataframe_by_entity) + result_by_variable, markdown_section_by_variable, markdown_summary_section_by_variable = self.compute_divergence( input_dataframe_by_entity = None, # To force load the data_table from hdf file target_dataframe_by_entity = target_dataframe_by_entity, @@ -552,7 +553,11 @@ def create_report(self, markdown_section_by_variable, markdown_summary_section_b with open(figures_directory / "table_agregats.md", "r", encoding = 'utf-8') as table_agregats_md_file: table_agregats_markdown = table_agregats_md_file.read() - front_sections = [messages_markdown_section, table_agregats_markdown] + if PurePath.joinpath(figures_directory, "distribution_comparison_md").exists(): + with open(figures_directory / "distribution_comparison_md", "r", encoding = 'utf-8') as distribution_comparison_md_file: + distribution_comparison_markdown = distribution_comparison_md_file.read() + + front_sections = [messages_markdown_section, table_agregats_markdown, distribution_comparison_markdown] sections_by_filename = { "variables": markdown_section_by_variable, "summary_variables": markdown_summary_section_by_variable @@ -588,8 +593,11 @@ def filter(self, data_frame): log.info(log_message) self.messages.append(log_message + "\n") - def get_survey_scenario(self, data = None): - survey_name = self.survey_name + def get_survey_scenario(self, data = None, survey_name = None): + + if self.survey_scenario is not None: + return self.survey_scenario + return get_survey_scenario( year = str(self.period), data = data, From ac912415c5e2e3a6e6a162ea3d46324acd93ff91 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 26 Jun 2023 18:06:39 +0200 Subject: [PATCH 073/140] Fix missing distribution_comparison --- openfisca_france_data/comparator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index e98e4eef..905e27ea 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -553,6 +553,7 @@ def create_report(self, markdown_section_by_variable, markdown_summary_section_b with open(figures_directory / "table_agregats.md", "r", encoding = 'utf-8') as table_agregats_md_file: table_agregats_markdown = table_agregats_md_file.read() + distribution_comparison_markdown = None if PurePath.joinpath(figures_directory, "distribution_comparison_md").exists(): with open(figures_directory / "distribution_comparison_md", "r", encoding = 'utf-8') as distribution_comparison_md_file: distribution_comparison_markdown = distribution_comparison_md_file.read() From baa9089a1d0df65667d487318aabf21214cab308 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 29 Jun 2023 16:23:39 +0200 Subject: [PATCH 074/140] wip --- openfisca_france_data/comparator.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/openfisca_france_data/comparator.py b/openfisca_france_data/comparator.py index 905e27ea..e1db9d22 100644 --- a/openfisca_france_data/comparator.py +++ b/openfisca_france_data/comparator.py @@ -356,7 +356,7 @@ def get_test_dataframes(self, rebuild = False, noindivs = None): } return input_dataframe_by_entity, target_dataframe_by_entity - def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False): + def compare(self, browse, load, verbose, debug, target_variables = None, period = None, rebuild = False, summary = False, compute_divergence = False): """Compare actual data with openfisca-france-data computation.""" log.setLevel(level = logging.DEBUG if verbose else logging.WARNING) @@ -368,6 +368,8 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period if target_variables is None: target_variables = self.default_target_variables + self.target_variables = target_variables + if period is not None: period = int(period) @@ -400,22 +402,25 @@ def compare(self, browse, load, verbose, debug, target_variables = None, period self.compute_distibution_comparison(input_dataframe_by_entity = input_dataframe_by_entity) - result_by_variable, markdown_section_by_variable, markdown_summary_section_by_variable = self.compute_divergence( - input_dataframe_by_entity = None, # To force load the data_table from hdf file - target_dataframe_by_entity = target_dataframe_by_entity, - target_variables = target_variables, - period = period, - summary = summary, - ) + if compute_divergence: + result_by_variable, markdown_section_by_variable, markdown_summary_section_by_variable = self.compute_divergence( + input_dataframe_by_entity = None, # To force load the data_table from hdf file + target_dataframe_by_entity = target_dataframe_by_entity, + target_variables = target_variables, + period = period, + summary = summary, + ) # Deal with markdown_section - self.create_report(markdown_section_by_variable, markdown_summary_section_by_variable) + self.create_report(markdown_section_by_variable, markdown_summary_section_by_variable) - if result_by_variable is None: - return + if result_by_variable is None: + return - result = pd.concat(result_by_variable, ignore_index = True) + result = pd.concat(result_by_variable, ignore_index = True) + else: + self.create_report(None,None) log.debug(f"Eveyrthing has been computed in {datetime.datetime.now() - start_time}") del input_dataframe_by_entity, target_dataframe_by_entity From f0194b5f39e46dcd91606c1f032316bf2df7b6ac Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 19 Jul 2023 10:27:49 +0200 Subject: [PATCH 075/140] Bump --- CHANGELOG.md | 6 ++++++ setup.py | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f13f957e..dc02a384 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# 1.3.0 [#227](https://github.com/openfisca/openfisca-france-data/pull/227) +* Technical changes +- Adapte le comparator pour effectuer plus de tests en distribution +- Permet l'utilisation d'un SurveyScenario facilement personnalisable + + # 1.2.0 [#226](https://github.com/openfisca/openfisca-france-data/pull/226) * Technical changes - Adapte le comparator pour permettre de changer les agrégats cibles. diff --git a/setup.py b/setup.py index 02833496..e67439f2 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "1.2.0", + version = "1.3.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -53,7 +53,6 @@ "mypy >= 0.670, < 1.0.0", "pypandoc", 'pytest >= 7.2.2, < 8.0', - # "pytest-cov >= 2.6.0, < 3.0.0", "scipy >= 1.2.1, < 2.0.0", "toolz >= 0.9.0, < 1.0.0", ], From 6e689d18802ce5d5cdb76b3023db4c9a7b771c11 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 20 Jul 2023 18:17:43 +0200 Subject: [PATCH 076/140] =?UTF-8?q?Ajoute=20les=20donn=C3=A9es=20d'agr?= =?UTF-8?q?=C3=A9gats=20dans=20le=20paquet?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e67439f2..dce0d830 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,9 @@ "Programming Language :: Python :: 3.7", "Topic :: Scientific/Engineering :: Information Analysis", ], - package_data = { - 'openfisca_france_data/assets': ['openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'], + package_data = { + 'openfisca_france_data': ['assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx', + 'assets/aggregats/ines/ines_2019.json'], }, entry_points = { 'console_scripts': [ From f4edad5e1c3fdbcfd6b0c781811bd5b50c48b262 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 20 Jul 2023 18:24:01 +0200 Subject: [PATCH 077/140] bump --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc02a384..8c559eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# 1.3.1 [#228](https://github.com/openfisca/openfisca-france-data/pull/228) +* Technical changes +- Ajoute de deux jeux de données utilisées pour les tests au paquet : + - 'assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx' + - `'assets/aggregats/ines/ines_2019.json' + # 1.3.0 [#227](https://github.com/openfisca/openfisca-france-data/pull/227) * Technical changes - Adapte le comparator pour effectuer plus de tests en distribution diff --git a/setup.py b/setup.py index dce0d830..e1afc26d 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "1.3.0", + version = "1.3.1", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From e0bc5d78333d3668d8a5e338a9a020406562c591 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 21 Jul 2023 10:44:39 +0200 Subject: [PATCH 078/140] fix tests avant 2018 --- .../input_data_builder/step_06_final.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py index eaa28055..48e06e7c 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py @@ -48,16 +48,24 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene "traitement_indiciaire_brut", ] - var_menages = [ - 'idmen', - 'loyer', - 'statut_occupation_logement', - 'taxe_habitation', - 'wprm', - 'zone_apl', - 'logement_conventionne', - 'prest_precarite_hand' # on récupère la variable de montant de aah / caah pour pouvoir faire une imputation du handicap - ] + if year >= 2018: + var_menages = [ + 'idmen', + 'loyer', + 'statut_occupation_logement', + 'taxe_habitation', + 'wprm', + 'zone_apl', + 'logement_conventionne', + 'prest_precarite_hand' # on récupère la variable de montant de aah / caah pour pouvoir faire une imputation du handicap + ] + else: + var_menages = [ + 'idmen', + 'loyer', + 'taxe_habitation', + 'wprm', + ] individus = create_ids_and_roles(individus) individus = individus[var_individus].copy() From f3db7af65f140fbe42bba9f98e64a623a9b3819e Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 26 Jul 2023 10:45:11 +0200 Subject: [PATCH 079/140] =?UTF-8?q?Mise=20=C3=A0=20jour=20Python=20et=20Op?= =?UTF-8?q?enFisca-France?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++-- setup.py | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c559eca..0f2d09eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ # Changelog +# 2.0.0 [#229](https://github.com/openfisca/openfisca-france-data/pull/229) +* Technical changes +- Met à jour Python dans le setup pour compatibilité conda + # 1.3.1 [#228](https://github.com/openfisca/openfisca-france-data/pull/228) * Technical changes -- Ajoute de deux jeux de données utilisées pour les tests au paquet : - - 'assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx' +- Ajout de deux jeux de données utilisées pour les tests au paquet : + - 'assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx' - `'assets/aggregats/ines/ines_2019.json' # 1.3.0 [#227](https://github.com/openfisca/openfisca-france-data/pull/227) diff --git a/setup.py b/setup.py index e1afc26d..100c9ae5 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "1.3.1", + version = "2.0.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -20,7 +20,9 @@ "License :: OSI Approved :: GNU Affero General Public License v3", "Operating System :: POSIX", "Programming Language :: Python", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering :: Information Analysis", ], package_data = { @@ -34,12 +36,12 @@ 'create-test-erfs-fpr=openfisca_france_data.erfs_fpr.test_case_creation:create_test', ], }, - python_requires = ">= 3.7", + python_requires = ">= 3.9", install_requires = [ "click >= 8.0.0, < 9.0.0", "matplotlib >= 3.1.1, < 4.0.0", "multipledispatch >= 0.6.0, < 1.0.0", - "openFisca-france >= 149.0.0, < 150.0.0", + "openFisca-france >= 150.0.0, < 151.0.0", "openFisca-survey-manager >= 1, < 2.0.0", "wquantiles >= 0.3.0, < 1.0.0", # To compute weighted quantiles ], From e2818858a908645251a581b0846acc482a118f25 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:56:11 +0200 Subject: [PATCH 080/140] Add event: push in conda CI --- .github/workflows/workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index d956bdbf..c0412338 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -260,6 +260,7 @@ jobs: commit: ${{steps.last_pr_commit.outputs.result}} name: conda-build-${{ env.PACKAGE_VERSION }}-${{steps.last_pr_commit.outputs.result}} path: conda-build-tmp + event: push # To avoid conflict with PR workflow if_no_artifact_found: fail - name: Conda upload # This shell is made necessary by https://github.com/conda-incubator/setup-miniconda/issues/128 From 5ceaf4df7a2b024e5e32080cc7cef7198013eb08 Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Mon, 31 Jul 2023 16:30:13 +0200 Subject: [PATCH 081/140] bump version 2.0.0 -> 2.0.1 --- .conda/meta.yaml | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index 6fc30fa5..e8b167e0 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -4,7 +4,7 @@ ############################################################################### {% set name = "openfisca-france-data" %} -{% set version = "0.23.1" %} +{% set version = "2.0.1" %} package: name: {{ name|lower }} diff --git a/setup.cfg b/setup.cfg index 1b70d92b..87f080f8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ testpaths = tests python_files = **/*.py [bumpver] -current_version = "0.23.1" +current_version = "2.0.1" version_pattern = "MAJOR.MINOR.PATCH" commit_message = "bump version {old_version} -> {new_version}" commit = True diff --git a/setup.py b/setup.py index 100c9ae5..f0bf46e5 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.0", + version = "2.0.1", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 06fc98ae46acfcf16ca948fe8143c98fabd9548c Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Mon, 31 Jul 2023 16:31:14 +0200 Subject: [PATCH 082/140] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f2d09eb..926ae506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.1 [#232](https://github.com/openfisca/openfisca-france-data/pull/232) +* Technical changes +- Fix de #230 qui se produit quand le wokflow de CI a tourné deux fois pour le même commit, ce qui arrive quand on crée la PR. La solution est de limiter la recherche de l’artefact aux workflows lancés par un push. + # 2.0.0 [#229](https://github.com/openfisca/openfisca-france-data/pull/229) * Technical changes - Met à jour Python dans le setup pour compatibilité conda From 0bf24ecf46c57d3e951b6eda374c865da40ba353 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 16 Aug 2023 10:52:19 +0200 Subject: [PATCH 083/140] Aligne requirements sur france et survey-manager --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f0bf46e5..f4ef4155 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.1", + version = "2.0.2", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -47,10 +47,10 @@ ], extras_require = { "test": [ - "autopep8 >= 1.4.0, < 1.5.0", + "autopep8 >= 2.0.2, < 3", "bumpver >= 2022.1120", "dtale", - "flake8 >= 3.7.0, < 4.0.0", + "flake8 >= 6.0.0, < 7.0.0", "ipdb >=0.13, <1.0", "ipython >= 7.5.0, < 8.0.0", "mypy >= 0.670, < 1.0.0", From 4b5c6a47b554fef7fb9206e92c28aec8c51f0a85 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 16 Aug 2023 11:05:59 +0200 Subject: [PATCH 084/140] Changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 926ae506..daee8c7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.2 [#234](https://github.com/openfisca/openfisca-france-data/pull/234) +* Technical changes +- Update autopep8 et flake8 pour avoir des requirements compatibles avec les dernières versions d'openfisca-france et openfisca-survey-manager + ### 2.0.1 [#232](https://github.com/openfisca/openfisca-france-data/pull/232) * Technical changes - Fix de #230 qui se produit quand le wokflow de CI a tourné deux fois pour le même commit, ce qui arrive quand on crée la PR. La solution est de limiter la recherche de l’artefact aux workflows lancés par un push. From ea58f4dc70671fbd6785fc977e177eb01911ef94 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Fri, 18 Aug 2023 17:04:22 +0200 Subject: [PATCH 085/140] Passe conda dep en jinja --- .conda/meta.yaml | 29 ++++++++++------------------- CHANGELOG.md | 4 ++++ setup.py | 2 +- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/.conda/meta.yaml b/.conda/meta.yaml index e8b167e0..208e6770 100644 --- a/.conda/meta.yaml +++ b/.conda/meta.yaml @@ -1,10 +1,11 @@ ############################################################################### -## Fichier de description du package pour Anaconda.org -## Le numéro de version est mis à jour automatiquement par BumpVer +## File for Anaconda.org +## It use Jinja2 templating code to retreive information from setup.py ############################################################################### {% set name = "openfisca-france-data" %} -{% set version = "2.0.1" %} +{% set data = load_setup_py_data() %} +{% set version = data.get('version') %} package: name: {{ name|lower }} @@ -23,13 +24,9 @@ requirements: - python - pip run: - - python >=3.7,<4.0 - - click >=8.0.0,<9.0.0 - - matplotlib >=3.1.1,<4.0.0 - - multipledispatch >=0.6.0,<1.0.0 - - openFisca-france >=113.0.0,<120.0.0 # Max 120 because of a bug in OF : https://github.com/openfisca/openfisca-france/issues/1996 - - openFisca-survey-manager >=0.44.2,<1.0.0 - - wquantiles >=0.3.0,<1.0.0 + {% for req in data.get('install_requires', []) %} + - {{ req }} + {% endfor %} test: imports: @@ -50,15 +47,9 @@ outputs: host: - python run: - - autopep8 >=1.4.0,<1.5.0 - - flake8 >=3.7.0,<3.8.0 - - ipython >=7.5.0,<8.0.0 - - mypy >=0.670,<1.0.0 - - pytest >=4.3.0,<5.0.0 - - pytest-cov >=2.6.0,<3.0.0 - - scipy >=1.2.1,<2.0.0 - - toolz >=0.9.0,<1.0.0 - - bumpver + {% for req in data.get('extras_require', []) %} + - {{ req }} + {% endfor %} - {{ pin_subpackage('openfisca-france-data', exact=True) }} about: diff --git a/CHANGELOG.md b/CHANGELOG.md index daee8c7f..fb2aa524 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.3 [#235](https://github.com/openfisca/openfisca-france-data/pull/235) +* Technical changes +- Passe conda dep en Jinja en suivant Openfisca-France + ### 2.0.2 [#234](https://github.com/openfisca/openfisca-france-data/pull/234) * Technical changes - Update autopep8 et flake8 pour avoir des requirements compatibles avec les dernières versions d'openfisca-france et openfisca-survey-manager diff --git a/setup.py b/setup.py index f4ef4155..09df3934 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.2", + version = "2.0.3", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From ffd097d25663b3059c481041e751dc4ab081ccaf Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Fri, 18 Aug 2023 17:21:48 +0200 Subject: [PATCH 086/140] Enleve espace --- setup.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index 09df3934..a0910e07 100644 --- a/setup.py +++ b/setup.py @@ -36,28 +36,28 @@ 'create-test-erfs-fpr=openfisca_france_data.erfs_fpr.test_case_creation:create_test', ], }, - python_requires = ">= 3.9", + python_requires = ">=3.9", install_requires = [ - "click >= 8.0.0, < 9.0.0", - "matplotlib >= 3.1.1, < 4.0.0", - "multipledispatch >= 0.6.0, < 1.0.0", - "openFisca-france >= 150.0.0, < 151.0.0", - "openFisca-survey-manager >= 1, < 2.0.0", - "wquantiles >= 0.3.0, < 1.0.0", # To compute weighted quantiles + "click >=8.0.0, <9.0.0", + "matplotlib >=3.1.1, <4.0.0", + "multipledispatch >=0.6.0, <1.0.0", + "openFisca-france >=150.0.0, <151.0.0", + "openFisca-survey-manager >=1, <2.0.0", + "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { "test": [ - "autopep8 >= 2.0.2, < 3", - "bumpver >= 2022.1120", + "autopep8 >=2.0.2, <3", + "bumpver >=2022.1120", "dtale", - "flake8 >= 6.0.0, < 7.0.0", + "flake8 >=6.0.0, <7.0.0", "ipdb >=0.13, <1.0", - "ipython >= 7.5.0, < 8.0.0", - "mypy >= 0.670, < 1.0.0", + "ipython >=7.5.0, <8.0.0", + "mypy >=0.670, <1.0.0", "pypandoc", - 'pytest >= 7.2.2, < 8.0', - "scipy >= 1.2.1, < 2.0.0", - "toolz >= 0.9.0, < 1.0.0", + 'pytest >=7.2.2, <8.0', + "scipy >=1.2.1, <2.0.0", + "toolz >=0.9.0, <1.0.0", ], }, packages = find_namespace_packages(exclude = ("docs", "tests")), From 8a58391f7cc0a9889526e9ebbae4268b7c4d380d Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Thu, 24 Aug 2023 14:54:20 +0200 Subject: [PATCH 087/140] Deplace dependance click vers test --- CHANGELOG.md | 4 ++++ setup.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb2aa524..c1bd4343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.4 [#237](https://github.com/openfisca/openfisca-france-data/pull/237) +* Technical changes +- Deplace click en dependance de test + ### 2.0.3 [#235](https://github.com/openfisca/openfisca-france-data/pull/235) * Technical changes - Passe conda dep en Jinja en suivant Openfisca-France diff --git a/setup.py b/setup.py index a0910e07..d77c2545 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.3", + version = "2.0.4", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -38,7 +38,6 @@ }, python_requires = ">=3.9", install_requires = [ - "click >=8.0.0, <9.0.0", "matplotlib >=3.1.1, <4.0.0", "multipledispatch >=0.6.0, <1.0.0", "openFisca-france >=150.0.0, <151.0.0", @@ -47,6 +46,7 @@ ], extras_require = { "test": [ + "click >=8.0.0, <9.0.0", "autopep8 >=2.0.2, <3", "bumpver >=2022.1120", "dtale", From 979acc9d2516718a99d80a0826a5a74d5d6ab338 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Tue, 29 Aug 2023 09:41:37 +0200 Subject: [PATCH 088/140] Enleve matplotlib des dep --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d77c2545..8806d815 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,6 @@ }, python_requires = ">=3.9", install_requires = [ - "matplotlib >=3.1.1, <4.0.0", "multipledispatch >=0.6.0, <1.0.0", "openFisca-france >=150.0.0, <151.0.0", "openFisca-survey-manager >=1, <2.0.0", From 36b427cadd589dd418ea10c6e8308fc38442c0b5 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 31 Aug 2023 09:34:08 +0200 Subject: [PATCH 089/140] Corrige le calcul du smic annuel en fonction du smic horaire --- .../erfs_fpr/input_data_builder/step_04_famille.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py b/openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py index 0b16d313..743772c1 100644 --- a/openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py +++ b/openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py @@ -215,10 +215,10 @@ def famille_1(indivi = None, kind = 'erfs_fpr', enfants_a_naitre = None, skip_en if kind == 'erfs_fpr': base['salaire_de_base'].fillna(0, inplace = True) - base['smic55'] = base.salaire_de_base >= (smic * 12 * 0.55) # 55% du smic mensuel brut + base['smic55'] = base.salaire_de_base >= (smic * 151 * 12 * 0.55) # 55% du smic mensuel brut else: base['ztsai'].fillna(0, inplace = True) - base['smic55'] = base.ztsai >= (smic * 12 * 0.55) # 55% du smic mensuel brut + base['smic55'] = base.ztsai >= (smic * 151 * 12 * 0.55) # 55% du smic mensuel brut base['famille'] = 0 base['kid'] = False @@ -630,7 +630,7 @@ def famille_5(base = None, famille = None, kind = 'erfs_fpr', year = None): fip['moins_de_15_ans_inclus'] = (fip.agepf < 16) fip['jeune_non_eligible_rsa'] = ((fip.agepf >= 16) & (fip.agepf < AGE_RSA)) fip['jeune_eligible_rsa'] = (fip.agepf >= AGE_RSA) - fip['smic55'] = (fip.ztsai >= smic * 12 * 0.55) + fip['smic55'] = (fip.ztsai >= smic * 151 * 12 * 0.55) fip['famille'] = 0 fip['kid'] = False for series_name in ['kid', 'moins_de_15_ans_inclus', 'jeune_non_eligible_rsa', 'jeune_eligible_rsa', 'smic55']: From ce9540ba8f115a8b50def5100cfe822fde5979d0 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 31 Aug 2023 09:56:15 +0200 Subject: [PATCH 090/140] Bump and Changelog --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1bd4343..e0a51c5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.5 [#238(https://github.com/openfisca/openfisca-france-data/pull/238) +* Technical changes +- Corrige le calcul du smic annuel en fonction du smic horaire dans openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py + ### 2.0.4 [#237](https://github.com/openfisca/openfisca-france-data/pull/237) * Technical changes - Deplace click en dependance de test diff --git a/setup.py b/setup.py index 8806d815..e270467f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.4", + version = "2.0.5", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 1f86d7d6ed2398df2253f7e72d0446324c554cbb Mon Sep 17 00:00:00 2001 From: benoit-cty <6603048+benoit-cty@users.noreply.github.com> Date: Fri, 8 Sep 2023 10:02:56 +0200 Subject: [PATCH 091/140] Bump OpenFisca-France to v153 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e270467f..9543aa55 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ python_requires = ">=3.9", install_requires = [ "multipledispatch >=0.6.0, <1.0.0", - "openFisca-france >=150.0.0, <151.0.0", + "OpenFisca-France >=150.0.0, <154.0.0", "openFisca-survey-manager >=1, <2.0.0", "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], From 48ab4e9b504f268f3f092fc71df1d6d1bb491cd8 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 11:32:46 +0200 Subject: [PATCH 092/140] =?UTF-8?q?rapratrie=20des=20anciennes=20r=C3=A9fo?= =?UTF-8?q?rmes=20d'openfisca=20france?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openfisca_france_data/base_survey.py | 4 +- .../old_openfisca_france_reforms/__init__.py | 0 .../allocations_familiales_imposables.py | 96 +++++++++ .../cesthra_invalidee.py | 66 ++++++ .../de_net_a_brut.py | 60 ++++++ .../allocations_familiales_imposables.yaml | 8 + .../parameters/cesthra_invalidite.yaml | 15 ++ .../parameters/landais_piketty_saez.yaml | 107 ++++++++++ .../parameters/plf2015.yaml | 17 ++ .../parameters/plf2016.yaml | 17 ++ .../parameters/plfr2014.yaml | 27 +++ .../parameters/plfrss2014.yaml | 188 ++++++++++++++++++ .../parameters/trannoy_wasmer.yaml | 23 +++ .../old_openfisca_france_reforms/plf2015.py | 42 ++++ .../plf2016_ayrault_muet.py | 132 ++++++++++++ .../old_openfisca_france_reforms/plfr2014.py | 84 ++++++++ .../trannoy_wasmer.py | 55 +++++ tests/erfs_fpr/integration/test_aggregates.py | 2 +- .../erfs_fpr/unit/test_get_survey_scenario.py | 2 +- tests/test_fake_survey_simulation.py | 2 +- 20 files changed, 942 insertions(+), 5 deletions(-) create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/__init__.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/allocations_familiales_imposables.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/cesthra_invalidite.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/landais_piketty_saez.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2015.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2016.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfr2014.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfrss2014.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/trannoy_wasmer.yaml create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py diff --git a/openfisca_france_data/base_survey.py b/openfisca_france_data/base_survey.py index cfa1ccb1..1c10f8ff 100644 --- a/openfisca_france_data/base_survey.py +++ b/openfisca_france_data/base_survey.py @@ -5,7 +5,7 @@ from openfisca_france.entities import Famille from openfisca_france_data import france_data_tax_benefit_system -from openfisca_france.reforms import ( +from openfisca_france_data.reforms.old_openfisca_france_reforms import ( allocations_familiales_imposables, cesthra_invalidee, # inversion_directe_salaires, # We use a custom inversion_directe_salaires, not merged in the openfisca-france yet @@ -37,7 +37,7 @@ scipy = None if scipy is not None: - from openfisca_france.reforms import de_net_a_brut + from openfisca_france_data.reforms.old_openfisca_france_reforms import de_net_a_brut reform_list['de_net_a_brut'] = de_net_a_brut.de_net_a_brut reform_by_full_key = {} diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/__init__.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py new file mode 100644 index 00000000..cce1f5fb --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py @@ -0,0 +1,96 @@ +import os + +from ..model.base import * + + +dir_path = os.path.join(os.path.dirname(__file__), 'parameters') + + +def modify_parameters(parameters): + file_path = os.path.join(dir_path, 'allocations_familiales_imposables.yaml') + reform_parameters_subtree = load_parameter_file(name='allocations_familiales_imposables', file_path=file_path) + parameters.add_child('allocations_familiales_imposables', reform_parameters_subtree) + return parameters + + +class allocations_familiales_imposables(Reform): + name = 'Allocations familiales imposables' + + class rbg(Variable): + label = 'Nouveau revenu brut global intégrant les allocations familiales' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + allocations_familiales_imposables = foyer_fiscal('allocations_familiales_imposables', period, options = [ADD]) + deficit_ante = foyer_fiscal('deficit_ante', period) + f6gh = foyer_fiscal('f6gh', period) + + nacc_pvce_i = foyer_fiscal.members('nacc_pvce', period) + nbic_impm_i = foyer_fiscal.members('nbic_impm', period) + + revenu_categoriel = foyer_fiscal('revenu_categoriel', period) + cga = parameters(period).impot_revenu.calcul_revenus_imposables.rpns.cga_taux2 + + nacc_pvce = foyer_fiscal.sum(nacc_pvce_i) + return max_( + 0, + allocations_familiales_imposables + + revenu_categoriel + + f6gh + + (foyer_fiscal.sum(nbic_impm_i) + nacc_pvce) + * (1 + cga) + - deficit_ante + ) + + class rfr(Variable): + label = 'Nouveau revenu fiscal de référence intégrant les allocations familiales' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + allocations_familiales_imposables = foyer_fiscal('allocations_familiales_imposables') + abattements_plus_values = foyer_fiscal('abattements_plus_values') + f3vi_holder = foyer_fiscal.members('f3vi') + f3vz = foyer_fiscal('f3vz') + rfr_cd = foyer_fiscal('rfr_cd') + rni = foyer_fiscal('rni') + rpns_exon_holder = foyer_fiscal.members('rpns_exon') + rfr_rvcm_abattements_a_reintegrer = foyer_fiscal('rfr_rvcm_abattements_a_reintegrer') # Supprimée en 2018 + revenus_capitaux_prelevement_liberatoire = foyer_fiscal('revenus_capitaux_prelevement_liberatoire', period, options = [ADD]) # Supprimée en 2018 + revenus_capitaux_prelevement_forfaitaire_unique_ir = foyer_fiscal('revenus_capitaux_prelevement_forfaitaire_unique_ir', period, options = [ADD]) # Existe à partir de 2018 + microentreprise = foyer_fiscal('microentreprise') + + f3vi = foyer_fiscal.sum(f3vi_holder) + rpns_exon = foyer_fiscal.sum(rpns_exon_holder) + + return ( + max_(0, rni - allocations_familiales_imposables) + + rfr_cd + + rfr_rvcm_abattements_a_reintegrer + + revenus_capitaux_prelevement_liberatoire + + revenus_capitaux_prelevement_forfaitaire_unique_ir + + f3vi + + rpns_exon + + abattements_plus_values + + f3vz + + microentreprise + ) + + # TO CHECK : f3vb after 2015 (abattements sur moins-values = interdits) + + class allocations_familiales_imposables(Variable): + value_type = float + entity = FoyerFiscal + label = 'Allocations familiales imposables' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + imposition = parameters(period).allocations_familiales_imposables.imposition + af = foyer_fiscal.declarant_principal.famille('af', period, options = [ADD]) + + return af * imposition + + def apply(self): + self.update_variable(self.rbg) + self.update_variable(self.rfr) + self.add_variable(self.allocations_familiales_imposables) + self.modify_parameters(modifier_function = modify_parameters) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py new file mode 100644 index 00000000..9b4472f9 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py @@ -0,0 +1,66 @@ +import os + +from openfisca_france.model.base import * + +from .. import entities + + +dir_path = os.path.join(os.path.dirname(__file__), 'parameters') + + +def modify_parameters(parameters): + file_path = os.path.join(dir_path, 'cesthra_invalidite.yaml') + reform_parameters_subtree = load_parameter_file(name='cesthra', file_path=file_path) + parameters.add_child('cesthra', reform_parameters_subtree) + return parameters + + +class cesthra(Variable): + value_type = float + entity = entities.FoyerFiscal + label = "Contribution exceptionnelle de solidarité sur les très hauts revenus d'activité" + definition_period = YEAR + # PLF 2013 (rejeté) : 'taxe à 75%' + + def formula(foyer_fiscal, period, parameters): + salaire_imposable_i = foyer_fiscal.members('salaire_imposable', period, options = [ADD]) + _cesthra = parameters(period).cesthra + + cesthra_i = max_(salaire_imposable_i - _cesthra.seuil, 0) * _cesthra.taux + + return foyer_fiscal.sum(cesthra_i) + + +class irpp(Variable): + label = 'Impôt sur le revenu des personnes physiques (réformée pour intégrer la cesthra)' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + ''' + Montant après seuil de recouvrement (hors ppe) + ''' + iai = foyer_fiscal('iai', period) + credits_impot = foyer_fiscal('credits_impot', period) + acomptes_ir = foyer_fiscal('acomptes_ir', period) + contribution_exceptionnelle_hauts_revenus = foyer_fiscal('contribution_exceptionnelle_hauts_revenus', period) + cesthra = foyer_fiscal('cesthra', period = period) + recouvrement = parameters(period).impot_revenu.calcul_impot_revenu.recouvrement + + pre_result = iai - credits_impot - acomptes_ir + contribution_exceptionnelle_hauts_revenus + cesthra + + return ( + (iai > recouvrement.seuil) + * ((pre_result < recouvrement.min) * (pre_result > 0) * iai * 0 + + ((pre_result <= 0) + (pre_result >= recouvrement.min)) * (- pre_result)) + + (iai <= recouvrement.seuil) * ((pre_result < 0) * (-pre_result) + + (pre_result >= 0) * 0 * iai) + ) + + +class cesthra_invalidee(Reform): + name = "Contribution execptionnelle sur les très hauts revenus d'activité (invalidée par le CC)" + + def apply(self): + self.add_variable(cesthra) + self.update_variable(irpp) + self.modify_parameters(modifier_function = modify_parameters) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py new file mode 100644 index 00000000..b6d7cb61 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py @@ -0,0 +1,60 @@ +from openfisca_core.reforms import Reform + +try: + from scipy.optimize import fsolve +except ImportError: + fsolve = None + +from .. import entities +from ..model.base import * + + +def calculate_net_from(salaire_de_base, individu, period): + + # We're not wanting to calculate salaire_de_base again, but instead manually set it as an input variable + individu.get_holder('salaire_de_base').put_in_cache(salaire_de_base, period) + + # Work in isolation + temp_simulation = individu.simulation.clone() + temp_individu = temp_simulation.individu + + # Force recomputing of salaire_net + temp_individu.get_holder('salaire_net_a_payer').delete_arrays() + + net = temp_individu('salaire_net_a_payer', period)[0] + + return net + + +class salaire_de_base(Variable): + value_type = float + entity = entities.Individu + label = 'Salaire brut' + definition_period = MONTH + + def formula(individu, period, parameters): + # Calcule le salaire brut à partir du salaire net par inversion numérique. + + net = individu.get_holder('salaire_net_a_payer').get_array(period) + + if net is None: + return individu.empty_array() + + def solve_func(net): + def innerfunc(essai): + return calculate_net_from(essai, individu, period) - net + return innerfunc + brut_calcule = fsolve( + solve_func(net), + net * 1.5, # on entend souvent parler cette méthode... + xtol = 1 / 10 # précision + ) + + return brut_calcule + + +class de_net_a_brut(Reform): + name = 'Inversion du calcul brut -> net' + + def apply(self): + self.update_variable(salaire_de_base) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/allocations_familiales_imposables.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/allocations_familiales_imposables.yaml new file mode 100644 index 00000000..3a78d6f4 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/allocations_familiales_imposables.yaml @@ -0,0 +1,8 @@ +description: Intégration au revenu imposable des allocations familiales +imposition: + description: "Indicatrice d'imposition" + values: + '2015-01-01': + value: null + '2000-01-01': + value: true diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/cesthra_invalidite.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/cesthra_invalidite.yaml new file mode 100644 index 00000000..9e455203 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/cesthra_invalidite.yaml @@ -0,0 +1,15 @@ +description: Contribution execptionnelle sur les très hauts revenus d'activité +seuil: + description: Seuil + values: + '2014-01-01': + value: null + '2012-01-01': + value: 1000000 +taux: + description: Taux + values: + '2014-01-01': + value: null + '2012-01-01': + value: 0.75 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/landais_piketty_saez.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/landais_piketty_saez.yaml new file mode 100644 index 00000000..e79b3cdf --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/landais_piketty_saez.yaml @@ -0,0 +1,107 @@ +description: Impôt à base large proposé par Landais Piketty et Saez +bareme: + unit: currency + description: Barème de l'impôt + brackets: + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.02 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 1100 + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.1 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 2200 + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.13 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 5000 + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.25 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 10000 + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.5 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 40000 + - average_rate: + '2015-01-01': + value: null + '2000-01-01': + value: 0.6 + threshold: + '2015-01-01': + value: null + '2000-01-01': + value: 100000 +imposition: + description: "Indicatrice d'imposition" + values: + '2015-01-01': + value: null + '2000-01-01': + value: true +credit_enfant: + description: "Crédit d'impôt forfaitaire par enfant" + values: + '2015-01-01': + value: null + '2000-01-01': + value: 0 +reduc_enfant: + description: "Réduction d'impôt forfaitaire par enfant" + values: + '2015-01-01': + value: null + '2000-01-01': + value: 0 +abatt_enfant: + description: Abattement forfaitaire sur le revenu par enfant + values: + '2015-01-01': + value: null + '2000-01-01': + value: 0 +reduc_conj: + description: "Réduction d'impôt forfaitaire si conjoint" + values: + '2015-01-01': + value: null + '2000-01-01': + value: 0 +abatt_conj: + description: Abattement forfaitaire sur le revenu si conjoint + values: + '2015-01-01': + value: null + '2000-01-01': + value: 0 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2015.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2015.yaml new file mode 100644 index 00000000..fbfecdcc --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2015.yaml @@ -0,0 +1,17 @@ +description: PLF 2015 sur revenus 2013 (Décote) +seuil_celib: + description: Seuil de la décote pour un célibataire + unit: currency + values: + '2014-01-01': + value: null + '2013-01-01': + value: 1135 +seuil_couple: + description: Seuil de la décote pour un couple + unit: currency + values: + '2014-01-01': + value: null + '2013-01-01': + value: 1870 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2016.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2016.yaml new file mode 100644 index 00000000..a3b59162 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plf2016.yaml @@ -0,0 +1,17 @@ +description: PLF 2016 sur revenus 2014 +decote_seuil_celib: + description: Seuil de la décôte pour un célibataire + unit: currency + values: + '2015-01-01': + value: null + '2014-01-01': + value: 1165 +decote_seuil_couple: + description: Seuil de la décôte pour un couple + unit: currency + values: + '2015-01-01': + value: null + '2014-01-01': + value: 1920 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfr2014.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfr2014.yaml new file mode 100644 index 00000000..afeec5a6 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfr2014.yaml @@ -0,0 +1,27 @@ +description: Projet de loi de finance rectificative 2014 +reduction_impot_exceptionnelle: + description: Réduction d'impôt exceptionnelle + montant_plafond: + description: Montant plafond par part pour les deux premières parts + unit: currency + values: + '2015-01-01': + value: null + '2013-01-01': + value: 350 + seuil: + description: Seuil (à partir duquel la réduction décroît) par part pour les deux premières parts + unit: currency + values: + '2015-01-01': + value: null + '2013-01-01': + value: 13795 + majoration_seuil: + description: Majoration du seuil par demi-part supplémentaire + unit: currency + values: + '2015-01-01': + value: null + '2013-01-01': + value: 3536 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfrss2014.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfrss2014.yaml new file mode 100644 index 00000000..79717e75 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/plfrss2014.yaml @@ -0,0 +1,188 @@ +description: Projet de loi de financement de la sécurité sociale rectificative 2014 +exonerations_bas_salaires: + description: Exonérations de cotisations salariées sur les bas salaires + prive: + description: Salariés du secteur privé + taux: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.03 + seuil: + description: Seuil (en SMIC) + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 1.3 + public: + description: Salariés du secteur public + taux_1: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.02 + seuil_1: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 312 + taux_2: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.018 + seuil_2: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 328 + taux_3: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.016 + seuil_3: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 343 + taux_4: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.014 + seuil_4: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 359 + taux_5: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.012 + seuil_5: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 375 + taux_6: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.01 + seuil_6: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 390 + taux_7: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.008 + seuil_7: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 406 + taux_8: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.007 + seuil_8: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 421 + taux_9: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.006 + seuil_9: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 437 + taux_10: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.005 + seuil_10: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 453 + taux_11: + description: Taux + unit: /1 + values: + '2015-01-01': + value: null + '2014-01-01': + value: 0.002 + seuil_11: + description: Indice majoré plafond + values: + '2015-01-01': + value: null + '2014-01-01': + value: 468 diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/trannoy_wasmer.yaml b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/trannoy_wasmer.yaml new file mode 100644 index 00000000..62150540 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/trannoy_wasmer.yaml @@ -0,0 +1,23 @@ +description: Charge de loyer +active: + description: Activation de la charge + values: + '2014-01-01': + value: null + '2002-01-01': + value: true +plaf: + description: Plafond mensuel + unit: currency + values: + '2014-01-01': + value: null + '2002-01-01': + value: 1000 +plaf_nbp: + description: Ajuster le plafond au nombre de part + values: + '2014-01-01': + value: null + '2002-01-01': + value: false diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py new file mode 100644 index 00000000..72d9ce4f --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py @@ -0,0 +1,42 @@ +import os + +from ..model.base import * + + +dir_path = os.path.join(os.path.dirname(__file__), 'parameters') + + +def modify_parameters(parameters): + reform_year = 2013 + reform_period = period(reform_year) + + file_path = os.path.join(dir_path, 'plf2015.yaml') + reform_parameters_subtree = load_parameter_file(name='plf2015', file_path=file_path) + parameters.add_child('plf2015', reform_parameters_subtree) + + parameters.impot_revenu.bareme_ir_depuis_1945.bareme[1].rate.update(period=reform_period, value=0) + parameters.impot_revenu.bareme_ir_depuis_1945.bareme[2].threshold.update(period=reform_period, value=9690) + + return parameters + + +class decote(Variable): + label = 'Décote IR 2015 appliquée sur IR 2014 (revenus 2013)' + definition_period = YEAR + + def formula_2013_01_01(foyer_fiscal, period, parameters): + ir_plaf_qf = foyer_fiscal('ir_plaf_qf', period) + nb_adult = foyer_fiscal('nb_adult', period) + plf = parameters(period).plf2015 + + decote_celib = (ir_plaf_qf < plf.seuil_celib) * (plf.seuil_celib - ir_plaf_qf) + decote_couple = (ir_plaf_qf < plf.seuil_couple) * (plf.seuil_couple - ir_plaf_qf) + return (nb_adult == 1) * decote_celib + (nb_adult == 2) * decote_couple + + +class plf2015(Reform): + name = 'Projet de Loi de Finances 2015 appliquée aux revenus 2013' + + def apply(self): + self.update_variable(decote) + self.modify_parameters(modifier_function = modify_parameters) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py new file mode 100644 index 00000000..3cc73c4c --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py @@ -0,0 +1,132 @@ +import os + +from ..model.base import * + + +dir_path = os.path.dirname(__file__) + + +# Réforme de l'amendement Ayrault-Muet + +def ayrault_muet_modify_parameters(parameters): + # TODO: inflater les paramètres de la décote le barème de l'IR + inflator = 1 + for inflation in [2.8, 0.1, 1.5, 2.1, 2, 0.9, 0.5, 0.1]: + inflator = inflator * (1 + inflation / 100) + del inflation + + elig1 = Parameter('elig1', {'values': {'2015-01-01': {'value': round(16251 * inflator)}, '2016-01-01': {'value': None}}}) + elig2 = Parameter('elig2', {'values': {'2015-01-01': {'value': round(32498 * inflator)}, '2016-01-01': {'value': None}}}) + elig3 = Parameter('elig3', {'values': {'2015-01-01': {'value': round(4490 * inflator)}, '2016-01-01': {'value': None}}}) + parameters.impot_revenu.credits_impots.ppe.add_child('elig1', elig1) + parameters.impot_revenu.credits_impots.ppe.add_child('elig2', elig2) + parameters.impot_revenu.credits_impots.ppe.add_child('elig3', elig3) + return parameters + + +class variator(Variable): + value_type = float + default_value = 1 + entity = FoyerFiscal + label = 'Multiplicateur du seuil de régularisation' + definition_period = YEAR + + +class reduction_csg(Variable): + value_type = float + entity = Individu + label = 'Réduction dégressive de CSG' + definition_period = YEAR + + def formula_2015_01_01(individu, period, parameters): + smic_proratise = individu('smic_proratise', period, options = [ADD]) + assiette_csg_abattue = individu('assiette_csg_abattue', period, options = [ADD]) + + seuil = 1.34 + coefficient_correctif = .9 + taux_csg = ( + parameters(period).csg.activite.imposable.taux + + parameters(period).csg.activite.deductible.taux + ) + tx_max = coefficient_correctif * taux_csg + ratio_smic_salaire = smic_proratise / (assiette_csg_abattue + 1e-16) + # règle d'arrondi: 4 décimales au dix-millième le plus proche + taux_allegement_csg = tx_max * min_(1, max_(seuil - 1 / ratio_smic_salaire, 0) / (seuil - 1)) + # Montant de l'allegment + return taux_allegement_csg * assiette_csg_abattue + + +class reduction_csg_foyer_fiscal(Variable): + entity = FoyerFiscal + label = 'Réduction dégressive de CSG des memebres du foyer fiscal' + value_type = float + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + reduction_csg = foyer_fiscal('reduction_csg', period) + return foyer_fiscal.sum(reduction_csg) + + +class reduction_csg_nette(Variable): + value_type = float + entity = Individu + label = 'Réduction dégressive de CSG' + definition_period = YEAR + + def formula_2015_01_01(individu, period): + reduction_csg = individu('reduction_csg', period) + ppe_elig_bis = individu.foyer_fiscal('ppe_elig_bis', period) + return reduction_csg * ppe_elig_bis + + +class ppe_elig_bis(Variable): + value_type = bool + entity = FoyerFiscal + label = 'ppe_elig_bis' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + ''' + PPE: eligibilité à la ppe, condition sur le revenu fiscal de référence + 'foy' + ''' + rfr = foyer_fiscal('rfr', period) + ppe_coef = foyer_fiscal('ppe_coef', period) + maries_ou_pacses = foyer_fiscal('maries_ou_pacses', period) + veuf = foyer_fiscal('veuf', period) + celibataire_ou_divorce = foyer_fiscal('celibataire_ou_divorce', period) + nbptr = foyer_fiscal('nbptr', period) + variator = foyer_fiscal('variator', period) + ppe = parameters(period).impot_revenu.credits_impots.ppe + seuil = (veuf | celibataire_ou_divorce) * (ppe.seuils_rfr_eligibilite.personne_seule + 2 * max_(nbptr - 1, 0) * ppe.seuils_rfr_eligibilite.increment_par_demi_part)\ + + maries_ou_pacses * (ppe.seuils_rfr_eligibilite.couple_marie_pacse + 2 * max_(nbptr - 2, 0) * ppe.seuils_rfr_eligibilite.increment_par_demi_part) + return (rfr * ppe_coef) <= (seuil * variator) + + +class regularisation_reduction_csg(Variable): + value_type = float + entity = FoyerFiscal + label = 'Régularisation complète réduction dégressive de CSG' + definition_period = YEAR + + def formula_2015_01_01(foyer_fiscal, period, parameters): + reduction_csg = foyer_fiscal('reduction_csg_foyer_fiscal', period) + ppe_elig_bis = foyer_fiscal('ppe_elig_bis', period) + return not_(ppe_elig_bis) * (reduction_csg > 1) + + +class ayrault_muet(Reform): + name = 'Amendement Ayrault-Muet au PLF2016' + key = 'ayrault_muet' + + def apply(self): + for variable in [ + reduction_csg, + regularisation_reduction_csg, + reduction_csg_foyer_fiscal, + reduction_csg_nette, + ppe_elig_bis, + variator, + ]: + self.update_variable(variable) + self.modify_parameters(modifier_function = ayrault_muet_modify_parameters) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py new file mode 100644 index 00000000..2396c18f --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py @@ -0,0 +1,84 @@ +import os + +from ..model.base import * + + +dir_path = os.path.join(os.path.dirname(__file__), 'parameters') + + +# TODO: les baisses de charges n'ont pas été codées car annulées (toute ou en partie ?) +# par le Conseil constitutionnel + +class plfr2014(Reform): + name = 'Projet de Loi de Finances Rectificative 2014' + + class reduction_impot_exceptionnelle(Variable): + definition_period = YEAR + + def formula_2013_01_01(foyer_fiscal, period, parameters): + janvier = period.first_month + + nb_adult = foyer_fiscal('nb_adult', period) + nb_parents = foyer_fiscal.declarant_principal.famille('nb_parents', period = janvier) + rfr = foyer_fiscal('rfr', period) + params = parameters(period).plfr2014.reduction_impot_exceptionnelle + plafond = params.seuil * nb_adult + (nb_parents - nb_adult) * 2 * params.majoration_seuil + montant = params.montant_plafond * nb_adult + return min_(max_(plafond + montant - rfr, 0), montant) + + class reductions(Variable): + label = "Somme des réductions d'impôt à intégrer pour l'année 2013" + definition_period = YEAR + + def formula_2013_01_01(foyer_fiscal, period, parameters): + accult = foyer_fiscal('accult', period) + frais_de_comptabilite = foyer_fiscal('frais_de_comptabilite', period) + cappme = foyer_fiscal('cappme', period) + creaen = foyer_fiscal('creaen', period) + accueil_dans_etablissement_personnes_agees = foyer_fiscal('accueil_dans_etablissement_personnes_agees', period) + defense_forets_contre_incendies = foyer_fiscal('defense_forets_contre_incendies', period) + dfppce = foyer_fiscal('dfppce', period) + doment = foyer_fiscal('doment', period) + domlog = foyer_fiscal('domlog', period) + duflot = foyer_fiscal('duflot_pinel_denormandie_metropole', period) + duflot_om = foyer_fiscal('duflot_pinel_denormandie_om', period) + reduction_enfants_scolarises = foyer_fiscal('reduction_enfants_scolarises', period) + gardenf = foyer_fiscal('gardenf', period) + interets_paiements_differes_agriculteurs = foyer_fiscal('interets_paiements_differes_agriculteurs', period) + investissement_forestier = foyer_fiscal('ri_investissement_forestier', period) + invlst = foyer_fiscal('invlst', period) + ip_net = foyer_fiscal('ip_net', period) + location_meublee = foyer_fiscal('location_meublee', period) + mecena = foyer_fiscal('mecena', period) + mohist = foyer_fiscal('mohist', period) + protection_patrimoine_naturel = foyer_fiscal('protection_patrimoine_naturel', period) + prestations_compensatoires = foyer_fiscal('prestations_compensatoires', period) + reduction_impot_exceptionnelle = foyer_fiscal('reduction_impot_exceptionnelle', period) + interets_emprunt_reprise_societe = foyer_fiscal('interets_emprunt_reprise_societe', period) + restauration_patrimoine_bati = foyer_fiscal('restauration_patrimoine_bati', period) + rente_survie = foyer_fiscal('rente_survie', period) + saldom = foyer_fiscal('ri_saldom', period) + scelli = foyer_fiscal('scelli', period) + sofica = foyer_fiscal('sofica', period) + souscriptions_parts_fcpi_fip = foyer_fiscal('souscriptions_parts_fcpi_fip', period) + total_reductions = accult + frais_de_comptabilite + cappme + creaen + accueil_dans_etablissement_personnes_agees + defense_forets_contre_incendies + dfppce + doment + domlog +\ + duflot + duflot_om + reduction_enfants_scolarises + gardenf + interets_paiements_differes_agriculteurs + investissement_forestier + invlst + location_meublee + mecena + mohist + protection_patrimoine_naturel +\ + prestations_compensatoires + interets_emprunt_reprise_societe + restauration_patrimoine_bati + rente_survie + saldom + scelli + sofica + souscriptions_parts_fcpi_fip + reduction_impot_exceptionnelle + return min_(ip_net, total_reductions) + + def apply(self): + for variable in [self.reduction_impot_exceptionnelle, self.reductions]: + self.update_variable(variable) + self.modify_parameters(modifier_function = modify_parameters) + + +def modify_parameters(parameters): + file_path = os.path.join(dir_path, 'plfr2014.yaml') + plfr2014_parameters_subtree = load_parameter_file(name='plfr2014', file_path=file_path) + + file_path = os.path.join(dir_path, 'plfrss2014.yaml') + plfrss2014_parameters_subtree = load_parameter_file(name='plfrss2014', file_path=file_path) + + parameters.add_child('plfr2014', plfr2014_parameters_subtree) + parameters.add_child('plfrss2014', plfrss2014_parameters_subtree) + return parameters diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py new file mode 100644 index 00000000..5b5c6392 --- /dev/null +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py @@ -0,0 +1,55 @@ +import os + +from ..model.base import * + + +dir_path = os.path.join(os.path.dirname(__file__), 'parameters') + + +def modify_parameters(parameters): + file_path = os.path.join(dir_path, 'trannoy_wasmer.yaml') + reform_parameters_subtree = load_parameter_file(name = 'trannoy_wasmer', file_path = file_path) + + parameters.add_child('charge_loyer', reform_parameters_subtree) + return parameters + + +class charges_deduc(Variable): + label = 'Charge déductibles intégrant la charge pour loyer (Trannoy-Wasmer)' + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + cd1 = foyer_fiscal('cd1', period) + cd2 = foyer_fiscal('cd2', period) + charge_loyer = foyer_fiscal('charge_loyer', period) + + return cd1 + cd2 + charge_loyer + + +class charge_loyer(Variable): + value_type = float + entity = FoyerFiscal + label = "Charge déductible pour paiement d'un loyer" + definition_period = YEAR + + def formula(foyer_fiscal, period, parameters): + nbptr = foyer_fiscal('nbptr', period) + + loyer = foyer_fiscal.declarant_principal.menage('loyer', period, options = [ADD]) + + charge_loyer = parameters(period).charge_loyer + + plaf = charge_loyer.plaf + plaf_nbp = charge_loyer.plaf_nbp + plafond = plaf * (not_(plaf_nbp) + plaf * nbptr * plaf_nbp) + + return 12 * min_(loyer / 12, plafond) + + +class trannoy_wasmer(Reform): + name = 'Loyer comme charge déductible (Trannoy-Wasmer)' + + def apply(self): + self.update_variable(charges_deduc) + self.add_variable(charge_loyer) + self.modify_parameters(modifier_function = modify_parameters) diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index aad235f9..c583c440 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -61,7 +61,7 @@ def test_erfs_fpr_aggregates_reform(): """Tests aggregates value with data. :param year: year of data and simulation to test agregates - :param reform: optional argument, put an openfisca_france.refoms object, default None + :param reform: optional argument, put an openfisca_france_data.refoms.old_openfisca_france_reforms object, default None """ tax_benefit_system = france_data_tax_benefit_system year = 2014 diff --git a/tests/erfs_fpr/unit/test_get_survey_scenario.py b/tests/erfs_fpr/unit/test_get_survey_scenario.py index 262cc33c..f1576f39 100644 --- a/tests/erfs_fpr/unit/test_get_survey_scenario.py +++ b/tests/erfs_fpr/unit/test_get_survey_scenario.py @@ -3,7 +3,7 @@ from openfisca_core.reforms import Reform # type: ignore from openfisca_france import FranceTaxBenefitSystem as TaxBenefitSystem # type: ignore -from openfisca_france.reforms.plf2015 import plf2015 # type: ignore +from openfisca_france_data.reforms.old_openfisca_france_reforms.plf2015 import plf2015 # type: ignore from openfisca_france_data import france_data_tax_benefit_system from openfisca_france_data.erfs_fpr.get_survey_scenario import ( diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 9e15db3d..47694fcf 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -7,7 +7,7 @@ from openfisca_core.tools import assert_near # type: ignore from openfisca_france_data.erfs.scenario import ErfsSurveyScenario # type: ignore from openfisca_survey_manager.calibration import Calibration # type: ignore -from openfisca_france.reforms.plf2015 import plf2015 # type: ignore +from openfisca_france_data.reforms.old_openfisca_france_reforms.plf2015 import plf2015 # type: ignore @pytest.fixture From a40a8dbcd017ae78625ef65ba18cfaa469b57a9c Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 11:38:40 +0200 Subject: [PATCH 093/140] fix imports --- .../allocations_familiales_imposables.py | 2 +- .../reforms/old_openfisca_france_reforms/cesthra_invalidee.py | 2 +- .../reforms/old_openfisca_france_reforms/de_net_a_brut.py | 4 ++-- .../reforms/old_openfisca_france_reforms/plf2015.py | 2 +- .../old_openfisca_france_reforms/plf2016_ayrault_muet.py | 2 +- .../reforms/old_openfisca_france_reforms/plfr2014.py | 2 +- .../reforms/old_openfisca_france_reforms/trannoy_wasmer.py | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py index cce1f5fb..f7ee0764 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/allocations_familiales_imposables.py @@ -1,6 +1,6 @@ import os -from ..model.base import * +from openfisca_france.model.base import * dir_path = os.path.join(os.path.dirname(__file__), 'parameters') diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py index 9b4472f9..66a5ecea 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py @@ -2,7 +2,7 @@ from openfisca_france.model.base import * -from .. import entities +from openfisca_france import entities dir_path = os.path.join(os.path.dirname(__file__), 'parameters') diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py index b6d7cb61..b5fb1773 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/de_net_a_brut.py @@ -5,8 +5,8 @@ except ImportError: fsolve = None -from .. import entities -from ..model.base import * +from openfisca_france import entities +from openfisca_france.model.base import * def calculate_net_from(salaire_de_base, individu, period): diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py index 72d9ce4f..08b76f86 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py @@ -1,6 +1,6 @@ import os -from ..model.base import * +from openfisca_france.model.base import * dir_path = os.path.join(os.path.dirname(__file__), 'parameters') diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py index 3cc73c4c..a12f46bb 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2016_ayrault_muet.py @@ -1,6 +1,6 @@ import os -from ..model.base import * +from openfisca_france.model.base import * dir_path = os.path.dirname(__file__) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py index 2396c18f..7d0996cd 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plfr2014.py @@ -1,6 +1,6 @@ import os -from ..model.base import * +from openfisca_france.model.base import * dir_path = os.path.join(os.path.dirname(__file__), 'parameters') diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py index 5b5c6392..27150aa7 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/trannoy_wasmer.py @@ -1,6 +1,6 @@ import os -from ..model.base import * +from openfisca_france.model.base import * dir_path = os.path.join(os.path.dirname(__file__), 'parameters') From f8a0bd7c495b1391c4f13cae7b41f1906afecf53 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 11:46:54 +0200 Subject: [PATCH 094/140] bump --- CHANGELOG.md | 6 +++++- setup.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e0a51c5f..124f350d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -### 2.0.5 [#238(https://github.com/openfisca/openfisca-france-data/pull/238) +### 2.0.6 [#240](https://github.com/openfisca/openfisca-france-data/pull/240) +* Technical changes +- Rapatrie certaines réformes qui ont été supprimés dans openfisca france par la (PR 2177)[https://github.com/Supprime vieilles réformes non utilisées openfisca-france#2177] mais qui sont encore utilisées dans ce dépôt + +### 2.0.5 [#238](https://github.com/openfisca/openfisca-france-data/pull/238) * Technical changes - Corrige le calcul du smic annuel en fonction du smic horaire dans openfisca_france_data/erfs_fpr/input_data_builder/step_04_famille.py diff --git a/setup.py b/setup.py index 9543aa55..6ab34d3a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.5", + version = "2.0.6", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 3a09a2a31ae8f7c76ed8db902f6ee9c5a781d0e3 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 12:13:03 +0200 Subject: [PATCH 095/140] wip --- .../reforms/old_openfisca_france_reforms/plf2015.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py index 08b76f86..b0253db1 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py @@ -10,6 +10,7 @@ def modify_parameters(parameters): reform_year = 2013 reform_period = period(reform_year) + print(dir_path) file_path = os.path.join(dir_path, 'plf2015.yaml') reform_parameters_subtree = load_parameter_file(name='plf2015', file_path=file_path) parameters.add_child('plf2015', reform_parameters_subtree) From cf8bfcc723cdf6712e4f982a4c4085a63c7c345b Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 14:00:27 +0200 Subject: [PATCH 096/140] add init --- .../reforms/old_openfisca_france_reforms/parameters/__init__.py | 0 .../reforms/old_openfisca_france_reforms/plf2015.py | 1 - 2 files changed, 1 deletion(-) create mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py index b0253db1..08b76f86 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/plf2015.py @@ -10,7 +10,6 @@ def modify_parameters(parameters): reform_year = 2013 reform_period = period(reform_year) - print(dir_path) file_path = os.path.join(dir_path, 'plf2015.yaml') reform_parameters_subtree = load_parameter_file(name='plf2015', file_path=file_path) parameters.add_child('plf2015', reform_parameters_subtree) From 6efd91afec072229b9adc1d3f68717f37b4c6475 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 8 Sep 2023 14:07:27 +0200 Subject: [PATCH 097/140] Add yaml files in setup --- .../old_openfisca_france_reforms/parameters/__init__.py | 0 setup.py | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/parameters/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/setup.py b/setup.py index 6ab34d3a..cc7fee5a 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,8 @@ ], package_data = { 'openfisca_france_data': ['assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx', - 'assets/aggregats/ines/ines_2019.json'], + 'assets/aggregats/ines/ines_2019.json', + 'reforms/old_openfisca_france_reforms/parameters/*.yaml'], }, entry_points = { 'console_scripts': [ From d029238f4b1580df28d2b2d317c296a824889bdc Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 1 Sep 2023 16:29:56 +0200 Subject: [PATCH 098/140] =?UTF-8?q?Ajoute=20des=20agr=C3=A9gats=20pour=20l?= =?UTF-8?q?a=20France=20enti=C3=A8re=20pour=20les=20ann=C3=A9es=20r=C3=A9c?= =?UTF-8?q?entes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../france_entiere_2019.json" | 179 +++++++++++++++++ .../france_entiere_2020.json" | 179 +++++++++++++++++ .../france_entiere_2021.json" | 176 +++++++++++++++++ .../france_entiere_2022.json" | 179 +++++++++++++++++ .../france_entiere_2023.json" | 173 ++++++++++++++++ .../france_entiere_template.json" | 184 ++++++++++++++++++ .../assets/aggregats/ines/ines_2019.json | 6 +- .../assets/aggregats/ines/ines_2020.json | 158 +++++++++++++++ 8 files changed, 1231 insertions(+), 3 deletions(-) create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" create mode 100644 "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" create mode 100644 openfisca_france_data/assets/aggregats/ines/ines_2020.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" new file mode 100644 index 00000000..f8d7fb14 --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" @@ -0,0 +1,179 @@ +{ + "documentation":"", + "data":[ + { + "variable":"salaire_imposable", + "actual_beneficiaries": 24918000, + "actual_amount": 743500000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)", + "note": "salaire et chomage" + }, + { + "variable":"chomage_imposable", + "actual_beneficiaries": 24918000, + "actual_amount": 743500000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)", + "note": "salaire et chomage" + }, + { + "variable":"retraite_imposable", + "actual_beneficiaries": 14768000, + "actual_amount": 327344000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"rpns_imposables", + "actual_beneficiaries": 3189000, + "actual_amount": 77837000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_foncier", + "actual_beneficiaries": 5019000, + "actual_amount": 28242000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_capital", + "actual_beneficiaries": 15235000, + "actual_amount": 356330000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_plus_values", + "actual_beneficiaries": 1600000, + "actual_amount": 18812000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"af", + "actual_beneficiaries": 5101000, + "actual_amount": 12719000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 273000, + "actual_amount": 922000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 321000, + "actual_amount": 1060000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"asf", + "actual_beneficiaries": 803000, + "actual_amount": 1771000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 1619000, + "actual_amount": 3374000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 46400000, + "actual_amount": 553000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"cf", + "actual_beneficiaries": 871000, + "actual_amount":2331000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"ars", + "actual_beneficiaries": 3132000, + "actual_amount": 2034000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 910000, + "actual_amount":6329000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 0, + "actual_amount": 16270006022, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2020, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/2020_Rapport%20du%20Dcf%20sur%20les%20comptes.pdf)" + }, + { + "variable":"apl", + "actual_beneficiaries": 7385478511, + "actual_amount": 16270006022, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2020, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/2020_Rapport%20du%20Dcf%20sur%20les%20comptes.pdf)" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount":10370937843, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2020, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/2020_Rapport%20du%20Dcf%20sur%20les%20comptes.pdf)" + }, + { + "variable":"aspa", + "actual_beneficiaries":0, + "actual_amount": 0 + }, + { + "variable":"asi", + "actual_beneficiaries":0, + "actual_amount":0 + }, + { + "variable":"rsa", + "actual_beneficiaries": 0, + "actual_amount":10927045125, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2020, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/2020_Rapport%20du%20Dcf%20sur%20les%20comptes.pdf)" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 9374056880, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2020, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/2020_Rapport%20du%20Dcf%20sur%20les%20comptes.pdf)" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 75706000000, + "source": "Comptes de la sécurité sociale, résultats 2019 et prévision 2020 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2020/RAPPORT%20CCSS%20JUIN%202020.pdf)" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 14098000000, + "source": "Comptes de la sécurité sociale, résultats 2019 et prévision 2020 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2020/RAPPORT%20CCSS%20JUIN%202020.pdf)" + }, + { + "variable":"csg_retraite", + "actual_beneficiaries":0, + "actual_amount":20799000000, + "source": "Comptes de la sécurité sociale, résultats 2019 et prévision 2020 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2020/RAPPORT%20CCSS%20JUIN%202020.pdf)" + }, + { + "variable":"csg_chomage", + "actual_beneficiaries":0, + "actual_amount":577000000, + "source": "Comptes de la sécurité sociale, résultats 2019 et prévision 2020 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2020/RAPPORT%20CCSS%20JUIN%202020.pdf)" + }, + { + "variable":"irpp", + "actual_beneficiaries":39332000, + "actual_amount":76911000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"ifi", + "actual_beneficiaries":0, + "actual_amount":0 + } + ] +} \ No newline at end of file diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" new file mode 100644 index 00000000..619edb0c --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" @@ -0,0 +1,179 @@ +{ + "documentation":"Pour les prestations sociales, les cotisations et prélèvements sociaux, note de validation INES 2019, pour l'ir note de validation 2019 et 2020", + "data":[ + { + "variable":"salaire_imposable", + "actual_beneficiaries": 25236000, + "actual_amount": 758742000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)", + "note": "salaire et chomage" + }, + { + "variable":"chomage_imposable", + "actual_beneficiaries": 25236000, + "actual_amount": 758742000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)", + "note": "salaire et chomage" + }, + { + "variable":"retraite_imposable", + "actual_beneficiaries": 14862000, + "actual_amount": 335220000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"rpns_imposables", + "actual_beneficiaries": 3275000, + "actual_amount": 75284000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_foncier", + "actual_beneficiaries": 5045000, + "actual_amount": 27152000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_capital", + "actual_beneficiaries": 14701000, + "actual_amount": 34590000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_plus_values", + "actual_beneficiaries": 1600000, + "actual_amount": 16559000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"af", + "actual_beneficiaries": 5091000, + "actual_amount": 12719000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 255000, + "actual_amount": 868000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 346000, + "actual_amount": 1162000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"asf", + "actual_beneficiaries": 813000, + "actual_amount": 1794000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 1555000, + "actual_amount": 3140000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 44900, + "actual_amount": 542000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"cf", + "actual_beneficiaries": 872000, + "actual_amount":2349000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"ars", + "actual_beneficiaries": 3132000, + "actual_amount": 2576000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 845000, + "actual_amount":6381000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 16320024194, + "actual_amount": 0, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"apl", + "actual_beneficiaries": 7120293737, + "actual_amount": 0, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount":11093058684, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"aspa", + "actual_beneficiaries":635000, + "actual_amount": 3530000000, + "source":"CCSS éclairage Juillet 2022 L'ASPA et le minimum vieillesse (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2022/Fiches%20%c3%a9clairages%20-%20Juillet%202022/CCSS-Fiche%20eclairage-Juillet%202022-L%e2%80%99allocation%20de%20solidarit%c3%a9%20aux%20personnes%20%c3%a2g%c3%a9es%20(Aspa)%20et%20le%20minimum%20vieillesse.pdf)" + }, + { + "variable":"asi", + "actual_beneficiaries":0, + "actual_amount":0 + }, + { + "variable":"rsa", + "actual_beneficiaries":11205404082, + "actual_amount":0, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 9592354920, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 72628000000, + "source": "Comptes de la sécurité sociale, résultats 2020, prévisions 2021 et 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2021/Rapport%20CCSS-Septembre2021.pdf)" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 11771000000, + "source": "Comptes de la sécurité sociale, résultats 2020, prévisions 2021 et 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2021/Rapport%20CCSS-Septembre2021.pdf)" + }, + { + "variable":"csg_retraite", + "actual_beneficiaries":0, + "actual_amount":20944000000, + "source": "Comptes de la sécurité sociale, résultats 2020, prévisions 2021 et 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2021/Rapport%20CCSS-Septembre2021.pdf)" + }, + { + "variable":"csg_chomage", + "actual_beneficiaries":0, + "actual_amount":1623000000, + "source": "Comptes de la sécurité sociale, résultats 2020, prévisions 2021 et 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2021/Rapport%20CCSS-Septembre2021.pdf)" + }, + { + "variable":"irpp", + "actual_beneficiaries":39888000, + "actual_amount":73979000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"ifi", + "actual_beneficiaries":0, + "actual_amount":0 + } + ] +} \ No newline at end of file diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" new file mode 100644 index 00000000..4440b74a --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" @@ -0,0 +1,176 @@ +{ + "documentation":"Pour les prestations sociales, les cotisations et prélèvements sociaux, note de validation INES 2019, pour l'ir note de validation 2019 et 2020", + "data":[ + { + "variable":"salaire_imposable", + "actual_beneficiaries": 23622000, + "actual_amount": 685645000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"chomage_imposable", + "actual_beneficiaries": 5601000, + "actual_amount": 37876000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"retraite_imposable", + "actual_beneficiaries": 14967000, + "actual_amount": 341810000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"rpns_imposables", + "actual_beneficiaries": 3476000, + "actual_amount": 85747000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_foncier", + "actual_beneficiaries": 5045000, + "actual_amount": 30261000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_capital", + "actual_beneficiaries": 15511000, + "actual_amount": 45881000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"revenu_categoriel_plus_values", + "actual_beneficiaries": 216000, + "actual_amount": 216000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition des revenus catégoriels déclarés entre imposés et non-imposés (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"af", + "actual_beneficiaries": 0, + "actual_amount": 12660000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 0, + "actual_amount": 770000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 0, + "actual_amount": 1225000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"asf", + "actual_beneficiaries": 0, + "actual_amount": 174000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 0, + "actual_amount": 2986000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 0, + "actual_amount": 729000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"cf", + "actual_beneficiaries": 0, + "actual_amount":2361000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"ars", + "actual_beneficiaries": 0, + "actual_amount": 2047000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 0, + "actual_amount":5909000000, + "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 0, + "actual_amount": 15262994463, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount":11222712959, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"apl", + "actual_beneficiaries": 6743703175, + "actual_amount": 0, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"aspa", + "actual_beneficiaries":0, + "actual_amount": 0 + }, + { + "variable":"asi", + "actual_beneficiaries":0, + "actual_amount":0 + }, + { + "variable":"rsa", + "actual_beneficiaries": 0, + "actual_amount":10849396420, + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 9468691928 , + "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 78091000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 14527000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_retraite", + "actual_beneficiaries":0, + "actual_amount":21414000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_chomage", + "actual_beneficiaries":0, + "actual_amount":973000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"irpp", + "actual_beneficiaries":40257000, + "actual_amount":80826000000, + "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" + }, + { + "variable":"ifi", + "actual_beneficiaries":0, + "actual_amount":0 + } + ] +} \ No newline at end of file diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" new file mode 100644 index 00000000..9bf93e25 --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" @@ -0,0 +1,179 @@ +{ + "documentation":"", + "data":[ + { + "variable":"salaire_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"chomage_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"retraite_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"rpns_imposables", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_foncier", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_capital", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_plus_values", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"af", + "actual_beneficiaries": 0, + "actual_amount": 13038682622, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 0, + "actual_amount": 757550577, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 0, + "actual_amount": 1311533249, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"asf", + "actual_beneficiaries": 0, + "actual_amount": 2012315236, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 0, + "actual_amount": 3004670266, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 0, + "actual_amount": 535273024, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"cf", + "actual_beneficiaries": 0, + "actual_amount": 2401233483, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"ars", + "actual_beneficiaries": 0, + "actual_amount": 2131128426, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 0, + "actual_amount": 6756309799, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 0, + "actual_amount": 15075235808, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"apl", + "actual_beneficiaries": 0, + "actual_amount": 6693675898, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount": 11937472725, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"aspa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"asi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"rsa", + "actual_beneficiaries": 0, + "actual_amount": 9914084209, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 9655667966, + "source": "Le rapport du Directeur comptable et financier de la CNAF, exercice 2022(https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/Textes%20de%20r%C3%A9f%C3%A9rence/comptes%20et%20financements/Point%203%20Rapport%20du%20Dcf%20sur%20les%20comptes%202022.pdf)" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 84090000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 16102000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_retraite", + "actual_beneficiaries":0, + "actual_amount":22639000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_chomage", + "actual_beneficiaries":0, + "actual_amount":576000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"irpp", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ifi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + } + ] +} \ No newline at end of file diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" new file mode 100644 index 00000000..c66ec02a --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" @@ -0,0 +1,173 @@ +{ + "documentation":"", + "data":[ + { + "variable":"salaire_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"chomage_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"retraite_imposable", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"rpns_imposables", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_foncier", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_capital", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"revenu_categoriel_plus_values", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"af", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"asf", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"cf", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ars", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aspa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"asi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"rsa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 88313000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 16292000000, + "source": "Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_retraite", + "actual_beneficiaries":0, + "actual_amount":23600000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"csg_chomage", + "actual_beneficiaries":0, + "actual_amount":614000000, + "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" + }, + { + "variable":"irpp", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ifi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + } + ] +} \ No newline at end of file diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" new file mode 100644 index 00000000..d3e4da85 --- /dev/null +++ "b/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" @@ -0,0 +1,184 @@ +{ + "documentation":"", + "data":[ + { + "variable":"af", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aeeh", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"asf", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_base", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"cf", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ars", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aah", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"aspa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"asi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"garantie_jeunes", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"rsa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ppa", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"cotisations_salariales", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"cotisations_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"cotisations_employeur", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"allegement_fillon", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"csg_salaire", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"csg_remplacement", + "actual_beneficiaries":0, + "actual_amount":24017000000 + }, + { + "variable":"crds_salaire", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"crds_non_salarie", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"crds_remplacement", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"irpp", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"prelevement_forfaitaire_unique_ir", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"prelevement_forfaitaire_liberatoire", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + }, + { + "variable":"ifi", + "actual_beneficiaries": 0, + "actual_amount": 0, + "source": "" + } + ] +} \ No newline at end of file diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2019.json b/openfisca_france_data/assets/aggregats/ines/ines_2019.json index e4fcd8b7..18a87b06 100644 --- a/openfisca_france_data/assets/aggregats/ines/ines_2019.json +++ b/openfisca_france_data/assets/aggregats/ines/ines_2019.json @@ -18,7 +18,7 @@ }, { "variable":"asf", - "actual_beneficiaries": 700000, + "actual_beneficiaries": 900000, "actual_amount": 1500000000 }, { @@ -43,7 +43,7 @@ }, { "variable":"paje_cmg", - "actual_beneficiaries": 1000000, + "actual_beneficiaries": 1200000, "actual_amount":6200000000 }, { @@ -54,7 +54,7 @@ { "variable":"aah", "actual_beneficiaries": 1100000, - "actual_amount":10400000000 + "actual_amount":8600000000 }, { "variable":"aspa", diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2020.json b/openfisca_france_data/assets/aggregats/ines/ines_2020.json new file mode 100644 index 00000000..b18f7d82 --- /dev/null +++ b/openfisca_france_data/assets/aggregats/ines/ines_2020.json @@ -0,0 +1,158 @@ +{ + "documentation":"Pour les prestations sociales, les cotisations et prélèvements sociaux, note de validation INES 2020", + "data":[ + { + "variable":"af", + "actual_beneficiaries": 5100000, + "actual_amount": 12000000000 + }, + { + "variable":"paje_prepare", + "actual_beneficiaries": 400000, + "actual_amount": 80000000 + }, + { + "variable":"aeeh", + "actual_beneficiaries": 400000, + "actual_amount": 1000000000 + }, + { + "variable":"asf", + "actual_beneficiaries": 900000, + "actual_amount": 1500000000 + }, + { + "variable":"paje_base", + "actual_beneficiaries": 1900000, + "actual_amount": 2900000000 + }, + { + "variable":"paje_naissance", + "actual_beneficiaries": 500000, + "actual_amount": 500000 + }, + { + "variable":"cf", + "actual_beneficiaries": 1000000, + "actual_amount":2200000000 + }, + { + "variable":"ars", + "actual_beneficiaries": 2900000, + "actual_amount": 2400000000 + }, + { + "variable":"paje_cmg", + "actual_beneficiaries": 1100000, + "actual_amount":5800000000 + }, + { + "variable":"aides_logement", + "actual_beneficiaries": 6700000, + "actual_amount": 14000000000 + }, + { + "variable":"aah", + "actual_beneficiaries": 1100000, + "actual_amount":9200000000 + }, + { + "variable":"aspa", + "actual_beneficiaries":500000, + "actual_amount": 2800000000 + }, + { + "variable":"asi", + "actual_beneficiaries":100000, + "actual_amount":200000000 + }, + { + "variable":"garantie_jeunes", + "actual_beneficiaries": 200000, + "actual_amount":400000000 + }, + { + "variable":"rsa", + "actual_beneficiaries": 2000000, + "actual_amount":8600000000 + }, + { + "variable":"ppa", + "actual_beneficiaries": 5800000, + "actual_amount": 9200000000 + }, + { + "variable":"cotisations_salariales", + "actual_beneficiaries":0, + "actual_amount": 75600000000 + }, + { + "variable":"cotisations_non_salarie", + "actual_beneficiaries":0, + "actual_amount":19600000000 + }, + { + "variable":"cotisations_employeur", + "actual_beneficiaries":0, + "actual_amount":215000000000 + }, + { + "variable":"allegement_fillon", + "actual_beneficiaries":0, + "actual_amount":24700000000 + }, + { + "variable":"csg_salaire", + "actual_beneficiaries":0, + "actual_amount":72100000000 + }, + { + "variable":"csg_non_salarie", + "actual_beneficiaries":0, + "actual_amount":8700000000 + }, + { + "variable":"csg_remplacement", + "actual_beneficiaries":0, + "actual_amount":22600000000 + }, + { + "variable":"crds_salaire", + "actual_beneficiaries":0, + "actual_amount":3900000000 + }, + { + "variable":"crds_non_salarie", + "actual_beneficiaries":0, + "actual_amount":500000000 + }, + { + "variable":"crds_remplacement", + "actual_beneficiaries":0, + "actual_amount":19000000000 + }, + { + "variable":"irpp", + "actual_beneficiaries":0, + "actual_amount":0, + "notes":"somme de prélèvement à la source réalisés en 2019 sur revenus 2019 (note 2019) 69,2 + impot résiduel sur le revenu dû au titre des revenus 2019, net de crédits et réduction (note 2020) -8.1 sans prendre en compte le crédit d'impôt de prélèvement forfaitaire non libératoire de N-1 de 3,4 soit 3.4 - 8.1 + 69.2 " + }, + { + "variable":"prelevement_forfaitaire_unique_ir", + "actual_beneficiaries":0, + "actual_amount":0, + "notes":"prélèvement forfaitaire unique libératoire sur revenus 2019 (note 2020)" + }, + { + "variable":"prelevement_forfaitaire_liberatoire", + "actual_beneficiaries":0, + "actual_amount":0, + "notes":"prélèvement forfaitaire libératoire sur revenus 2019 (note 2019)" + }, + { + "variable":"ifi", + "actual_beneficiaries":0, + "actual_amount":0 + } + ] +} \ No newline at end of file From be0e129af7064f144c10ddfa5ca83ef05a8a4d64 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 1 Sep 2023 16:44:41 +0200 Subject: [PATCH 099/140] renomme dossier --- .../assets/aggregats/france_entiere/france_entiere_2019.json | 0 .../assets/aggregats/france_entiere/france_entiere_2020.json | 0 .../assets/aggregats/france_entiere/france_entiere_2021.json | 0 .../assets/aggregats/france_entiere/france_entiere_2022.json | 0 .../assets/aggregats/france_entiere/france_entiere_2023.json | 0 .../assets/aggregats/france_entiere/france_entiere_template.json | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json (100%) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json (100%) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json (100%) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json (100%) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json (100%) rename "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" => openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json (100%) diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2019.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2020.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2021.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2022.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_2023.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json diff --git "a/openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json similarity index 100% rename from "openfisca_france_data/assets/aggregats/france_enti\303\250re/france_entiere_template.json" rename to openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json From da06ff3b297d88cc1368c44a1b8d5f481590ecff Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 1 Sep 2023 16:44:54 +0200 Subject: [PATCH 100/140] ajoute france_enitre dans aggregates --- openfisca_france_data/aggregates.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 3d0bb7e8..dcfca474 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -107,4 +107,23 @@ def load_actual_data(self, year = None): result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") + elif target_source == "france_entiere": + ines_aggregates_file = Path( + pkg_resources.get_distribution("openfisca-france_data").location, + "openfisca_france_data", + "assets", + "aggregats", + "france_entiere", + f"france_entiere_{year}.json" + ) + + with open(ines_aggregates_file, 'r') as f: + data = json.load(f) + + result = pd.DataFrame(data['data']).drop(['source', 'notes'], axis = 1) + result['actual_beneficiaries'] = result. actual_beneficiaries / self.beneficiaries_unit + result['actual_amount'] = result. actual_amount / self.amount_unit + + result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") + return result From d6969732a4c5d9c77771c72aff1f9537bb796761 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 1 Sep 2023 17:48:19 +0200 Subject: [PATCH 101/140] Ajoute des variables de csg dans model/common --- openfisca_france_data/model/common.py | 20 ++++++++++++++++++++ setup.py | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index f9730a7f..37516265 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -96,6 +96,26 @@ def formula(individu, period): + individu('csg_imposable_chomage', period) + individu('csg_deductible_chomage', period) ) + +class csg_retraite(Variable): + value_type = float + entity = Individu + label = 'Contribution sociale généralisée sur les non salariés' + definition_period = YEAR + def formula(individu, period): + csg_imposable_retraite = individu('csg_imposable_retraite', period, options = [ADD]) + csg_deductible_retraite = individu('csg_deductible_retraite', period, options = [ADD]) + return csg_deductible_retraite + csg_imposable_retraite + +class csg_chomage(Variable): + value_type = float + entity = Individu + label = 'Contribution sociale généralisée sur les non salariés' + definition_period = YEAR + def formula(individu, period): + csg_imposable_chomage = individu('csg_imposable_chomage', period, options = [ADD]) + csg_deductible_chomage = individu('csg_deductible_chomage', period, options = [ADD]) + return csg_deductible_chomage + csg_imposable_chomage class impot_revenu(Variable): diff --git a/setup.py b/setup.py index cc7fee5a..1bd7c282 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,9 @@ package_data = { 'openfisca_france_data': ['assets/aggregats/taxipp/agregats_tests_taxipp_2_0.xlsx', 'assets/aggregats/ines/ines_2019.json', - 'reforms/old_openfisca_france_reforms/parameters/*.yaml'], + 'reforms/old_openfisca_france_reforms/parameters/*.yaml', + 'assets/aggregats/ines/ines_2020.json', + 'assets/aggregats/france_entiere/*.json'], }, entry_points = { 'console_scripts': [ From 121daa002382752ebc5ce0966b409bed1045f9a3 Mon Sep 17 00:00:00 2001 From: cgl Date: Mon, 4 Sep 2023 09:34:14 +0200 Subject: [PATCH 102/140] fix --- openfisca_france_data/aggregates.py | 28 +++++++++++++++++-- .../france_entiere/france_entiere_2021.json | 4 +-- openfisca_france_data/model/common.py | 4 +-- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index dcfca474..ae47df5d 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -6,7 +6,8 @@ import numpy as np import pandas as pd import pkg_resources - +import os +from datetime import datetime from openfisca_survey_manager.aggregates import AbstractAggregates from openfisca_france_data import AGGREGATES_DEFAULT_VARS # type: ignore @@ -39,7 +40,7 @@ def __init__(self, survey_scenario = None, target_source = None): def load_actual_data(self, year = None): target_source = self.target_source - assert target_source in ["ines", "taxipp"], "les options possible pour source_cible sont ines ou taxipp" + assert target_source in ["ines", "taxipp", "france_entiere"], "les options possible pour source_cible sont ines, taxipp ou france_entiere" assert year is not None if target_source == "taxipp": @@ -120,10 +121,31 @@ def load_actual_data(self, year = None): with open(ines_aggregates_file, 'r') as f: data = json.load(f) - result = pd.DataFrame(data['data']).drop(['source', 'notes'], axis = 1) + result = pd.DataFrame(data['data']).drop(['source'], axis = 1) result['actual_beneficiaries'] = result. actual_beneficiaries / self.beneficiaries_unit result['actual_amount'] = result. actual_amount / self.amount_unit result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") return result + + def to_csv(self, path = None, absolute = True, amount = True, beneficiaries = True, default = 'actual', + relative = True, target = "reform"): + """Saves the table to csv.""" + assert path is not None + + if os.path.isdir(path): + now = datetime.now() + file_path = os.path.join(path, 'Aggregates_%s_%s_%s.%s' % (self.target_source,self.year,now.strftime('%d-%m-%Y'), "csv")) + else: + file_path = path + + df = self.get_data_frame( + absolute = absolute, + amount = amount, + beneficiaries = beneficiaries, + default = default, + relative = relative, + target = target, + ) + df.to_csv(file_path, index = False, header = True) diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json index 4440b74a..06411f68 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json @@ -111,8 +111,8 @@ }, { "variable":"apl", - "actual_beneficiaries": 6743703175, - "actual_amount": 0, + "actual_beneficiaries": 0, + "actual_amount": 6743703175, "source": "Rapport du Directeur comptable et financier sur les comptes de l'exercice 2021, CAF (https://www.caf.fr/sites/default/files/medias/cnaf/Nous_connaitre/qui%20sommes%20nous/presentation/Comptes-et-financements/2021/Rapport_Dcf_comptes_exercice2021.pdf)" }, { diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 37516265..13049ba8 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -100,7 +100,7 @@ def formula(individu, period): class csg_retraite(Variable): value_type = float entity = Individu - label = 'Contribution sociale généralisée sur les non salariés' + label = 'CSG sur les retraites' definition_period = YEAR def formula(individu, period): csg_imposable_retraite = individu('csg_imposable_retraite', period, options = [ADD]) @@ -110,7 +110,7 @@ def formula(individu, period): class csg_chomage(Variable): value_type = float entity = Individu - label = 'Contribution sociale généralisée sur les non salariés' + label = 'CSG sur le chomage' definition_period = YEAR def formula(individu, period): csg_imposable_chomage = individu('csg_imposable_chomage', period, options = [ADD]) From b4520c86df09dbc199eabd9b79b71c3978d2ab8c Mon Sep 17 00:00:00 2001 From: cgl Date: Mon, 4 Sep 2023 14:38:25 +0200 Subject: [PATCH 103/140] Corrige des agregats --- .../aggregats/france_entiere/france_entiere_2019.json | 5 +++-- .../aggregats/france_entiere/france_entiere_2021.json | 7 ++++--- openfisca_france_data/assets/aggregats/ines/ines_2019.json | 6 +++--- openfisca_france_data/assets/aggregats/ines/ines_2020.json | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json index f8d7fb14..55d37846 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json @@ -120,8 +120,9 @@ }, { "variable":"aspa", - "actual_beneficiaries":0, - "actual_amount": 0 + "actual_beneficiaries":600000, + "actual_amount": 3500000000, + "source": "Panorama DREES les retraites et les retraités, édition 2021" }, { "variable":"asi", diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json index 06411f68..2d67b16a 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json @@ -64,7 +64,7 @@ { "variable":"asf", "actual_beneficiaries": 0, - "actual_amount": 174000000, + "actual_amount": 1740000000, "source": "Dossier statistique des prestations familiales de la DSS, 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2022/Cahier_stat_Prestations_familiales_2022.pdf)" }, { @@ -117,8 +117,9 @@ }, { "variable":"aspa", - "actual_beneficiaries":0, - "actual_amount": 0 + "actual_beneficiaries":660000, + "actual_amount": 3600000000, + "source": "Panorama DREES les retraites et les retraités, édition 2023" }, { "variable":"asi", diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2019.json b/openfisca_france_data/assets/aggregats/ines/ines_2019.json index 18a87b06..a7273d98 100644 --- a/openfisca_france_data/assets/aggregats/ines/ines_2019.json +++ b/openfisca_france_data/assets/aggregats/ines/ines_2019.json @@ -1,5 +1,5 @@ { - "documentation":"Pour les prestations sociales, les cotisations et prélèvements sociaux, note de validation INES 2019, pour l'ir note de validation 2019 et 2020", + "documentation":"Note de validation INES 2019", "data":[ { "variable":"af", @@ -134,8 +134,8 @@ { "variable":"irpp", "actual_beneficiaries":0, - "actual_amount":64500000000, - "notes":"somme de prélèvement à la source réalisés en 2019 sur revenus 2019 (note 2019) 69,2 + impot résiduel sur le revenu dû au titre des revenus 2019, net de crédits et réduction (note 2020) -8.1 sans prendre en compte le crédit d'impôt de prélèvement forfaitaire non libératoire de N-1 de 3,4 soit 3.4 - 8.1 + 69.2 " + "actual_amount":69400000000, + "notes":"Impot payé en 2019, en parti sur revenus 2019 et 2018" }, { "variable":"prelevement_forfaitaire_unique_ir", diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2020.json b/openfisca_france_data/assets/aggregats/ines/ines_2020.json index b18f7d82..ba52c2f4 100644 --- a/openfisca_france_data/assets/aggregats/ines/ines_2020.json +++ b/openfisca_france_data/assets/aggregats/ines/ines_2020.json @@ -134,8 +134,8 @@ { "variable":"irpp", "actual_beneficiaries":0, - "actual_amount":0, - "notes":"somme de prélèvement à la source réalisés en 2019 sur revenus 2019 (note 2019) 69,2 + impot résiduel sur le revenu dû au titre des revenus 2019, net de crédits et réduction (note 2020) -8.1 sans prendre en compte le crédit d'impôt de prélèvement forfaitaire non libératoire de N-1 de 3,4 soit 3.4 - 8.1 + 69.2 " + "actual_amount":73900000000, + "notes":"Impot payé en 2020, en parti sur revenus 2020 et 2019" }, { "variable":"prelevement_forfaitaire_unique_ir", From 70bde7b9e3d652f93901e3a07bb09b36ea92e163 Mon Sep 17 00:00:00 2001 From: cgl Date: Mon, 4 Sep 2023 14:47:39 +0200 Subject: [PATCH 104/140] Bump --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 124f350d..8fa3c413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 2.0.7 [#239](https://github.com/openfisca/openfisca-france-data/pull/239/files) +* New features +- Ajoute des nouveaux agrégats pour FranceAggregates + ### 2.0.6 [#240](https://github.com/openfisca/openfisca-france-data/pull/240) * Technical changes - Rapatrie certaines réformes qui ont été supprimés dans openfisca france par la (PR 2177)[https://github.com/Supprime vieilles réformes non utilisées openfisca-france#2177] mais qui sont encore utilisées dans ce dépôt diff --git a/setup.py b/setup.py index 1bd7c282..47c9ea66 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.6", + version = "2.0.7", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 275eb4cfe815d732f79dd45fa33bd133241504c7 Mon Sep 17 00:00:00 2001 From: cgl Date: Wed, 6 Sep 2023 09:36:25 +0200 Subject: [PATCH 105/140] =?UTF-8?q?Ajoute=20les=20agr=C3=A9gats=20france?= =?UTF-8?q?=5Fentiere=20dans=20la=20ci?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/erfs_fpr/integration/test_aggregates.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/erfs_fpr/integration/test_aggregates.py b/tests/erfs_fpr/integration/test_aggregates.py index c583c440..2b07ffd1 100644 --- a/tests/erfs_fpr/integration/test_aggregates.py +++ b/tests/erfs_fpr/integration/test_aggregates.py @@ -40,6 +40,7 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in ) aggregates_taxipp = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') aggregates_ines = Aggregates(survey_scenario = survey_scenario, target_source = 'ines') + aggregates_france_entiere = Aggregates(survey_scenario = survey_scenario, target_source = 'france_entiere') if False: mtr_rd = survey_scenario.compute_marginal_tax_rate(target_variable = 'revenu_disponible', period = year, use_baseline = True) @@ -54,7 +55,7 @@ def test_erfs_fpr_survey_simulation_aggregates(year = REFERENCE_YEAR, rebuild_in np.quantile(mtr_rd, q = np.arange(0, 1.1, .1)) - return survey_scenario, aggregates_taxipp.get_data_frame(), aggregates_ines.get_data_frame() + return survey_scenario, aggregates_taxipp.get_data_frame(), aggregates_ines.get_data_frame(), aggregates_france_entiere.get_data_frame() def test_erfs_fpr_aggregates_reform(): @@ -112,7 +113,7 @@ def main(year, configfile = None, verbose = False): years = [year] for year in years: - survey_scenario, aggregates_taxipp, aggregates_ines = test_erfs_fpr_survey_simulation_aggregates( + survey_scenario, aggregates_taxipp, aggregates_ines, aggregates_france_entiere = test_erfs_fpr_survey_simulation_aggregates( year = year, rebuild_input_data = False, use_marginal_tax_rate = True, @@ -121,8 +122,9 @@ def main(year, configfile = None, verbose = False): ) df = pd.concat({ - "taxipp": aggregates_taxipp, "ines": aggregates_ines, + "france_entiere": aggregates_france_entiere, + "taxipp": aggregates_taxipp }) df.to_csv(f'aggregates_erfs_fpr_{year}.csv') From e0849648a2577033db2483c391971a2bf54e2eed Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 21 Jul 2023 18:24:53 +0200 Subject: [PATCH 106/140] Remove pkg_resources --- openfisca_france_data/__init__.py | 10 ++++++++-- openfisca_france_data/aggregates.py | 11 ++++------- .../erfs/input_data_builder/step_08_final.py | 4 ++-- openfisca_france_data/erfs/old/datatable.py | 6 +++--- tests/test_calibration.py | 7 ++++--- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 81ef2499..969e5a53 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -1,8 +1,9 @@ import inspect +import importlib import logging import os -import pkg_resources import pandas +from pathlib import Path from openfisca_core import reforms # type: ignore @@ -13,6 +14,11 @@ from openfisca_france_data.model.base import * # noqa analysis:ignore +openfisca_france_data_location = Path( + importlib.metadata.distribution('openfisca-survey-manager').files[0] + ).parent + + log = logging.getLogger(__name__) @@ -206,7 +212,7 @@ def apply(self): COUNTRY_DIR = os.path.dirname(os.path.abspath(__file__)) DATA_DIR = os.path.join( - pkg_resources.get_distribution('openfisca-france-data').location, + openfisca_france_data_location, 'openfisca_france_data', 'plugins', 'aggregates', diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index ae47df5d..9730239f 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -5,12 +5,9 @@ import numpy as np import pandas as pd -import pkg_resources -import os -from datetime import datetime from openfisca_survey_manager.aggregates import AbstractAggregates -from openfisca_france_data import AGGREGATES_DEFAULT_VARS # type: ignore +from openfisca_france_data import openfisca_france_data_location, AGGREGATES_DEFAULT_VARS # type: ignore log = logging.getLogger(__name__) @@ -45,7 +42,7 @@ def load_actual_data(self, year = None): if target_source == "taxipp": taxipp_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", @@ -91,7 +88,7 @@ def load_actual_data(self, year = None): elif target_source == "ines": ines_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", @@ -128,7 +125,7 @@ def load_actual_data(self, year = None): result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") return result - + def to_csv(self, path = None, absolute = True, amount = True, beneficiaries = True, default = 'actual', relative = True, target = "reform"): """Saves the table to csv.""" diff --git a/openfisca_france_data/erfs/input_data_builder/step_08_final.py b/openfisca_france_data/erfs/input_data_builder/step_08_final.py index 3ff828a7..4f7c160c 100644 --- a/openfisca_france_data/erfs/input_data_builder/step_08_final.py +++ b/openfisca_france_data/erfs/input_data_builder/step_08_final.py @@ -10,6 +10,7 @@ from openfisca_survey_manager.temporary import temporary_store_decorator +from openfisca_france_data import openfisca_france_data_location from openfisca_france_data.utils import ( check_structure, control, @@ -20,6 +21,7 @@ set_variables_default_value, ) + log = logging.getLogger(__name__) @@ -202,8 +204,6 @@ def final(temporary_store = None, year = None, check = True): print_id(final2) # # TODO: merging with patrimoine log.info(' traitement des zones apl') - import pkg_resources - openfisca_france_data_location = pkg_resources.get_distribution('openfisca-france-data').location zone_apl_imputation_data_file_path = os.path.join( openfisca_france_data_location, 'openfisca_france_data', diff --git a/openfisca_france_data/erfs/old/datatable.py b/openfisca_france_data/erfs/old/datatable.py index b2eac78b..3de1bee4 100644 --- a/openfisca_france_data/erfs/old/datatable.py +++ b/openfisca_france_data/erfs/old/datatable.py @@ -1,5 +1,4 @@ import os -import pkg_resources import sys import gc @@ -18,8 +17,9 @@ #from openfisca_france.data.sources.config import DATA_DIR -openfisca_france_location = pkg_resources.get_distribution('openfisca-france-data').location -CONFIG_DIR = os.path.join(openfisca_france_location) +from openfisca_france_data import openfisca_france_data_location + +CONFIG_DIR = os.path.join(openfisca_france_data_location) #ERF_HDF5_DATA_DIR = os.path.join(SRC_PATH,'countries','france','data', 'erf') diff --git a/tests/test_calibration.py b/tests/test_calibration.py index c56f0308..dee2ee60 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -1,14 +1,16 @@ -import pkg_resources import os import pytest from openfisca_survey_manager.calibration import Calibration # type: ignore +from openfisca_france_data import openfisca_france_data_location + + @pytest.fixture def location() -> str: - return pkg_resources.get_distribution("openfisca-france-data").location + return openfisca_france_data_location def test_calibration(survey_scenario, fake_input_data, location, year: int = 2009): @@ -34,4 +36,3 @@ def test_calibration(survey_scenario, fake_input_data, location, year: int = 200 pre_cal_weight = survey_scenario.calculate_variable("wprm", period = year) calibration.set_calibrated_weights() assert pre_cal_weight * 1.123 == survey_scenario.calculate_variable("wprm", period = year) - From 0782168b5e2e46c355e22bde0fb4de575549b759 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 21 Jul 2023 19:17:51 +0200 Subject: [PATCH 107/140] Fix importlib metadata import --- openfisca_france_data/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 969e5a53..035770e3 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -1,5 +1,5 @@ import inspect -import importlib +from importlib import metadata import logging import os import pandas @@ -15,7 +15,7 @@ openfisca_france_data_location = Path( - importlib.metadata.distribution('openfisca-survey-manager').files[0] + metadata.distribution('openfisca-survey-manager').files[0] ).parent From 07c027991abd8a09c78fe8faa1e352500d7a74e2 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 25 Jul 2023 12:09:53 +0200 Subject: [PATCH 108/140] Improve package location --- openfisca_france_data/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 035770e3..84388307 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -14,9 +14,7 @@ from openfisca_france_data.model.base import * # noqa analysis:ignore -openfisca_france_data_location = Path( - metadata.distribution('openfisca-survey-manager').files[0] - ).parent +openfisca_france_data_location = Path(__file__).parent.parent log = logging.getLogger(__name__) From 7b84173922fe445192937118e0f07fe51e6d4deb Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 25 Jul 2023 12:17:29 +0200 Subject: [PATCH 109/140] Upgrade survey-manager dep --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 47c9ea66..b45b2cac 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager >=1, <2.0.0", + "openFisca-survey-manager >= 1.1.5, < 2.0.0", "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { From 2c522897701dda04100abadd08452cc9a82eb1b8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 6 Jul 2023 11:03:19 +0200 Subject: [PATCH 110/140] Use specific survey-manager-branch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b45b2cac..fb5c0dac 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager >= 1.1.5, < 2.0.0", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { From 25b90c3939bc336d2e42d6638479f44abe77bec9 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 8 Aug 2023 18:01:51 +0200 Subject: [PATCH 111/140] Change percenile calculation --- openfisca_france_data/model/common.py | 6 +++--- openfisca_france_data/model/survey_variables.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 13049ba8..dad888f6 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -190,10 +190,10 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) + method = 2 + decile, values = mark_weighted_percentiles(niveau_de_vie, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) # Alternative method - # method = 2 - # decile, values = mark_weighted_percentiles(niveau_de_vie, labels, pondmen, method, return_quantiles = True) - decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) + # decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) return decile diff --git a/openfisca_france_data/model/survey_variables.py b/openfisca_france_data/model/survey_variables.py index e9cde28c..16ef928b 100644 --- a/openfisca_france_data/model/survey_variables.py +++ b/openfisca_france_data/model/survey_variables.py @@ -4,7 +4,7 @@ class menage_ordinaire(Variable): value_type = int is_period_size_independent = True - default_value = True + default_value = 1 entity = Menage definition_period = YEAR From 973d897aa232340af539ec613067a05364382fc8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 8 Aug 2023 18:06:54 +0200 Subject: [PATCH 112/140] Typo --- openfisca_france_data/model/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index dad888f6..e3591bd3 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -191,7 +191,7 @@ def formula(foyer_fiscal, period): menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) method = 2 - decile, values = mark_weighted_percentiles(niveau_de_vie, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) + decile, values = mark_weighted_percentiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) # Alternative method # decile, values = weighted_quantiles(rfr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) return decile From 01a80ccf27bee276eb1bc6f8d2a111f0c9b108af Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 10 Aug 2023 13:00:42 +0200 Subject: [PATCH 113/140] Remove wquantiles dep --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index fb5c0dac..4e15c3f4 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,6 @@ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", - "wquantiles >=0.3.0, <1.0.0", # To compute weighted quantiles ], extras_require = { "test": [ From 7b5d45dbf6427a75c5202d8d763e89ef65069158 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 9 Sep 2023 08:06:26 +0200 Subject: [PATCH 114/140] Use standard method to compute decile_rfr_par_part --- openfisca_france_data/model/common.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index e3591bd3..336aa83a 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -228,11 +228,9 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) - # Alternative method # method = 2 - # decile, values = mark_weighted_percentiles(niveau_de_vie, labels, pondmen, method, return_quantiles = True) - decile, values = weighted_quantiles( - rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, return_quantiles = True) + decile, values = mark_weighted_percentiles( + rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) return decile From e9458856ec92ea23e9b68f5a5ac62783ce34a57c Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Sat, 9 Sep 2023 08:48:59 +0200 Subject: [PATCH 115/140] Fix missing method for decile_rfr_par_part --- openfisca_france_data/model/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/model/common.py b/openfisca_france_data/model/common.py index 336aa83a..c798e5f8 100644 --- a/openfisca_france_data/model/common.py +++ b/openfisca_france_data/model/common.py @@ -228,7 +228,7 @@ def formula(foyer_fiscal, period): weight_foyers = foyer_fiscal('weight_foyers', period) menage_ordinaire_foyers_fiscaux = foyer_fiscal('menage_ordinaire_foyers_fiscaux', period) labels = arange(1, 11) - # method = 2 + method = 2 decile, values = mark_weighted_percentiles( rfr / nbptr, labels, weight_foyers * menage_ordinaire_foyers_fiscaux, method, return_quantiles = True) return decile From ab1d32a1fbe8346b65f34d84cbdcef23a70582b1 Mon Sep 17 00:00:00 2001 From: cgl Date: Sat, 9 Sep 2023 10:18:25 +0200 Subject: [PATCH 116/140] Fix update error in tax and benefits system reform --- openfisca_france_data/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 84388307..b0c9b6f6 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -6,6 +6,7 @@ from pathlib import Path from openfisca_core import reforms # type: ignore +from openfisca_core.errors import VariableNameConflictError import openfisca_france # type: ignore @@ -145,7 +146,8 @@ def apply(self): continue try: self.add_variable(variable) - except AttributeError: + except VariableNameConflictError: + log.warning(f"{variable.__name__} has been updated") self.update_variable(variable) From f6fc7a47f11363b75094e8546b3f87f7c74259bc Mon Sep 17 00:00:00 2001 From: cgl Date: Mon, 11 Sep 2023 11:05:31 +0200 Subject: [PATCH 117/140] corrige log --- openfisca_france_data/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index b0c9b6f6..43389933 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -147,7 +147,7 @@ def apply(self): try: self.add_variable(variable) except VariableNameConflictError: - log.warning(f"{variable.__name__} has been updated") + # log.debug(f"{variable.__name__} has been updated in openfisca-france-data") self.update_variable(variable) From 30c6509a98966030aa327090facc822120fc7bea Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 13 Oct 2023 09:50:32 +0200 Subject: [PATCH 118/140] =?UTF-8?q?mets=20=C3=A0=20jour=20la=20d=C3=A9pens?= =?UTF-8?q?e=20=C3=A0=20openfisca-survey-manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- openfisca_france_data/surveys.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index a595570d..e1853070 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -7,7 +7,7 @@ from openfisca_core.model_api import Enum # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore from openfisca_france_data import base_survey as base # type: ignore -from openfisca_survey_manager.scenarios import AbstractSurveyScenario # type: ignore +from openfisca_survey_manager.scenarios.abstract_scenario import AbstractSurveyScenario # type: ignore log = logging.getLogger(__name__) diff --git a/setup.py b/setup.py index 4e15c3f4..d0bc3d89 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@modularize_scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-sceanrio#egg=openfisca-survey-manager", ], extras_require = { "test": [ From a774ef6dca61c03034b79cd6ef3a9e6d42368314 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 13 Oct 2023 10:28:02 +0200 Subject: [PATCH 119/140] fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d0bc3d89..1f84143b 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-sceanrio#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { "test": [ From 40766271ea65ca76129f3148ffb333b25a2ec11f Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Fri, 13 Oct 2023 21:16:45 +0100 Subject: [PATCH 120/140] Adapt ErfsSurveyScenario --- openfisca_france_data/surveys.py | 20 +++++++++++++------- tests/test_calibration.py | 17 ++++++++--------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index e1853070..5d20955b 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -7,12 +7,13 @@ from openfisca_core.model_api import Enum # type: ignore from openfisca_core.taxbenefitsystems import TaxBenefitSystem # type: ignore from openfisca_france_data import base_survey as base # type: ignore -from openfisca_survey_manager.scenarios.abstract_scenario import AbstractSurveyScenario # type: ignore +from openfisca_survey_manager.scenarios.reform_scenario import ReformScenario # type: ignore + log = logging.getLogger(__name__) -class AbstractErfsSurveyScenario(AbstractSurveyScenario): +class AbstractErfsSurveyScenario(ReformScenario): """ Parties communes entre ERFS et ERFS PFR @@ -116,10 +117,15 @@ def create( else: survey_scenario = cls(year = year) - survey_scenario.set_tax_benefit_systems( - tax_benefit_system = tax_benefit_system, - baseline_tax_benefit_system = baseline_tax_benefit_system, - ) + if baseline_tax_benefit_system: + survey_scenario.set_tax_benefit_systems(dict( + reform = tax_benefit_system, + baseline = baseline_tax_benefit_system, + )) + else: + survey_scenario.set_tax_benefit_systems(dict( + baseline = tax_benefit_system, + )) survey_scenario.year = year survey_scenario.period = year @@ -162,7 +168,7 @@ def custom_initialize(self, simulation): assert variable in self.used_as_input_variables, \ f"{variable} is not a in the input_varaibles to be used {self.used_as_input_variables}" # noqa: E501 - if self.tax_benefit_system.variables[variable].value_type == Enum: + if simulation.tax_benefit_system.variables[variable].value_type == Enum: permanent_value = simulation.calculate( variable, period = periods.period(self.year).first_month, diff --git a/tests/test_calibration.py b/tests/test_calibration.py index dee2ee60..a9293435 100644 --- a/tests/test_calibration.py +++ b/tests/test_calibration.py @@ -24,15 +24,14 @@ def test_calibration(survey_scenario, fake_input_data, location, year: int = 200 survey_scenario.init_from_data(data = dict(input_data_frame = input_data)) # On fait la calibration - calibration = Calibration(survey_scenario) - calibration.parameters["method"] = "linear" - calibration.total_population = calibration.initial_total_population * 1.123 + parameters = dict( + method = "logit", + invlo = 3, + up = 3, + ) - calibration.set_parameters("invlo", 3) - calibration.set_parameters("up", 3) - calibration.set_parameters("method", "logit") - - calibration.calibrate() pre_cal_weight = survey_scenario.calculate_variable("wprm", period = year) - calibration.set_calibrated_weights() + target_entity_count = pre_cal_weight.sum() * 1.123 + survey_scenario.calibrate(target_entity_count = target_entity_count, entity = "menage", parameters = parameters, period = year) + assert pre_cal_weight * 1.123 == survey_scenario.calculate_variable("wprm", period = year) From 75cc1c816d8ec9e237951d7fe7db57b680d4ce67 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 16 Oct 2023 09:58:16 +0100 Subject: [PATCH 121/140] Lint --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f84143b..03c26a56 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <154.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { "test": [ From dd2c78a3b4850f18a8e0f4d0f4fcb76007b50f4d Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Mon, 16 Oct 2023 15:11:15 +0100 Subject: [PATCH 122/140] Remove unused variable use_baseline --- openfisca_france_data/surveys.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index 5d20955b..e6ec6f95 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -189,7 +189,6 @@ def custom_initialize(self, simulation): except ValueError as e: log.debug(f"Dealing with: {e}") if sum(simulation.calculate_add(variable, simulation_period.offset(offset))) != sum(permanent_value): - use_baseline = self.baseline_simulation == simulation simulation.delete_arrays(variable, simulation_period.offset(offset)) simulation.set_input( variable, From 14ea2d2d6226d3f48f46dd3bdff94b181d292faa Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 17 Oct 2023 15:08:37 +0100 Subject: [PATCH 123/140] Migrate aggregate attribute --- openfisca_france_data/aggregates.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 9730239f..e3e6b4e7 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -35,10 +35,10 @@ def __init__(self, survey_scenario = None, target_source = None): super().__init__(survey_scenario = survey_scenario) self.target_source = target_source - def load_actual_data(self, year = None): + def load_actual_data(self, period = None): target_source = self.target_source assert target_source in ["ines", "taxipp", "france_entiere"], "les options possible pour source_cible sont ines, taxipp ou france_entiere" - assert year is not None + assert period is not None if target_source == "taxipp": taxipp_aggregates_file = Path( @@ -59,15 +59,15 @@ def load_actual_data(self, year = None): .rename(columns = {"unnamed: 0": "description"}) .dropna(subset = ["annee 2019", "annee 2018", "annee 2017", "annee 2016"], how = "all") ) - if f"annee {year}" not in df: + if f"annee {period}" not in df: return df = ( - df[["variable_openfisca", f"annee {year}"]] + df[["variable_openfisca", f"annee {period}"]] .dropna() .rename(columns = { "variable_openfisca": "variable", - f"annee {year}": year, + f"annee {period}": period, }) ) @@ -75,13 +75,13 @@ def load_actual_data(self, year = None): df.loc[df.variable.str.startswith("nombre")] .set_index("variable") .rename(index = lambda x : x.replace("nombre_", "")) - .rename(columns = {year: "actual_beneficiaries"}) + .rename(columns = {period: "actual_beneficiaries"}) ) / self.beneficiaries_unit amounts = ( df.loc[~df.variable.str.startswith("nombre")] .set_index("variable") - .rename(columns = {year: "actual_amount"}) + .rename(columns = {period: "actual_amount"}) ) / self.amount_unit result = amounts.merge(beneficiaries, on = "variable", how = "outer").drop("PAS SIMULE") @@ -93,7 +93,7 @@ def load_actual_data(self, year = None): "assets", "aggregats", "ines", - f"ines_{year}.json" + f"ines_{period}.json" ) with open(ines_aggregates_file, 'r') as f: @@ -112,7 +112,7 @@ def load_actual_data(self, year = None): "assets", "aggregats", "france_entiere", - f"france_entiere_{year}.json" + f"france_entiere_{period}.json" ) with open(ines_aggregates_file, 'r') as f: @@ -133,7 +133,7 @@ def to_csv(self, path = None, absolute = True, amount = True, beneficiaries = Tr if os.path.isdir(path): now = datetime.now() - file_path = os.path.join(path, 'Aggregates_%s_%s_%s.%s' % (self.target_source,self.year,now.strftime('%d-%m-%Y'), "csv")) + file_path = os.path.join(path, 'Aggregates_%s_%s_%s.%s' % (self.target_source, self.period, now.strftime('%d-%m-%Y'), "csv")) else: file_path = path From 763210e0f8641b89f59e38f6f05b830853d9dae8 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 16:52:45 +0200 Subject: [PATCH 124/140] Fix tests --- openfisca_france_data/erfs/scenario.py | 4 +- .../erfs_fpr/get_survey_scenario.py | 4 +- openfisca_france_data/erfs_fpr/scenario.py | 5 +- openfisca_france_data/surveys.py | 25 ++-- tests/conftest.py | 2 +- .../erfs_fpr/integration/test_pivot_table.py | 2 +- tests/test_aggregate.py | 16 +-- tests/test_fake_survey_simulation.py | 124 ++++++++++-------- 8 files changed, 94 insertions(+), 88 deletions(-) diff --git a/openfisca_france_data/erfs/scenario.py b/openfisca_france_data/erfs/scenario.py index 46ad488b..d3d6148a 100644 --- a/openfisca_france_data/erfs/scenario.py +++ b/openfisca_france_data/erfs/scenario.py @@ -41,5 +41,5 @@ class ErfsSurveyScenario(AbstractErfsSurveyScenario): 'zone_apl', ] - def __init__(self, year: int) -> None: - self.year = year + def __init__(self, period: int) -> None: + self.period = period diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 448467fc..6b2ff551 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -126,14 +126,14 @@ def get_survey_scenario( survey_scenario = ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, baseline_tax_benefit_system = baseline_tax_benefit_system, - year = year, + period = year, ) else: assert varying_variable is not None, "You need to specify the varying variable." survey_scenario = ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, baseline_tax_benefit_system = baseline_tax_benefit_system, - year = year, + period = year, ) # taux marginaux !! survey_scenario.variation_factor = variation_factor diff --git a/openfisca_france_data/erfs_fpr/scenario.py b/openfisca_france_data/erfs_fpr/scenario.py index 7e9711d3..eef10cd5 100644 --- a/openfisca_france_data/erfs_fpr/scenario.py +++ b/openfisca_france_data/erfs_fpr/scenario.py @@ -63,8 +63,9 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario): "wprm_init", ] - def __init__(self, year: int) -> None: - self.year = year + def __init__(self, period: int) -> None: + # self.year = period + self.period = period @classmethod def build_input_data(cls, year: int) -> None: diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index e6ec6f95..1fb0db71 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -62,7 +62,7 @@ def build_input_data_from_test_case(self, test_case_scenario): ) array_by_variable = dict() - period = periods.period(str(self.year)) + period = periods.period(str(self.period)) for variable in self.used_as_input_variables: array_by_variable[variable] = simulation.calculate_add( @@ -88,10 +88,10 @@ def create( input_data_type = None, reform = None, reform_key = None, - year: int = None, + period: int = None, ): - assert year is not None + assert period is not None assert not ((reform is not None) and (reform_key is not None)) reform_is_provided = (reform is not None) or (reform_key is not None) @@ -112,10 +112,10 @@ def create( tax_benefit_system = reform if input_data_type is not None: - survey_scenario = cls(input_data_type = input_data_type, year = year) + survey_scenario = cls(input_data_type = input_data_type, period = period) else: - survey_scenario = cls(year = year) + survey_scenario = cls(period = period) if baseline_tax_benefit_system: survey_scenario.set_tax_benefit_systems(dict( @@ -127,8 +127,7 @@ def create( baseline = tax_benefit_system, )) - survey_scenario.year = year - survey_scenario.period = year + survey_scenario.period = period return survey_scenario @@ -156,27 +155,27 @@ def custom_initialize(self, simulation): "primes_fonction_publique", "retraite_brute", "retraite_imposable", - "salaire_de_base", + "salaire_imposable", "traitement_indiciaire_brut", # 'chomage_brut', ] - three_year_span_variables = input_variables + computed_variables_used_as_input + three_period_span_variables = input_variables + computed_variables_used_as_input - simulation_period = periods.period(self.year) - for variable in three_year_span_variables: + simulation_period = periods.period(self.period) + for variable in three_period_span_variables: assert variable in self.used_as_input_variables, \ f"{variable} is not a in the input_varaibles to be used {self.used_as_input_variables}" # noqa: E501 if simulation.tax_benefit_system.variables[variable].value_type == Enum: permanent_value = simulation.calculate( variable, - period = periods.period(self.year).first_month, + period = periods.period(self.period).first_month, ) else: permanent_value = simulation.calculate_add( variable, - period = self.year, + period = self.period, ) for offset in [-1, -2]: diff --git a/tests/conftest.py b/tests/conftest.py index 1b6cc218..933e6350 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,7 +35,7 @@ def survey_scenario(tax_benefit_system: TaxBenefitSystem): def _survey_scenario(year: int) -> ErfsFprSurveyScenario: return ErfsFprSurveyScenario.create( tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) return _survey_scenario diff --git a/tests/erfs_fpr/integration/test_pivot_table.py b/tests/erfs_fpr/integration/test_pivot_table.py index 4ba92652..81b0bc04 100644 --- a/tests/erfs_fpr/integration/test_pivot_table.py +++ b/tests/erfs_fpr/integration/test_pivot_table.py @@ -60,4 +60,4 @@ def get_survey_scenario(kind = 'erfs_fpr', year = None): year = 2009 SurveyScenario = ErfsSurveyScenario - return SurveyScenario.create(year = year) + return SurveyScenario.create(period = year) diff --git a/tests/test_aggregate.py b/tests/test_aggregate.py index 61adb5a4..1e6d3fc0 100644 --- a/tests/test_aggregate.py +++ b/tests/test_aggregate.py @@ -8,12 +8,13 @@ ) -@pytest.mark.skip( - reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", - ) def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 2009): # On ititialise le survey scenario - survey_scenario = ErfsFprSurveyScenario.create(year) + survey_scenario = ErfsFprSurveyScenario.create( + tax_benefit_system = france_data_tax_benefit_system, + + period = year, + ) # On charge les données input_data = fake_input_data(year) @@ -23,16 +24,13 @@ def test_erfs_survey_simulation(survey_scenario, fake_input_data, year: int = 20 # On calcule les agrégats aggregates = Aggregates(survey_scenario = survey_scenario, target_source = 'taxipp') - aggregates.compute_aggregates(use_baseline = False) + aggregates.compute_aggregates() return aggregates.base_data_frame -@pytest.mark.skip( - reason = "FileNotFoundError: [Errno 2] No such file or directory: '/opt/hostedtoolcache/Python/3.9.9/x64/lib/python3.9/site-packages/openfisca_france_data/assets/agregats_tests_taxipp_2_0.xlsx'", - ) def test_erfs_fpr_aggregates_reform(fake_input_data, year:int = 2013): survey_scenario = ErfsFprSurveyScenario.create( - year = year, + period = year, reform_key = 'plf2015', baseline_tax_benefit_system = france_data_tax_benefit_system, ) diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 47694fcf..5faa7b79 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -10,29 +10,13 @@ from openfisca_france_data.reforms.old_openfisca_france_reforms.plf2015 import plf2015 # type: ignore -@pytest.fixture -def fake_calibration(tax_benefit_system): - def _fake_calibration(fake_input_data: pandas.DataFrame, year: int) -> Calibration: - input_data_frame = fake_input_data(year) - - survey_scenario = ErfsSurveyScenario.create( - tax_benefit_system = tax_benefit_system, - year = year, - ) - - survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) - - calibration = Calibration(survey_scenario = survey_scenario, period = year) - calibration.set_parameters("invlo", 3) - calibration.set_parameters("up", 3) - calibration.set_parameters("method", "logit") - - return calibration - - return _fake_calibration +parameters = dict( + invlo = 3, + up = 3, + method = "logit", + ) -@pytest.mark.skip(reason = "AssertionError: [0. 0.] != [20000. 10000.]") def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = 2006): input_data_frame = fake_input_data(year) assert input_data_frame.salaire_imposable.loc[0] == 20000 @@ -40,7 +24,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = survey_scenario = ErfsSurveyScenario.create( tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) @@ -48,7 +32,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert (input_data_frame.salaire_imposable.loc[0] == 20000).all() assert (input_data_frame.salaire_imposable.loc[1] == 10000).all() - simulation = survey_scenario.simulation + simulation = survey_scenario.simulations["baseline"] salaire_imposable = simulation.calculate_add("salaire_imposable", period = year) @@ -70,7 +54,10 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert (sal_2005 == sal_2006).all() for year, month in itertools.product(range(2003, 2004), range(1, 13)): - period = f"{year}-{month}" + if month < 10: + period = f"{year}-0{month}" + else: + period = f"{year}-{month}" assert ( simulation @@ -78,11 +65,13 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = ).all() for year, month in itertools.product(range(2004, 2007), range(1, 13)): - period = f"{year}-{month}" + if month < 10: + period = f"{year}-0{month}" + else: + period = f"{year}-{month}" assert ( - simulation - .calculate("salaire_imposable", period = period) == sal_2006 / 12 + simulation.calculate("salaire_imposable", period = period) == sal_2006 / 12 ).all() data_frame_by_entity = survey_scenario.create_data_frame_by_entity( @@ -106,66 +95,85 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "weight_familles", "revenu_disponible", "impo", - ] + ], ) return data_frame_by_entity, simulation -@pytest.mark.skip( - reason = "AttributeError: 'Calibration' object has no attribute 'simulation'", - ) -def test_fake_calibration_float(fake_calibration, year: int = 2006): - calibration = fake_calibration(year) - calibration.total_population = calibration.initial_total_population * 1.123 +def test_fake_calibration_float(tax_benefit_system, fake_input_data, year: int = 2006): + input_data_frame = fake_input_data(year) + survey_scenario = ErfsSurveyScenario.create( + tax_benefit_system = tax_benefit_system, + period = year, + ) + survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) + + initial_count = survey_scenario.compute_aggregate("revenu_disponible", aggfunc = "count", period = year) + target_entity_count = initial_count * 1.123 - revenu_disponible_target = 7e6 - calibration.set_target_margin("revenu_disponible", revenu_disponible_target) - calibration.calibrate() - calibration.set_calibrated_weights() + revenu_disponible_target = 7.7e6 + target_margins_by_variable = dict(revenu_disponible = revenu_disponible_target) - simulation = calibration.survey_scenario.simulation + survey_scenario.calibrate( + period = year, + target_margins_by_variable = target_margins_by_variable, + target_entity_count = target_entity_count, + parameters=parameters, + ) assert_near( - simulation.calculate("wprm", period=year).sum(), - calibration.total_population, + survey_scenario.calculate_variable("wprm", period=year).sum(), + target_entity_count, absolute_error_margin = None, relative_error_margin = 0.00001, ) assert_near( ( - + simulation.calculate("revenu_disponible", period = year) - * simulation.calculate("wprm", period = year) + + survey_scenario.calculate_variable("revenu_disponible", period = year) + * survey_scenario.calculate_variable("wprm", period = year) ).sum(), revenu_disponible_target, absolute_error_margin = None, relative_error_margin = 0.00001, ) - @pytest.mark.skip( - reason = "ValueError: Length of values does not match length of index", + reason = "ValueError: Unable to compute variable 'age' for period 2006: 'age' must be computed for a whole month. Should use an ge with year definition period", ) -def test_fake_calibration_age(fake_calibration, fake_input_data, year: int = 2006): - calibration = fake_calibration(fake_input_data, year) - survey_scenario = calibration.survey_scenario - calibration.total_population = calibration.initial_total_population * 1.123 - calibration.set_target_margin("age", [95, 130]) - calibration.calibrate() - calibration.set_calibrated_weights() +def test_fake_calibration_age(tax_benefit_system, fake_input_data, year: int = 2006): + input_data_frame = fake_input_data(year) + period = year + survey_scenario = ErfsSurveyScenario.create( + tax_benefit_system = tax_benefit_system, + period = period, + ) + survey_scenario.init_from_data(data = dict(input_data_frame = input_data_frame)) - simulation = survey_scenario.simulation + initial_count = survey_scenario.compute_aggregate("wprm", aggfunc = "count", period = year) + target_entity_count = initial_count * 1.123 + + target_margins_by_variable = { + "age": [95, 130] + } + + survey_scenario.calibrate( + period = period, + target_margins_by_variable = target_margins_by_variable, + target_entity_count = target_entity_count, + parameters=parameters, + ) assert_near( - simulation.calculate("wprm").sum(), - calibration.total_population, + survey_scenario.calculate_variable("wprm", period=period).sum(), + target_entity_count, absolute_error_margin = None, relative_error_margin = 0.00001, ) - age = (survey_scenario.simulation.calculate("age"),) - weight_individus = survey_scenario.simulation.calculate("weight_individus") + age = survey_scenario.calculate_variable("age", period = period) + weight_individus = survey_scenario.calculate_variable("weight_individus", period = year) for category, target in calibration.margins_by_variable["age"]["target"].items(): actual = ((age == category) * weight_individus).sum() / weight_individus.sum() @@ -197,7 +205,7 @@ def test_calculate_irpp_before_and_after_plf2015( survey_scenario = ErfsSurveyScenario.create( tax_benefit_system = reform, baseline_tax_benefit_system = tax_benefit_system, - year = year, + period = year, ) # On charge les données From cce63a79128560544dfb16bf5356b58438e36d95 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 17:01:03 +0200 Subject: [PATCH 125/140] Fix tests --- openfisca_france_data/surveys.py | 2 +- tests/test_fake_survey_simulation.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/openfisca_france_data/surveys.py b/openfisca_france_data/surveys.py index 1fb0db71..32e9b156 100644 --- a/openfisca_france_data/surveys.py +++ b/openfisca_france_data/surveys.py @@ -155,7 +155,7 @@ def custom_initialize(self, simulation): "primes_fonction_publique", "retraite_brute", "retraite_imposable", - "salaire_imposable", + "salaire_de_base", "traitement_indiciaire_brut", # 'chomage_brut', ] diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 5faa7b79..cf0f38d2 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -44,14 +44,14 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert age[0] == 77 assert age[1] == 37 - sal_2003 = simulation.calculate_add("salaire_imposable", period = 2003) - sal_2004 = simulation.calculate_add("salaire_imposable", period = 2004) - sal_2005 = simulation.calculate_add("salaire_imposable", period = 2005) - sal_2006 = simulation.calculate_add("salaire_imposable", period = 2006) + retraite_2003 = simulation.calculate_add("retraite_imposable", period = 2003) + retraite_2004 = simulation.calculate_add("retraite_imposable", period = 2004) + retraite_2005 = simulation.calculate_add("retraite_imposable", period = 2005) + retraite_2006 = simulation.calculate_add("retraite_imposable", period = 2006) - assert (sal_2003 == 0).all() - assert (sal_2004 == sal_2006).all() - assert (sal_2005 == sal_2006).all() + assert (retraite_2003 == 0).all() + assert (retraite_2004 == retraite_2006).all() + assert (retraite_2005 == retraite_2006).all() for year, month in itertools.product(range(2003, 2004), range(1, 13)): if month < 10: @@ -61,7 +61,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = assert ( simulation - .calculate_add("salaire_imposable", period = period) == 0 + .calculate_add("retraite_imposable", period = period) == 0 ).all() for year, month in itertools.product(range(2004, 2007), range(1, 13)): @@ -71,7 +71,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = period = f"{year}-{month}" assert ( - simulation.calculate("salaire_imposable", period = period) == sal_2006 / 12 + simulation.calculate("retraite_imposable", period = period) == retraite_2006 / 12 ).all() data_frame_by_entity = survey_scenario.create_data_frame_by_entity( @@ -80,6 +80,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "activite", "rsa_base_ressources_i", "menage_ordinaire_individus", + "retraite_imposable", "salaire_imposable", "salaire_net", "autonomie_financiere", @@ -97,7 +98,7 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "impo", ], ) - + BIM return data_frame_by_entity, simulation From 54bab67225f2f2ec0703ff107d264cb693af5c8c Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Wed, 18 Oct 2023 17:40:50 +0200 Subject: [PATCH 126/140] Remove leftover --- tests/test_fake_survey_simulation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index cf0f38d2..9eacf4d0 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -98,7 +98,6 @@ def test_fake_survey_simulation(tax_benefit_system, fake_input_data, year: int = "impo", ], ) - BIM return data_frame_by_entity, simulation From 16868db0df1709b49f4ceeb69065a12875563166 Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Thu, 19 Oct 2023 14:35:59 +0200 Subject: [PATCH 127/140] Fix path --- openfisca_france_data/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index e3e6b4e7..91ac656f 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -107,7 +107,7 @@ def load_actual_data(self, period = None): elif target_source == "france_entiere": ines_aggregates_file = Path( - pkg_resources.get_distribution("openfisca-france_data").location, + openfisca_france_data_location, "openfisca_france_data", "assets", "aggregats", From e67d037e143754de4ffbf10fa851b8e6ae260103 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 24 Oct 2023 15:24:03 +0200 Subject: [PATCH 128/140] fix import os --- openfisca_france_data/aggregates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 91ac656f..79d0ab02 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -3,7 +3,7 @@ import json from pathlib import Path -import numpy as np +import os import pandas as pd from openfisca_survey_manager.aggregates import AbstractAggregates From b5d1182a87e97e157400f50a0dc45c58eb351b30 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 24 Oct 2023 15:28:43 +0200 Subject: [PATCH 129/140] fix import datetime --- openfisca_france_data/aggregates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 79d0ab02..1b1b9351 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -4,6 +4,7 @@ from pathlib import Path import os +from datetime import datetime import pandas as pd from openfisca_survey_manager.aggregates import AbstractAggregates From d9796bf28d016c0193c004e7947bf9ec12d1940d Mon Sep 17 00:00:00 2001 From: Mahdi Ben Jelloul Date: Tue, 24 Oct 2023 15:52:10 +0200 Subject: [PATCH 130/140] Fix typo (space). --- openfisca_france_data/aggregates.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openfisca_france_data/aggregates.py b/openfisca_france_data/aggregates.py index 1b1b9351..f46a5ebc 100644 --- a/openfisca_france_data/aggregates.py +++ b/openfisca_france_data/aggregates.py @@ -121,9 +121,13 @@ def load_actual_data(self, period = None): result = pd.DataFrame(data['data']).drop(['source'], axis = 1) result['actual_beneficiaries'] = result. actual_beneficiaries / self.beneficiaries_unit - result['actual_amount'] = result. actual_amount / self.amount_unit + result['actual_amount'] = result.actual_amount / self.amount_unit - result = result[["variable","actual_amount","actual_beneficiaries"]].set_index("variable") + result = result[[ + "variable", + "actual_amount", + "actual_beneficiaries", + ]].set_index("variable") return result From 49a873c29a9b9e17f906bdb29babae75dfbce19d Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 2 Nov 2023 19:11:43 +0100 Subject: [PATCH 131/140] =?UTF-8?q?update=20d=C3=A9pendance=20openfisca-fr?= =?UTF-8?q?ance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 03c26a56..20eff165 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ python_requires = ">=3.9", install_requires = [ "multipledispatch >=0.6.0, <1.0.0", - "OpenFisca-France >=150.0.0, <154.0.0", + "OpenFisca-France >=150.0.0, <155.0.0", "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", ], extras_require = { From af576e69f344cf85469cac79cd75c4c10371a5f8 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 7 Nov 2023 17:30:07 +0100 Subject: [PATCH 132/140] =?UTF-8?q?bump=20+=20update=20d=C3=A9pendance=20o?= =?UTF-8?q?penfisca-survey-manager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 8 ++++++++ setup.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fa3c413..bfd72c1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +### 3.0.0 [#241](https://github.com/openfisca/openfisca-france-data/pull/241) +- Breaking changes + +Adapte le dépôt au passage à openfisca-survey-manager 2.0.0 qui constitue une refactorisation de l'objet survey-scenario et des simulations qui sont dedans. Cela concerne donc les parties de ce dépôts qui héritent d'objets d'openfisca-survey-manager : +- `openfisca_france_data/aggregates.py` +- `openfisca_france_data/surveys.py` +Les autres modifications sont des adaptions syntaxique mineurs du fait de cette adaptation + ### 2.0.7 [#239](https://github.com/openfisca/openfisca-france-data/pull/239/files) * New features - Ajoute des nouveaux agrégats pour FranceAggregates diff --git a/setup.py b/setup.py index 20eff165..a49ead52 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "2.0.7", + version = "3.0.0", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -43,7 +43,7 @@ install_requires = [ "multipledispatch >=0.6.0, <1.0.0", "OpenFisca-France >=150.0.0, <155.0.0", - "openFisca-survey-manager @ git+https://git@github.com/openfisca/openfisca-survey-manager@refacto-scenario#egg=openfisca-survey-manager", + "openFisca-survey-manager >=2.0.0, <2.1.0", ], extras_require = { "test": [ From ea591f1546370ccd6e0e1747aea6b540927aaea6 Mon Sep 17 00:00:00 2001 From: cgl Date: Tue, 7 Nov 2023 21:31:16 +0100 Subject: [PATCH 133/140] fix config file directory --- openfisca_france_data/erfs_fpr/get_survey_scenario.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openfisca_france_data/erfs_fpr/get_survey_scenario.py b/openfisca_france_data/erfs_fpr/get_survey_scenario.py index 6b2ff551..ed626143 100644 --- a/openfisca_france_data/erfs_fpr/get_survey_scenario.py +++ b/openfisca_france_data/erfs_fpr/get_survey_scenario.py @@ -10,6 +10,7 @@ from openfisca_france_data.erfs_fpr.scenario import ErfsFprSurveyScenario from openfisca_france_data import france_data_tax_benefit_system +from openfisca_survey_manager import default_config_files_directory from openfisca_france_data.model.id_variables import ( idmen_original, @@ -102,6 +103,7 @@ def get_survey_scenario( variation_factor: float = 0.03, varying_variable: str = None, survey_name: str = "input", + config_files_directory : str = default_config_files_directory, ) -> ErfsFprSurveyScenario: """Helper pour créer un `ErfsFprSurveyScenario`. @@ -153,6 +155,8 @@ def get_survey_scenario( input_data_table_by_entity_by_period = input_data_table_by_entity_by_period, survey = survey_name ) + data["config_files_directory"] = config_files_directory + # Les données peuvent venir en différents formats : # From 1f89020c9dac100b8d4a0aaf139ff4b6f401b4d9 Mon Sep 17 00:00:00 2001 From: cgl Date: Thu, 9 Nov 2023 16:40:48 +0100 Subject: [PATCH 134/140] maj dependance --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a49ead52..e6348601 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "3.0.0", + version = "3.0.1", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", @@ -42,7 +42,7 @@ python_requires = ">=3.9", install_requires = [ "multipledispatch >=0.6.0, <1.0.0", - "OpenFisca-France >=150.0.0, <155.0.0", + "OpenFisca-France >=155.0.0, <156.0.0", "openFisca-survey-manager >=2.0.0, <2.1.0", ], extras_require = { From 429f91be30059a313fe0e439da76ea5e48a42c01 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 10 Nov 2023 14:58:44 +0100 Subject: [PATCH 135/140] renomme irpp en impot_revenu_restant_a_payer --- openfisca_france_data/__init__.py | 2 +- .../aggregats/france_entiere/france_entiere_2019.json | 2 +- .../aggregats/france_entiere/france_entiere_2020.json | 2 +- .../aggregats/france_entiere/france_entiere_2021.json | 2 +- .../aggregats/france_entiere/france_entiere_2022.json | 2 +- .../aggregats/france_entiere/france_entiere_2023.json | 2 +- .../aggregats/france_entiere/france_entiere_template.json | 2 +- .../assets/aggregats/ines/ines_2019.json | 2 +- .../assets/aggregats/ines/ines_2020.json | 2 +- openfisca_france_data/erfs/__init__.py | 2 +- .../old_openfisca_france_reforms/cesthra_invalidee.py | 4 ++-- tests/erfs_fpr/integration/test_impot_revenu.py | 6 +++--- tests/erfs_fpr/integration/test_pivot_table.py | 8 ++++---- tests/test_fake_survey_simulation.py | 4 ++-- 14 files changed, 21 insertions(+), 21 deletions(-) diff --git a/openfisca_france_data/__init__.py b/openfisca_france_data/__init__.py index 43389933..9b8d426c 100644 --- a/openfisca_france_data/__init__.py +++ b/openfisca_france_data/__init__.py @@ -174,7 +174,7 @@ def apply(self): 'csg', 'crds', 'impot_revenu', - 'irpp', + 'impot_revenu_restant_a_payer', 'prelevement_forfaitaire_unique_ir', 'prelevement_forfaitaire_liberatoire', 'taxe_habitation', diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json index 55d37846..656e2d7d 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2019.json @@ -166,7 +166,7 @@ "source": "Comptes de la sécurité sociale, résultats 2019 et prévision 2020 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2020/RAPPORT%20CCSS%20JUIN%202020.pdf)" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries":39332000, "actual_amount":76911000000, "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json index 619edb0c..d2cbdf55 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2020.json @@ -165,7 +165,7 @@ "source": "Comptes de la sécurité sociale, résultats 2020, prévisions 2021 et 2022 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/CCSS/2021/Rapport%20CCSS-Septembre2021.pdf)" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries":39888000, "actual_amount":73979000000, "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json index 2d67b16a..ca3c69cd 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2021.json @@ -163,7 +163,7 @@ "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries":40257000, "actual_amount":80826000000, "source": "Statistiques DGFIP, Impôt sur le revenu : répartition par région (https://www.impots.gouv.fr/statistiques-impots-des-particuliers)" diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json index 9bf93e25..538ca5ad 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2022.json @@ -164,7 +164,7 @@ "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries": 0, "actual_amount": 0, "source": "" diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json index c66ec02a..0c7e2cc7 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_2023.json @@ -158,7 +158,7 @@ "source":"Comptes de la sécurité sociale, résultats 2022 et prévisions 2023 (https://www.securite-sociale.fr/files/live/sites/SSFR/files/medias/DSS/2023/CCSS-Mai2023.pdf)" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries": 0, "actual_amount": 0, "source": "" diff --git a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json index d3e4da85..3adf627d 100644 --- a/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json +++ b/openfisca_france_data/assets/aggregats/france_entiere/france_entiere_template.json @@ -157,7 +157,7 @@ "source": "" }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries": 0, "actual_amount": 0, "source": "" diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2019.json b/openfisca_france_data/assets/aggregats/ines/ines_2019.json index a7273d98..2d7549a2 100644 --- a/openfisca_france_data/assets/aggregats/ines/ines_2019.json +++ b/openfisca_france_data/assets/aggregats/ines/ines_2019.json @@ -132,7 +132,7 @@ "actual_amount":17000000000 }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries":0, "actual_amount":69400000000, "notes":"Impot payé en 2019, en parti sur revenus 2019 et 2018" diff --git a/openfisca_france_data/assets/aggregats/ines/ines_2020.json b/openfisca_france_data/assets/aggregats/ines/ines_2020.json index ba52c2f4..2534af82 100644 --- a/openfisca_france_data/assets/aggregats/ines/ines_2020.json +++ b/openfisca_france_data/assets/aggregats/ines/ines_2020.json @@ -132,7 +132,7 @@ "actual_amount":19000000000 }, { - "variable":"irpp", + "variable":"impot_revenu_restant_a_payer", "actual_beneficiaries":0, "actual_amount":73900000000, "notes":"Impot payé en 2020, en parti sur revenus 2020 et 2019" diff --git a/openfisca_france_data/erfs/__init__.py b/openfisca_france_data/erfs/__init__.py index 2debdb5e..124290a8 100644 --- a/openfisca_france_data/erfs/__init__.py +++ b/openfisca_france_data/erfs/__init__.py @@ -3,7 +3,7 @@ def get_of2erf(year=None): of2erf["csg"] = "csgim" # imposable, et "csgdm", déductible #of2erf["csgd"] = "csgdm" of2erf["crds"] = "crdsm" - of2erf["irpp"] = "zimpom" + of2erf["impot_revenu_restant_a_payer"] = "zimpom" of2erf["ppe"] = "m_ppem" of2erf["af"] = "m_afm" #af_base diff --git a/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py index 66a5ecea..505bcac5 100644 --- a/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py +++ b/openfisca_france_data/reforms/old_openfisca_france_reforms/cesthra_invalidee.py @@ -31,7 +31,7 @@ def formula(foyer_fiscal, period, parameters): return foyer_fiscal.sum(cesthra_i) -class irpp(Variable): +class impot_revenu_restant_a_payer(Variable): label = 'Impôt sur le revenu des personnes physiques (réformée pour intégrer la cesthra)' definition_period = YEAR @@ -62,5 +62,5 @@ class cesthra_invalidee(Reform): def apply(self): self.add_variable(cesthra) - self.update_variable(irpp) + self.update_variable(impot_revenu_restant_a_payer) self.modify_parameters(modifier_function = modify_parameters) diff --git a/tests/erfs_fpr/integration/test_impot_revenu.py b/tests/erfs_fpr/integration/test_impot_revenu.py index c3f8fa68..08c828ca 100644 --- a/tests/erfs_fpr/integration/test_impot_revenu.py +++ b/tests/erfs_fpr/integration/test_impot_revenu.py @@ -5,7 +5,7 @@ data_frame_by_entity = survey_scenario.create_data_frame_by_entity( variables = [ - 'irpp', + 'impot_revenu_restant_a_payer', 'maries_ou_pacses', 'nbptr', 'rfr', @@ -31,8 +31,8 @@ assert 25e6 < (foyer_fiscal.weight_foyers).sum() -print(foyer_fiscal.groupby('nbptr').apply(lambda x: (x.irpp.astype('float') * x.weight_foyers.astype('float')).sum())) -print((foyer_fiscal.irpp.astype('float') * foyer_fiscal.weight_foyers.astype('float')).sum() / 1e9) +print(foyer_fiscal.groupby('nbptr').apply(lambda x: (x.impot_revenu_restant_a_payer.astype('float') * x.weight_foyers.astype('float')).sum())) +print((foyer_fiscal.impot_revenu_restant_a_payer.astype('float') * foyer_fiscal.weight_foyers.astype('float')).sum() / 1e9) # survey_scenario.summarize_variable('salaire_imposable') # salaire_imposable: 1 periods * 127126 cells * item size 4 (float32, default = 0) = 496.6K diff --git a/tests/erfs_fpr/integration/test_pivot_table.py b/tests/erfs_fpr/integration/test_pivot_table.py index 81b0bc04..0e867df8 100644 --- a/tests/erfs_fpr/integration/test_pivot_table.py +++ b/tests/erfs_fpr/integration/test_pivot_table.py @@ -10,7 +10,7 @@ def test_pivot_table_1d_mean(): survey_scenario = get_survey_scenario(kind = 'erfs_fpr', year = year) pivot_table = survey_scenario.compute_pivot_table( columns = ['decile_rfr'], - values = ['irpp'], + values = ['impot_revenu_restant_a_payer'], period = year, ) return pivot_table @@ -22,7 +22,7 @@ def test_pivot_table_1d_sum(): pivot_table = survey_scenario.compute_pivot_table( aggfunc = 'sum', columns = ['decile_rfr'], - values = ['irpp'], + values = ['impot_revenu_restant_a_payer'], period = year, ) return pivot_table @@ -34,7 +34,7 @@ def test_pivot_table_1d_count(): pivot_table = survey_scenario.compute_pivot_table( aggfunc = 'count', columns = ['decile_rfr'], - values = ['irpp'], + values = ['impot_revenu_restant_a_payer'], period = year, ) return pivot_table @@ -46,7 +46,7 @@ def test_pivot_table_2d_2values(): pivot_table = survey_scenario.compute_pivot_table( columns = ['decile_rfr'], index = ['nbptr'], - values = ['irpp', 'rfr'], + values = ['impot_revenu_restant_a_payer', 'rfr'], period = year, ) return pivot_table diff --git a/tests/test_fake_survey_simulation.py b/tests/test_fake_survey_simulation.py index 9eacf4d0..7f7f8595 100644 --- a/tests/test_fake_survey_simulation.py +++ b/tests/test_fake_survey_simulation.py @@ -221,7 +221,7 @@ def test_calculate_irpp_before_and_after_plf2015( # IRPP avant la réforme assert_near( - survey_scenario.calculate_variable("irpp", use_baseline = True, period = year), + survey_scenario.calculate_variable("impot_revenu_restant_a_payer", use_baseline = True, period = year), [-10124, -869], error_margin, ) @@ -229,7 +229,7 @@ def test_calculate_irpp_before_and_after_plf2015( # IRPP après la réforme # -911.4 + (1135 - 911.4) = -686.8 assert_near( - survey_scenario.calculate_variable("irpp", period = year), + survey_scenario.calculate_variable("impot_revenu_restant_a_payer", period = year), [-10118, -911.4 + (1135 - 911.4)], error_margin, ) From 2c21826c5b2a514a25e760a0412fce473cccf141 Mon Sep 17 00:00:00 2001 From: cgl Date: Fri, 10 Nov 2023 15:06:33 +0100 Subject: [PATCH 136/140] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfd72c1a..babc6c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.0.1 [#242](https://github.com/openfisca/openfisca-france-data/pull/242) +* Technical changes: +- Passage à Openfsca-France >= 155.0.0 + ### 3.0.0 [#241](https://github.com/openfisca/openfisca-france-data/pull/241) - Breaking changes From 6f0898616fe656731372c43e562d7c618df674e5 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 29 Nov 2023 13:40:29 +0100 Subject: [PATCH 137/140] Permet autre tax and benefit system dans utils --- openfisca_france_data/utils.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openfisca_france_data/utils.py b/openfisca_france_data/utils.py index 91cf6ce2..51d5c22b 100644 --- a/openfisca_france_data/utils.py +++ b/openfisca_france_data/utils.py @@ -231,8 +231,7 @@ def check_structure(dataframe): return False, erroneous_ids_by_entity -def build_cerfa_fields_by_column_name(year, sections_cerfa): - tax_benefit_system = openfisca_france_tax_benefit_system +def build_cerfa_fields_by_column_name(year, sections_cerfa, tax_benefit_system = openfisca_france_tax_benefit_system): cerfa_fields_by_column_name = dict() for name, column in tax_benefit_system.variables.items(): for section_cerfa in sections_cerfa: @@ -248,8 +247,7 @@ def build_cerfa_fields_by_column_name(year, sections_cerfa): return cerfa_fields_by_column_name -def build_cerfa_fields_by_variable(year): - tax_benefit_system = openfisca_france_tax_benefit_system +def build_cerfa_fields_by_variable(year, tax_benefit_system = openfisca_france_tax_benefit_system): cerfa_fields_by_variable = dict() for name, variable in sorted(tax_benefit_system.variables.items()): if variable.cerfa_field is None: @@ -318,8 +316,7 @@ def normalizes_roles_in_entity(dataframe, entity_id_name, entity_role_name): dataframe[entity_role_name] = dataframe[entity_role_name].astype('int') -def set_variables_default_value(dataframe, year): - tax_benefit_system = openfisca_france_tax_benefit_system +def set_variables_default_value(dataframe, year, tax_benefit_system = openfisca_france_tax_benefit_system): for column_name, column in tax_benefit_system.variables.items(): if column_name in dataframe.columns: From 148b0a93c9d8c9269a6def0826a33455b80b99f5 Mon Sep 17 00:00:00 2001 From: Duchesne Sylvain Date: Wed, 29 Nov 2023 14:38:14 +0100 Subject: [PATCH 138/140] Bump --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index babc6c37..64018a34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.0.2 [#243](https://github.com/openfisca/openfisca-france-data/pull/243) +* Technical changes: +- Ajoute la possibilité d'utiliser un tax and benefit system plus en aval dans les fonctions de utils.py + ### 3.0.1 [#242](https://github.com/openfisca/openfisca-france-data/pull/242) * Technical changes: - Passage à Openfsca-France >= 155.0.0 diff --git a/setup.py b/setup.py index e6348601..5a8f2c95 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "3.0.1", + version = "3.0.2", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown", From 1bc73ec808547e3e78c02077cdda6162fda9344c Mon Sep 17 00:00:00 2001 From: Lola Josseran Date: Thu, 11 Jan 2024 14:38:39 +0100 Subject: [PATCH 139/140] Correction parameters (OF france) create_taux_csg_remplacement --- .../input_data_builder/create_variables_individuelles.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openfisca_france_data/felin/input_data_builder/create_variables_individuelles.py b/openfisca_france_data/felin/input_data_builder/create_variables_individuelles.py index 16e80f13..9a653369 100644 --- a/openfisca_france_data/felin/input_data_builder/create_variables_individuelles.py +++ b/openfisca_france_data/felin/input_data_builder/create_variables_individuelles.py @@ -19,12 +19,12 @@ def create_taux_csg_remplacement(individus, period, tax_benefit_system, sigma = def compute_taux_csg_remplacement(rfr, nbptr): parameters = tax_benefit_system.get_parameters_at_instant(period.start) - seuils = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.pensions_de_retraite_et_d_invalidite - seuil_exoneration = seuils.seuil_de_rfr_1 + (nbptr - 1) * seuils.demi_part_suppl - seuil_reduction = seuils.seuil_de_rfr_2 + (nbptr - 1) * seuils.demi_part_suppl + seuils = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.seuils + seuil_exoneration = seuils.seuil_de_rfr_1 + (nbptr - 1) * seuils.seuil_rfr1.demi_part_suppl_rfr1 + seuil_reduction = seuils.seuil_de_rfr_2 + (nbptr - 1) * seuils.seuil_rfr2.demi_part_suppl_rfr2 taux_csg_remplacement = 0.0 * rfr if period.start.year >= 2019: - seuil_taux_intermediaire = seuils.seuil_rfr3 + (nbptr - 1) * seuils.demi_part_suppl_rfr3 + seuil_taux_intermediaire = seuils.seuil_rfr3 + (nbptr - 1) * seuils.seuil_rfr3.demi_part_suppl_rfr3 taux_csg_remplacement = np.where( rfr <= seuil_exoneration, 1, From 89dd432f778e7fa798627ec6780937dceedd0a0f Mon Sep 17 00:00:00 2001 From: Lola Josseran Date: Thu, 11 Jan 2024 14:45:38 +0100 Subject: [PATCH 140/140] changelog et version --- CHANGELOG.md | 4 ++++ setup.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64018a34..26247c16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.0.3 [#244](https://github.com/openfisca/openfisca-france-data/pull/244) +* Technical changes: +- Correction dees liens des paramètres dans la fonction create_taux_csg_remplacement + ### 3.0.2 [#243](https://github.com/openfisca/openfisca-france-data/pull/243) * Technical changes: - Ajoute la possibilité d'utiliser un tax and benefit system plus en aval dans les fonctions de utils.py diff --git a/setup.py b/setup.py index 5a8f2c95..d357ba3f 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name = "OpenFisca-France-Data", - version = "3.0.2", + version = "3.0.3", description = "OpenFisca-France-Data module to work with French survey data", long_description = long_description, long_description_content_type="text/markdown",