Merge pull request #209 from openfisca/tests_inversion

Tests inversion revenus remplacement

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 3.1.0 [#209](https://github.com/openfisca/openfisca-france-data/pull/209)
+
+New features
+    Introduce testing for income "inversion" (deduce gross from net)
+Update features
+    Corrects the inversion functions + extends the inversion of unemployment benefit's taxation
+
 ### 3.0.6 [#248](https://github.com/openfisca/openfisca-france-data/pull/248)
 * Technical changes
 - Correction d'une typo dans la PR précédente

openfisca_france_data/common.py

@@ -104,7 +104,7 @@ def create_salaire_de_base(individus, period = None, revenu_type = 'imposable',
             if name not in target:
                 baremes_to_remove.append(name)
 
-        # We split since we cannot remove from dict while iterating
+        # We split since we cannot remove from dict while iterating
         for name in baremes_to_remove:
             del baremes_collection._children[name]
 
@@ -400,10 +400,14 @@ def create_revenus_remplacement_bruts(individus, period, tax_benefit_system):
 
     individus.chomage_imposable.fillna(0, inplace = True)
     individus.retraite_imposable.fillna(0, inplace = True)
+    individus.salaire_net.fillna(0, inplace = True)
 
     parameters = tax_benefit_system.get_parameters_at_instant(period.start)
     csg = parameters.prelevements_sociaux.contributions_sociales.csg
     csg_deductible_chomage = csg.remplacement.allocations_chomage.deductible
+    pss = parameters.prelevements_sociaux.pss.plafond_securite_sociale_annuel
+    taux_abattement_csg_chomage = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.allocations_chomage.deductible.abattement.rates[0]
+    seuil_abattement_csg_chomage = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.allocations_chomage.deductible.abattement.thresholds[1]
     taux_plein = csg_deductible_chomage.taux_plein
     taux_reduit = csg_deductible_chomage.taux_reduit
     seuil_chomage_net_exoneration = (
@@ -412,17 +416,28 @@ def create_revenus_remplacement_bruts(individus, period, tax_benefit_system):
             (individus.taux_csg_remplacement == 2) / (1 - taux_reduit)
             + (individus.taux_csg_remplacement >= 3) / (1 - taux_plein)
             )
-        )
+        ) - individus.salaire_net
     exonere_csg_chomage = (
         (individus.taux_csg_remplacement < 2)
         | (individus.chomage_imposable <= seuil_chomage_net_exoneration)
         )
+    taux_csg_chomage = np.where(
+        individus.taux_csg_remplacement < 2,
+        0,
+        (individus.taux_csg_remplacement == 2) * taux_reduit
+        + (individus.taux_csg_remplacement >= 3) * taux_plein
+        )
+    threshold = seuil_abattement_csg_chomage * pss * (1 - (taux_csg_chomage * (1 - taux_abattement_csg_chomage)))
+    base_csg_chomage = np.where(
+        individus.chomage_imposable <= threshold,
+        individus.chomage_imposable * (1 - taux_abattement_csg_chomage) / (1 - (taux_csg_chomage * (1 - taux_abattement_csg_chomage))),
+        (individus.chomage_imposable - seuil_abattement_csg_chomage * taux_abattement_csg_chomage * pss) / (1 - taux_csg_chomage)
+        )
     individus['chomage_brut'] = np.where(
         exonere_csg_chomage,
         individus.chomage_imposable,
-        (individus.taux_csg_remplacement == 2) * individus.chomage_imposable / (1 - taux_reduit)
-        + (individus.taux_csg_remplacement >= 3) * individus.chomage_imposable / (1 - taux_plein)
-        )
+        individus.chomage_imposable + (base_csg_chomage * taux_csg_chomage)
+    )
     assert individus['chomage_brut'].notnull().all()
 
     csg_deductible_retraite = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.pensions_retraite_invalidite.deductible

openfisca_france_data/erfs_fpr/comparison.py

@@ -55,6 +55,17 @@ class ErfsFprtoInputComparator(AbstractComparator):
        "statut_occupation_logement",
         ]
 
+    from openfisca_france_data.erfs_fpr.get_survey_scenario import menage_projected_variables
+
+    target_menage_projected_variables = [
+        f"{menage_projected_variable}_menage"
+        for menage_projected_variable
+        in menage_projected_variables
+        ]
+
+    default_target_variables += target_menage_projected_variables
+
+
     def compute_test_dataframes(self):
         erfs_fpr_survey_collection = SurveyCollection.load(collection = "erfs_fpr")
         # infer names of the survey and data tables

openfisca_france_data/erfs_fpr/get_survey_scenario.py

@@ -33,6 +33,88 @@
 ]
 
 
+class erfs_fpr_plugin(Reform):
+    name = "ERFS-FPR ids plugin"
+
+    def apply(self):
+
+        for variable in variables_converted_to_annual:
+            class_name =  f"{variable}_annuel"
+            label = f"{variable} sur l'année entière"
+
+            def annual_formula_creator(variable):
+                def formula(individu, period):
+                    result = individu(variable, period, options = [ADD])
+                    return result
+
+                formula.__name__ = 'formula'
+
+                return formula
+
+            variable_instance = type(class_name, (Variable,), dict(
+                value_type = float,
+                entity = self.variables[variable].entity,
+                label = label,
+                definition_period = YEAR,
+                formula = annual_formula_creator(variable),
+                ))
+
+            self.add_variable(variable_instance)
+            del variable_instance
+
+        for variable in menage_projected_variables:
+            class_name =  f"{variable}_menage"
+            label = f"{variable} agrégée à l'échelle du ménage"
+
+            def projection_formula_creator(variable):
+                def formula(menage, period):
+                    result_i = menage.members.foyer_fiscal(variable, period, options = [ADD])
+                    result = menage.sum(result_i, role = FoyerFiscal.DECLARANT_PRINCIPAL)
+                    return result
+
+                formula.__name__ = 'formula'
+
+                return formula
+
+            variable_instance = type(class_name, (Variable,), dict(
+                value_type = float,
+                entity = Menage,
+                label = label,
+                definition_period = YEAR,
+                formula = projection_formula_creator(variable),
+                ))
+
+            self.add_variable(variable_instance)
+            del variable_instance
+
+
+        self.add_variable(idmen_original)
+        self.add_variable(noindiv)
+
+
+from openfisca_france_data.model.id_variables import (
+    idmen_original,
+    noindiv,
+    )
+
+variables_converted_to_annual = [
+    "salaire_imposable",
+    "chomage_imposable",
+    "retraite_imposable",
+    "salaire_net",
+    "chomage_net",
+    "retraite_nette",
+    "ppa",
+    ]
+
+
+menage_projected_variables = [
+    # "rev_financier_prelev_lib_imputes",
+    "revenu_categoriel_foncier",
+    "revenus_capitaux_prelevement_forfaitaire_unique_ir",
+    ]
+
+
 class erfs_fpr_plugin(Reform):
     name = "ERFS-FPR ids plugin"
 
@@ -144,6 +226,7 @@ def get_survey_scenario(
     # S'il n'y a pas de données, on sait où les trouver.
     if data is None:
         input_data_table_by_entity = dict(
+            foyer_fiscal = f"foyer_fiscal_{year}",
             individu = f"individu_{year}",
             menage = f"menage_{year}",
             )

openfisca_france_data/erfs_fpr/input_data_builder/__init__.py

@@ -49,7 +49,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None:
     # - On merge les tables individus / menages
     #
     # Note : c'est ici où on objectivise les hypothèses, step 1
-    log.info('\n [[[ Year {} - Step 1 / 5 ]]] \n'.format(year))
+    log.info('\n [[[ Year {} - Step 1 / 6 ]]] \n'.format(year))
     preprocessing.build_merged_dataframes(year = year)
 
     # Step 02 : Si on veut calculer les allocations logement, il faut faire le matching avec une autre enquête (ENL)
@@ -62,7 +62,7 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None:
     menage.build_variables_menage(year = year)
 
     # Step 03 : on commence par les variables indivuelles
-    log.info('\n [[[ Year {} - Step 3 / 5 ]]] \n'.format(year))
+    log.info('\n [[[ Year {} - Step 3 / 6 ]]] \n'.format(year))
     variables_individuelles.build_variables_individuelles(year = year)
 
     # Step 04 : ici on va constituer foyer et famille à partir d'invididu et ménage
@@ -71,15 +71,18 @@ def build(year: int, export_flattened_df_filepath: str = None) -> None:
     # - On va faire des suppositions pour faire les familles
     # - On va faire les foyers fiscaux à partir des familles
     # - On va faire de suppositions pour faire les foyers fiscaux
-    log.info('\n [[[ Year {} - Step 4 / 5 ]]] \n'.format(year))
+    log.info('\n [[[ Year {} - Step 4 / 6 ]]] \n'.format(year))
     famille.build_famille(year = year)
 
+    log.info('\n [[[ Year {} - Step 5 / 6 ]]] \n'.format(year))
+    foyer.build_variables_foyers_fiscal(year = year)
+
     # Affreux ! On injectait tout dans un même DataFrame !!!
     # C'est très moche !
     #
     # On crée une df par entité par période.
     # Elles sont stockées dans un fichier h5
-    log.info('\n [[[ Year {} - Step 5 / 5 ]]] \n'.format(year))
+    log.info('\n [[[ Year {} - Step 6 / 6 ]]] \n'.format(year))
     final.create_input_data_frame(year = year, export_flattened_df_filepath = export_flattened_df_filepath)
 
 

openfisca_france_data/erfs_fpr/input_data_builder/step_02_menage.py

@@ -6,7 +6,6 @@
 
 from openfisca_survey_manager.temporary import temporary_store_decorator
 
-
 log = logging.getLogger(__name__)
 
 

openfisca_france_data/erfs_fpr/input_data_builder/step_03_variables_individuelles.py

@@ -77,6 +77,7 @@ def create_variables_individuelles(individus, year, survey_year = None, revenu_t
     create_contrat_de_travail(individus, period = period, salaire_type = revenu_type)
     create_categorie_salarie(individus, period = period, survey_year = survey_year)
     create_categorie_non_salarie(individus)
+
     # inversion des revenus pour retrouver le brut
     # pour les revenus de remplacement on a la csg et la crds dans l'erfs-fpr donc on peut avoir le brut directement
     create_revenus_remplacement_bruts(individus)

openfisca_france_data/erfs_fpr/input_data_builder/step_05_foyer.py

@@ -0,0 +1,57 @@
+import logging
+import pandas as pd
+
+
+from openfisca_survey_manager.temporary import temporary_store_decorator  # type: ignore
+
+
+log = logging.getLogger(__name__)
+
+
+@temporary_store_decorator(file_name = 'erfs_fpr')
+def build_variables_foyers_fiscal(temporary_store = None, year = None):
+
+    assert temporary_store is not None
+    assert year is not None
+
+    individus = temporary_store['individus_{}'.format(year)]
+    menages = temporary_store['menages_{}'.format(year)]
+
+    individus['idfoy'] = individus['idfam'].copy()
+    individus['quifoy'] = individus['quifam'].copy()
+
+    foyers_fiscaux = individus[['idfoy','ident',]].drop_duplicates()
+    foyers_fiscaux = pd.merge(
+        menages[[
+            'ident',
+            'rev_financier_prelev_lib_imputes',
+            'rev_fonciers_bruts',
+            'rev_valeurs_mobilieres_bruts',
+            'wprm',
+            ]],
+        foyers_fiscaux,
+        how = 'inner',
+        on = 'ident'
+        )
+    # première version pour splitter les revenus du capital du ménage dans les foyers fiscaux
+    # on attribue l'ensemble des revenus du capital du ménage au foyer avec la personne ayant les plus hauts revenus
+    # procédure à améliorer
+    idfoy = (individus
+        .sort_values(
+            [
+                'ident',
+                'salaire_de_base',
+                'traitement_indiciaire_brut',
+                'retraite_brute'
+                ],
+            ascending = False
+            )
+        .groupby('ident')
+        .first()
+        .idfoy
+        )
+    foyers_fiscaux['revenu_categoriel_foncier'] = foyers_fiscaux['rev_fonciers_bruts'] * foyers_fiscaux.idfoy.isin(idfoy)
+    foyers_fiscaux['revenus_capitaux_prelevement_forfaitaire_unique_ir'] = foyers_fiscaux['rev_valeurs_mobilieres_bruts'] * foyers_fiscaux.idfoy.isin(idfoy)
+    foyers_fiscaux['rev_financier_prelev_lib_imputes'] = foyers_fiscaux['rev_financier_prelev_lib_imputes'] * foyers_fiscaux.idfoy.isin(idfoy)
+
+    temporary_store[f"foyers_fiscaux_{year}"] = foyers_fiscaux

openfisca_france_data/erfs_fpr/input_data_builder/step_06_final.py

@@ -47,7 +47,6 @@ def create_input_data_frame(temporary_store = None, year = None, export_flattene
         "primes_fonction_publique",
         "traitement_indiciaire_brut",
         ]
-
     if year >= 2018:
         var_menages = [
             'idmen',

openfisca_france_data/erfs_fpr/scenario.py

@@ -28,6 +28,9 @@ class ErfsFprSurveyScenario(AbstractErfsSurveyScenario):
         "rag",
         "retraite_brute",
         "retraite_imposable",
+        # "rev_financier_prelev_lib_imputes",
+        "revenu_categoriel_foncier",
+        "revenus_capitaux_prelevement_forfaitaire_unique_ir",
         "ric",
         "rnc",
         "rpns_imposables",

openfisca_france_data/felin/input_data_builder/create_variables_individuelles.py

@@ -20,11 +20,11 @@ def create_taux_csg_remplacement(individus, period, tax_benefit_system, sigma =
     def compute_taux_csg_remplacement(rfr, nbptr):
         parameters = tax_benefit_system.get_parameters_at_instant(period.start)
         seuils = parameters.prelevements_sociaux.contributions_sociales.csg.remplacement.seuils
-        seuil_exoneration = seuils.seuil_de_rfr_1 + (nbptr - 1) * seuils.seuil_rfr1.demi_part_suppl_rfr1
-        seuil_reduction = seuils.seuil_de_rfr_2 + (nbptr - 1) * seuils.seuil_rfr2.demi_part_suppl_rfr2
+        seuil_exoneration = seuils.seuil_rfr1.seuil_rfr1 + (nbptr - 1)*2 * seuils.seuil_rfr1.demi_part_suppl_rfr1
+        seuil_reduction = seuils.seuil_rfr2.seuil_rfr2 + (nbptr - 1)*2 * seuils.seuil_rfr2.demi_part_suppl_rfr2
         taux_csg_remplacement = 0.0 * rfr
         if period.start.year >= 2019:
-            seuil_taux_intermediaire = seuils.seuil_rfr3 + (nbptr - 1) * seuils.seuil_rfr3.demi_part_suppl_rfr3
+            seuil_taux_intermediaire = seuils.seuil_rfr3.seuil_rfr3 + (nbptr - 1)*2 * seuils.seuil_rfr3.demi_part_suppl_rfr3
             taux_csg_remplacement = np.where(
                 rfr <= seuil_exoneration,
                 1,

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name = "OpenFisca-France-Data",
-    version = "3.0.6",
+    version = "3.1.0",
     description = "OpenFisca-France-Data module to work with French survey data",
     long_description = long_description,
     long_description_content_type="text/markdown",

tests/fixtures/formulas/af.yaml

@@ -28,7 +28,7 @@
     - id: "enfant2"
       age_en_mois:
         2015-01: 9 * 12
-  output_variables:
+  output:
     autonomie_financiere:
       2015-01:
         - false