-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add source for data edited by us
- Loading branch information
Showing
15 changed files
with
315 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 43 additions & 1 deletion
44
pipeline/dbt/models/intermediate/data_inclusion/_data_inclusion__models.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,46 @@ | ||
| version: 2 | ||
|
|
||
| models: | ||
| - name: int_siretisation__annotations | ||
| - name: int_siretisation__annotations | ||
|
|
||
| - name: int_data_inclusion__adresses | ||
| columns: | ||
| - name: id | ||
| tests: | ||
| - unique | ||
| - not_null | ||
| - dbt_utils.not_empty_string | ||
|
|
||
| - name: int_data_inclusion__services | ||
| columns: | ||
| - name: id | ||
| tests: | ||
| - unique | ||
| - not_null | ||
| - dbt_utils.not_empty_string | ||
| - name: structure_id | ||
| tests: | ||
| - not_null | ||
| - relationships: | ||
| to: ref('int_data_inclusion__structures') | ||
| field: id | ||
| - name: adresse_id | ||
| tests: | ||
| - not_null | ||
| - relationships: | ||
| to: ref('int_data_inclusion__adresses') | ||
| field: id | ||
|
|
||
| - name: int_data_inclusion__structures | ||
| columns: | ||
| - name: id | ||
| tests: | ||
| - unique | ||
| - not_null | ||
| - dbt_utils.not_empty_string | ||
| - name: adresse_id | ||
| tests: | ||
| - not_null | ||
| - relationships: | ||
| to: ref('int_data_inclusion__adresses') | ||
| field: id |
43 changes: 43 additions & 0 deletions
43
pipeline/dbt/models/intermediate/data_inclusion/int_data_inclusion__adresses.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| WITH structures AS ( | ||
| SELECT * FROM {{ ref('stg_data_inclusion__structures') }} | ||
| ), | ||
|
|
||
| services AS ( | ||
| SELECT * FROM {{ ref('stg_data_inclusion__services') }} | ||
| ), | ||
|
|
||
| structure_adresses AS ( | ||
| SELECT | ||
| id AS "id", | ||
| longitude AS "longitude", | ||
| latitude AS "latitude", | ||
| _di_source_id AS "source", | ||
| complement_adresse AS "complement_adresse", | ||
| commune AS "commune", | ||
| adresse AS "adresse", | ||
| code_postal AS "code_postal", | ||
| code_insee AS "code_insee" | ||
| FROM structures | ||
| ), | ||
|
|
||
| service_adresses AS ( | ||
| SELECT | ||
| id AS "id", | ||
| longitude AS "longitude", | ||
| latitude AS "latitude", | ||
| _di_source_id AS "source", | ||
| complement_adresse AS "complement_adresse", | ||
| commune AS "commune", | ||
| adresse AS "adresse", | ||
| code_postal AS "code_postal", | ||
| code_insee AS "code_insee" | ||
| FROM services | ||
| ), | ||
|
|
||
| final AS ( | ||
| SELECT * FROM structure_adresses | ||
| UNION ALL | ||
| SELECT * FROM service_adresses | ||
| ) | ||
|
|
||
| SELECT * FROM final |
58 changes: 58 additions & 0 deletions
58
pipeline/dbt/models/intermediate/data_inclusion/int_data_inclusion__services.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| WITH services AS ( | ||
| SELECT * FROM {{ ref('stg_data_inclusion__services') }} | ||
| ), | ||
|
|
||
| di_profil_by_dora_profil AS ( | ||
| -- dora's thematiques are not yet normalized | ||
| SELECT x.* | ||
| FROM ( | ||
| VALUES | ||
| ('Adultes', 'adultes'), | ||
| ('Femmes', 'femmes'), | ||
| ('Public bénéficiaire du Revenu de Solidarité Active (RSA)', 'beneficiaire-rsa'), | ||
| ('Demandeur d''emploi', 'demandeur-demploi') | ||
| ) AS x (dora_profil, di_profil) | ||
| ), | ||
|
|
||
| final AS ( | ||
| SELECT | ||
| id AS "adresse_id", | ||
| contact_public AS "contact_public", | ||
| NULL AS "contact_nom_prenom", | ||
| NULL AS "courriel", | ||
| cumulable AS "cumulable", | ||
| date_creation::DATE AS "date_creation", | ||
| date_maj::DATE AS "date_maj", | ||
| date_suspension::DATE AS "date_suspension", | ||
| formulaire_en_ligne AS "formulaire_en_ligne", | ||
| frais_autres AS "frais_autres", | ||
| id AS "id", | ||
| justificatifs AS "justificatifs", | ||
| NULL AS "lien_source", -- ignored | ||
| modes_accueil AS "modes_accueil", | ||
| NULL::TEXT [] AS "modes_orientation_accompagnateur", | ||
| NULL::TEXT [] AS "modes_orientation_beneficiaire", | ||
| nom AS "nom", | ||
| presentation_resume AS "presentation_resume", | ||
| presentation_detail AS "presentation_detail", | ||
| prise_rdv AS "prise_rdv", | ||
| ARRAY( | ||
| SELECT di_profil_by_dora_profil.di_profil | ||
| FROM di_profil_by_dora_profil | ||
| WHERE di_profil_by_dora_profil.dora_profil = ANY(services.profils) | ||
| )::TEXT [] AS "profils", | ||
| recurrence AS "recurrence", | ||
| _di_source_id AS "source", | ||
| structure_id AS "structure_id", | ||
| NULL AS "telephone", | ||
| thematiques AS "thematiques", | ||
| types AS "types", | ||
| zone_diffusion_code AS "zone_diffusion_code", | ||
| zone_diffusion_nom AS "zone_diffusion_nom", | ||
| zone_diffusion_type AS "zone_diffusion_type", | ||
| pre_requis AS "pre_requis", | ||
| ARRAY[frais] AS "frais" | ||
| FROM services | ||
| ) | ||
|
|
||
| SELECT * FROM final |
30 changes: 30 additions & 0 deletions
30
pipeline/dbt/models/intermediate/data_inclusion/int_data_inclusion__structures.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| WITH structures AS ( | ||
| SELECT * FROM {{ ref('stg_data_inclusion__structures') }} | ||
| ), | ||
|
|
||
| final AS ( | ||
| SELECT | ||
| accessibilite AS "accessibilite", | ||
| id AS "adresse_id", | ||
| antenne AS "antenne", | ||
| courriel AS "courriel", | ||
| date_maj::DATE AS "date_maj", | ||
| horaires_ouverture AS "horaires_ouverture", | ||
| id AS "id", | ||
| labels_autres AS "labels_autres", | ||
| labels_nationaux AS "labels_nationaux", | ||
| lien_source AS "lien_source", | ||
| nom AS "nom", | ||
| presentation_detail AS "presentation_detail", | ||
| presentation_resume AS "presentation_resume", | ||
| rna AS "rna", | ||
| siret AS "siret", | ||
| site_web AS "site_web", | ||
| _di_source_id AS "source", | ||
| telephone AS "telephone", | ||
| NULL::TEXT [] AS "thematiques", | ||
| typologie AS "typologie" | ||
| FROM structures | ||
| ) | ||
|
|
||
| SELECT * FROM final |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
pipeline/dbt/models/staging/data_inclusion/_data_inclusion__models.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| version: 2 | ||
|
|
||
| models: | ||
| - name: stg_data_inclusion__services | ||
| config: | ||
| tags: data_inclusion | ||
| columns: | ||
| - name: id | ||
| tests: | ||
| - unique | ||
| - not_null | ||
| - dbt_utils.not_empty_string | ||
|
|
||
| - name: stg_data_inclusion__structures | ||
| config: | ||
| tags: data_inclusion | ||
| columns: | ||
| - name: id | ||
| tests: | ||
| - unique | ||
| - not_null | ||
| - dbt_utils.not_empty_string |
48 changes: 48 additions & 0 deletions
48
pipeline/dbt/models/staging/data_inclusion/stg_data_inclusion__services.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| WITH source AS ( | ||
| SELECT * FROM {{ source('data_inclusion_extra', 'services') }} | ||
| ), | ||
|
|
||
| final AS ( | ||
| SELECT | ||
| _di_source_id AS "_di_source_id", | ||
| (data ->> 'contact_public')::BOOLEAN AS "contact_public", | ||
| (data ->> 'cumulable')::BOOLEAN AS "cumulable", | ||
| (data ->> 'date_creation')::TIMESTAMP WITH TIME ZONE AS "date_creation", | ||
| (data ->> 'date_maj')::TIMESTAMP WITH TIME ZONE AS "date_maj", | ||
| (data ->> 'date_suspension')::TIMESTAMP WITH TIME ZONE AS "date_suspension", | ||
| (data ->> 'latitude')::FLOAT AS "latitude", | ||
| (data ->> 'longitude')::FLOAT AS "longitude", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'modes_accueil'))::TEXT [] AS "modes_accueil", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'profils'))::TEXT [] AS "profils", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'thematiques'))::TEXT [] AS "thematiques", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'types'))::TEXT [] AS "types", | ||
| NULLIF(TRIM(data ->> 'justificatifs'), '') AS "justificatifs", | ||
| NULLIF(TRIM(data ->> 'pre_requis'), '') AS "pre_requis", | ||
| data ->> 'adresse' AS "adresse", | ||
| data ->> 'code_insee' AS "code_insee", | ||
| data ->> 'code_postal' AS "code_postal", | ||
| data ->> 'commune' AS "commune", | ||
| data ->> 'complement_adresse' AS "complement_adresse", | ||
| NULLIF(TRIM(data ->> 'contact_nom'), '') AS "contact_nom", | ||
| NULLIF(TRIM(data ->> 'contact_prenom'), '') AS "contact_prenom", | ||
| NULLIF(TRIM(data ->> 'courriel'), '') AS "courriel", | ||
| data ->> 'formulaire_en_ligne' AS "formulaire_en_ligne", | ||
| data ->> 'frais_autres' AS "frais_autres", | ||
| data ->> 'frais' AS "frais", | ||
| data ->> 'id' AS "id", | ||
| data ->> 'lien_source' AS "lien_source", | ||
| data ->> 'nom' AS "nom", | ||
| data ->> 'presentation_resume' AS "presentation_resume", | ||
| data ->> 'presentation_detail' AS "presentation_detail", | ||
| data ->> 'prise_rdv' AS "prise_rdv", | ||
| data ->> 'recurrence' AS "recurrence", | ||
| data ->> 'source' AS "source", | ||
| data ->> 'structure_id' AS "structure_id", | ||
| NULLIF(TRIM(data ->> 'telephone'), '') AS "telephone", | ||
| NULLIF(TRIM(data ->> 'zone_diffusion_code'), '') AS "zone_diffusion_code", | ||
| NULLIF(TRIM(data ->> 'zone_diffusion_nom'), '') AS "zone_diffusion_nom", | ||
| data ->> 'zone_diffusion_type' AS "zone_diffusion_type" | ||
| FROM source | ||
| ) | ||
|
|
||
| SELECT * FROM final |
37 changes: 37 additions & 0 deletions
37
pipeline/dbt/models/staging/data_inclusion/stg_data_inclusion__structures.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| WITH source AS ( | ||
| SELECT * FROM {{ source('data_inclusion_extra', 'structures') }} | ||
| ), | ||
|
|
||
| final AS ( | ||
| SELECT | ||
| _di_source_id AS "_di_source_id", | ||
| (data ->> 'antenne')::BOOLEAN AS "antenne", | ||
| (data ->> 'date_maj')::TIMESTAMP WITH TIME ZONE AS "date_maj", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'labels_autres'))::TEXT [] AS "labels_autres", | ||
| ARRAY(SELECT * FROM JSONB_ARRAY_ELEMENTS_TEXT(data -> 'labels_nationaux'))::TEXT [] AS "labels_nationaux", | ||
| (data ->> 'latitude')::FLOAT AS "latitude", | ||
| (data ->> 'longitude')::FLOAT AS "longitude", | ||
| (data ->> 'thematiques')::TEXT [] AS "thematiques", | ||
| data ->> 'accessibilite' AS "accessibilite", | ||
| data ->> 'adresse' AS "adresse", | ||
| data ->> 'code_insee' AS "code_insee", | ||
| data ->> 'code_postal' AS "code_postal", | ||
| data ->> 'commune' AS "commune", | ||
| NULLIF(TRIM(data ->> 'complement_adresse'), '') AS "complement_adresse", | ||
| data ->> 'courriel' AS "courriel", | ||
| data ->> 'horaires_ouverture' AS "horaires_ouverture", | ||
| data ->> 'id' AS "id", | ||
| data ->> 'lien_source' AS "lien_source", | ||
| NULLIF(TRIM(data ->> 'nom'), '') AS "nom", | ||
| data ->> 'presentation_detail' AS "presentation_detail", | ||
| data ->> 'presentation_resume' AS "presentation_resume", | ||
| data ->> 'rna' AS "rna", | ||
| data ->> 'siret' AS "siret", | ||
| data ->> 'site_web' AS "site_web", | ||
| data ->> 'source' AS "source", | ||
| data ->> 'telephone' AS "telephone", | ||
| data ->> 'typologie' AS "typologie" | ||
| FROM source | ||
| ) | ||
|
|
||
| SELECT * FROM final |