diff --git a/src/dev_config.yaml b/src/dev_config.yaml index a6a0b97ad..a9b94be4a 100644 --- a/src/dev_config.yaml +++ b/src/dev_config.yaml @@ -73,7 +73,8 @@ network_paths: # updated_snapshot_path: "R:/BERD Results System Development 2023/DAP_emulation/spp_snapshots/2022_snapshots/snapshot-202212-002-83b5bacd-7c99-45cf-b989-d43d762dd054.json" # Freezing data paths frozen_data_staged_output_path: "02_freezing/frozen_data_staged/" - frozen_data_staged_path: "02_freezing/frozen_data_staged/2023_FROZEN_staged_full_responses_25-01-21_v397.csv" + berd_frozen_data_staged_path: "02_freezing/frozen_data_staged/2023_FROZEN_staged_full_responses_25-01-21_v397.csv" + pnp_frozen_data_staged_path: "02_freezing/frozen_data_staged/PNP_2023_FROZEN_staged_full_responses_25-01-21_v397.csv" freezing_changes_to_review_path: "02_freezing/changes_to_review/" freezing_additions_path: "R:/BERD Results System Development 2023/DAP_emulation/2023_surveys/BERD/02_freezing/freezing_updates/2023_freezing_additions_to_review_24-09-12_v1324.csv" freezing_amendments_path: "R:/BERD Results System Development 2023/DAP_emulation/2023_surveys/BERD/02_freezing/freezing_updates/2023_freezing_amendments_to_review_24-09-12_v1324.csv" diff --git a/src/freezing/freezing_main.py b/src/freezing/freezing_main.py index be3abe27a..7cdddcef5 100644 --- a/src/freezing/freezing_main.py +++ b/src/freezing/freezing_main.py @@ -106,7 +106,15 @@ def read_frozen_csv(config: dict, read_csv: Callable) -> pd.DataFrame: Returns: pd.DataFrame: The frozen data csv. """ - frozen_data_staged_path = config["freezing_paths"]["frozen_data_staged_path"] + if config["survey"]["survey_type"] == "BERD": + frozen_data_staged_path = config["freezing_paths"][ + "berd_frozen_data_staged_path" + ] + elif config["survey"]["survey_type"] == "PNP": + frozen_data_staged_path = config["freezing_paths"][ + "pnp_frozen_data_staged_path" + ] + FreezingLogger.info("Loading frozen data...") frozen_csv = read_csv(frozen_data_staged_path) validate_data_with_schema(frozen_csv, "./config/frozen_data_staged_schema.toml") diff --git a/src/user_config.yaml b/src/user_config.yaml index 0610b2cd6..9aed06803 100644 --- a/src/user_config.yaml +++ b/src/user_config.yaml @@ -2,7 +2,7 @@ config_validation: validate: True path: src/user_config_schema.yaml survey: - survey_type: "BERD" + survey_type: "PNP" survey_year: 2023 global: # Staging and validation settings diff --git a/src/utils/config.py b/src/utils/config.py index 0a4fd2476..bdfdf251f 100644 --- a/src/utils/config.py +++ b/src/utils/config.py @@ -332,11 +332,20 @@ def validate_freezing_config_settings(config: dict): ) = validate_freezing_run_config(config) snapshot_path = config["staging_paths"]["snapshot_path"] - frozen_data_staged_path = config["freezing_paths"]["frozen_data_staged_path"] updated_snapshot_path = config["staging_paths"]["updated_snapshot_path"] freezing_additions_path = config["freezing_paths"]["freezing_additions_path"] freezing_amendments_path = config["freezing_paths"]["freezing_amendments_path"] + # Frozen data path is dependent on the survey type + if config["survey"]["survey_type"] == "BERD": + frozen_data_staged_path = config["freezing_paths"][ + "berd_frozen_data_staged_path" + ] + elif config["survey"]["survey_type"] == "PNP": + frozen_data_staged_path = config["freezing_paths"][ + "pnp_frozen_data_staged_path" + ] + if run_with_snapshot: if snapshot_path is None: raise ValueError( diff --git a/src/utils/path_helpers.py b/src/utils/path_helpers.py index 50d95552a..f327f5a13 100644 --- a/src/utils/path_helpers.py +++ b/src/utils/path_helpers.py @@ -1,6 +1,5 @@ """This module contains helper functions for creating paths.""" -import re import os import logging @@ -48,6 +47,8 @@ def get_paths(config: dict) -> dict: platform = config["global"]["platform"] survey = config["survey"]["survey_type"] + print(platform) + # select either network_paths or s3_paths from the config, depending on platform, paths = config[f"{platform}_paths"] paths["year"] = config["survey"]["survey_year"] @@ -218,18 +219,14 @@ def create_freezing_config(config: dict) -> dict: paths = get_paths(config) survey_path = paths["survey_path"] - if ( - config["survey"]["survey_type"] == "PNP" - and "PNP_" not in paths["frozen_data_staged_path"] - ): - pattern = "frozen_data_staged/" - insert_string = "frozen_data_staged/PNP_" - paths["frozen_data_staged_path"] = re.sub( - pattern, insert_string, paths["frozen_data_staged_path"] + if config["survey"]["survey_type"] == "BERD": + freezing_dict["frozen_data_staged_path"] = os.path.join( + survey_path, paths["berd_frozen_data_staged_path"] + ) + elif config["survey"]["survey_type"] == "PNP": + freezing_dict["frozen_data_staged_path"] = os.path.join( + survey_path, paths["pnp_frozen_data_staged_path"] ) - freezing_dict["frozen_data_staged_path"] = os.path.join( - survey_path, paths["frozen_data_staged_path"] - ) freezing_dict["freezing_changes_to_review_path"] = os.path.join( survey_path, paths["freezing_changes_to_review_path"] )