diff --git a/global-api/importer-mage/cc-mage/data_exporters/load_waste_emissionfactor.py b/global-api/importer-mage/cc-mage/data_exporters/load_waste_emissionfactor.py new file mode 100644 index 000000000..dfd84a43c --- /dev/null +++ b/global-api/importer-mage/cc-mage/data_exporters/load_waste_emissionfactor.py @@ -0,0 +1,31 @@ +from mage_ai.settings.repo import get_repo_path +from mage_ai.io.config import ConfigFileLoader +from mage_ai.io.postgres import Postgres +from pandas import DataFrame +from os import path + +if 'data_exporter' not in globals(): + from mage_ai.data_preparation.decorators import data_exporter + + +@data_exporter +def export_data_to_postgres(df: DataFrame, **kwargs) -> None: + """ + Template for exporting data to a PostgreSQL database. + Specify your configuration settings in 'io_config.yaml'. + + Docs: https://docs.mage.ai/design/data-loading#postgresql + """ + schema_name = 'raw_data' # Specify the name of the schema to export data to + table_name = 'ipcc_waste_emissionsfactor' # Specify the name of the table to export data to + config_path = path.join(get_repo_path(), 'io_config.yaml') + config_profile = 'default' + + with Postgres.with_config(ConfigFileLoader(config_path, config_profile)) as loader: + loader.export( + df, + schema_name, + table_name, + index=False, # Specifies whether to include index in exported table + if_exists='replace', # Specify resolution policy if table name already exists + ) diff --git a/global-api/importer-mage/cc-mage/data_loaders/load_s3_ipcc_ef.py b/global-api/importer-mage/cc-mage/data_loaders/load_s3_ipcc_ef.py new file mode 100644 index 000000000..05b5fab1f --- /dev/null +++ b/global-api/importer-mage/cc-mage/data_loaders/load_s3_ipcc_ef.py @@ -0,0 +1,61 @@ +from mage_ai.settings.repo import get_repo_path +from mage_ai.io.config import ConfigFileLoader +from mage_ai.data_preparation.shared.secrets import get_secret_value +from mage_ai.io.s3 import S3 +from mage_ai.io.duckdb import DuckDB +from os import path +import boto3 +import duckdb +if 'data_loader' not in globals(): + from mage_ai.data_preparation.decorators import data_loader +if 'test' not in globals(): + from mage_ai.data_preparation.decorators import test + + +@data_loader +def load_from_s3_bucket(*args, **kwargs): + """ + load excel document in s3 + """ + aws_access_key_id = get_secret_value('AWS_ACCESS_KEY_ID') + aws_secret_access_key = get_secret_value('AWS_SECRET_ACCESS_KEY') + aws_region = 'us-east-1' + + config_path = path.join(get_repo_path(), 'io_config.yaml') + config_profile = 'default' + + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region + ) + + s3_filename = 's3://global-api-raw-data/ipcc/EFDB_output.xlsx' + + conn = duckdb.connect() + + query = f""" + install spatial; + load spatial; + + SET s3_region='{aws_region}'; + SET s3_access_key_id='{aws_access_key_id}'; + SET s3_secret_access_key='{aws_secret_access_key}'; + + CREATE OR REPLACE TABLE ipcc_emissionfactor as + SELECT * FROM st_read('{s3_filename}'); + """ + + duckdb_loader = DuckDB.with_config(ConfigFileLoader(config_path, config_profile)) + results = duckdb_loader.execute(query) + + return 1 + + + +@test +def test_output(output, *args) -> None: + """ + Template code for testing the output of the block. + """ + assert output is not None, 'The output is undefined' diff --git a/global-api/importer-mage/cc-mage/data_loaders/waste_duckdb_rawdata.sql b/global-api/importer-mage/cc-mage/data_loaders/waste_duckdb_rawdata.sql new file mode 100644 index 000000000..80951331a --- /dev/null +++ b/global-api/importer-mage/cc-mage/data_loaders/waste_duckdb_rawdata.sql @@ -0,0 +1,21 @@ +select gpc_sector, + gas_name, + gas, + parameter_code, + gpc_subsector, + technical_reference_year, + parameter_subcategory_type1, + parameter_subcategory_typename1, + parameter_subcategory_type2, + parameter_subcategory_typename2, + emissionsfactor_value, + emissionsfactor_units, + region, + data_source, + ef_id as ipcc_ef_id, + rank() over(partition by parameter_code,parameter_subcategory_type1, parameter_subcategory_typename1,parameter_subcategory_type2, parameter_subcategory_typename2 + order by technical_reference_year desc) as rnk +from waste_default_values +where parameter_code is not null +and region is not null +order by parameter_code,parameter_subcategory_type1, parameter_subcategory_typename1,parameter_subcategory_type2, parameter_subcategory_typename2, technical_reference_year desc; \ No newline at end of file diff --git a/global-api/importer-mage/cc-mage/pipelines/ipcc_emissions_factor/__init__.py b/global-api/importer-mage/cc-mage/pipelines/ipcc_emissions_factor/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/global-api/importer-mage/cc-mage/pipelines/ipcc_emissions_factor/metadata.yaml b/global-api/importer-mage/cc-mage/pipelines/ipcc_emissions_factor/metadata.yaml new file mode 100755 index 000000000..80ebe3b0b --- /dev/null +++ b/global-api/importer-mage/cc-mage/pipelines/ipcc_emissions_factor/metadata.yaml @@ -0,0 +1,156 @@ +blocks: +- all_upstream_blocks_executed: true + color: null + configuration: {} + downstream_blocks: + - waste_default_values + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: load_s3_ipcc_ef + retry_config: null + status: updated + timeout: null + type: data_loader + upstream_blocks: [] + uuid: load_s3_ipcc_ef +- all_upstream_blocks_executed: false + color: null + configuration: + data_provider: duckdb + data_provider_profile: default + dbt: {} + disable_query_preprocessing: false + export_write_policy: append + limit: 1000 + use_raw_sql: true + downstream_blocks: + - waste_unpackrecord + executor_config: null + executor_type: local_python + has_callback: false + language: sql + name: waste_default_values + retry_config: null + status: executed + timeout: null + type: transformer + upstream_blocks: + - load_s3_ipcc_ef + uuid: waste_default_values +- all_upstream_blocks_executed: false + color: null + configuration: + data_provider: duckdb + data_provider_profile: default + dbt: {} + disable_query_preprocessing: false + export_write_policy: append + limit: 1000 + use_raw_sql: true + downstream_blocks: + - waste_cleandata + executor_config: null + executor_type: local_python + has_callback: false + language: sql + name: waste_unpackrecord + retry_config: null + status: updated + timeout: null + type: transformer + upstream_blocks: + - waste_default_values + uuid: waste_unpackrecord +- all_upstream_blocks_executed: false + color: null + configuration: + data_provider: duckdb + data_provider_profile: default + dbt: {} + disable_query_preprocessing: false + export_write_policy: append + limit: 1000 + use_raw_sql: true + downstream_blocks: [] + executor_config: null + executor_type: local_python + has_callback: false + language: sql + name: waste_cleandata + retry_config: null + status: updated + timeout: null + type: transformer + upstream_blocks: + - waste_unpackrecord + uuid: waste_cleandata +- all_upstream_blocks_executed: true + color: null + configuration: + data_provider: duckdb + data_provider_profile: default + dbt: {} + disable_query_preprocessing: false + export_write_policy: append + limit: 1000 + use_raw_sql: true + downstream_blocks: + - load_waste_emissionfactor + executor_config: null + executor_type: local_python + has_callback: false + language: sql + name: waste_duckdb_rawdata + retry_config: null + status: updated + timeout: null + type: data_loader + upstream_blocks: [] + uuid: waste_duckdb_rawdata +- all_upstream_blocks_executed: false + color: null + configuration: + data_provider: duckdb + data_provider_profile: default + export_write_policy: append + downstream_blocks: [] + executor_config: null + executor_type: local_python + has_callback: false + language: python + name: load_waste_emissionfactor + retry_config: null + status: executed + timeout: null + type: data_exporter + upstream_blocks: + - waste_duckdb_rawdata + uuid: load_waste_emissionfactor +cache_block_output_in_memory: false +callbacks: [] +concurrency_config: {} +conditionals: [] +created_at: '2024-07-22 11:07:36.325251+00:00' +data_integration: null +description: null +executor_config: {} +executor_count: 1 +executor_type: null +extensions: {} +name: ipcc_emissions_factor +notification_config: {} +remote_variables_dir: null +retry_config: {} +run_pipeline_in_one_process: false +settings: + triggers: null +spark_config: {} +tags: [] +type: python +uuid: ipcc_emissions_factor +variables: + bucket_name: global-api-raw-data +variables_dir: /home/src/mage_data/cc-mage +widgets: [] diff --git a/global-api/importer-mage/cc-mage/transformers/waste_cleandata.sql b/global-api/importer-mage/cc-mage/transformers/waste_cleandata.sql new file mode 100644 index 000000000..4befd8e94 --- /dev/null +++ b/global-api/importer-mage/cc-mage/transformers/waste_cleandata.sql @@ -0,0 +1,214 @@ +CREATE OR REPLACE TABLE waste_default_values AS +SELECT + ef_id, + ipcc_sector, + CASE + WHEN ipcc_sector = '6A - Solid Waste Disposal on Land' THEN 'Solid Waste' + WHEN ipcc_sector IN ('6B - Wastewater Handling', '6B2 - Domestic and Commercial Wastewater') THEN 'Wastewater' + WHEN ipcc_sector = '6B1 - Industrial Wastewater' THEN 'Wastewater' + WHEN ipcc_sector = '6C - Waste Incineration' THEN 'Waste Incineration' + END AS gpc_sector, + lower(gas) AS gas_name, + CASE + WHEN gas = 'METHANE' THEN 'CH4' + WHEN gas = 'NITROUS OXIDE' THEN 'N2O' + WHEN gas = 'CARBON DIOXIDE' THEN 'CO2' + END AS gas, + description AS description, + CASE + WHEN gpc_sector = 'Solid Waste' THEN + CASE + WHEN description LIKE '%DOCf%' OR description LIKE '%Fraction%DOC%' OR description LIKE '%DOC%fraction%' THEN 'DOCf' + WHEN description LIKE '%DDOCm%' THEN 'DDOCm' + WHEN description LIKE '%DOC%' AND lower(description) NOT LIKE '%fraction%' AND lower(description) NOT LIKE '%frction%' THEN 'DOC' + WHEN description LIKE '%MCF%' THEN 'MCF' + WHEN description LIKE '%fraction of methane%' THEN 'F' + WHEN description LIKE '%oxidation factor%' THEN 'OX' + WHEN description LIKE '%(k)%' THEN 'k' + ELSE NULL + END + WHEN gpc_sector LIKE '%Wastewater%' THEN + CASE + WHEN lower(description) LIKE '%wastewater generation%' THEN 'W_i' + WHEN lower(description) LIKE '%cod%' THEN 'COD_i' + WHEN lower(description) LIKE '% bo' THEN 'B0' + WHEN lower(description) LIKE '%mcf%' THEN 'MCF' + WHEN Description LIKE '%BOD%' THEN 'BOD' + ELSE NULL + END + END AS parameter_code, + CASE + WHEN gpc_sector = 'Solid Waste' THEN + CASE + WHEN description LIKE '%MSW%' THEN 'Domestic' + WHEN lower(description) LIKE '%industrial%' OR description LIKE '%industry%' THEN 'Industrial' + WHEN description LIKE '%clinical%' THEN 'Clinical' + ELSE NULL + END + ELSE NULL + END AS gpc_subsector, + type_parameter, + CASE + WHEN regexp_extract(technical_reference, '([0-9]{4})') = '' OR regexp_extract(technical_reference, '([0-9]{4})') IS NULL THEN regexp_extract(type_parameter, '([0-9]{4})') + ELSE regexp_extract(technical_reference, '([0-9]{4})') + END AS technical_reference_year, + CASE + WHEN parameter_code IN ('DOCf', 'DOC') THEN 'waste_form' + WHEN parameter_code IN ('MCF', 'OX') AND gpc_sector = 'Solid Waste' THEN 'treatment_type' + WHEN parameter_code = 'k' THEN 'climate' + WHEN parameter_code IN ('COD_i', 'W_i') THEN 'industry_type' + WHEN parameter_code = 'MCF' AND gpc_sector LIKE '%water%' THEN 'treatment_type' + ELSE NULL + END AS parameter_subcategory_type1, + CASE + WHEN parameter_code IN ('DOCf', 'DOC') THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%wet waste%' OR emissionsfactor_units LIKE '%wet%' THEN 'wet waste' + WHEN lower(emissionfactor_details) LIKE '%dry waste%' OR emissionsfactor_units LIKE '%dry%' THEN 'dry waste' + ELSE 'unclassified' + END + WHEN parameter_code = 'MCF' AND gpc_sector = 'Solid Waste' THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%managed%anaerobic%' THEN 'Managed – Anaerobic' + WHEN lower(emissionfactor_details) LIKE '%semi%aerobic%well%' THEN 'Managed Well – Semi-Aerobic' + WHEN lower(emissionfactor_details) LIKE '%semi%aerobic%poor%' THEN 'Managed Poorly – Semi-Aerobic' + WHEN lower(emissionfactor_details) LIKE '%active%aeration%poor%' THEN 'Managed Poorly – Active Aeration' + WHEN lower(emissionfactor_details) LIKE '%active%aeration%' THEN 'Managed Well – Active Aeration' + WHEN lower(emissionfactor_details) LIKE '%managed%greater%5%' OR lower(emissionfactor_details) LIKE '%unmanaged-deep%' THEN 'Unmanaged Waste Sites - Deep SWDS' + WHEN lower(emissionfactor_details) LIKE '%managed%less%5%' THEN 'Unmanaged Waste Sites - Shallow SWDS' + WHEN lower(emissionfactor_details) LIKE '%uncategorised%' OR lower(emissionfactor_details) LIKE '%uncategorized%' THEN 'Uncategorized Waste Sites' + ELSE 'Other' + END + WHEN parameter_code = 'OX' THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%unmanaged%' THEN 'Unmanaged' + WHEN lower(emissionfactor_details) LIKE '%well-managed%' THEN 'Managed' + ELSE NULL + END + WHEN parameter_code = 'k' THEN + CASE + WHEN (emissionfactor_details LIKE '%< 1%' OR emissionfactor_details LIKE '%< 1000 mm%') AND lower(emissionfactor_details) LIKE '%dry%' THEN 'Temperate - Dry' + WHEN (emissionfactor_details LIKE '%< 1%' OR emissionfactor_details LIKE '%< 1000 mm%') AND lower(emissionfactor_details) LIKE '%wet%' THEN 'Temperate - Wet' + WHEN lower(emissionfactor_details) LIKE '%moist%' THEN 'Tropical - Moist' + WHEN (emissionfactor_details LIKE '%> 1%' OR emissionfactor_details LIKE '%> 1000 mm%') AND lower(emissionfactor_details) LIKE '%dry%' THEN 'Tropical - Dry' + WHEN lower(emissionfactor_details) LIKE '%dry%' THEN 'Dry' + WHEN lower(emissionfactor_details) LIKE '%wet%' THEN 'Wet' + END + WHEN parameter_code IN ('COD_i', 'W_i') THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%starch%' THEN 'Starch Production' + WHEN lower(emissionfactor_details) LIKE '%leather%' THEN 'Leather Tanning' + WHEN lower(emissionfactor_details) LIKE '%potato%' THEN 'Potato Processing' + WHEN lower(emissionfactor_details) LIKE '%textile%' THEN 'Textiles' + WHEN lower(emissionfactor_details) LIKE '%vegetable oil%' THEN 'Vegetable Oils' + WHEN lower(emissionfactor_details) LIKE '%beer%' OR lower(emissionfactor_details) LIKE '%malt%' THEN 'Beer & Malt' + WHEN lower(emissionfactor_details) LIKE '%fish%' THEN 'Fish Processing' + WHEN lower(emissionfactor_details) LIKE '%plastics%' OR lower(emissionfactor_details) LIKE '%resins%' THEN 'Plastics & Resins' + WHEN lower(emissionfactor_details) LIKE '%sugar%' OR lower(emissionfactor_details) LIKE '%suger%' THEN 'Sugar Refining' + WHEN lower(emissionfactor_details) LIKE '%dairy%' THEN 'Dairy Products' + WHEN lower(emissionfactor_details) LIKE '%pulp & paper%' OR lower(emissionfactor_details) LIKE '%paper%' THEN 'Pulp & Paper' + WHEN lower(emissionfactor_details) LIKE '%soft drink%' THEN 'Soft Drinks' + WHEN lower(emissionfactor_details) LIKE '%apple%' THEN 'Apple Processing' + WHEN lower(emissionfactor_details) LIKE '%distilled%' OR lower(emissionfactor_details) LIKE '%ethanol%' THEN 'Distilled & Ethanol Beverages' + WHEN lower(emissionfactor_details) LIKE '%organic chemicals%' THEN 'Organic Chemicals' + WHEN lower(emissionfactor_details) LIKE '%alcohol%' THEN 'Alcohol Refining' + WHEN lower(emissionfactor_details) LIKE '%wine%' OR lower(emissionfactor_details) LIKE '%vinegar%' THEN 'Wine & Vinegar' + WHEN lower(emissionfactor_details) LIKE '%frozen food%' THEN 'Frozen Food Processing' + WHEN lower(emissionfactor_details) LIKE '%seasoning%' THEN 'Seasoning' + WHEN lower(emissionfactor_details) LIKE '%meat%' OR lower(emissionfactor_details) LIKE '%poultry%' THEN 'Meat & Poultry' + WHEN lower(emissionfactor_details) LIKE '%soap%' OR lower(emissionfactor_details) LIKE '%detergents%' THEN 'Soap & Detergents' + WHEN lower(emissionfactor_details) LIKE '%petroleum%' OR lower(emissionfactor_details) LIKE '%refineries%' THEN 'Petroleum Refining' + WHEN lower(emissionfactor_details) LIKE '%fruits%' OR lower(emissionfactor_details) LIKE '%vegetables%' OR lower(emissionfactor_details) LIKE '%cannery%' THEN 'Fruits & Vegetables Processing' + WHEN lower(emissionfactor_details) LIKE '%paints%' THEN 'Paints' + WHEN lower(emissionfactor_details) LIKE '%coffee%' THEN 'Coffee' + WHEN lower(emissionfactor_details) LIKE '%iron%' OR lower(emissionfactor_details) LIKE '%steel%' THEN 'Iron & Steel Manufacturing' + WHEN lower(emissionfactor_details) LIKE '%drugs%' OR lower(emissionfactor_details) LIKE '%medicines%' OR lower(emissionfactor_details) LIKE '%pharmaceuticals%' THEN 'Pharmaceuticals' + WHEN lower(emissionfactor_details) LIKE '%petroleum production%' THEN 'Petroleum Production' + WHEN lower(emissionfactor_details) LIKE '%coke%' THEN 'Coke Production' + WHEN lower(emissionfactor_details) LIKE '%ice cream%' THEN 'Ice Cream Production' + WHEN lower(emissionfactor_details) LIKE '%animal feed%' THEN 'Animal Feed Production' + WHEN lower(emissionfactor_details) LIKE '%rubber%' THEN 'Rubber Manufacturing' + WHEN lower(emissionfactor_details) LIKE '%nitrogen%' THEN 'Nitrogen Fertiliser Production' + WHEN lower(emissionfactor_details) LIKE '%canneries%' THEN 'Canneries' + WHEN lower(emissionfactor_details) LIKE '%tannery%' THEN 'Tannery' + WHEN lower(emissionfactor_details) LIKE '%grapes%' THEN 'Grapes Processing' + WHEN lower(emissionfactor_details) LIKE '%flour%' THEN 'Flour Products' + WHEN lower(emissionfactor_details) LIKE '%chemical%' THEN 'Chemical Products' + WHEN lower(emissionfactor_details) LIKE '%citrus%' THEN 'Citrus Processing' + WHEN lower(emissionfactor_details) LIKE '%domestic wastewater%' THEN 'Domestic Wastewater Treatment' + WHEN lower(emissionfactor_details) LIKE '%petrochemical%' THEN 'Petrochemical Products' + WHEN lower(emissionfactor_details) LIKE '%food - oils%' THEN 'Food Oils' + WHEN lower(emissionfactor_details) LIKE '%other vegetable processing%' THEN 'Other Vegetable Processing' + WHEN lower(emissionfactor_details) LIKE '%non-citrus%' THEN 'Non-Citrus Processing' + ELSE 'Other' + END + WHEN parameter_code IN ('MCF') AND gpc_sector LIKE '%water%' THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%flowing sewer%' THEN 'Flowing Sewer' + WHEN lower(emissionfactor_details) LIKE '%sea, river, lake discharge%' THEN 'Flowing Water' + WHEN lower(emissionfactor_details) LIKE '%stagnant sewer%' THEN 'Stagnant Sewer' + WHEN lower(emissionfactor_details) LIKE '%anaerobic reactor%' + OR lower(emissionfactor_details) LIKE '%a2o%' THEN 'Anaerobic Reactor' + WHEN lower(emissionfactor_details) LIKE '%septic%' THEN 'Septic Tank' + WHEN lower(emissionfactor_details) LIKE '%centralized% aerobic treatment%' + OR lower(emissionfactor_details) LIKE '%centralised% aerobic treatment%' THEN 'Centralized Aerobic Treatment Plant' + WHEN lower(emissionfactor_details) LIKE '%anaerobic shallow lagoon%' + OR lower(emissionfactor_details) LIKE '%facultative lagoons%' THEN 'Anaerobic Shallow Lagoon' + WHEN lower(emissionfactor_details) LIKE '%bardenpho%' THEN 'Bardenpho Treatment' + WHEN lower(emissionfactor_details) LIKE '%biological nutrient removal%' THEN 'Biological Nutrient Removal' + WHEN lower(emissionfactor_details) LIKE '%latrine%' THEN 'Latrine' + WHEN lower(emissionfactor_details) LIKE '%untreated%' THEN 'Untreated System' + WHEN lower(emissionfactor_details) LIKE '%treated anaerobic%' + OR lower(emissionfactor_details) LIKE '%anaerobic digester%' + OR lower(emissionfactor_details) LIKE '%anaerobic shallow lagoon%' + OR lower(emissionfactor_details) LIKE '%anaerobic deep lagoon%' THEN 'Anaerobic Treatment' + WHEN lower(emissionfactor_details) LIKE '%treated aerobic%' + OR lower(emissionfactor_details) LIKE '%well-managed aerobic%' THEN 'Aerobic Treatment Plant - Well Managed' + WHEN lower(emissionfactor_details) LIKE '%overloaded aerobic%' THEN 'Aerobic Treatment Plant - Overloaded' + WHEN lower(emissionfactor_details) LIKE '%discharge to aquatic%' + OR lower(emissionfactor_details) LIKE '%reservoir%' THEN 'Discharge to Aquatic Environments' + WHEN lower(emissionfactor_details) LIKE '%activated sludge%' THEN 'Activated Sludge' + ELSE 'Other' + END + END AS parameter_subcategory_typename1, + CASE + WHEN parameter_code IN ('DOCf', 'DOC', 'k') THEN 'waste_type' + ELSE NULL + END AS parameter_subcategory_type2, + CASE + WHEN parameter_code IN ('DOCf', 'DOC') THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%food%' THEN 'Food Waste' + WHEN lower(emissionfactor_details) LIKE '%textile%' THEN 'Textile' + WHEN lower(emissionfactor_details) LIKE '%paper%' THEN 'Paper/Cardboard' + WHEN lower(emissionfactor_details) LIKE '%wood%' THEN 'Wood' + WHEN lower(emissionfactor_details) LIKE '%garden%' THEN 'Garden and Park Waste' + WHEN lower(emissionfactor_details) LIKE '%nappies%' THEN 'Nappies' + WHEN lower(emissionfactor_details) LIKE '%rubber%' THEN 'Rubber and Leather' + WHEN lower(emissionfactor_details) LIKE '%clinical%' THEN 'Clinical Waste' + ELSE NULL + END + WHEN parameter_code = 'k' THEN + CASE + WHEN lower(emissionfactor_details) LIKE '%slow%' THEN 'Slowly Degrading Waste' + WHEN lower(emissionfactor_details) LIKE '%rapid%' THEN 'Rapidly Degrading Waste' + WHEN lower(emissionfactor_details) LIKE '%bulk%' OR lower(emissionfactor_details) LIKE '%mixed%' THEN 'Bulk MSW or Industrial Waste' + ELSE NULL + END + ELSE NULL + END AS parameter_subcategory_typename2, + emissionsfactor_value, + emissionsfactor_units, + CASE + WHEN lower(region) LIKE '%latin america%' THEN 'Latin America' + WHEN lower(region) LIKE '%brazil%' THEN 'Brazil' + WHEN lower(region) LIKE '%chile%' THEN 'Chile' + WHEN lower(region) LIKE '%argentina%' THEN 'Argentina' + WHEN region IS NULL OR region = 'Region: Generic' THEN 'Global' + ELSE NULL + END AS region, + data_source, + technical_reference, + emissionfactor_details +FROM waste_ef +; diff --git a/global-api/importer-mage/cc-mage/transformers/waste_default_values.sql b/global-api/importer-mage/cc-mage/transformers/waste_default_values.sql new file mode 100644 index 000000000..62ac698c5 --- /dev/null +++ b/global-api/importer-mage/cc-mage/transformers/waste_default_values.sql @@ -0,0 +1,25 @@ +CREATE OR REPLACE TABLE waste_ef AS +SELECT "EF ID" AS ef_id, +REPLACE(REGEXP_REPLACE("IPCC 1996 Source/Sink Category", '\n', '|'), CHR(10), '|') AS ipcc_sector_multi, +REPLACE(REGEXP_REPLACE("Gas", '\n', '|'), CHR(10), '|') AS gas_multi, +"Fuel 1996" AS fuel_1996, +"Fuel 2006" AS fuel_2006, +"C pool" c_pool, +"Type of parameter" AS type_parameter, +Description, +"Technologies / Practices" AS technologies_paractises, +"Parameters / Conditions" AS parameters_conditions, +"Region / Regional Conditions" AS region, +"Abatement / Control Technologies" AS control_paractises, +"Other properties" AS properties, +Value AS emissionsfactor_value, +Unit AS emissionsfactor_units, +Equation AS ipcc_equation, +"IPCC Worksheet" as ipcc_worksheet, +"Technical Reference" as technical_reference, +"Source of data" as dataset_name, +"Data provider" AS data_source +FROM ipcc_emissionfactor +WHERE lower("IPCC 2006 Source/Sink Category") LIKE '%waste%' +AND regexp_matches("IPCC 1996 Source/Sink Category", '\n') +; \ No newline at end of file diff --git a/global-api/importer-mage/cc-mage/transformers/waste_unpackrecord.sql b/global-api/importer-mage/cc-mage/transformers/waste_unpackrecord.sql new file mode 100644 index 000000000..b206d0f76 --- /dev/null +++ b/global-api/importer-mage/cc-mage/transformers/waste_unpackrecord.sql @@ -0,0 +1,66 @@ +CREATE OR REPLACE TABLE waste_ef AS +WITH waste_ef_split AS ( +SELECT ef_id, + UNNEST(STRING_SPLIT(ipcc_sector_multi, '|')) AS ipcc_sector, + gas_multi, + fuel_1996, + fuel_2006, + c_pool, + type_parameter, + Description, + technologies_paractises, + parameters_conditions, + region, + control_paractises, + properties, + emissionsfactor_value, + emissionsfactor_units, + ipcc_equation, + data_source, + ipcc_worksheet, + technical_reference, + dataset_name +FROM waste_ef), +waste_ef_split_gas AS ( +SELECT ef_id, + ipcc_sector, + UNNEST(STRING_SPLIT(gas_multi, '|')) AS gas, + fuel_1996, + fuel_2006, + c_pool, + type_parameter, + Description, + technologies_paractises, + parameters_conditions, + region, + control_paractises, + properties, + emissionsfactor_value, + emissionsfactor_units, + ipcc_equation, + data_source, + ipcc_worksheet, + technical_reference, + dataset_name +FROM waste_ef_split +WHERE ipcc_sector != '') +SELECT ef_id, + ipcc_sector, + gas, + type_parameter, + description, + technologies_paractises, + parameters_conditions, + region, + control_paractises, + properties, + emissionsfactor_value, + emissionsfactor_units, + ipcc_equation, + data_source, + ipcc_worksheet, + technical_reference, + dataset_name, + trim(COALESCE(technologies_paractises, '') || ' ' || COALESCE(parameters_conditions,'') || '' || COALESCE(control_paractises,'') || ' '|| COALESCE(properties,'')) as emissionfactor_details +FROM waste_ef_split_gas +WHERE gas != '' \ No newline at end of file diff --git a/global-api/importer-mage/docker-compose.yml b/global-api/importer-mage/docker-compose.yml index e4d6ac302..89d403476 100644 --- a/global-api/importer-mage/docker-compose.yml +++ b/global-api/importer-mage/docker-compose.yml @@ -6,6 +6,8 @@ services: build: context: . dockerfile: Dockerfile + #env_file: + # - .env ports: - 6789:6789 volumes: diff --git a/global-api/importer-mage/my_db b/global-api/importer-mage/my_db new file mode 100644 index 000000000..6023998ae Binary files /dev/null and b/global-api/importer-mage/my_db differ diff --git a/global-api/importer-mage/my_db.wal b/global-api/importer-mage/my_db.wal new file mode 100644 index 000000000..58e049eb2 Binary files /dev/null and b/global-api/importer-mage/my_db.wal differ