From d0b349c14b0d96abbf311241877bc528163f942e Mon Sep 17 00:00:00 2001 From: Bipin Abraham Date: Thu, 23 Jan 2025 09:17:26 +0000 Subject: [PATCH] Added download script and modified pvmap --- statvar_imports/USA/DOL_Wages/README.md | 14 ++- .../DOL_Wages/USA_State_Places_Resolved.csv | 59 ---------- .../USA/DOL_Wages/US_Dol_Wages_metadata.csv | 31 +++--- .../DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv | 105 +++++++++++------- 4 files changed, 89 insertions(+), 120 deletions(-) delete mode 100644 statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv diff --git a/statvar_imports/USA/DOL_Wages/README.md b/statvar_imports/USA/DOL_Wages/README.md index 725a618c7e..5393d66e54 100644 --- a/statvar_imports/USA/DOL_Wages/README.md +++ b/statvar_imports/USA/DOL_Wages/README.md @@ -2,19 +2,23 @@ - source: https://www.dol.gov/agencies/whd/state/minimum-wage/history, -- how to download data: Manual download from source. +- how to download data: Download script (download_script/main.py). - type of place: Country and State. - statvars: Economy -- years: 1968 to 2023 +- years: 1968 to 2024 -- place_resolution: Resolved state name to state geoId using places resolved csv. +- place_resolution: Places resolved to geoId in pvmap itself. ### How to run: -`python3 stat_var_processor.py --input_data=.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --places_resolved_csv=statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv --output_path=--output_path=` +`python3 stat_var_processor.py --input_data=.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --output_path=--output_path=` #### Example -`python3 stat_var_processor.py --input_data=statvar_imports/USA/DOL_Wages/test_data/sample_input/US_Dol_Wages_data.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv --config=statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv --output_path=--output_path=statvar_imports/USA/DOL_Wages/test_data/sample_output/US_DOL_Wages` \ No newline at end of file +#### Download : +`python3 main.py` + +#### Processing +`python3 stat_var_processor.py --input_data=statvar_imports/USA/DOL_Wages/download_script/input_files/final_data.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --output_path=--output_path=statvar_imports/USA/DOL_Wages/test_data/sample_output/US_DOL_Wages` \ No newline at end of file diff --git a/statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv b/statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv deleted file mode 100644 index 97f56aa441..0000000000 --- a/statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv +++ /dev/null @@ -1,59 +0,0 @@ -place_name,dcid -Alabama,geoId/01 -Alaska,geoId/02 -Arizona,geoId/04 -Arkansas,geoId/05 -California,geoId/06 -Colorado,geoId/08 -Connecticut,geoId/09 -Delaware,geoId/10 -District of Columbia,geoId/11 -Florida,geoId/12 -Georgia,geoId/13 -Hawaii,geoId/15 -Idaho,geoId/16 -Illinois,geoId/17 -Indiana,geoId/18 -Iowa,geoId/19 -Kansas,geoId/20 -Kentucky,geoId/21 -Louisiana,geoId/22 -Maine,geoId/23 -Maryland,geoId/24 -Massachusetts,geoId/25 -Michigan,geoId/26 -Minnesota,geoId/27 -Mississippi,geoId/28 -Missouri,geoId/29 -Montana,geoId/30 -Nebraska,geoId/31 -Nevada,geoId/32 -New Hampshire,geoId/33 -New Jersey,geoId/34 -New Mexico,geoId/35 -New York,geoId/36 -North Carolina,geoId/37 -North Dakota,geoId/38 -Ohio,geoId/39 -Oklahoma,geoId/40 -Oregon,geoId/41 -Pennsylvania,geoId/42 -Rhode Island,geoId/44 -South Carolina,geoId/45 -South Dakota,geoId/46 -Tennessee,geoId/47 -Texas,geoId/48 -Utah,geoId/49 -Vermont,geoId/50 -Virginia,geoId/51 -Washington,geoId/53 -West Virginia,geoId/54 -Wisconsin,geoId/55 -Wyoming,geoId/56 -AS,geoId/60 -Guam,geoId/66 -MP,geoId/69 -Puerto Rico,geoId/72 -UM,geoId/74 -U.S. Virgin Islands,geoId/78 -Federal (FLSA),country/USA \ No newline at end of file diff --git a/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv b/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv index e5bd0b7702..a295db9984 100644 --- a/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv +++ b/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv @@ -1,16 +1,15 @@ -parameter,value -url,https://www.dol.gov/agencies/whd/state/minimum-wage/history -description,Minimum wages in USA country -#place_type,State -#places_within,country/USA -start_date,1968 -end_date,2023 -release_frequency,Yearly -process, -comments, -output_columns,"observationAbout,observationDate,variableMeasured,value,unit,scalingFactor" -places_resolved_csv,statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv -drop_statvars_without_svobs,0 -header_rows,1 -mapped_rows,1 -mapped_columns,1 \ No newline at end of file +parameter,value +url,https://www.dol.gov/agencies/whd/state/minimum-wage/history +description,Minimum wages in USA country +#place_type,State +#places_within,country/USA +start_date,1968 +end_date,2024 +release_frequency,Yearly +process, +comments, +output_columns,"observationAbout,observationDate,variableMeasured,value,unit,scalingFactor" +drop_statvars_without_svobs,0 +header_rows,1 +mapped_rows,1 +mapped_columns,1 diff --git a/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv b/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv index 3a16d9e85d..cd9a2a677f 100644 --- a/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv +++ b/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv @@ -1,40 +1,65 @@ -key,p1,v1,p2,v2,p3,v3,p4,v4 -state,observationAbout,{Data},populationType,Person,measuredProperty,minimumWage,unit,USDollarPerHour -1968 (a),observationDate,1968,value,{Number},,,, -1970 (a),observationDate,1970,value,{Number},,,, -1972,observationDate,1972,value,{Number},,,, -1976 (a),observationDate,1976,value,{Number},,,, -1979,observationDate,1979,value,{Number},,,, -1980,observationDate,1980,value,{Number},,,, -1981,observationDate,1981,value,{Number},,,, -1988,observationDate,1988,value,{Number},,,, -1991,observationDate,1991,value,{Number},,,, -1992,observationDate,1992,value,{Number},,,, -1994,observationDate,1994,value,{Number},,,, -1996,observationDate,1996,value,{Number},,,, -1997,observationDate,1997,value,{Number},,,, -1998,observationDate,1998,value,{Number},,,, -2000,observationDate,2000,value,{Number},,,, -2001,observationDate,2001,value,{Number},,,, -2002,observationDate,2002,value,{Number},,,, -2003,observationDate,2003,value,{Number},,,, -2004,observationDate,2004,value,{Number},,,, -2005,observationDate,2005,value,{Number},,,, -2006,observationDate,2006,value,{Number},,,, -2007,observationDate,2007,value,{Number},,,, -2008,observationDate,2008,value,{Number},,,, -2009,observationDate,2009,value,{Number},,,, -2010,observationDate,2010,value,{Number},,,, -2011,observationDate,2011,value,{Number},,,, -2012,observationDate,2012,value,{Number},,,, -2013,observationDate,2013,value,{Number},,,, -2014,observationDate,2014,value,{Number},,,, -2015,observationDate,2015,value,{Number},,,, -2016,observationDate,2016,value,{Number},,,, -2017,observationDate,2017,value,{Number},,,, -2018,observationDate,2018,value,{Number},,,, -2019,observationDate,2019,value,{Number},,,, -2020,observationDate,2020,value,{Number},,,, -2021,observationDate,2021,value,{Number},,,, -2022,observationDate,2022,value,{Number},,,, -2023,observationDate,2023,value,{Number},,,, \ No newline at end of file +key,p1,v1,p2,v2,p3,v3,p4,v4 +State or otherjurisdiction,observationDate,{Number},populationType,Person,measuredProperty,minimumWage,unit,USDollarPerHour +,,,,,,,, +Alabama,observationAbout,geoId/01,value,{Number},,,, +Alaska,observationAbout,geoId/02,value,{Number},,,, +Arizona,observationAbout,geoId/04,value,{Number},,,, +Arkansas,observationAbout,geoId/05,value,{Number},,,, +California,observationAbout,geoId/06,value,{Number},,,, +Colorado,observationAbout,geoId/08,value,{Number},,,, +Connecticut,observationAbout,geoId/09,value,{Number},,,, +Delaware,observationAbout,geoId/10,value,{Number},,,, +District of Columbia,observationAbout,geoId/11,value,{Number},,,, +Florida,observationAbout,geoId/12,value,{Number},,,, +Georgia,observationAbout,geoId/13,value,{Number},,,, +Hawaii,observationAbout,geoId/15,value,{Number},,,, +Idaho,observationAbout,geoId/16,value,{Number},,,, +Illinois,observationAbout,geoId/17,value,{Number},,,, +Indiana,observationAbout,geoId/18,value,{Number},,,, +Iowa,observationAbout,geoId/19,value,{Number},,,, +Kansas,observationAbout,geoId/20,value,{Number},,,, +Kentucky,observationAbout,geoId/21,value,{Number},,,, +Louisiana,observationAbout,geoId/22,value,{Number},,,, +Maine,observationAbout,geoId/23,value,{Number},,,, +Maryland,observationAbout,geoId/24,value,{Number},,,, +Massachusetts,observationAbout,geoId/25,value,{Number},,,, +Michigan,observationAbout,geoId/26,value,{Number},,,, +Minnesota,observationAbout,geoId/27,value,{Number},,,, +Mississippi,observationAbout,geoId/28,value,{Number},,,, +Missouri,observationAbout,geoId/29,value,{Number},,,, +Montana,observationAbout,geoId/30,value,{Number},,,, +Nebraska,observationAbout,geoId/31,value,{Number},,,, +Nevada,observationAbout,geoId/32,value,{Number},,,, +New Hampshire,observationAbout,geoId/33,value,{Number},,,, +New Jersey,observationAbout,geoId/34,value,{Number},,,, +New Mexico,observationAbout,geoId/35,value,{Number},,,, +New York,observationAbout,geoId/36,value,{Number},,,, +North Carolina,observationAbout,geoId/37,value,{Number},,,, +North Dakota,observationAbout,geoId/38,value,{Number},,,, +Ohio,observationAbout,geoId/39,value,{Number},,,, +Oklahoma,observationAbout,geoId/40,value,{Number},,,, +Oregon,observationAbout,geoId/41,value,{Number},,,, +Pennsylvania,observationAbout,geoId/42,value,{Number},,,, +Rhode Island,observationAbout,geoId/44,value,{Number},,,, +South Carolina,observationAbout,geoId/45,value,{Number},,,, +South Dakota,observationAbout,geoId/46,value,{Number},,,, +Tennessee,observationAbout,geoId/47,value,{Number},,,, +Texas,observationAbout,geoId/48,value,{Number},,,, +Utah,observationAbout,geoId/49,value,{Number},,,, +Vermont,observationAbout,geoId/50,value,{Number},,,, +Virginia,observationAbout,geoId/51,value,{Number},,,, +Washington,observationAbout,geoId/53,value,{Number},,,, +West Virginia,observationAbout,geoId/54,value,{Number},,,, +Wisconsin,observationAbout,geoId/55,value,{Number},,,, +Wyoming,observationAbout,geoId/56,value,{Number},,,, +AS,observationAbout,geoId/60,value,{Number},,,, +Guam,observationAbout,geoId/66,value,{Number},,,, +MP,observationAbout,geoId/69,value,{Number},,,, +Puerto Rico,observationAbout,geoId/72,value,{Number},,,, +UM,observationAbout,geoId/74,value,{Number},,,, +U.S. Virgin Islands,observationAbout,geoId/78,value,{Number},,,, +Federal (FLSA),observationAbout,country/USA,value,{Number},,,, +,,,,,,,, +1968 (a),observationDate,1968,,,,,, +1970 (a),observationDate,1970,,,,,, +1976 (a),observationDate,1976,,,,,,