Added download script and modified pvmap

datacommonsorg · Jan 23, 2025 · d0b349c · d0b349c
1 parent 08052bf
commit d0b349c
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 120 deletions.
diff --git a/statvar_imports/USA/DOL_Wages/README.md b/statvar_imports/USA/DOL_Wages/README.md
@@ -2,19 +2,23 @@
 
 - source: https://www.dol.gov/agencies/whd/state/minimum-wage/history, 
 
-- how to download data: Manual download from source.
+- how to download data: Download script (download_script/main.py).
 
 - type of place: Country and State.
 
 - statvars: Economy
 
-- years: 1968 to 2023
+- years: 1968 to 2024
 
-- place_resolution: Resolved state name to state geoId using places resolved csv.
+- place_resolution: Places resolved to geoId in pvmap itself.
 
 ### How to run:
 
-`python3 stat_var_processor.py --input_data=<input_file>.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/<filename>_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --places_resolved_csv=statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv --output_path=--output_path=<filepath/filename>`
+`python3 stat_var_processor.py --input_data=<input_file>.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/<filename>_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --output_path=--output_path=<filepath/filename>`
 
 #### Example
-`python3 stat_var_processor.py --input_data=statvar_imports/USA/DOL_Wages/test_data/sample_input/US_Dol_Wages_data.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv --config=statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv --output_path=--output_path=statvar_imports/USA/DOL_Wages/test_data/sample_output/US_DOL_Wages`
+#### Download : 
+`python3 main.py`
+
+#### Processing
+`python3 stat_var_processor.py --input_data=statvar_imports/USA/DOL_Wages/download_script/input_files/final_data.csv --pv_map=statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv --config=statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv --output_path=--output_path=statvar_imports/USA/DOL_Wages/test_data/sample_output/US_DOL_Wages`
diff --git a/statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv b/statvar_imports/USA/DOL_Wages/USA_State_Places_Resolved.csv
diff --git a/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv b/statvar_imports/USA/DOL_Wages/US_Dol_Wages_metadata.csv
@@ -1,16 +1,15 @@
-parameter,value
-url,https://www.dol.gov/agencies/whd/state/minimum-wage/history
-description,Minimum wages in USA country
-#place_type,State
-#places_within,country/USA
-start_date,1968
-end_date,2023
-release_frequency,Yearly
-process,
-comments,
-output_columns,"observationAbout,observationDate,variableMeasured,value,unit,scalingFactor"
-places_resolved_csv,statvar_imports/USA/DOL_Wages/USA_State_Places_resolved.csv
-drop_statvars_without_svobs,0
-header_rows,1
-mapped_rows,1
-mapped_columns,1
+parameter,value
+url,https://www.dol.gov/agencies/whd/state/minimum-wage/history
+description,Minimum wages in USA country
+#place_type,State
+#places_within,country/USA
+start_date,1968
+end_date,2024
+release_frequency,Yearly
+process,
+comments,
+output_columns,"observationAbout,observationDate,variableMeasured,value,unit,scalingFactor"
+drop_statvars_without_svobs,0
+header_rows,1
+mapped_rows,1
+mapped_columns,1
diff --git a/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv b/statvar_imports/USA/DOL_Wages/pv_map/US_Dol_Wages_pvmap.csv
@@ -1,40 +1,65 @@
-key,p1,v1,p2,v2,p3,v3,p4,v4
-state,observationAbout,{Data},populationType,Person,measuredProperty,minimumWage,unit,USDollarPerHour
-1968 (a),observationDate,1968,value,{Number},,,,
-1970 (a),observationDate,1970,value,{Number},,,,
-1972,observationDate,1972,value,{Number},,,,
-1976 (a),observationDate,1976,value,{Number},,,,
-1979,observationDate,1979,value,{Number},,,,
-1980,observationDate,1980,value,{Number},,,,
-1981,observationDate,1981,value,{Number},,,,
-1988,observationDate,1988,value,{Number},,,,
-1991,observationDate,1991,value,{Number},,,,
-1992,observationDate,1992,value,{Number},,,,
-1994,observationDate,1994,value,{Number},,,,
-1996,observationDate,1996,value,{Number},,,,
-1997,observationDate,1997,value,{Number},,,,
-1998,observationDate,1998,value,{Number},,,,
-2000,observationDate,2000,value,{Number},,,,
-2001,observationDate,2001,value,{Number},,,,
-2002,observationDate,2002,value,{Number},,,,
-2003,observationDate,2003,value,{Number},,,,
-2004,observationDate,2004,value,{Number},,,,
-2005,observationDate,2005,value,{Number},,,,
-2006,observationDate,2006,value,{Number},,,,
-2007,observationDate,2007,value,{Number},,,,
-2008,observationDate,2008,value,{Number},,,,
-2009,observationDate,2009,value,{Number},,,,
-2010,observationDate,2010,value,{Number},,,,
-2011,observationDate,2011,value,{Number},,,,
-2012,observationDate,2012,value,{Number},,,,
-2013,observationDate,2013,value,{Number},,,,
-2014,observationDate,2014,value,{Number},,,,
-2015,observationDate,2015,value,{Number},,,,
-2016,observationDate,2016,value,{Number},,,,
-2017,observationDate,2017,value,{Number},,,,
-2018,observationDate,2018,value,{Number},,,,
-2019,observationDate,2019,value,{Number},,,,
-2020,observationDate,2020,value,{Number},,,,
-2021,observationDate,2021,value,{Number},,,,
-2022,observationDate,2022,value,{Number},,,,
-2023,observationDate,2023,value,{Number},,,,
+key,p1,v1,p2,v2,p3,v3,p4,v4
+State or otherjurisdiction,observationDate,{Number},populationType,Person,measuredProperty,minimumWage,unit,USDollarPerHour
+,,,,,,,,
+Alabama,observationAbout,geoId/01,value,{Number},,,,
+Alaska,observationAbout,geoId/02,value,{Number},,,,
+Arizona,observationAbout,geoId/04,value,{Number},,,,
+Arkansas,observationAbout,geoId/05,value,{Number},,,,
+California,observationAbout,geoId/06,value,{Number},,,,
+Colorado,observationAbout,geoId/08,value,{Number},,,,
+Connecticut,observationAbout,geoId/09,value,{Number},,,,
+Delaware,observationAbout,geoId/10,value,{Number},,,,
+District of Columbia,observationAbout,geoId/11,value,{Number},,,,
+Florida,observationAbout,geoId/12,value,{Number},,,,
+Georgia,observationAbout,geoId/13,value,{Number},,,,
+Hawaii,observationAbout,geoId/15,value,{Number},,,,
+Idaho,observationAbout,geoId/16,value,{Number},,,,
+Illinois,observationAbout,geoId/17,value,{Number},,,,
+Indiana,observationAbout,geoId/18,value,{Number},,,,
+Iowa,observationAbout,geoId/19,value,{Number},,,,
+Kansas,observationAbout,geoId/20,value,{Number},,,,
+Kentucky,observationAbout,geoId/21,value,{Number},,,,
+Louisiana,observationAbout,geoId/22,value,{Number},,,,
+Maine,observationAbout,geoId/23,value,{Number},,,,
+Maryland,observationAbout,geoId/24,value,{Number},,,,
+Massachusetts,observationAbout,geoId/25,value,{Number},,,,
+Michigan,observationAbout,geoId/26,value,{Number},,,,
+Minnesota,observationAbout,geoId/27,value,{Number},,,,
+Mississippi,observationAbout,geoId/28,value,{Number},,,,
+Missouri,observationAbout,geoId/29,value,{Number},,,,
+Montana,observationAbout,geoId/30,value,{Number},,,,
+Nebraska,observationAbout,geoId/31,value,{Number},,,,
+Nevada,observationAbout,geoId/32,value,{Number},,,,
+New Hampshire,observationAbout,geoId/33,value,{Number},,,,
+New Jersey,observationAbout,geoId/34,value,{Number},,,,
+New Mexico,observationAbout,geoId/35,value,{Number},,,,
+New York,observationAbout,geoId/36,value,{Number},,,,
+North Carolina,observationAbout,geoId/37,value,{Number},,,,
+North Dakota,observationAbout,geoId/38,value,{Number},,,,
+Ohio,observationAbout,geoId/39,value,{Number},,,,
+Oklahoma,observationAbout,geoId/40,value,{Number},,,,
+Oregon,observationAbout,geoId/41,value,{Number},,,,
+Pennsylvania,observationAbout,geoId/42,value,{Number},,,,
+Rhode Island,observationAbout,geoId/44,value,{Number},,,,
+South Carolina,observationAbout,geoId/45,value,{Number},,,,
+South Dakota,observationAbout,geoId/46,value,{Number},,,,
+Tennessee,observationAbout,geoId/47,value,{Number},,,,
+Texas,observationAbout,geoId/48,value,{Number},,,,
+Utah,observationAbout,geoId/49,value,{Number},,,,
+Vermont,observationAbout,geoId/50,value,{Number},,,,
+Virginia,observationAbout,geoId/51,value,{Number},,,,
+Washington,observationAbout,geoId/53,value,{Number},,,,
+West Virginia,observationAbout,geoId/54,value,{Number},,,,
+Wisconsin,observationAbout,geoId/55,value,{Number},,,,
+Wyoming,observationAbout,geoId/56,value,{Number},,,,
+AS,observationAbout,geoId/60,value,{Number},,,,
+Guam,observationAbout,geoId/66,value,{Number},,,,
+MP,observationAbout,geoId/69,value,{Number},,,,
+Puerto Rico,observationAbout,geoId/72,value,{Number},,,,
+UM,observationAbout,geoId/74,value,{Number},,,,
+U.S. Virgin Islands,observationAbout,geoId/78,value,{Number},,,,
+Federal (FLSA),observationAbout,country/USA,value,{Number},,,,
+,,,,,,,,
+1968 (a),observationDate,1968,,,,,,
+1970 (a),observationDate,1970,,,,,,
+1976 (a),observationDate,1976,,,,,,