Adding industry composition at a census tract level

.gitignore

@@ -23,3 +23,6 @@ tests/.coverage
 .vscode/launch.json
 data/sql/counties_database.db
 data/sql/msa_database.db
+
+
+**/.Rproj.user/** 

cities/utils/cleaning_pipeline.py

@@ -14,6 +14,7 @@
     clean_income_distribution,
 )
 from cities.utils.cleaning_scripts.clean_industry import clean_industry
+from cities.utils.cleaning_scripts.clean_industry_ct import clean_industry_CT
 from cities.utils.cleaning_scripts.clean_industry_ma import clean_industry_ma
 from cities.utils.cleaning_scripts.clean_industry_ts import clean_industry_ts
 from cities.utils.cleaning_scripts.clean_population import clean_population
@@ -31,6 +32,7 @@
 )
 from cities.utils.cleaning_scripts.clean_transport import clean_transport
 from cities.utils.cleaning_scripts.clean_unemployment import clean_unemployment
+from cities.utils.cleaning_scripts.clean_unemployment_ct import clean_unemployment_CT
 from cities.utils.cleaning_scripts.clean_urbanicity_ct import clean_urbanicity_CT
 from cities.utils.cleaning_scripts.clean_urbanicity_ma import clean_urbanicity_ma
 from cities.utils.cleaning_scripts.clean_urbanization import clean_urbanization
@@ -41,6 +43,10 @@
 
 # clean_health() lost of another 15-ish fips
 
+clean_industry_CT()
+
+clean_unemployment_CT()
+
 clean_urbanicity_CT()
 
 clean_population_CT()

cities/utils/cleaning_scripts/clean_industry_ct.py

@@ -0,0 +1,22 @@
+from cities.utils.clean_variable import VariableCleanerCT
+from cities.utils.data_grabber import find_repo_root
+
+root = find_repo_root()
+
+
+def clean_industry_CT():
+    cleaner = VariableCleanerCT(
+        variable_name="industry_pre2020_CT",
+        path_to_raw_csv=f"{root}/data/raw/industry_pre2020_ct.csv",
+        year_or_category_column_label="Category",
+        time_interval="pre2020",
+    )
+    cleaner.clean_variable()
+
+    cleaner2 = VariableCleanerCT(
+        variable_name="industry_post2020_CT",
+        path_to_raw_csv=f"{root}/data/raw/industry_post2020_ct.csv",
+        year_or_category_column_label="Category",
+        time_interval="post2020",
+    )
+    cleaner2.clean_variable()

cities/utils/cleaning_scripts/clean_unemployment_ct.py

@@ -0,0 +1,22 @@
+from cities.utils.clean_variable import VariableCleanerCT
+from cities.utils.data_grabber import find_repo_root
+
+root = find_repo_root()
+
+
+def clean_unemployment_CT():
+    cleaner = VariableCleanerCT(
+        variable_name="unemployment_pre2020_CT",
+        path_to_raw_csv=f"{root}/data/raw/unemployment_pre2020_ct.csv",
+        year_or_category_column_label="Year",
+        time_interval="pre2020",
+    )
+    cleaner.clean_variable()
+
+    cleaner2 = VariableCleanerCT(
+        variable_name="unemployment_post2020_CT",
+        path_to_raw_csv=f"{root}/data/raw/unemployment_post2020_ct.csv",
+        year_or_category_column_label="Year",
+        time_interval="post2020",
+    )
+    cleaner2.clean_variable()

cities/utils/scraping/scrape_industry_ct.py

@@ -0,0 +1,215 @@
+import numpy as np
+import pandas as pd
+import requests
+from us import states
+
+from cities.utils.data_grabber import find_repo_root
+
+root = find_repo_root()
+variables = (
+    "NAME,"
+    "DP03_0004E,"
+    "DP03_0033E,"
+    "DP03_0034E,"
+    "DP03_0035E,"
+    "DP03_0036E,"
+    "DP03_0037E,"
+    "DP03_0038E,"
+    "DP03_0039E,"
+    "DP03_0040E,"
+    "DP03_0041E,"
+    "DP03_0042E,"
+    "DP03_0043E,"
+    "DP03_0044E,"
+    "DP03_0045E"
+)
+
+
+county_fips = "*"  # all counties
+tract = "*"  # all tracts
+api_key = "077d857d6c12d5b9b3aeafa07d2c1916ba12a86c"
+# private api key required to access the data https://api.census.gov/data/key_signup.html
+
+interval = [2019, 2022]
+dfs = []
+
+for year in interval:
+    for x in range(
+        0, len(states.STATES)
+    ):  # in this call it's not possible to use the '*' wildcard to access all states, so we need to iterate over all states
+        fips = states.STATES[x].fips
+
+        url = (
+            f"https://api.census.gov/data/{year}/acs/acs5/profile?"
+            f"get={variables}&for=tract:{tract}&in=state:{fips}&in=county:{county_fips}&key={api_key}"
+        )
+
+        response = requests.get(url)
+
+        assert (
+            response.status_code == 200
+        ), "The data retrieval went wrong"  # 200 means success
+
+        print(f"{fips} fips for year {year} done")
+
+        data = response.json()
+
+        df = pd.DataFrame(data[1:], columns=data[0])
+        df["Year"] = year  # Add the year column
+
+        dfs.append(df)
+
+combined_df = pd.concat(dfs, ignore_index=True)
+
+
+industry = combined_df.copy()
+
+column_name_mapping = {
+    "DP03_0004E": "employed_sum",
+    "DP03_0033E": "agri_forestry_mining",
+    "DP03_0034E": "construction",
+    "DP03_0035E": "manufacturing",
+    "DP03_0036E": "wholesale_trade",
+    "DP03_0037E": "retail_trade",
+    "DP03_0038E": "transport_utilities",
+    "DP03_0039E": "information",
+    "DP03_0040E": "finance_real_estate",
+    "DP03_0041E": "prof_sci_mgmt_admin",
+    "DP03_0042E": "education_health",
+    "DP03_0043E": "arts_entertainment",
+    "DP03_0044E": "other_services",
+    "DP03_0045E": "public_admin",
+}
+
+industry.rename(columns=column_name_mapping, inplace=True)
+
+state_abbreviations = {
+    "Alabama": "AL",
+    "Alaska": "AK",
+    "Arizona": "AZ",
+    "Arkansas": "AR",
+    "California": "CA",
+    "Colorado": "CO",
+    "Connecticut": "CT",
+    "Delaware": "DE",
+    "Florida": "FL",
+    "Georgia": "GA",
+    "Hawaii": "HI",
+    "Idaho": "ID",
+    "Illinois": "IL",
+    "Indiana": "IN",
+    "Iowa": "IA",
+    "Kansas": "KS",
+    "Kentucky": "KY",
+    "Louisiana": "LA",
+    "Maine": "ME",
+    "Maryland": "MD",
+    "Massachusetts": "MA",
+    "Michigan": "MI",
+    "Minnesota": "MN",
+    "Mississippi": "MS",
+    "Missouri": "MO",
+    "Montana": "MT",
+    "Nebraska": "NE",
+    "Nevada": "NV",
+    "New Hampshire": "NH",
+    "New Jersey": "NJ",
+    "New Mexico": "NM",
+    "New York": "NY",
+    "North Carolina": "NC",
+    "North Dakota": "ND",
+    "Ohio": "OH",
+    "Oklahoma": "OK",
+    "Oregon": "OR",
+    "Pennsylvania": "PA",
+    "Rhode Island": "RI",
+    "South Carolina": "SC",
+    "South Dakota": "SD",
+    "Tennessee": "TN",
+    "Texas": "TX",
+    "Utah": "UT",
+    "Vermont": "VT",
+    "Virginia": "VA",
+    "Washington": "WA",
+    "West Virginia": "WV",
+    "Wisconsin": "WI",
+    "Wyoming": "WY",
+}
+
+industry["GeoFIPS"] = industry.apply(
+    lambda row: f"{row['state']}{row['county']}{row['tract']}", axis=1
+).astype(np.int64)
+
+industry.drop(["state", "county", "tract"], axis=1, inplace=True)
+
+
+def parse_geo_name(name):
+    if ";" in name:
+        parts = name.split(";")
+    else:
+        parts = name.split(",")
+
+    if len(parts) >= 3:
+        county = parts[1].strip().replace(" County", "")
+        state_full = parts[2].strip()
+        state_abbr = state_abbreviations.get(state_full, state_full)
+        return f"{county}, {state_abbr} (CT)"
+    return "Unknown"
+
+
+industry["GeoName"] = industry["NAME"].apply(parse_geo_name).astype(str)
+
+assert (
+    industry[industry["GeoName"] == "Unknown"].shape[0] == 0
+), "There are Unknown GeoNames"
+
+industry = industry.drop(["NAME"], axis=1)
+
+
+rows1 = industry.shape[0]
+industry.dropna(how="any", inplace=True)  # Drop NaN values inplace
+rows2 = industry.shape[0]
+print(f"This many rows were removed because of NaNs: {rows1 - rows2}")
+
+
+industry.sort_values(by=["GeoFIPS", "GeoName"], inplace=True)
+
+cols_to_save = industry.shape[1] - 2
+industry = industry[["GeoFIPS", "GeoName"] + list(industry.columns[0:cols_to_save])]
+industry = industry.reset_index(drop=True)
+
+industry_pre2020 = industry[industry["Year"] < 2020]
+industry_post2020 = industry[industry["Year"] >= 2020]
+
+
+industry_list = [industry_pre2020, industry_post2020]
+
+for i in range(len(industry_list)):
+    industry_singl = industry_list[i]
+
+    industry_singl = industry_singl.drop(columns=["Year"])
+
+    columns_to_convert = industry_singl.columns[2:]
+    industry_singl[columns_to_convert] = industry_singl[columns_to_convert].astype(
+        float
+    )
+
+    industry_list[i] = industry_singl.reset_index(drop=True)
+
+
+industry_pre2020, industry_post2020 = industry_list
+
+for i in range(len(industry_list)):
+    industry_singl = industry_list[i]
+
+    row_sums = industry_singl.iloc[:, 3:].sum(axis=1)
+
+    industry_singl.iloc[:, 3:] = industry_singl.iloc[:, 3:].div(row_sums, axis=0)
+    industry_singl = industry_singl.drop(["employed_sum"], axis=1)
+
+    industry_list[i] = industry_singl
+
+industry_pre2020, industry_post2020 = industry_list
+
+industry_pre2020.to_csv(f"{root}/data/raw/industry_pre2020_ct.csv", index=False)
+industry_post2020.to_csv(f"{root}/data/raw/industry_post2020_ct.csv", index=False)